浅谈NLP：从词向量到Transformer| 技术创作特训营第一期

分词是自然语言处理（NLP）中的一个重要步骤，它是将连续的文本序列切分成一系列的独立单元。
在英文中，分词相对简单，因为单词之间通常由空格分隔。但在一些语言中，如中文，文本是连续的，
没有明显的单词分隔符，这就需要分词技术来将句子切分成有意义的词汇单元。

分词的意义主要体现在以下几个方面：

信息检索：在搜索引擎中，用户输入的查询通常是一串连续的文本，分词可以帮助系统理解用户的查询意图，
提高搜索的准确性。

机器翻译：在机器翻译中，分词可以帮助系统更好地理解源语言，提高翻译的准确性。

文本挖掘：在文本挖掘中，分词可以帮助系统提取出文本中的关键信息，如关键词、主题等。
分词根据实现原理和特点，主要分为以下3个类别：
1、基于规则分词算法
该算法是按照一定的策略将待匹配的字符串和一个已建立好的“充分大的”词典中的词进行匹配，若找到某个词条，则说明
匹配成功。常见的基于词典的分词算法分为以下几种：正向最大匹配法、逆向最大匹配法和双向匹配分词法等。
2、基于统计的机器学习算法
这类方法主要是通过统计语料库中词出现的频率来进行分词。常用的是算法是HMM、CRF、SVM。
常见的分词器都是使用机器学习算法和词典相结合，一方面能够提高分词准确率，另一方面能够改善领域适应性。
3、基于深度学习的分词算法
这种方法主要是通过训练神经网络模型来进行分词。例如，双向长短期记忆网络（Bi-LSTM）和变压器（Transformer）
模型就是两种常用的基于深度学习的分词方法。

text='魑'
seqs = tokenizer._tokenize(text, add_dummy_prefix=False)
输出：
seq ['<0xE9>', '<0xAD>', '<0x91>']

wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd],
         initializer=tf.random_normal_initializer(stddev=0.02))

from langchain.vectorstores import Chroma
from langchain.embeddings.huggingface import HuggingFaceEmbeddings
import sentence_transformers
embedding_model_dict = {
    "ernie-tiny": "nghuyong/ernie-3.0-nano-zh",
    "ernie-base": "nghuyong/ernie-3.0-base-zh",
    "text2vec": "GanymedeNil/text2vec-large-chinese",
    "text2vec2": "uer/sbert-base-chinese-nli",
    "text2vec3": "shibing624/text2vec-base-chinese",
}
EMBEDDING_MODEL = "text2vec3"
embeddings = HuggingFaceEmbeddings(model_name=embedding_model_dict[EMBEDDING_MODEL], )
embeddings.client = sentence_transformers.SentenceTransformer(
        embeddings.model_name, device='cuda')
db = Chroma(persist_directory="./data/doc_embeding", embedding_function=embeddings)
print(db)

def get_info_by_zoey(model):
    """ 查看参数个数
    """
    Total_params = 0.0
    Trainable_params = 0.0
    for param in model.parameters():
        mulValue = np.prod(param.size())
        if param.requires_grad:
            Trainable_params += mulValue
    print(f'Trainable params: {Trainable_params}, {Trainable_params / 1e9}B')
    
tokenizer = AutoTokenizer.from_pretrained(r"F:\gpt\chatGLM\ChatGLM-6B\model_file\chatglm-6b-int4", trust_remote_code=True)
model = AutoModel.from_pretrained(r"F:\gpt\chatGLM\ChatGLM-6B\model_file\chatglm-6b-int4", trust_remote_code=True).half().cuda()
get_info(model.transformer)

model = AutoModel.from_pretrained(r"F:\gpt\chatGLM\ChatGLM-6B\model_file\chatglm-6b", trust_remote_code=True).half().cuda()
get_info(model.transformer)

class QuantizedLinear(Linear):
    def __init__(self, weight_bit_width: int, weight_tensor=None, bias_tensor=None, quantized_weight=None,
                 quantized_weight_scale=None, quantization_cache=None, empty_init=False, *args, **kwargs):
        super(QuantizedLinear, self).__init__(*args, **kwargs)
        self.weight_bit_width = weight_bit_width
        self.quantization_cache = quantization_cache

        if (quantized_weight is not None) and (quantized_weight_scale is not None):
            del self.weight
            self.weight = Parameter(quantized_weight.to(kwargs["device"]), requires_grad=False)
            self.weight_scale = Parameter(quantized_weight_scale.to(kwargs["device"]), requires_grad=False)
        else:
            shape = self.weight.shape
            del self.weight
            # 注意：这里逻辑导致int4参数量减少
            if weight_tensor is None or empty_init:
                self.weight = torch.empty(
                    shape[0], shape[1] * weight_bit_width // 8, dtype=torch.int8, device=kwargs["device"]
                )
                self.weight_scale = torch.empty(shape[0], dtype=kwargs["dtype"], device=kwargs["device"])
            else:
                self.weight_scale = (weight_tensor.abs().max(dim=-1).values / ((2 ** (weight_bit_width - 1)) - 1)).to(
                    kwargs["dtype"])
                self.weight = torch.round(weight_tensor / self.weight_scale[:, None]).to(torch.int8)
                if weight_bit_width == 4:
                    self.weight = compress_int4_weight(self.weight)
            print("real shape", self.weight.shape)
            self.weight = Parameter(self.weight.to(kwargs["device"]), requires_grad=False)
            self.weight_scale = Parameter(self.weight_scale.to(kwargs["device"]), requires_grad=False)

{
  "_name_or_path": "THUDM/chatglm-6b-int4",
  "architectures": [
    "ChatGLMModel"
  ],
  "auto_map": {
    "AutoConfig": "configuration_chatglm.ChatGLMConfig",
    "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
    "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
  },
  "bos_token_id": 130004,
  "eos_token_id": 130005,
  "mask_token_id": 130000,
  "gmask_token_id": 130001,
  "pad_token_id": 3,
  "hidden_size": 4096,
  "inner_hidden_size": 16384,
  "layernorm_epsilon": 1e-05,
  "max_sequence_length": 2048,
  "model_type": "chatglm",
  "num_attention_heads": 32,
  "num_layers": 28,
  "position_encoding_2d": true,
  "quantization_bit": 4,
  "quantization_embeddings": false,
  "torch_dtype": "float16",
  "transformers_version": "4.27.1",
  "use_cache": true,
  "vocab_size": 130528
}

class GLMBlock(torch.nn.Module):
    def __init__(
            self,
            hidden_size,
            num_attention_heads,
            layernorm_epsilon,
            layer_id,
            inner_hidden_size=None,
            hidden_size_per_attention_head=None,
            layernorm=LayerNorm,
            use_bias=True,
            params_dtype=torch.float,
            num_layers=28,
            position_encoding_2d=True,
            empty_init=True
    ):
        super(GLMBlock, self).__init__()
        # Set output layer initialization if not provided.

        self.layer_id = layer_id

        # Layernorm on the input data.
        self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)

        self.position_encoding_2d = position_encoding_2d

        # Self attention.
        self.attention = SelfAttention(
            hidden_size,
            num_attention_heads,
            layer_id,
            hidden_size_per_attention_head=hidden_size_per_attention_head,
            bias=use_bias,
            params_dtype=params_dtype,
            position_encoding_2d=self.position_encoding_2d,
            empty_init=empty_init
        )

        # Layernorm on the input data.
        self.post_attention_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)

        self.num_layers = num_layers

        # GLU
        self.mlp = GLU(
            hidden_size,
            inner_hidden_size=inner_hidden_size,
            bias=use_bias,
            layer_id=layer_id,
            params_dtype=params_dtype,
            empty_init=empty_init
        )

    def forward(
            self,
            hidden_states: torch.Tensor,
            position_ids,
            attention_mask: torch.Tensor,
            layer_id,
            layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
            use_cache: bool = False,
            output_attentions: bool = False,
    ):
        """
        hidden_states: [seq_len, batch, hidden_size]
        attention_mask: [(1, 1), seq_len, seq_len]
        """

        # Layer norm at the begining of the transformer layer.
        # [seq_len, batch, hidden_size]
        attention_input = self.input_layernorm(hidden_states)

        # Self attention.
        attention_outputs = self.attention(
            attention_input,
            position_ids,
            attention_mask=attention_mask,
            layer_id=layer_id,
            layer_past=layer_past,
            use_cache=use_cache,
            output_attentions=output_attentions
        )

        attention_output = attention_outputs[0]

        outputs = attention_outputs[1:]

        # Residual connection.
        alpha = (2 * self.num_layers) ** 0.5
        hidden_states = attention_input * alpha + attention_output

        mlp_input = self.post_attention_layernorm(hidden_states)

        # MLP.
        mlp_output = self.mlp(mlp_input)

        # Second residual connection.
        output = mlp_input * alpha + mlp_output

        if use_cache:
            outputs = (output,) + outputs
        else:
            outputs = (output,) + outputs[1:]

        return outputs  # hidden_states, present, attentions