NLP 自然语言处理的发展历程

# 代码示例：使用深度学习进行文本分类
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, Flatten, Dense

# 假设有一个包含文本和标签的数据集
texts = ["This is a positive example.", "Another example with negative sentiment.", ...]
labels = [1, 0, ...]  # 1 for positive, 0 for negative

# 文本预处理
tokenizer = Tokenizer()
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
data = pad_sequences(sequences)

# 构建深度学习模型
model = Sequential()
model.add(Embedding(input_dim=len(word_index) + 1, output_dim=100, input_length=data.shape[1]))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# 模型训练
model.fit(data, labels, epochs=10, validation_split=0.2)

# 代码示例：迁移学习在NLP中的应用
import tensorflow as tf
from tensorflow.keras.layers import Embedding, LSTM, Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import train_test_split

# 假设有一个包含大量通用文本数据的预训练模型
pretrained_model = tf.keras.applications.MobileNetV2(weights='imagenet', include_top=False)

# 准备任务特定的文本数据
texts_task_specific = ["Task-specific text data 1.", "Task-specific text data 2.", ...]
labels_task_specific = [1, 0, ...]

# 划分训练集和测试集
train_texts, test_texts, train_labels, test_labels = train_test_split(texts_task_specific, labels_task_specific, test_size=0.2, random_state=42)


# 构建迁移学习模型
model = Sequential()
model.add(pretrained_model)
model.add(LSTM(64))
model.add(Dense(1, activation='sigmoid'))

# 编译模型
model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])

# 模型训练
model.fit(train_texts, train_labels, epochs=5, validation_data=(test_texts, test_labels))

# 代码示例：注意力机制在NLP中的应用
import tensorflow as tf
from tensorflow.keras.layers import Input, Embedding, LSTM, Dense, Attention
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam

# 构建带有注意力机制的NLP模型
input_layer = Input(shape=(max_sequence_length,))
embedding_layer = Embedding(input_dim=vocab_size, output_dim=embedding_dim)(input_layer)
lstm_layer = LSTM(units=64, return_sequences=True)(embedding_layer)
attention_layer = Attention()([lstm_layer, lstm_layer])
attended_lstm = tf.keras.layers.Add()([lstm_layer, attention_layer])
output_layer = Dense(num_classes, activation='softmax')(attended_lstm)

# 构建模型
model_with_attention = Model(inputs=input_layer, outputs=output_layer)

# 编译模型
model_with_attention.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])

# 打印模型结构
model_with_attention.summary()

# 代码示例：命名实体识别任务的数据处理
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize

# 假设有一个包含文本和对应实体标注的数据集
corpus = ["Steve Jobs was the co-founder of Apple.", "Apple Inc. is headquartered in Cupertino.", ...]
entity_labels = [["B-PER", "I-PER", "O", "O", "B-ORG", "O", "B-LOC", "O"],
                 ["B-ORG", "I-ORG", "O", "O", "O", "O", "B-LOC", "O", "O", "O"]]

# 分句和分词
tokenized_corpus = [word_tokenize(sent) for sent in sent_tokenize(corpus)]

# 将实体标签转换为数值形式
tag_to_index = {"O": 0, "B-PER": 1, "I-PER": 2, "B-ORG": 3, "I-ORG": 4, "B-LOC": 5, "I-LOC": 6}
indexed_entity_labels = [[tag_to_index[tag] for tag in sent] for sent in entity_labels]

# 打印处理后的数据
print("Tokenized Corpus:", tokenized_corpus)
print("Indexed Entity Labels:", indexed_entity_labels)