智能知识图谱：大模型如何实现高效实体识别与关系抽取

from transformers import BertTokenizer, BertForTokenClassification
import torch

# 加载预训练的BERT模型和分词器
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForTokenClassification.from_pretrained('bert-base-uncased', num_labels=9)

# 输入文本
text = "Apple is a technology company headquartered in Cupertino, California."

# 分词并转换为模型输入
inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True)
tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])

# 模型预测
outputs = model(**inputs).logits
predictions = torch.argmax(outputs, dim=2)

# 输出实体识别结果
entities = []
for token, prediction in zip(tokens, predictions[0]):
    if prediction != 0:  # 过滤非实体
        entities.append((token, prediction.item()))
print("识别到的实体：", entities)

+-------------------+       +-------------------+       +-------------------+
|   输入文本         |       |   文本预处理       |       |   模型输入         |
|                   | ----> |                   | ----> |                   |
| "Apple is a       |       | 分词、去停用词等   |       | 将文本编码为模型    |
| technology company|       |                   |       | 输入格式           |
| headquartered in  |       |                   |       |                   |
| Cupertino,        |       |                   |       |                   |
| California."      |       |                   |       |                   |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+       +-------------------+       +-------------------+
|   实体识别模型     |       |   实体标注         |       |   实体提取         |
|                   | ----> |                   | ----> |                   |
| 使用BERT等模型进行 |       | 输出每个词的实体   |       | 提取并过滤实体     |
| 实体标注           |       | 标签              |       |                   |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+
|   输出实体列表     |
|                   |
| ORG: Apple        |
| LOC: Cupertino,   |
| California        |
+-------------------+

from transformers import GPT2Tokenizer, GPT2LMHeadModel

# 加载预训练的GPT模型和分词器
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# 输入文本
text = "Steve Jobs founded Apple in 1976."

# 生成关系抽取提示
prompt = f"Extract relations from the text: {text}\nRelations:"

# 模型生成
inputs = tokenizer(prompt, return_tensors="pt", truncation=True, padding=True)
outputs = model.generate(**inputs, max_length=50)

# 解码输出
relations = tokenizer.decode(outputs[0], skip_special_tokens=True)
print("抽取到的关系：", relations)

+-------------------+       +-------------------+       +-------------------+
|   输入文本         |       |   文本预处理       |       |   实体识别         |
|                   | ----> |                   | ----> |                   |
| "Steve Jobs founded|       | 分词、去停用词等   |       | 识别实体：Steve Jobs|
| Apple in 1976."    |       |                   |       | 和 Apple           |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+       +-------------------+       +-------------------+
|   关系抽取模型     |       |   关系分类         |       |   输出关系三元组   |
|                   | ----> |                   | ----> |                   |
| 生成候选关系       |       | 分类关系：founded  |       | (Steve Jobs,       |
|                   |       |                   |       |  founded, Apple)   |
+-------------------+       +-------------------+       +-------------------+

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans

# 假设已有知识图谱数据
existing_entities = ["Apple", "Steve Jobs", "Cupertino"]
new_entities = ["Tim Cook", "iPhone", "California"]

# 合并新旧实体
all_entities = existing_entities + new_entities

# 使用TF-IDF向量化
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(all_entities)

# 聚类以发现新关系
kmeans = KMeans(n_clusters=2)
kmeans.fit(X)

# 输出聚类结果
print("实体聚类结果：", kmeans.labels_)

+-------------------+       +-------------------+       +-------------------+
|   输入新数据       |       |   数据预处理       |       |   实体识别         |
|                   | ----> |                   | ----> |                   |
| "Tim Cook is the  |       | 清洗、分词、去停用词|       | 识别实体：Tim Cook |
| CEO of Apple."    |       | 等操作             |       | 和 Apple           |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+       +-------------------+       +-------------------+
|   关系抽取         |       |   知识图谱查询     |       |   冲突检测与消解   |
|                   | ----> |                   | ----> |                   |
| 抽取关系：         |       | 检查新实体和关系   |       | 检测冲突并消解     |
| (Tim Cook, CEO of, |       | 是否已存在         |       |                   |
| Apple)            |       |                   |       |                   |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+       +-------------------+
|   图谱更新         |       |   输出更新后的知识图谱 |
|                   | ----> |                   |
| 添加新实体和关系   |       | 包含新增的实体和关系 |
+-------------------+       +-------------------+