用深度学习解决nlp中的命名实体识别(ner)问题(深度学习入门项目)

6       O
月      O
油      O
印      O
的      O
《      O
北      B-LOC
京      I-LOC
文      O
物      O
保      O
存      O
保      O
管      O
状      O
态      O
之      O
调      O
查      O
报      O
告      O
》      O
，      O

调      O
查      O
范      O
围      O
涉      O
及      O
故      B-LOC
宫      I-LOC
、      O
历      B-LOC
博      I-LOC
、      O
古      B-ORG
研      I-ORG
所      I-ORG
、      O
北      B-LOC
大      I-LOC
清      I-LOC
华      I-LOC
图      I-LOC
书      I-LOC
馆      I-LOC
、      O
北      B-LOC
图      I-LOC
、      O
日      B-LOC
伪      O
资      O
料      O

# 对传入目录下的训练和测试文件进行 char 级别的编码，以及加载已有的编码文件，
# 只有在更换训练文件之后才需要 gen, 其他时间直接 load 即可。
class Word2Id:
    def __init__(self, file):
        self.file = file

    def gen_save(self):
        data_file = [args.train_data, args.test_data]
        all_char = []
        for f in data_file:
            file = open(f, "rb")
            data = file.read().decode("utf-8")
            data = data.split("\n\n")
            data = [token.split("\n") for token in data]
            data = [[j.split() for j in i] for i in data]
            data.pop()
            all_char.extend([char[0] if char else 'unk' for sen in data for char in sen])
        chars = set(all_char)
        word2id = {char: id_ + 1 for id_, char in enumerate(chars)}
        word2id["unk"] = 0
        with open(self.file, "wb") as f:
            f.write(json.dumps(word2id, ensure_ascii=False).encode('utf-8'))

    def load(self):
        return json.load(open(self.file, 'r'))

# BILSTM-CRF 模型
class Ner:
    def __init__(self, vocab, labels_category, Embedding_dim=200):
        self.Embedding_dim = Embedding_dim
        self.vocab = vocab
        self.labels_category = labels_category
        self.model = self.build_model()

    # 构建模型
    def build_model(self):
        model = Sequential()
        # embedding 层
        model.add(Embedding(len(self.vocab), self.Embedding_dim, mask_zero=True))  # Random embedding
        # bilstm 层
        model.add(Bidirectional(LSTM(100, return_sequences=True)))
        # crf 层
        crf = CRF(len(self.labels_category), sparse_target=True)
        model.add(crf)
        model.summary()
        model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
        return model

    # 训练方法
    def train(self, data, label, EPOCHS):
        self.model.fit(data, label, batch_size=args.batch_size, callbacks=[CallBack()], epochs=EPOCHS)

    # 加载已有的模型进行训练
    def retrain(self, model_path, data, label, epoch):
        model = self.load_model_fromfile(model_path)
        print("load model, evaluate it.")
        loss, accuracy = model.evaluate(data, label)
        print("load model, loss = %s, acc =%s ." % (loss, accuracy))
        model.fit(data, label, batch_size=124, callbacks=[CallBack()], epochs=epoch)

    # 从给定的目录加载一个模型
    def load_model_fromfile(self, model_path):
        crf = CRF(len(self.labels_category), sparse_target=True)
        return load_model(model_path, custom_objects={"CRF": CRF, 'crf_loss': crf.loss_function,
                                                      'crf_viterbi_accuracy': crf.accuracy})
    
    # 预测，主要用于交互式的测试某些样本的预测结果。我个人习惯在训练完成之后手动测试一些常见的 case,
    def predict(self, model_path, data, maxlen):
        model = self.model
        char2id = [self.vocab.get(i) for i in data]
        input_data = pad_sequences([char2id], maxlen)
        model.load_weights(model_path)
        result = model.predict(input_data)[0][-len(data):]
        result_label = [np.argmax(i) for i in result]
        return result_label

    # 测试，可以用某个测试集跑一下模型，看看效果
    def test(self, model_path, data, label):
        model = self.load_model_fromfile(model_path)
        loss, acc = model.evaluate(data, label)
        return loss, acc

# 处理数据集
class DataSet:
    def __init__(self, data_path, labels):
        with open(data_path, "rb") as f:
            self.data = f.read().decode("utf-8")
        self.process_data = self.process_data()
        self.labels = labels

    def process_data(self):
        # 读取样本并分割
        train_data = self.data.split("\n\n")
        train_data = [token.split("\n") for token in train_data]
        train_data = [[j.split() for j in i] for i in train_data]
        train_data.pop()
        return train_data

    def generate_data(self, vocab, maxlen):
        char_data_sen = [[token[0] for token in i] for i in self.process_data]
        label_sen = [[token[1] for token in i] for i in self.process_data]
        # 对样本进行编码
        sen2id = [[vocab.get(char, 0) for char in sen] for sen in char_data_sen]
        # 对样本中的标签进行编码
        label2id = {label: id_ for id_, label in enumerate(self.labels)}
        lab_sen2id = [[label2id.get(lab, 0) for lab in sen] for sen in label_sen]
        # padding
        sen_pad = pad_sequences(sen2id, maxlen)
        lab_pad = pad_sequences(lab_sen2id, maxlen, value=-1)
        lab_pad = np.expand_dims(lab_pad, 2)
        return sen_pad, lab_pad

    def export_saved_model(self, saved_dir, epoch):
        model_version = epoch
        model_signature = tf.saved_model.signature_def_utils.predict_signature_def(
            inputs={'input': self.model.input}, outputs={'output': self.model.output})
        export_path = os.path.join(compat.as_bytes(saved_dir), compat.as_bytes(str(model_version)))
        builder = tf.saved_model.builder.SavedModelBuilder(export_path)
        builder.add_meta_graph_and_variables(
            sess=K.get_session(),
            tags=[tf.saved_model.tag_constants.SERVING],
            clear_devices=True,
            signature_def_map={
                tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
                    model_signature
            })
        builder.save()

cmd="./tensorflow_model_server \
        --port=4590 \
        --rest_api_port=4591  \
        --model_config_file=model/ \
        --tensorflow_session_parallelism=40 \
        --per_process_gpu_memory_fraction=0.2"

[
    [0,1,0,0,0,0,0]
    [0,0,1,0,0,0,0]
]

    public static void main(String[] args) {
        // 构造请求
        ManagedChannel channel = ManagedChannelBuilder.forAddress("192.168.1.251", 7010).usePlaintext(true).build();
        PredictionServiceGrpc.PredictionServiceBlockingStub stub = PredictionServiceGrpc.newBlockingStub(channel);
        Predict.PredictRequest.Builder predictRequestBuilder = Predict.PredictRequest.newBuilder();
        Model.ModelSpec.Builder modelSpecBuilder = Model.ModelSpec.newBuilder();
        // 你的模型的名字
        modelSpecBuilder.setName("model");
        modelSpecBuilder.setSignatureName("");
        predictRequestBuilder.setModelSpec(modelSpecBuilder);
        TensorProto.Builder tensorProtoBuilder = TensorProto.newBuilder();
        // 模型接受的数据类型
        tensorProtoBuilder.setDtype(DataType.DT_FLOAT);
        TensorShapeProto.Builder tensorShapeBuilder = TensorShapeProto.newBuilder();
        // 接受数据的 shape, 几维的数组，每一维多少个。我的测试数据是三个。
        tensorShapeBuilder.addDim(TensorShapeProto.Dim.newBuilder().setSize(1));
        tensorShapeBuilder.addDim(TensorShapeProto.Dim.newBuilder().setSize(3));

        // 我的测试数据，这里需要把输入的字符串进行编码。比如在我的编码下，比如将 : 呼延十 编码成下面三个数字。
        String s = "呼延十";
        List<Float> ret = new ArrayList<>();
        ret.add(348.0f);
        ret.add(3848.0f);
        ret.add(2557.0f);

        tensorProtoBuilder.setTensorShape(tensorShapeBuilder.build());
        tensorProtoBuilder.addAllFloatVal(ret);
        predictRequestBuilder.putInputs("input", tensorProtoBuilder.build());
        Predict.PredictResponse predictResponse = stub.predict(predictRequestBuilder.build());

        //  这里拿到的是一个 (1,1,3) 的矩阵。所以我们需要把他解码成我们想要的 tag. 涉及到你的 tag 列表。
        List<Float> output = predictResponse.getOutputsMap().get("output").getFloatValList();
        List<String> tags = Arrays.asList("O", "B-PER", "I-PER", "B-LOC", "I-LOC", "B-ORG", "i-ORG");
        List<String> rets = phraseFrom(s, output, tags);
        System.out.println(rets);

    }

    private static List<String> phraseFrom(String q, List<Float> output, List<String> tags) {
        List<List<Float>> partition = Lists.partition(output, tags.size());
        List<Integer> idx = new ArrayList<>();
        for (List<Float> floats : partition) {
            for (int j = 0; j < floats.size(); j++) {
                if (floats.get(j) == 1.0f) {
                    idx.add(j);
                    break;
                }
            }
        }
        assert q.length() != idx.size();
        // 从 query 和每个字的 tag 解析成词语的意图。
        StringBuilder sb = new StringBuilder();
        char[] chars = q.toCharArray();
        List<String> rets = new ArrayList<>();
        for (int i = 0; i < chars.length; i++) {
            Integer tag = idx.get(i);
            if ((tag & 1) == 1 && sb.length() != 0) {
                String item = sb.toString();
                String ret = tags.get(idx.get(i - 1));
                rets.add(ret);
                sb.setLength(0);
                sb.append(chars[i]);
            } else {
                sb.append(chars[i]);
            }
        }
        if (sb.length() != 0) {
            String ret = tags.get(idx.get(q.length() - 1));
            rets.add(ret);
        }
        return rets;
    }