【建议收藏】|信息抽取与经济学文本量化分析

{"postag": [{"word": "查尔斯", "pos": "nr"}, {"word": "·", "pos": "w"},
{"word": "阿兰基斯", "pos": "nr"}, {"word": "（", "pos": "w"}, {"word": "Charles Aránguiz", "pos": "nz"},
{"word": "）", "pos": "w"}, {"word": "，", "pos": "w"}, {"word": "1989年4月17日", "pos": "t"},
{"word": "出生", "pos": "v"}, {"word": "于", "pos": "p"}, {"word": "智利圣地亚哥", "pos": "ns"},
{"word": "，", "pos": "w"}, {"word": "智利", "pos": "ns"}, {"word": "职业", "pos": "n"},
{"word": "足球", "pos": "n"}, {"word": "运动员", "pos": "n"}, {"word": "，", "pos": "w"}, {"word": "司职", "pos": "v"},
{"word": "中场", "pos": "n"}, {"word": "，", "pos": "w"}, {"word": "效力", "pos": "v"}, {"word": "于", "pos": "p"},
{"word": "德国", "pos": "ns"}, {"word": "足球", "pos": "n"}, {"word": "甲级", "pos": "a"}, {"word": "联赛", "pos": "n"},
{"word": "勒沃库森足球俱乐部", "pos": "nt"}],
"text": "查尔斯·阿兰基斯（Charles Aránguiz），1989年4月17日出生于智利圣地亚哥，智利职业足球运动员，司职中场，效力于德国足球甲级联赛勒沃库森足球俱乐部",
"spo_list": [{"predicate": "出生地", "object_type": "地点", "subject_type": "人物", "object": "圣地亚哥", "subject": "查尔斯·阿兰基斯"},
{"predicate": "出生日期", "object_type": "Date", "subject_type": "人物", "object": "1989年4月17日", "subject": "查尔斯·阿兰基斯"}]}

{"id": 865,
"text": "自2012年二季度开始,整个家禽养殖业就进入下行亏损通道,随后2012年末爆发的“速成鸡事件”与2013年的“H7N9”等不可抗力因素导致了家禽业进入深度亏损状态,在2013年上半年同期,“圣农发展”的净利润为亏损2.33亿元",
"relations": [{"id": 2472, "from_id": 3652, "to_id": 3654, "type": "Influence"},
              {"id": 2473, "from_id": 3653, "to_id": 3654, "type": "Influence"}],
"entities": [{"id": 3652, "start_offset": 40, "end_offset": 47, "label": "event"},
             {"id": 3653, "start_offset": 54, "end_offset": 60, "label": "event"},
             {"id": 3654, "start_offset": 70, "end_offset": 81, "label": "event"}]}

import importlib
import os
# encoding: utf-8
import sys

# pip uninstall pdfminer.six
importlib.reload(sys)
from pdfminer.pdfparser import PDFParser, PDFDocument
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LTTextBoxHorizontal, LAParams

import ray

ray.init(num_cpus=os.cpu_count()*2)


@ray.remote
def parse(path, out_path):
    if os.path.exists(out_path + ".txt"):
        return "ok"
    fp = open(path, 'rb')
    # 用文件对象来创建一个pdf文档分析器PDFParser
    praser = PDFParser(fp)
    # 创建一个PDF文档PDFDocument
    doc = PDFDocument()
    # 连接分析器 与文档对象
    praser.set_document(doc)
    doc.set_parser(praser)

    # 提供初始化密码,如果没有密码 就创建一个空的字符串
    doc.initialize()

    # 检测文档是否提供txt转换，不提供就忽略
    if not doc.is_extractable:
        # raise PDFTextExtractionNotAllowed
        return "ok"
    else:
        # 创建PDf 资源管理器 来管理共享资源PDFResourceManager
        rsrcmgr = PDFResourceManager()
        # 创建一个PDF设备对象LAParams
        laparams = LAParams()
        # 创建聚合器,用于读取文档的对象PDFPageAggregator
        device = PDFPageAggregator(rsrcmgr, laparams=laparams)
        # 创建一个PDF解释器对象,对文档编码，解释成Python能够识别的格式：PDFPageInterpreter
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        try:
            # 循环遍历列表，每次处理一个page的内容
            for page in doc.get_pages():  # doc.get_pages() 获取page列表
                # 利用解释器的process_page()方法解析读取单独页数
                try:
                    interpreter.process_page(page)
                except:
                    continue
                # 这里layout是一个LTPage对象,里面存放着这个page解析出的各种对象,一般包括LTTextBox, LTFigure, LTImage, LTTextBoxHorizontal等等,想要获取文本就获得对象的text属性，
                # 使用聚合器get_result()方法获取页面内容
                layout = device.get_result()
                for x in layout:
                    if (isinstance(x, LTTextBoxHorizontal)):
                        # 需要写出编码格式
                        with open(out_path + ".txt", 'a',
                                  encoding='utf-8') as f:
                            results = x.get_text()
                            f.write(results + '\n')
            return "ok"
        except:
            return "ok"


if __name__ == '__main__':
    base_path = "../../上市公司年报"
    out_path = "../../公司年报txt"
    first_path = os.listdir(os.path.join(base_path))

    futures = [parse.remote(os.path.join(base_path, first_path),
                            os.path.join(out_path, first_path)) for first_path in first_path]
    print(ray.get(futures))  # [0, 1, 4, 9]

import os

base_path = "./2021年报_text_dir"
base_path_list = os.listdir(base_path)
word_list = []
for base_path_one in base_path_list:
    try:
        base_path_two = os.listdir(os.path.join(base_path,base_path_one))

        base_path_data = open(os.path.join(base_path,base_path_one,base_path_two[0]),"r").read().replace("\n", " ")
    except:
        continue
    words = ""
    for i in base_path_data.split("。"):
        if "年年度报告" in i:
            continue
        if len(i) < 10:
            continue
        if len(words + i) < 128:
            words+=i.replace(" ","")+"。"
        if len(words+i) > 128:
            if len(words):
                word_list.append(words)
            words = ""

def normalize(text):
    """简单的文本格式化函数
    """
    return ' '.join(text.split())


def load_data(filename):
    """加载数据
    单条格式：{'text': text, 'spo_list': [(s, p, o)]}
    """
    D = []
    with open(filename, encoding='utf-8') as f:
        for l in f:
            l = json.loads(l)
            D.append({
                'text': normalize(l['text']),
                'spoes': [(
                    normalize(spo['subject']), spo['predicate'],
                    normalize(spo['object'])
                ) for spo in l['spo_list']]
            })
    return D


# 加载数据集
train_data = load_data('../小说人物关系抽取/train_data.json')
valid_data = load_data('../小说人物关系抽取/dev_data.json')
predicate2id, id2predicate = {}, {}

with open('../小说人物关系抽取/all_50_schemas') as f:
    for l in f:
        l = json.loads(l)
        if l['predicate'] not in predicate2id:
            id2predicate[len(predicate2id)] = l['predicate']
            predicate2id[l['predicate']] = len(predicate2id)

def load_data(filename):
    """加载数据
    单条格式：{'text': text, 'spo_list': [(s, p, o)]}
    """
    D = []
    id2predicate = {}
    predicate2id = {}
    with open(filename, encoding='utf-8') as f:
        for l in f:
            l = json.loads(l)
            entities_mapping = {}
            for i in l["entities"]:
                entities_mapping[i["id"]]=l['text'][i["start_offset"]:i["end_offset"]]
            D.append({
                'text': l['text'],
                'spo_list': [(entities_mapping[spo['from_id']], spo['type'], entities_mapping[spo['to_id']])
                             for spo in l['relations']]
            })
            for spo in l["relations"]:
                if spo['type'] not in predicate2id:
                    id2predicate[len(predicate2id)] = spo['type']
                    predicate2id[spo['type']] = len(predicate2id)

    return D,id2predicate,predicate2id


# 加载数据集
all_data,id2predicate,predicate2id = load_data('untitled.txt')
train_data = all_data[:int(len(all_data)*0.8)]
valid_data = all_data[int(len(all_data)*0.8):]

model.load_weights('best_model.weights')
def evaluate_data(data):
    """评估函数，计算f1、precision、recall
    """
    X, Y, Z = 1e-10, 1e-10, 1e-10
    f = open('dev_pred.jsonl', 'w', encoding='utf-8')
    pbar = tqdm()
    for d in data:
        R = set([SPO(spo) for spo in extract_spoes(d)])
        if len(R) < 1:
            continue
        T = set()
        X += len(R & T)
        Y += len(R)
        Z += len(T)
        f1, precision, recall = 2 * X / (Y + Z), X / Y, X / Z
        pbar.update()
        pbar.set_description(
            'f1: %.5f, precision: %.5f, recall: %.5f' % (f1, precision, recall)
        )
        s = json.dumps({
            'text': d,
            'spo_list': list(R),
        },
                       ensure_ascii=False)
        f.write(s + '\n')
    pbar.close()
    f.close()
    return f1, precision, recall
evaluate_data(word_list_10000)

{"text": "公司与员工签订劳动合同，规范劳动关系管理工作。公司制定了合理的薪酬管理制度，对公司 员工薪酬设计和管理原则、福利与年假、定级和调薪等方面进行了明确规定。公司高度重视对员工的培训 工作，提升员工素质，实现员工与企业共同成长。",
 "spo_list": [["提升员工素质", "Influence", "实现员工与企业共同成长"]]}

import json
dev_pred = open("dev_pred_50000.jsonl").readlines()
all_data = []
for dev_pred_one in dev_pred:
    all_data.append(json.loads(dev_pred_one))

import networkx as nx
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['font.sans-serif'] = ['FZSongYi-Z13S'] # 指定默认字体
from matplotlib import font_manager

# fname中选择一个你本机查询出来的字体 若没有中文字体则需要你本人手动安装

font = font_manager.FontProperties(fname="./simhei.ttf")
plt.figure(figsize=(30, 30))
G = nx.DiGraph()
all_nodes = []
all_relation = []
for one_data in all_data:
    for spo_one in one_data["spo_list"]:
        all_nodes.append(spo_one[0].replace(" ",""))
        all_nodes.append(spo_one[2].replace(" ",""))
        all_relation.append((spo_one[0].replace(" ",""),spo_one[2].replace(" ","")))
all_set_nodes = list(set(all_nodes))
# G.add_nodes_from(all_set_nodes[:200])   # 添加节点 1 2 3

G.add_edges_from(all_relation)   # 添加多条边

pagerank_list = nx.pagerank(G, alpha=1)

a = sorted(pagerank_list.items(),key = lambda x:x[1],reverse = True)
a[:30]

[('对公司的经营业绩产生不利影响', 0.0008828635052164171),
 ('增加公司的经营风险', 0.000840754589394436),
 ('对公司经营业绩产生不利影响', 0.000629184609583069),
 ('提高生产效率', 0.0005525447685727118),
 ('公司的盈利能力', 0.0005452975937731509),
 ('降低生产成本', 0.0005247477884018558),
 ('投资者在证券交易中遭受损失', 0.0005004412738005438),
 ('利润', 0.000495572338004653),
 ('差异', 0.0004899555161635981),
 ('业绩同比较大增长', 0.0004629239604155248),
 ('公司的经营业绩', 0.00039319747216364984),
 ('提高客户的满意度和忠诚度', 0.0003917398023592222),
 ('研发费用变动', 0.0003686641308238447),
 ('促进公司的新能源汽车业务持续发展', 0.00036793980326920686),
 ('对公司的经营业绩造成不利影响', 0.00036793529592163085),
 ('促进企业与员工和谐共赢、共同发展', 0.00036501995631277545),
 ('影响公司盈利能力', 0.00036501995631277545),
 ('医药行业在国民经济的比重日益扩大', 0.00036501995631277545),
 ('促进公司经营业绩的提升', 0.00036501995631277545),
 ('筹资活动产生的现金流量净额变动', 0.0003423087022285049),
 ('债务人信用风险的预期变动', 0.0003412154498751841),
 ('现金及现金等价物净增加额同比减少', 0.0003397577800707564),
 ('提升企业整体盈利水平', 0.00033609035859508547),
 ('提高企业市场竞争力', 0.0003178754563592766),
 ('机器人高精密减速机业务获得高速发展', 0.00031595327363316507),
 ('行业高质量发展水平持续提升', 0.00031595327363316507),
 ('夯实公司的行业领先地位', 0.00031595327363316507),
 ('提升经济效', 0.00031595327363316507),
 ('增强公司竞争力', 0.00031595327363316507),
 ('良好的基本面没有发生改变', 0.00031595327363316507)]

plt.figure(figsize=(90, 90))

nx.draw(G, pos=layout, node_size=10,node_color='r', with_labels=True)
plt.savefig('./2021企业年报因果抽取-pagerank.jpg', dpi=200, bbox_inches = 'tight')

from pyvis.network import Network
import os
import pandas as pd
import networkx as nx


G = nx.Graph()


nt = Network('1500px', '1500px', notebook=True, neighborhood_highlight=False, select_menu=True, filter_menu=False, )
for one_data in all_relation_data:
    try:
        if relation_counts[one_data[0].strip()] > 1:
            G.add_edge(one_data[0].strip(),one_data[2].strip(), label=one_data[1])
    except:
        continue
nt.from_nx(G)

nt.show_buttons(filter_=['physics'])

nt.show("2021企业事件因果可视化.html")

import json

all_relation = open("dev_pred_50000.jsonl").readlines()
all_relation_data = []
entities_id_mapping = {}
relation_count = []
for one in all_relation:
    relations = json.loads(one)["spo_list"]
    for relation in relations:
        all_relation_data.append((relation[0].replace(" ",""),
                                  relation[1],
                                  relation[2].replace(" ","")))
    relation_count.append(relation[0].replace(" ",""))
all_relation_data = list(set(all_relation_data))
len(all_relation_data)

from collections import Counter
count = Counter(relation_count)  # 统计词频
relation_counts = dict(count)

from pyvis.network import Network
import os
import pandas as pd
import networkx as nx


G = nx.Graph()


nt = Network('1500px', '1500px', notebook=True)
for one_data in all_relation_data:
    try:
        if relation_counts[one_data[0].strip()] > 5:
            G.add_edge(one_data[0].strip(),one_data[2].strip(), label="导致")
    except:
        continue
nt.from_nx(G)

# nt.show_buttons(filter_=['physics'])

nt.show("2021企业事件因果可视化.view.top.5.html")