几百个作者有相同的名字？| 智源-AMiner大赛，从EDA到冠军baseline

authors = [author for author in train_data]authors_num_person = [len(train_data[author].keys()) for author in train_data]print('训练集同名数量：', len(authors))print('消歧后实际作者数量：',  sum(authors_num_person))
# 以绘制训练集同名作者个体数量为例plt.figure(figsize=(40, 8), dpi=80)x = range(len(authors))
plt.bar(x, authors_num_person, width=0.5)plt.xticks(x, authors)plt.xticks(rotation=270)plt.xlabel('训练集同名作者')plt.ylabel('该名字同名作者数量（个）')for xl, yl in zip(x, authors_num_person):    plt.text(xl, yl+0.3, str(yl), ha='center', va='bottom', fontsize=10.5)
mean_person = int(np.mean(authors_num_person))plt.gca().hlines(mean_person,-1,225,linestyles='--',colors='red',label='平均值')plt.annotate(u"平均值:" + str(mean_person), xy = (225, mean_person), xytext = (225, mean_person+40),arrowprops=dict(facecolor='red',shrink=0.1,width=2))

训练集同名数量：221消歧后实际作者数量：22839涉及的论文总数（篇）：205498

paper id: zszavvJhpaper author name:  feifei chengpaper author name:  heng lipaper author name:  y w wangpaper author name:  martin skitmorepaper author name:  p j forsythebr

# 查看机构、协作作者、协作机构、常用关键词、发表的会议期刊是否有交集for aid1 in author_data:    for aid2 in author_data:        if aid1 != aid2:            if set(org[aid1]).intersection(set(org[aid2])):                print('%s与%s机构重叠:' % (aid1, aid2), set(org[aid1]).intersection(set(org[aid2])))            if set(co_author[aid1]).intersection(set(co_author[aid2])):                print('%s与%s协作者重叠:' % (aid1, aid2), set(co_author[aid1]).intersection(set(co_author[aid2])))            if set(co_org[aid1]).intersection(set(co_org[aid2])):                print('%s与%s协作机构重叠:' % (aid1, aid2), set(co_org[aid1]).intersection(set(co_org[aid2])))            if set(keywords[aid1]).intersection(set(keywords[aid2])):                print('%s与%s关键词重叠:' % (aid1, aid2), set(keywords[aid1]).intersection(set(keywords[aid2])))            if set(venue[aid1]).intersection(set(venue[aid2])):                print('%s与%s会议期刊重叠:' % (aid1, aid2), set(venue[aid1]).intersection(set(venue[aid2])))br

bXCEdRdj与WSFIcbxu关键词重叠: {'chemical composition'}bXCEdRdj与WSFIcbxu会议期刊重叠: {'Journal of Agricultural and Food Chemistry'}bXCEdRdj与gUSOTB2u会议期刊重叠: {'Journal of home economics'}OaaWMLWs与5Bxjg2tr协作者重叠: {'Shoei Sato'}OaaWMLWs与5Bxjg2tr会议期刊重叠: {'APSIPA'}WSFIcbxu与bXCEdRdj关键词重叠: {'chemical composition'}WSFIcbxu与bXCEdRdj会议期刊重叠: {'Journal of Agricultural and Food Chemistry'}CsE5vsJA与JvJ9opoq协作者重叠: {'Hiroki Sakaji'}gUSOTB2u与bXCEdRdj会议期刊重叠: {'Journal of home economics'}5Bxjg2tr与OaaWMLWs协作者重叠: {'Shoei Sato'}5Bxjg2tr与OaaWMLWs会议期刊重叠: {'APSIPA'}JvJ9opoq与CsE5vsJA协作者重叠: {'Hiroki Sakaji'}br

#正则去标点def etl(content):    content = re.sub("[\s+\.\!\/,;$%^*(+\"\')]+|[+——()?【】“”！，。？、~@#￥%……&*（）]+", " ", content)    content = re.sub(r" {2,}", " ", content)    return content
def get_org(co_authors, author_name):    for au in co_authors:        name = precessname(au['name'])        name = name.split('_')        if ('_'.join(name) == author_name or '_'.join(name[::-1]) == author_name) and 'org' in au:            return au['org']    return ''

# 统计论文的年份范围min_year = 3000max_year = 0pubs_no_year = 0for each in tqdm(author_pub.items()):    try:        year = int(each[1]['year'])    except:        pubs_no_year += 1        print("Paper {} has no year info".format(each[0]))        continue    if year<1500 or year>2100:        pubs_no_year +=1        print("Paper {} has wrong year info: {}".format(each[0], year))        continue    if year<min_year:        min_year = year    if year>max_year:        max_year = yearprint(min_year, max_year)print("共计{}份论文没有年份信息".format(pubs_no_year))

import seaborn as snsimport matplotlib.pyplot as pltsns.distplot(same_author_num,30,hist_kws={"rwidth":0.0001,'edgecolor':'black', 'alpha':1.0})br

sns.distplot(same_author_num,300,             hist_kws=dict(cumulative=True),             kde_kws=dict(cumulative=True))br

sns.distplot(same_author_num,3000,             hist_kws=dict(cumulative=True),             kde_kws=dict(cumulative=True))plt.xlim(0,100)br

sns.distplot(thesis_percent,30,hist_kws={"rwidth":0.001,'edgecolor':'black', 'alpha':1.0})br

sns.distplot(thesis_percent,300,             hist_kws=dict(cumulative=True),             kde_kws=dict(cumulative=True))br

for name in train_author.keys()[15:20]:    f, ax = plt.subplots(figsize=(8, 6))    plt.plot([len(train_author[name][author])for author in train_author[name].keys()])br

# 同名中各个作者发布论文区间count=0for name in train_author.keys()[15:20]:    f, ax = plt.subplots(figsize=(8, 6))    authors = train_author[name].keys()    for author in authors:        years=[]        for thesis in train_author[name][author]:            if 'year' in train_pub[thesis].keys():                years.append(train_pub[thesis]['year'])        plt.plot(np.sort(years))    plt.ylim(1990,2020)    if count==3:        print(name)    count+=1br

for author in validate_data:
    # print(author)    coauther_orgs = []    papers = validate_data[author]    if len(papers) == 0:        res_dict[author] = []        continue    # print(len(papers))    paper_dict = {}    for paper in papers:        authors = paper['authors']        names = [precessname(paper_author['name']) for paper_author in authors]        orgs = [preprocessorg(paper_author['org']) for paper_author in authors if 'org' in paper_author]        abstract = paper["abstract"] if 'abstract' in paper else ''        coauther_orgs.append(etl(' '.join(names + orgs) + ' '+ abstract))    tfidf = TfidfVectorizer().fit_transform(coauther_orgs)

    clf = DBSCAN(metric='cosine')    s = clf.fit_predict(tfidf)    # 每个样本所属的簇    for label, paper in zip(clf.labels_, papers):        if str(label) not in paper_dict:            paper_dict[str(label)] = [paper['id']]        else:            paper_dict[str(label)].append(paper['id'])    res_dict[author] = list(paper_dict.values())
    tfidf = PCA(n_components=2, random_state=0).fit_transform(tfidf.toarray())

res_dict = {}t0 = time()for author in validate_data:    # print(author)    coauther_orgs = []    papers = validate_data[author]    if len(papers) == 0:        res_dict[author] = []        continue    # print(len(papers))    paper_dict = {}    for paper in papers:        authors = paper['authors']        names = [precessname(paper_author['name']) for paper_author in authors]        orgs = [preprocessorg(paper_author['org']) for paper_author in authors if 'org' in paper_author]        abstract = paper["abstract"] if 'abstract' in paper else ''        coauther_orgs.append(etl(' '.join(names + orgs) + ' '+ abstract))    tfidf = TfidfVectorizer().fit_transform(coauther_orgs)
    # affinity_clustering，参数可以调整，此处进攻参考，未提交测试    clf = cluster.AffinityPropagation(damping=.9, preference=-20)    s = clf.fit_predict(tfidf)    # 每个样本所属的簇    for label, paper in zip(clf.labels_, papers):        if str(label) not in paper_dict:            paper_dict[str(label)] = [paper['id']]        else:            paper_dict[str(label)].append(paper['id'])    res_dict[author] = list(paper_dict.values())
t1 = time()print('running time: %.2fs'% (t1 - t0))
# 降维tfidf = PCA(n_components=2, random_state=0).fit_transform(tfidf.toarray())