可视化探索开源项目的 contributor 关系

select count() from system.contributors

select * from system.contributors limit 20;

# Copyright @Shinji-IkariG
from github import Github
from datetime import datetime
import sh
from sh import curl
import csv
import requests
import time

def main():
# 你的 GitHub ID
    GH_USER = 'xxx'
# 你的个人 token，可以前往 GitHub 设置中的 Developer settings 生成自己的 token
    GH_PAT = 'xxx'
    github = Github(GH_PAT)
# 你需要爬取的开源组织的组织名
    org = github.get_organization('vesoft-inc')
    repos = org.get_repos(type='all', sort='full_name', direction='asc')
# 命名存放爬下来的 pr 数据的文件
    with open('all-prs.csv', 'w', newline='') as csvfile:
# 爬取哪些数据
        fieldnames = ['pr num','repo','author', 'create date','close date','merged date','version','labels1','state','branch','assignee','reviewed(commented)','reviewd(approved)','request reviewer','code line(+)','code line(-)','files number']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()


        for repo in repos:
            print(repo)
            Apulls = repo.get_pulls(state='all', sort='created')
            prs = []
            for a in Apulls:
                prs.append(a)

            for i in prs:
                github = Github(GH_PAT)
                print('rate_limite' , github.rate_limiting[0])
                if github.rate_limiting[0] < 500:
                    if github.rate_limiting_resettime - time.time() > 0:
                        time.sleep(github.rate_limiting_resettime - time.time()+900)
                    else:time.sleep(3700)
                else:
                    print(i.number)
                    prUrl = 'https://api.github.com/repos/'+ str(repo.full_name) + '/pulls/' + str(i.number)
                    pr = requests.get(prUrl, auth=(GH_USER, GH_PAT))



                    assigneesList = []
                    if pr.json().get('assignees'):
                        for assignee in pr.json().get('assignees'):
                            assigneesList.append(assignee.get('login'))
                    else: ""



                    reviewerCList = []
                    reviewerAList = []
                    reviewers = requests.get(prUrl + '/reviews', auth=(GH_USER, GH_PAT))
                    if reviewers.json():
                        for reviewer in reviewers.json():
                            if reviewer.get('state') == 'COMMENTED':
                                if reviewer.get('user'): 
                                    reviewerCList.append(reviewer.get('user').get('login'))
                                else: reviewerCList.append('GHOST USER')
                            elif reviewer.get('state') == 'APPROVED':
                                if reviewer.get('user'): 
                                    reviewerAList.append(reviewer.get('user').get('login'))
                                else: reviewerAList.append('GHOST USER')
                            else : print(reviewer.get('state'), 'TYPE REVIEWS')
                    else: ""


                    reqReviewersList = []
                    reqReviewers = requests.get(prUrl + '/requested_reviewers', auth=(GH_USER, GH_PAT))
                    if reqReviewers.json().get('users'):
                        for reqReviewer in reqReviewers.json().get('users'):
                            reqReviewersList.append(reqReviewer.get('login'))
                        print(reqReviewersList)
                    else: ""



                    labelList = []
                    if pr.json().get('labels'):
                        for label in pr.json().get('labels'):
                            labelList.append(label.get('name'))
                    else: ""



                    milestone = pr.json().get('milestone').get('title') if pr.json().get('milestone') else ""



                    writer.writerow({'pr num': i.number,'repo': repo.full_name,'author': pr.json().get('user').get('login'), 'create date': pr.json().get('created_at'),'close date': pr.json().get('closed_at'),'merged date': pr.json().get('merged_at'),'version': milestone,'labels1': ",".join(labelList),'state': pr.json().get('state'),'branch': pr.json().get('base').get('ref'),'assignee': ",".join(assigneesList),'reviewed(commented)': ",".join(reviewerCList),'reviewd(approved)': ",".join(reviewerAList),'request reviewer': ",".join(reqReviewersList),'code line(+)': pr.json().get('additions'),'code line(-)': pr.json().get('deletions'),'files number': pr.json().get('changed_files')})

if __name__ == "__main__":
    main()

#pip3 install sh pygithub

# contributor.csv
wenhaocs,haowen,148,2021-09-24 16:53:33,1,nebula
lopn,lopn,149,2021-09-26 06:02:11,0,nebula-docs-cn
liwenhui-soul,liwenhui-soul,150,2021-09-26 13:38:20,1,nebula
Reid00,Reid00,151,2021-10-08 06:20:24,0,nebula-http-gateway
...

# pr.csv
nevermore3,nebula,4095,2022-03-29 11:23:15,2022-04-13 03:29:44,2022-04-13 03:29:44,1,2310,3979,31
cooper-lzy,docs_cn,1614,2022-03-30 03:21:35,2022-04-07 07:28:31,2022-04-07 07:28:31,1,107,2,4
wuxiaobai24,nebula,4098,2022-03-30 05:51:14,2022-04-11 10:54:04,2022-04-11 10:54:03,1,53,0,3
NicolaCage,website,876,2022-03-30 06:08:02,2022-03-30 06:09:21,2022-03-30 06:09:21,1,4,2,1
...

#repo.csv
clients,nebula-clients,vesoft-inc/nebula-clients,Java
common,nebula-common,vesoft-inc/nebula-common,C++
community,nebula-community,vesoft-inc/nebula-community,Markdown
console,nebula-console,vesoft-inc/nebula-console,Go
...

# nebula-contributor-2023 是这个图空间名字，其他默认；
CREATE SPACE 'nebula-contributor-2023'(partition_num = 10, vid_type = FIXED_STRING(32))

CREATE tag contributor (name string NULL, number int16 NULL, anniversary datetime NULL, is_vesoft bool NULL, first_merged string NULL) COMMENT = "贡献者"

# 创建 repo tag
CREATE tag repo (repo_name string NULL, language string NULL, path string NULL) COMMENT = "仓库"

# 创建 pr edge
CREATE edge pr (number int NULL, created_time datetime NULL, closed_time datetime NULL DEFAULT NULL, merged_time datetime NULL DEFAULT NULL, is_merged bool NULL, ins_code_line int NULL, des_code_line int NULL, file_changed_num int NULL)

match (v0:contributor) -[e:pr]-> (v1:repo) return e limit 15000

match (v0:contributor) -[e:pr]-> (v1:repo) where (v0.contributor.is_vesoft == false) return e limit 15000

# 查看各个查询语言的开源仓库贡献情况
match (v0:contributor) -[e:pr]-> (v1:repo) where (v1.repo.language == "C++") return e

match (v0:contributor) -[e:pr]-> (v1:repo) where (v1.repo.language == "Python") return e

match (v0:contributor) -[e:pr]-> (v1:repo) where (v1.repo.language == "Go") return e

match (v0:contributor) -[e:pr]-> (v1:repo) where (v1.repo.language == "Java") return e

# 内核仓 nebula 的非雇员贡献者

match (v0:contributor) -[e:pr]-> (v1:repo) where (v1.repo.repo_name == "nebula" and v0.contributor.is_vesoft == false) return e

# 2021 年诞生的非雇员 contributor
match (v0:contributor) -[e:pr]-> (v1:repo) where (v0.contributor.anniversary >= datetime("2021-01-01T00:00:00") and v0.contributor.anniversary < datetime("2022-01-01T00:00:00")  ) and v0.contributor.is_vesoft ==false return e

# 目前未被合并的 pr
match (v0:contributor) -[e:pr]-> (v1:repo) where (e.is_merged == false) return e