CV领域最HOT的职位！你猜对了吗？（附代码）

# 导入相关库
import requests
import pandas as pd
import time
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import random
import numpy as np

# 定义相关函数将csv格式id列表转为python list
def pd_to_list(pd):
    m, n = pd.shape
    return [int(pd.iloc[i,j])for i in range(m) for j in range(n) if pd.iloc[i,j] > 0]

bj01 = pd.read_csv('job_csv/bj01.csv')
bj02 = pd.read_csv('job_csv/bj02.csv')
bj03 = pd.read_csv('job_csv/bj03.csv')
bj04 = pd.read_csv('job_csv/bj04.csv')
bj05 = pd.read_csv('job_csv/bj05.csv')
bj06 = pd.read_csv('job_csv/bj06.csv')

list_bj01 = pd_to_list(bj01)
list_bj02 = pd_to_list(bj02)
list_bj03 = pd_to_list(bj03)
list_bj04 = pd_to_list(bj04)
list_bj05 = pd_to_list(bj05)
list_bj06 = pd_to_list(bj06)
len(list_bj01),len(list_bj02),len(list_bj03),len(list_bj04),len(list_bj05),len(list_bj06)

# 定义headers
co_zk = '_ga=GA1.2.696897302.1536903383; \
user_trace_token=20180914133622-1d558d49-b7e0-11e8-99fe-525400f775ce; \
LGUID=20180914133622-1d5596f3-b7e0-11e8-99fe-525400f775ce; \
showExpriedIndex=1; showExpriedCompanyHome=1; \
showExpriedMyPublish=1; hasDeliver=1; \
index_location_city=%E5%85%A8%E5%9B%BD; \
JSESSIONID=ABAAABAAAFCAAEG52161C740B05DA79D3CD7C30FAFE85A2; \
_gid=GA1.2.960840066.1537838157; \
X_HTTP_TOKEN=a949d9bc13e2814d18c7af9a6358088c; \
LG_LOGIN_USER_ID=dcdbad096b97504ea798f6af94679dff97decf2a465e3fdd3c844317a90ce628; \
_putrc=D8F61B9302D837FF123F89F2B170EADC; \
login=true; unick=%E5%BC%A0%E6%98%86; \
gate_login_token=9ea27fd87c922fcd35d70c6ec7b2632b7c9f5ceb0863337a2a3f316c35b78f8d; \
LGSID=20180925165120-2c1376fc-c0a0-11e8-a682-525400f775ce; \
Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1537857767,1537865480,1537865672,1537865689; \
TG-TRACK-CODE=search_code; SEARCH_ID=96538bec915549989cbcb229ab8e052e; \
LGRID=20180925173103-b885a3f4-c0a5-11e8-a685-525400f775ce; \
Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1537867864'

pcUserAgent = [
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0);",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
":Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"User-Agent:Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"User-Agent:Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
"User-Agent:Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
"User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"
]

# 定义headers
import random
headers_zk = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.9',
    'ache-Control': 'max-age=0',
    'Connection': 'colse',
    'Cookie':co_zk,
    'Host':'www.lagou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent':random.sample(pcUserAgent, 1)[0]
#     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, \
#                 like Gecko) Ubuntu Chromium/68.0.3440.106 Chrome/68.0.3440.106 Safari/537.36'  
}

id_list = list_bj01
job1 = list()
soups = list()
for i in range(0, len(id_list)):
    print('第{:>3}  个:'.format(i))
    res = requests.get('https://www.lagou.com/jobs/'+str(id_list[i])+'.html', headers=headers_zk)
    print('https://www.lagou.com/jobs/'+str(id_list[i])+'.html')
    print(res)
    res = res.content.decode('utf-8')
    # 网页转化为soup对象

    soup=BeautifulSoup(res,'html.parser')
    soups.append(soup)
    # 定义列表，存储 i 个公司信息
    job_i = []
    # 获取公司名
    try:
        company_name = soup.find('div', class_="company").string.split('招聘')[0]
        print(company_name)
        job_i.append(company_name) 
    except AttributeError:
        continue
    # 获取公司规模 
    list_num = len(soup.find_all('ul', class_='c_feature')[0].find_all('li'))
    if list_num == 4:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[2].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    else:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[3].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    job_i.append(guimo)  
    # 获取公司行业
    hangye = soup.find_all('ul', class_='c_feature')[0].find_all('li')[0].text.split('领域')[0].replace('\n', '').replace(' ', '')
    job_i.append(hangye)
    # 获取职位名
    job_name = soup.find_all('span', class_='name')[0].string
    job_i.append(job_name)
    # 5项列表
    five_list = [a.contents for a in soup.select('dd')[0].find_all('span')]
    # 获取薪资
    salary = five_list[0][0]
    job_i.append(salary)
    # 获取地址
    city = five_list[1][0].replace('/', '')
    job_i.append(city)
    # 获取经验
    jy = five_list[2][0].replace('/', '')
    job_i.append(jy)
    # 获取学历
    xl = five_list[3][0].replace('/', '')
    job_i.append(xl)
    # 获取工作类型
    leixing = five_list[4][0]
    job_i.append(leixing)
    # 获取工作详情
    jieshao = soup.select('dl')[0].text.split('工作地址')[0].replace('\n', '').replace('\xa0', '').split('职位描述：')[-1]
    job_i.append(jieshao)
    job1.append(job_i)
    time.sleep(0)

第0个:https://www.lagou.com/jobs/5133698.html<Response [200]>
第1个:https://www.lagou.com/jobs/5125993.html<Response [200]>
英特美迪研发部
第2个:https://www.lagou.com/jobs/5170395.html<Response [200]>
印迹
第3个:https://www.lagou.com/jobs/5195454.html<Response [200]>
第4个:https://www.lagou.com/jobs/3384529.html<Response [200]>
真机智能研发部

df = pd.DataFrame(data = job1 ,columns = ['公司名','公司规模', '公司行业', '职位名','薪资','地址','经验','学历','工作类型','职位要求'])   
df.to_csv('job_csv/job_bj01.csv',index = False)  
print('已保存为csv文件.')

已保存为csv文件。15-50In[122]:

id_list = list_bj02
job2 = list()
soups = list()
for i in range(0, len(id_list)):
    print('第{:>3}  个:'.format(i))
    res = requests.get('https://www.lagou.com/jobs/'+str(id_list[i])+'.html', headers=headers_zk)
    print('https://www.lagou.com/jobs/'+str(id_list[i])+'.html')
    print(res)
    res = res.content.decode('utf-8')
    # 网页转化为soup对象

    soup=BeautifulSoup(res,'html.parser')
    soups.append(soup)
    # 定义列表，存储 i 个公司信息
    job_i = []
    # 获取公司名
    try:
        company_name = soup.find('div', class_="company").string.split('招聘')[0]
        print(company_name)
        job_i.append(company_name) 
        print(job_i)
    except AttributeError:
        continue
    # 获取公司规模 
    list_num = len(soup.find_all('ul', class_='c_feature')[0].find_all('li'))
    if list_num == 4:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[2].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    else:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[3].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    job_i.append(guimo)  
    # 获取公司行业
    hangye = soup.find_all('ul', class_='c_feature')[0].find_all('li')[0].text.split('领域')[0].replace('\n', '').replace(' ', '')
    job_i.append(hangye)
    # 获取职位名
    job_name = soup.find_all('span', class_='name')[0].string
    job_i.append(job_name)
    # 5项列表
    five_list = [a.contents for a in soup.select('dd')[0].find_all('span')]
    # 获取薪资
    salary = five_list[0][0]
    job_i.append(salary)
    # 获取地址
    city = five_list[1][0].replace('/', '')
    job_i.append(city)
    # 获取经验
    jy = five_list[2][0].replace('/', '')
    job_i.append(jy)
    # 获取学历
    xl = five_list[3][0].replace('/', '')
    job_i.append(xl)
    # 获取工作类型
    leixing = five_list[4][0]
    job_i.append(leixing)
    # 获取工作详情
    jieshao = soup.select('dl')[0].text.split('工作地址')[0].replace('\n', '').replace('\xa0', '').split('职位描述：')[-1]
    job_i.append(jieshao)
    job2.append(job_i)
    time.sleep(0)

df = pd.DataFrame(data = job2 ,columns = ['公司名','公司规模', '公司行业', '职位名','薪资','地址','经验','学历','工作类型','职位要求'])   
df.to_csv('job_csv/job_bj02.csv',index = False)
print('已保存为csv文件.') 

已保存为csv文件。

id_list = list_bj03
job3 = list()
soups = list()
for i in range(0, len(id_list)):
    print('第{:>3}  个:'.format(i))
    res = requests.get('https://www.lagou.com/jobs/'+str(id_list[i])+'.html', headers=headers_zk)
    print('https://www.lagou.com/jobs/'+str(id_list[i])+'.html')
    print(res)
    res = res.content.decode('utf-8')
    # 网页转化为soup对象

    soup=BeautifulSoup(res,'html.parser')
    soups.append(soup)
    # 定义列表，存储 i 个公司信息
    job_i = []
    # 获取公司名
    try:
        company_name = soup.find('div', class_="company").string.split('招聘')[0]
        print(company_name)
        job_i.append(company_name) 
    except AttributeError:
        continue
    # 获取公司规模 
    list_num = len(soup.find_all('ul', class_='c_feature')[0].find_all('li'))
    if list_num == 4:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[2].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    else:
        guimo = soup.find_all('ul', class_='c_feature')[0].find_all('li')[3].text.split('领域')[0].replace('\n', '').replace(' ', '').split('人')[0]
    job_i.append(guimo)  
    # 获取公司行业
    hangye = soup.find_all('ul', class_='c_feature')[0].find_all('li')[0].text.split('领域')[0].replace('\n', '').replace(' ', '')
    job_i.append(hangye)
    # 获取职位名
    job_name = soup.find_all('span', class_='name')[0].string
    job_i.append(job_name)
    # 5项列表
    five_list = [a.contents for a in soup.select('dd')[0].find_all('span')]
    # 获取薪资
    salary = five_list[0][0]
    job_i.append(salary)
    # 获取地址
    city = five_list[1][0].replace('/', '')
    job_i.append(city)
    # 获取经验
    jy = five_list[2][0].replace('/', '')
    job_i.append(jy)
    # 获取学历
    xl = five_list[3][0].replace('/', '')
    job_i.append(xl)
    # 获取工作类型
    leixing = five_list[4][0]
    job_i.append(leixing)
    # 获取工作详情
    jieshao = soup.select('dl')[0].text.split('工作地址')[0].replace('\n', '').replace('\xa0', '').split('职位描述：')[-1]
    job_i.append(jieshao)
    job3.append(job_i)
    time.sleep(0)

df = pd.DataFrame(data = job3 ,columns = ['公司名','公司规模', '公司行业', '职位名','薪资','地址','经验','学历','工作类型','职位要求'])   
df.to_csv('job_csv/job_bj03.csv',index = False)  
print('已保存为csv文件.') 

已保存为csv文件。

'''导入相关的包'''
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from pylab import *
%matplotlib inline
warnings.filterwarnings("ignore")

'''将数据从 csv 到 pandas'''
a = pd.read_csv('job_total.csv')
print(a.shape)
%matplotlib inline
# 展示前5个数据
a.head()

(260, 11)

Out[2]:

'''定义相关搜索词'''
# 定义了9组搜索词 
# 如果是包含汉字的话，需要转化为英文表述

# 内容分类
test_01 = [['图像'],['视频'],['三维','3d'],['二维','2D']]
test_01_p = ['image','video','3D','2D']

# 方向分类
test_02 = ['分类','识别','检测','分割','跟踪','OCR','理解']
test_02_p = ['classification','recognition','detection','segmentation','tracking', \
             'OCR','understanding']
# 语言分类
test_03 = ['python','C/C++','matlab','opencv','gpu','linux','shell','cuda']

# 框架分类
test_04 = ['tensorflow','caffe','torch','paddle','theano','mxnet','currennt']

# 常见模型分类
test_05 = ['cnn','dnn','lstm','rnn',]

# 分类方向细分
test_06 = [['分类'], ['目标分类'], ['图像分类','图片分类'], ['物体分类','物品分类'], ['视频分类',], \
           ['动作分类','行为分类'], ['场景分类'], ['人脸分类']]
test_06_p = ['classification', 'target', 'Image', 'object', 'video', \
             'behavior', 'scenen', 'face']
# 识别方向细分
test_07 = [['识别'], ['目标识别'], ['图像识别','图片识别'], ['物体识别','物品识别'], \
           ['视频识别',], ['动作分类','行为分类'], ['场景分类'], \
           ['人脸识别'], ['模式识别'], ['特征点识别'], \
           ['生物特征识别'], ['车辆识别', '车牌识别', '车标识别', '车型识别',], ['字体', '手写']]
test_07_p = ['recognition', 'target', 'image', 'object', \
            'video', 'behavior', 'scenen', \
            'face', 'pattern', 'feature points', 
             'biological characteristics', 'vehicle', 'character']
# 检测方向细分
test_08 = [['检测'], ['目标检测'], ['图像检测','图片检测'], ['物体检测','物品检测'], \
           ['视频检测',], ['动作检测','行为检测', '姿态检测'], ['场景检测'], \
           ['人脸检测', '人脸属性检测'], ['人体检测'], ['关键点检测'], \
           ['车辆检测', '车牌检测', '车标检测', '车型检测',], ['字体', '手写']]
test_08_p = ['detection', 'target', 'image', 'object', \
            'video', 'behavior', 'scenen', \
            'face', 'human body', 'feature points', 
             'vehicle', 'character']
# 分割方向细分
test_09 = [['分割'], ['目标分割'], ['图像分割','图片分割'], ['物体分割','物品分割'], \
           ['语义分割',], ['影像分割'], ['场景分割'], \
           ['细胞分割']]
test_09_p = ['segmentation', 'target', 'image', 'object', \
            'semantic', 'imaging', 'scenen', \
            'cell']

# 设置相应的投票数都为0
n01 = [0]*len(test_01)
n02 = [0]*len(test_02)
n03 = [0]*len(test_03)
n04 = [0]*len(test_04)
n05 = [0]*len(test_05)
n06 = [0]*len(test_06)
n07 = [0]*len(test_07)
n08 = [0]*len(test_08)
n09 = [0]*len(test_09)

'''循环进行投票'''

for i in range(len(a.iloc[:,9])):
    # 01
    for idx, j in enumerate(test_01):
        for k in j:
            if k.lower() in a.iloc[:,9][i].lower():
                n01[idx] += 1
                break
    # 02
    for idx, j in enumerate(test_02):
        if j.lower() in a.iloc[:,9][i].lower():
            n02[idx] += 1  
    # 03
    for idx, j in enumerate(test_03):
        if j.lower() in a.iloc[:,9][i].lower():
            n03[idx] += 1
    # 04
    for idx, j in enumerate(test_04):
        if j.lower() in a.iloc[:,9][i].lower():
            n04[idx] += 1        
    # 05
    for idx, j in enumerate(test_05):
        if j.lower() in a.iloc[:,9][i].lower():
            n05[idx] += 1

    # 06
    for idx, j in enumerate(test_06):
        for k in j:
            if k.lower() in a.iloc[:,9][i].lower():
                n06[idx] += 1
                break           
    # 07
    for idx, j in enumerate(test_07):
        for k in j:
            if k.lower() in a.iloc[:,9][i].lower():
                n07[idx] += 1
                break       
    # 08
    for idx, j in enumerate(test_08):
        for k in j:
            if k.lower() in a.iloc[:,9][i].lower():
                n08[idx] += 1
                break    
    # 09
    for idx, j in enumerate(test_09):
        for k in j:
            if k.lower() in a.iloc[:,9][i].lower():
                n09[idx] += 1
                break   

'''01内容分类'''
plt.figure(figsize=(6, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n01)), n01, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n01)), test_01_p, rotation=90)
x = range(len(n01))
y = n01
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''02方向'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n02)), n02, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n02)), test_02_p, rotation=90)
x = range(len(n02))
y = n02
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''03语言'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n03)), n03, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n03)), test_03, rotation=90)
x = range(len(n03))
y = n03
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''04框架'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n04)), n04, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n04)), test_04, rotation=90)
x = range(len(n04))
y = n04
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''05模型'''
plt.figure(figsize=(6, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n05)), n05, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n05)), test_05, rotation=90)
x = range(len(n05))
y = n05
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''06分类'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n06)), n06, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n06)), test_06_p, rotation=90)
x = range(len(n06))
y = n06
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''07识别'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n07)), n07, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n07)), test_07_p, rotation=90)
x = range(len(n07))
y = n07
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()

'''08检测'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n08)), n08, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n08)), test_08_p, rotation=90)
x = range(len(n08))
y = n08
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.grid(axis='y', ls='--', alpha=0.3, )
plt.show()

'''09分割'''
plt.figure(figsize=(12, 4))
# mpl.rcParams['font.sans-serif'] = ['SimHei']
p2 = plt.bar(np.arange(len(n09)), n09, 0.1, label="rainfall", color="#87CEFA")
plt.xticks(np.arange(len(n09)), test_09_p, rotation=90)
x = range(len(n09))
y = n09
for aa, bb in zip (x, y):
    plt.text(aa, bb+0.05, '%.0f' % bb, ha='center', va= 'bottom',fontsize=8)
plt.show()