爬虫百战穿山甲（3）：全国高校的自我介绍，快来找找你的大学吧！！！

#coding:utf-8		
# 我们李老师说了，这个的意思是：告诉解释器，本段代码获取的数据皆以utf-8编码，解码的时候以utf-8解码即可


import requests    
import json
import time
from lxml import etree
import random
import openpyxl

#使用时请自行寻找四五个可用的请求头
header_list = ["Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"]


def get_html(url,sleep_time):
    '''
    获取网页源码
    :param url: 目标网址
    :param header_list: 请求头列表
    :param sleep_time:  友好时间
    :return: 网页源码，未经任何处理
    '''
    times = 3
    try:
        res = requests.get(url=url, headers={"User-Agent":random.choice(header_list)})
        if res.status_code >= 200 and res.status_code<300:
            time.sleep(sleep_time)
            return res
        else:
            return None
    except Exception as e:
        print(e)
        if times>0:
            print("机会次数："+str(times))
            get_html(url,sleep_time)
        else:
            print("无法爬取")


def get_xpath_data(html_data,xpath):
    '''
    解析网页源码，获取标签数据
    :param html_data: 网页源码
    :param xpath: Xpath路径
    :return: 标签数据列表
    '''
    data = html_data.content  # 这里要使用content，而不是text
    data = data.decode().replace("<!--","").replace("--!>","")  # 对数据进行转码操作
    tree = etree.HTML(data) # 创建element对象
    el_list = tree.xpath(xpath)
    return el_list

wb = openpyxl.Workbook()
ws = wb.active

ws.append(['address', 'admissions', 'answerurl', 'belong', 'central', 'city_id', 'city_name', 'code_enroll', 'colleges_level', 'county_id', 'county_name', 'department', 'doublehigh', 'dual_class', 'dual_class_name', 'f211', 'f985', 'id', 'is_recruitment', 'is_top', 'level', 'level_name', 'name', 'nature', 'nature_name', 'province_id', 'province_name', 'rank', 'rank_type', 'school_id', 'school_type', 'single_province', 'type', 'type_name', 'view_month', 'view_month_number', 'view_total', 'view_total_number', 'view_week', 'view_week_number', 'view_year'])

for i in range(1,143):
    url = 'https://api.eol.cn/gkcx/api/?access_token=&admissions=&central=&department=&dual_class=&f211=&f985=&is_doublehigh=&is_dual_class=&keyword=&nature=&page='+str(i)+'&province_id=&request_type=1&school_type=&signsafe=&size=20&sort=view_total&type=&uri=apidata/api/gk/school/lists'

    try:
        res = get_html(url,2)
        j_data = json.loads(res.content)["data"]['item']
        for data in j_data:
            ws.append([data['address'], data['admissions'], data['answerurl'], data['belong'], data['central'], data['city_id'],
                data['city_name'], data['code_enroll'],
                data['colleges_level'], data['county_id'], data['county_name'], data['department'], data['doublehigh'],
                data['dual_class'], data['dual_class_name'],
                data['f211'], data['f985'], data['id'], data['is_recruitment'], data['is_top'], data['level'],
                data['level_name'], data['name'], data['nature'],data[ 'nature_name'],data[ 'province_id'],data['province_name'],data['rank'],data[ 'rank_type'],data['school_id'],data['school_type'],
                data['single_province'],data['type'],data['type_name'],data['view_month'],data['view_month_number'],data['view_total'],
                data['view_total_number'],data['view_week'],data['view_week_number'],data['view_year']])

            print("Success")

    except:
        print(url)

wb.save('test.xlsx')
wb.close()