爬虫学习

conda install jupyter notebook

jupyter notebook -port port_number

jupyter notebook -no-browser

import requests
url = 'https://www.sogou.com/'
response_obj = requests.get(url=url)
page_text = response_obj.text
with open('./sougou.html', 'w', encoding='utf-8')as fp:
    fp.write(page_text)
​

# import requests
# url = 'https://www.sogou.com/web'
# # 封装参数
# can_shu = input('enter a word:')
# param = {
#     'query': can_shu
# }
# response_obj = requests.get(url=url, params=param)
# page_text = response_obj.content
# fileName = can_shu+'.html'
# with open(fileName, 'wb')as fp:
#     fp.write(page_text)
#     print('over')

# 案例2 爬取百度翻译结果
# 注意: 翻译的结果是局部刷新, 采用ajax异步请求
import requests
# post 请求
url = 'https://fanyi.baidu.com/sug'
can_shu = input('worlds:')
data = {
    'kw': can_shu
}
response_obj = requests.post(url=url, data=data)
# 返回的response对象以json数据类型展示.若以text形式为字符串, 若以content形式为二进制.
print(response_obj.json())

from requests
# get 请求
url = 'https://movie.douban.com/j/chart/top_list'
# 动态捕获电影, 设置一个url字典
param = {
    "type": "5",
    "interval_id": "100:90",
    "action": "",
    "start": "0", # 表示从第'0'索引位置开始
    "limit": "1" # 表示爬取一个
}
# 返回一个列表
move_data = requests.get(url=url, params=Parma).json()
print(move_data)

# 案例4
# 反扒机制: UA检测 --> 反反爬策略UA伪装.
# --- 请求载体的身份标识: User-Agent.请求载体不一样, 标识就不一样.基于浏览器和爬虫
#     的请求在意不一样, 所以需要伪装成某一浏览器请求.
import requests
id_list = []
# post 请求
url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
​
}
# 爬取多页
for page in range(1, 3):
    data = {
        'on': 'true',
        'page': str(page),
        'pageSize': '15',
        'productName': '',
        'conditionType': '1',
        'applyname': '',
        'applysn': ''
    }
data_obj = requests.post(url=url, data=data, headers=headers).json()
# 获取各个ID值
for dic in data_obj['list']:
    id = dic['ID']
    id_list.append(id)
print(id_list)
​
# 通过ID 获取各个公司的生产许可证相关信息
detail_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
for id in id_list:
    detail_data = {
        'id':id
    }
    detail_json = requests.post(url=detail_url, data=detail_data, headers=headers).json()
    print(detail_json)
​

# 爬取图片
import requests
url = 'http://d.ifengimg.com/w640_q75/p0.ifengimg.com/pmop/2018/0923/D7D76D3B007F024D2D0964DFE3AD909A68232483_size561_w1080_h1920.jpeg'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
​
}
# 图片为二进制数据,可以用content
img_data = requests.get(url=url, headers=headers).content
with open('./gua_jie.jpg', 'wb') as fp:
    fp.write(img_data)

# 爬取动态加载的图片
import requests
import re
import urllib
import os
# 拿到指定页码的图片
url = 'https://www.qiushibaike.com/pic/page/%d/?s=5171142'
start_page = int(input('start page'))
end_page = int(input('end page'))
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
if not os.path.exists('./qiutu'):
    os.mkdir('./qiutu')
for page in range(start_page, end_page):
    new_url = format(url%page)
    page_text = requests.get(url=new_url, headers=headers).text
    # 提取图片的src属性
    img_url_list = re.findall('<div class="thumb">.*?<img src="(.*?)" alt=.*?</div>', page_text, re.S)
    # 根据src属性爬取纯图片
    for img_url in img_url_list:
        img_url = 'http:'+img_url
        imgName = img_url.split('/')[-1]
        imgPath ='qiutu/'+imgName
        urllib.request.urlretrieve(url=img_url, filename=imgPath)
        print('well done!')
print('完成!!!') 

# bs4模块
from bs4 import BeautifulSoup
import requests
url = 'http://www.shicimingju.com/book/rulinwaishi.html'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text
# 加载要解析的源码   lxml 为解析器
soup = BeautifulSoup(page_text, 'lxml')
# 进行标签定位
a_list = soup.select('.book-mulu>ul>li>a')
fp = open('儒林外史.txt', 'w', encoding='utf-8')
# 获取到标签之间的有用数据, 并通过该数据下载想要所取得的内容
for a in a_list:
    title = a.string
    detail_url ='http://www.shicimingju.com'+a['href']
    detail_page_text = requests.get(url=detail_url, headers=headers).text
    
    soup = BeautifulSoup(detail_page_text, 'lxml')
    content = soup.find('div', class_='chapter_content').text
​
    fp.write(title+'\n'+content)
    print(title, '下载完毕')
print('well done!!')
fp.close()

# xpath模块爬取二手房信息
​
import requests
from lxml import etree
​
url = 'https://xy.58.com/ershoufang/?PGTID=0d000000-0000-00d2-8801-ee7a9683ca0f&ClickID=2/'
headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
# 获取页面源码数据
page_text = requests.get(url=url, headers=headers).text
# 实例化对象
tree = etree.HTML(page_text)
# li_list 的类型为Element 类型的列表对象
# 标签定位及内容捕获
li_list = tree.xpath('//ul[@class="house-list-wrap"]/li')
fp = open('fang.csv', 'w', encoding='utf-8')
for li in li_list:
    # 进行局部解析要加 '.'
    xin_xi = li.xpath('./div[2]/p[1]//text()')[1]
    price = li.xpath('./div[3]//text()')
    price = ''.join(price)
    fp.write(xin_xi+":"+price+"\n")
fp.close()
print('well done!!')

# 案例2
# 解析图片
import requests
from lxml import etree
import os
import urllib
​
url = 'http://pic.netbian.com/4kdongman/index_2.html'
headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
​
}
response = requests.get(url=url, headers=headers)
if not os.path.exists('./imgs'):
    os.mkdir('./imgs')
page_text = response.text
tree = etree.HTML(page_text)
print(tree)
li_list = tree.xpath('//div[@class="slist"]/ul/li')
print(li_list)
for li in li_list:
    img_name = li.xpath('./a/b/text()')[0]
    # 处理中文乱码
    img_name.encode('iso-8859-1').decode('gbk')
    img_url = 'http://pic.netbian.com'+li.xpath('./a/img/@src')[0]
    print(img_name)
    img_path = './img'+img_name+'.jpg'
    urllib.request.urlretrieve(url=img_url, filename=img_path)
    print(img_path, 'nice!!')
print('over!!')

# 爬取站长素材的简历模板
import random
import requests
from lxml import etree
headers ={
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
​
}
url = 'http://sc.chinaz.com/jianli/free_%d.html'
for page in range(1, 4):
    if page == 1:
        new_url = 'http://sc.chinaz.com/jianli/free.html'
    else:
        new_url = format(url%page)
# 获取源码数据
response = requests.get(url=new_url, headers=headers)
response.encoding = 'utf-8'
page_text = response.text
# 实例化etree对象
tree = etree.HTML(page_text)
# 定位标签
div_list = tree.xpath('//div[@id="container"]/div')
for div in  div_list:
    detail_url = div.xpath('./a/@href')[0]
    name = div.xpath('./a/img/@alt')[0]
    detail_page = requests.get(url=detail_url, headers=headers).text
    tree = etree.HTML(detail_page)
    # 锁定下载地址
    down_list = tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li/a/@href')
    # 随机选取一个下载地址
    down_url = random.choice(down_list)
    # 获取到下载内容的压缩包
    data = requests.get(url= detail_url, headers=headers).content
    fileName = name+'.rar'
    with open(fileName, 'wb') as fp:
        fp.write(data)
        print('下载成功!!')

解决方法:
 1, 当请求成功后马上断开该次请求. 为了及时释放请求池资源,
     ---- 'connection': 'close'
    若首次执行任然报错, 那就再次执行代码.
 2, 使用代理ip
 3, 使用sleep

# 解析所有的城市名称
import requests
from lxml import etree
headers = {
​
}
# 明确要爬取数据的url地址
url = 'https://www.aqistudy.cn/historydata/'
# 确定要爬取的整体数据信息
page_text = requests.get(url=url, headers=headers).text
# 把要爬取的数据生成在etree对象中
tree = etree.HTML(page_text)
# 锁定要爬取信息的具体页面标签, '|'该管道符表示左边成立则执行右边,管道符左右都要空格.
li_list = tree.xpath('//div[@class="bottom"]/ul/li | //div[@class="bottom"]/ul/div[2]/li')
for li in li_list:
    city_name = li.xpath('./a/text()')[0]
    print(city_name)

# 图片懒加载(一种反爬机制)
# 首先排除动态加载和url加密, 直接在Elements中找.
# 当图片还未在可视化范围内时, 图片标签是'src2', 加载后为'src',
# src2 是一个伪属性, 在爬取图片时可以直接用'src2'作为标签, 无需考虑是否在可视化范围内.

# 请求过于频繁, 有被封ip的风险, 可以使用代理ip
# 代理ip的类型必须要和请求url的协议头保持一致
import requests
url = 'https://www.baidu.com/s?cl=3&wd=ip'
headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
# proxies后为代理ip的键值对
page_text = requests.get(url=url, headers=headers, proxies={'https': '103.203.133.250:8080'}).text
with open('./ip.html', 'w', encoding='utf-8')as fp:
    fp.write(page_text)

from selenium import webdriver
from time import sleep
# 需要借助浏览器终端的驱动程序, 创建浏览器对象
bro = webdriver.Chrome(executable_path=r'D:\chromedriver\chromedriver.exe')
# 标明要使用的浏览器
bro.get(url='https://www.baidu.com/')
# 明确搜索框位置
text_input = bro.find_element_by_id('kw')
# 确定搜索框内要搜索的内容
text_input.send_keys('美少女战士')
# 点击搜索
bro.find_element_by_id('su').click()
sleep(4)
# 获取当前页面的源码数据(包括动态加载的数据)
print(bro.page_source)
# 退出
bro.quit()

# 获取更多的详情数据
from selenium import webdriver
from time import sleep
url = '想要获得数据的页面详细地址'
# 调用驱动程序
bro = webdriver.Chrome(executable_path=r'D:\chromedriver\chromedriver.exe')
bro.get(url)
sleep(3)
# 页面向下滚动的js代码(有些页面数据是在页面向下滚动时才刷新出来的)
bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')
sleep(2)
# 拿到所有页面数据
page_text = bro.page_source
with open('./douban.html', 'w',encoding='utf-8')as fp:
    fp.write(page_text)
sleep(2)
bro.quit()

# -*- coding: utf-8 -*-
import scrapy
​
# 该文件作用是进行数据的爬取和解析
class FirstSpider(scrapy.Spider):
    # 爬虫文件的名称: 根据爬虫名称可以定位到指定的爬虫文件
    name = 'first'
    # 允许的域名
    allowed_domains = ['www.baidu.com']
    # 起始URL列表(要爬取的url地址必须在允许的域名下,为了不冲突也可以注释掉域名)
    start_urls = ['https://www.baidu.com/']
​
    # 用于解析: response就是起始URL对应的对象,每执行一次起始url列表中的url,就会调用一次该方法.
    def parse(self, response):
        print(response)

执行输出指定格式进行存储：将爬取到的数据写入不同格式的文件中进行存储
    scrapy crawl 爬虫名称 -o xxx.json
    scrapy crawl 爬虫名称 -o xxx.xml
    scrapy crawl 爬虫名称 -o xxx.csv

# -*- coding: utf-8 -*-
import scrapy
​
# 该文件作用是进行数据的爬取和解析
class FirstSpider(scrapy.Spider):
    # 爬虫文件的名称: 根据爬虫名称可以定位到指定的爬虫文件
    name = 'first'
    # 允许的域名
    # allowed_domains = ['www.baidu.com']
    # 起始URL列表(要爬取的url地址必须在允许的域名下,为了不冲突也可以注释掉域名)
    start_urls = ['https://www.qiushibaike.com/text/']
​
    # 用于解析: response就是起始URL对应的对象,每执行一次起始url列表中的url
    # 就会调用一次该方法.
    def parse(self, response):
        all_data = []
        # xpath 返回的列表元素类型是Select类型
        div_list = response.xpath('//div[@id="content-left"]/div')
        for div in div_list:
            title = div.xpath('./div[1]/a[2]/h2/text() | ./div/span[2]/h2/text()')[0].extract()
            # 如果能保证列表内只有一个元素, 可以用extract_first
            # title = div.xpath('./div[1]/a[2]/h2/text() | ./div/span[2]/h2/text()').extract_first()
            print(title)
            content = div.xpath('./a/div/span/text()').extract()
            # 该种情况不能往数据库中存储
            dic = {
                'title': title,
                'content': content
            }
            all_data.append(dic)
    # 基于终端指令的持久化存储: 可以通过终端指令的形式将parse方法的返回      值中存储的数据进行本地磁盘的持久化存储.
        return all_data

    items.py：数据结构模板文件。定义数据属性。
    pipelines.py：管道文件。接收数据（items），进行持久化操作。
​
持久化流程：
    1.爬虫文件爬取到数据后，需要将数据封装到items对象中。
    2.使用yield关键字将items对象提交给pipelines管道进行持久化操作。
    3.在管道文件中的process_item方法中接收爬虫文件提交过来的item对象，然后编写持久化存储的代码将item对象中存储的数据进行持久化存储
    4.settings.py配置文件中开启管道

# -*- coding: utf-8 -*-
import scrapy
from secondblood.items import SecondbloodItem
​
class QiubaidemoSpider(scrapy.Spider):
    name = 'qiubaiDemo'
    allowed_domains = ['www.qiushibaike.com']
    start_urls = ['http://www.qiushibaike.com/']
​
    def parse(self, response):
        odiv = response.xpath('//div[@id="content-left"]/div')
        for div in odiv:
            # xpath函数返回的为列表，列表中存放的数据为Selector类型的数据。我们解析到的内容被封装在了Selector对象中，需要调用extract()函数将解析的内容从Selecor中取出。
            author = div.xpath('.//div[@class="author clearfix"]//h2/text()').extract_first()
            author = author.strip('\n')#过滤空行
            content = div.xpath('.//div[@class="content"]/span/text()').extract_first()
            content = content.strip('\n')#过滤空行
​
            #将解析到的数据封装至items对象中
            item = SecondbloodItem()
            item['author'] = author
            item['content'] = content
​
            yield item#提交item到管道文件（pipelines.py）

import scrapy
​
​
class SecondbloodItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    author = scrapy.Field() #存储作者
    content = scrapy.Field() #存储段子内容

# -*- coding: utf-8 -*-
​
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
​
​
class SecondbloodPipeline(object):
    #构造方法
    def __init__(self):
        self.fp = None  #定义一个文件描述符属性
　　#下列都是在重写父类的方法：
    #开始爬虫时，执行一次
    def open_spider(self,spider):
        print('爬虫开始')
        self.fp = open('./data.txt', 'w')
​
　　 #因为该方法会被执行调用多次，所以文件的开启和关闭操作写在了另外两个只会各自执行一次的方法中。
    def process_item(self, item, spider):
        #将爬虫程序提交的item进行持久化存储
        self.fp.write(item['author'] + ':' + item['content'] + '\n')
        return item
​
    #结束爬虫时，执行一次
    def close_spider(self,spider):
        self.fp.close()
        print('爬虫结束')

#开启管道
ITEM_PIPELINES = {
    'secondblood.pipelines.SecondbloodPipeline': 300, #300表示为优先级，值越小优先级越高
}

# -*- coding: utf-8 -*-
​
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
​
#导入数据库的类
import pymysql
class QiubaiproPipelineByMysql(object):
​
    conn = None  #mysql的连接对象声明
    cursor = None#mysql游标对象声明
    def open_spider(self,spider):
        print('开始爬虫')
        #链接数据库
        self.conn = pymysql.Connect(host='127.0.0.1',port=3306,user='root',password='123456',db='qiubai')
    #编写向数据库中存储数据的相关代码
    def process_item(self, item, spider):
        #1.链接数据库
        #2.执行sql语句
        sql = 'insert into qiubai values("%s","%s")'%(item['author'],item['content'])
        self.cursor = self.conn.cursor()
        #执行事务
        try:
            self.cursor.execute(sql)
            self.conn.commit()
        except Exception as e:
            print(e)
            self.conn.rollback()
​
        return item
    def close_spider(self,spider):
        print('爬虫结束')
        self.cursor.close()
        self.conn.close()

ITEM_PIPELINES = {
    'qiubaiPro.pipelines.QiubaiproPipelineByMysql': 300,
}

# -*- coding: utf-8 -*-
​
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
​
import redis
​
class QiubaiproPipelineByRedis(object):
    conn = None
    def open_spider(self,spider):
        print('开始爬虫')
        #创建链接对象
        self.conn = redis.Redis(host='127.0.0.1',port=6379)
    def process_item(self, item, spider):
        dict = {
            'author':item['author'],
            'content':item['content']
        }
        #写入redis中
        self.conn.lpush('data', dict)
        return item

ITEM_PIPELINES = {
    'qiubaiPro.pipelines.QiubaiproPipelineByRedis': 300,
}

#该类为管道类，该类中的process_item方法是用来实现持久化存储操作的。
class DoublekillPipeline(object):
​
    def process_item(self, item, spider):
        #持久化操作代码 （方式1：写入磁盘文件）
        return item
​
#如果想实现另一种形式的持久化操作，则可以再定制一个管道类：
class DoublekillPipeline_db(object):
​
    def process_item(self, item, spider):
        #持久化操作代码 （方式1：写入数据库）
        return item

#下列结构为字典，字典中的键值表示的是即将被启用执行的管道文件和其执行的优先级。
ITEM_PIPELINES = {
   'doublekill.pipelines.DoublekillPipeline': 300,
    'doublekill.pipelines.DoublekillPipeline_db': 200,
}
​
#上述代码中，字典中的两组键值分别表示会执行管道文件中对应的两个管道类中的process_item方法，实现两种不同形式的持久化操作。

# -*- coding: utf-8 -*-
import scrapy
from qiushibaike.items import QiushibaikeItem
# scrapy.http import Request
class QiushiSpider(scrapy.Spider):
    name = 'qiushi'
    allowed_domains = ['www.qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/']
​
    #爬取多页
    pageNum = 1 #起始页码
    url = 'https://www.qiushibaike.com/text/page/%s/' #每页的url
​
    def parse(self, response):
        div_list=response.xpath('//*[@id="content-left"]/div')
        for div in div_list:
            #//*[@id="qiushi_tag_120996995"]/div[1]/a[2]/h2
            author=div.xpath('.//div[@class="author clearfix"]//h2/text()').extract_first()
            author=author.strip('\n')
            content=div.xpath('.//div[@class="content"]/span/text()').extract_first()
            content=content.strip('\n')
            item=QiushibaikeItem()
            item['author']=author
            item['content']=content
​
            yield item #提交item到管道进行持久化
​
         #爬取所有页码数据
        if self.pageNum <= 13: #一共爬取13页（共13页）
            self.pageNum += 1
            url = format(self.url % self.pageNum)
​
            #递归爬取数据：callback参数的值为回调函数（将url请求后，得到的相应数据继续进行parse解析），递归调用parse函数
            yield scrapy.Request(url=url,callback=self.parse)

  def start_requests(self):
        for u in self.start_urls:
           yield scrapy.Request(url=u,callback=self.parse)

def start_requests(self):
        #请求的url
        post_url = 'http://fanyi.baidu.com/sug'
        # post请求参数
        formdata = {
            'kw': 'wolf',
        }
        # 发送post请求
        yield scrapy.FormRequest(url=post_url, formdata=formdata, callback=self.parse)

# -*- coding: utf-8 -*-
import scrapy
from moviePro.items import MovieproItem
​
class MovieSpider(scrapy.Spider):
    name = 'movie'
    allowed_domains = ['www.id97.com']
    start_urls = ['http://www.id97.com/']
​
    def parse(self, response):
        div_list = response.xpath('//div[@class="col-xs-1-5 movie-item"]')
​
        for div in div_list:
            item = MovieproItem()
            item['name'] = div.xpath('.//h1/a/text()').extract_first()
            item['score'] = div.xpath('.//h1/em/text()').extract_first()
            #xpath(string(.))表示提取当前节点下所有子节点中的数据值（.）表示当前节点
            item['kind'] = div.xpath('.//div[@class="otherinfo"]').xpath('string(.)').extract_first()
            item['detail_url'] = div.xpath('./div/a/@href').extract_first()
            #请求二级详情页面，解析二级页面中的相应内容,通过meta参数进行Request的数据传递
            yield scrapy.Request(url=item['detail_url'],callback=self.parse_detail,meta={'item':item})
​
    def parse_detail(self,response):
        #通过response获取item
        item = response.meta['item']
        item['actor'] = response.xpath('//div[@class="row"]//table/tr[1]/a/text()').extract_first()
        item['time'] = response.xpath('//div[@class="row"]//table/tr[7]/td[2]/text()').extract_first()
        item['long'] = response.xpath('//div[@class="row"]//table/tr[8]/td[2]/text()').extract_first()
        #提交item到管道
        yield item

# -*- coding: utf-8 -*-
​
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
​
import scrapy
​
​
class MovieproItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    score = scrapy.Field()
    time = scrapy.Field()
    long = scrapy.Field()
    actor = scrapy.Field()
    kind = scrapy.Field()
    detail_url = scrapy.Field()

# -*- coding: utf-8 -*-
​
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
​
import json
class MovieproPipeline(object):
    def __init__(self):
        self.fp = open('data.txt','w')
    def process_item(self, item, spider):
        dic = dict(item)
        print(dic)
        json.dump(dic,self.fp,ensure_ascii=False)
        return item
    def close_spider(self,spider):
        self.fp.close()

增加并发：
    默认scrapy开启的并发线程为32个，可以适当进行增加。在settings配置文件中修改CONCURRENT_REQUESTS = 100值为100,并发设置成了为100。
​
降低日志级别：
    在运行scrapy时，会有大量日志信息的输出，为了减少CPU的使用率。可以设置log输出信息为INFO或者ERROR即可。在配置文件中编写：LOG_LEVEL = ‘INFO’
​
禁止cookie：
    如果不是真的需要cookie，则在scrapy爬取数据时可以进制cookie从而减少CPU的使用率，提升爬取效率。在配置文件中编写：COOKIES_ENABLED = False
​
禁止重试：
    对失败的HTTP进行重新请求（重试）会减慢爬取速度，因此可以禁止重试。在配置文件中编写：RETRY_ENABLED = False
​
减少下载超时：
    如果对一个非常慢的链接进行爬取，减少下载超时可以能让卡住的链接快速被放弃，从而提升效率。在配置文件中进行编写：DOWNLOAD_TIMEOUT = 10 超时时间为10s

# -*- coding: utf-8 -*-
import scrapy
from xiaohua.items import XiaohuaItem
​
class XiahuaSpider(scrapy.Spider):
​
    name = 'xiaohua'
    allowed_domains = ['www.521609.com']
    start_urls = ['http://www.521609.com/daxuemeinv/']
​
    pageNum = 1
    url = 'http://www.521609.com/daxuemeinv/list8%d.html'
​
    def parse(self, response):
        li_list = response.xpath('//div[@class="index_img list_center"]/ul/li')
        for li in li_list:
            school = li.xpath('./a/img/@alt').extract_first()
            img_url = li.xpath('./a/img/@src').extract_first()
​
            item = XiaohuaItem()
            item['school'] = school
            item['img_url'] = 'http://www.521609.com' + img_url
​
            yield item
​
        if self.pageNum < 10:
            self.pageNum += 1
            url = format(self.url % self.pageNum)
            #print(url)
            yield scrapy.Request(url=url,callback=self.parse)

# -*- coding: utf-8 -*-
​
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
​
import scrapy
​
​
class XiaohuaItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    school=scrapy.Field()
    img_url=scrapy.Field()

# -*- coding: utf-8 -*-
​
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
​
import json
import os
import urllib.request
class XiaohuaPipeline(object):
    def __init__(self):
        self.fp = None
​
    def open_spider(self,spider):
        print('开始爬虫')
        self.fp = open('./xiaohua.txt','w')
​
    def download_img(self,item):
        url = item['img_url']
        fileName = item['school']+'.jpg'
        if not os.path.exists('./xiaohualib'):
            os.mkdir('./xiaohualib')
        filepath = os.path.join('./xiaohualib',fileName)
        urllib.request.urlretrieve(url,filepath)
        print(fileName+"下载成功")
​
    def process_item(self, item, spider):
        obj = dict(item)
        json_str = json.dumps(obj,ensure_ascii=False)
        self.fp.write(json_str+'\n')
​
        #下载图片
        self.download_img(item)
        return item
​
    def close_spider(self,spider):
        print('结束爬虫')
        self.fp.close()

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
​
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
​
# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 100
COOKIES_ENABLED = False
LOG_LEVEL = 'ERROR'
RETRY_ENABLED = False
DOWNLOAD_TIMEOUT = 3
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
DOWNLOAD_DELAY = 3