使用python批量爬取主流搜索引擎图片

from bs4 import BeautifulSoup
import urllib.request
import requests
import time
import json
import sys
import re
import os

#爬取目标网站url
CRAWL_TARGET_URL = 'https://cn.bing.com/images/async?q=%s&first=%d&count=%d&relp=%d&lostate=r&mmasync=1'
#每次抓取图片数量(35是此网页每次翻页请求数量)
NUMS_PER_CRAWL = 35
#抓取图片最小大小(单位字节)，小于此值抛弃
MIN_IMAGE_SIZE = 10


def get_image(url, path, count):
    try:
        u = urllib.request.urlopen(url, timeout=5)
        t = u.read()
        if sys.getsizeof(t) < MIN_IMAGE_SIZE:
            return -1
    except Exception as e:
        print(url, e)
        return -2
    #提取图片格式
    frmt = url[url.rfind('.'):]
    p = re.compile("^\\.[a-zA-Z]+")
    m = p.match(frmt)
    frmt = m.group(0)
    try:
        if not os.path.exists(path):
            os.mkdir(path)
        f = open(os.path.join(path, str(count)+frmt), 'wb')
        f.write(t)
        f.close()
    except Exception as e:
        print(os.path.join(path, str(count)+frmt), e)
        return -3
    return 0


def crawl_data(info, num):
    first = 0
    count = 0
    #创建一个会话
    s = requests.Session()
    #创建文件路径
    path="./"+info
    if not os.path.exists(path):
        os.mkdir(path)
    index=len(os.listdir(path))#文件中原有图片数
    while(count < num):
        u = CRAWL_TARGET_URL%(info, first, NUMS_PER_CRAWL, NUMS_PER_CRAWL)
        #3.05s为发送超时时间，10s为接收到数据超时时间
        req = s.get(url =u, timeout=(3.05, 10))
        bf = BeautifulSoup(req.text, "html.parser")
        imgtags = bf.find_all("a", class_ = "iusc")
        for e in imgtags:
            if count == num:
                return False
            urldict = json.loads(e.get('m'))
            if get_image(urldict["murl"], path, index) < 0:
                continue
            print("Downloaded %d picture"%(count+1))
            sys.stdout.flush()
            count =count+1
            index=index+1
            time.sleep(0.01)
        first = first + NUMS_PER_CRAWL
        time.sleep(0.1)

    return True

if __name__ == '__main__':
    
    # 关键词，可设置为多个
    key_words=['戴口罩',]
    # 下载的图片数量
    picture_num = 10000

    for i in range(len(key_words)):
        word=key_words[i]
        print(word)
        if crawl_data(word, picture_num):
            i=i+1

import json
import os
import requests

# 路径
BASE_URL = './戴口罩'
# 关键词
NAME = '戴口罩'

class PictureDownload(object):
    def __init__(self, q=None, sn=100):
        self.url = 'https://m.image.so.com/j?q={}&src=srp&pn=100&sn={}&kn=0&gn=0&cn=0'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
        }
        self.q = q
        self.sn = sn
        self.num = 0
        self.total = 2

    def makedir(self):
        if not os.path.exists(os.path.join(BASE_URL, self.q)):
            os.makedirs(os.path.join(BASE_URL, self.q))

    def parse_url(self):
        response = requests.get(self.url.format(self.q, self.num), headers=self.headers)
        return response.content.decode()

    def parse_image_list(self, html_json_str):
        image_list = json.loads(html_json_str)['list']
        total = json.loads(html_json_str)['total']
        return image_list, total

    def save_image(self, image_list):

        for item in image_list:
            response = requests.get(item['thumb'], headers=self.headers)
            with open(os.path.join(BASE_URL, '%s\%s.jpg' % (self.q, item['index'])), 'wb') as f:
                f.write(response.content)

    def run(self):
        self.makedir()
        while self.num < self.total:
            html_json_str = self.parse_url()
            image_list, self.total = self.parse_image_list(html_json_str)
            self.save_image(image_list)
            self.num += 100
            print(self.num)


if __name__ == '__main__':
    xxx = PictureDownload(NAME)
    xxx.run()

import requests
import json
import urllib

# 三个参数，你要获取整个图片集的名字，你要获取多少张，获取过来的放在哪里
def getSogoulmag(category,length,path):
    n=length
    cate=category
    # 获取的是图片所有信息
    imgs=requests.get('http://pic.sogou.com/pics/channel/getAllRecomPicByTag.jsp?category='+cate+
                      '&tag=%E5%85%A8%E9%83%A8&start=0&len='+str(n))

    # 转换成为json格式
    jd=json.loads(imgs.text)
    # all_items所有的图片
    jd=jd['all_items']

    imgs_t=[]
    for j in jd:
        # 通过定位bthumbUrl获取图片
        imgs_t.append(j['bthumbUrl'])
    m=0
    for img in imgs_t:
        # 打印某一张图片正在下载
        print(str(m)+'.jpg'+'Downlod......')
        # 用来把远程数据下载到本地
        urllib.request.urlretrieve(img,path+str(m)+'.jpg')
        m=m+1
    print('Complete!')
    
# 调用整个处理逻辑
# 三个参数，你要获取整个图片集的名字，你要获取多少张，获取过来的放在哪里
getSogoulmag('戴口罩',10,'./sougoutupian/')