《手把手带你学爬虫──初级篇》第4课 正则表达式以及re库的用法

import re

In [22]: ls = re.split(r'([1-9]\d{5})', '北京海淀100036 郑州高新450001')

In [23]: ls
Out[23]: ['北京海淀', '100036', ' 郑州高新', '450001', '']

In [31]: regex = re.compile(r'[1-9]\d{5}')

In [32]: regex.sub(repl=":邮编",string="北京海淀100036 郑州高新450001")
Out[32]: '北京海淀:邮编 郑州高新:邮编'

In [14]: match = re.match(r'[1-9]\d{5}', '100036 北京海淀')
In [15]: match.group(0)
Out[15]: '100036'
    ... ... ... ...
In [33]: type(match)
Out[33]: _sre.SRE_Match

In [34]: match.re
Out[34]: re.compile(r'[1-9]\d{5}', re.UNICODE)

In [35]: match.string
Out[35]: '100036 北京海淀'

In [36]: match.pos
Out[36]: 0

In [37]: match.endpos
Out[37]: 11

# 匹配的结构存储在这里
In [38]: match.group(0)
Out[38]: '100036'

In [39]: match.start()
Out[39]: 0

In [40]: match.end()
Out[40]: 6

In [41]: match.span()
Out[41]: (0, 6)

In [49]: match = re.search(r'one.*n','oneanbncndnen')

In [50]: match.group(0)
Out[50]: 'oneanbncndnen'

In [51]: match = re.search(r'one.*?n','oneanbncndnen')

In [52]: match.group(0)
Out[52]: 'onean'

https://search.jd.com/Search?keyword=书包&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=3&stock=1&page=1&s=1&click=0

{
    "keyword": "书包",
    "enc": utf-8",
    "qrst": 1",
    "rt": 1",
    "stop": 1",
    "vt": 2",
    "psort": 3",
    "stock": 1",
    "page": 1",
    "s": 1",
    "click: 0",
}

第1页 page=1
https://search.jd.com/Search?keyword=书包&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=3&stock=1&page=1&s=1&click=0

第2页 page=3
https://search.jd.com/Search?keyword=书包&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=3&stock=1&page=3&s=61&click=0

{
        'keyword': '书包',
        'enc': 'utf-8',
        'qrst': '1',
        'rt': '1',
        'stop': '1',
        'vt': '2',
        'psort': '3',
        'stock': '1',
        'page': '2',
        's': '31',
        'scrolling': 'y',
        'log_id': '1529759067.93124',
        'tpl': '3_M',
        'show_items': '2241345,4153317,10285134151,10503046182,13180766655,6117013,2330770,5168648,10285341597,26194798282,5471930,2804865,28561551177,1585517611,1025981997,6174172,11693982705,10540540570,5386778,11096315457,20682615258,23533395828,5248730,27306328004,3929123,5218868,12422531859,4208541,5634094,2330752'
    }

import requests
import re
from bs4 import BeautifulSoup
import time
import json


def get_html(url, params=None):
    headers = {
        'Referer': 'https://search.jd.com',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:58.0) Gecko/20100101 Firefox/58.0'
    }
    try:
        r = requests.get(url=url, timeout=30, headers=headers, params=params)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        print("### 正在请求的链接为：{}".format(r.url))
        return r.text
    except Exception as e:
        print(str(e))


def set_even_params(html, even_params):
    soup = BeautifulSoup(html, "lxml")
    li_list = soup.find_all(class_="gl-item")
    show_items = []
    for li in li_list:
        show_items.append(li.attrs['data-pid'])

    log_id = re.compile(r"log_id:'\d+.\d+'").search(html).group(0).replace('log_id:', '')

    even_params['show_items'] = ",".join(show_items)
    even_params['log_id'] = log_id


def parse_page(html):
    soup = BeautifulSoup(html, "lxml")
    li_list = soup.find_all(class_='gl-item')

    products = []

    for gl_item in li_list:
        product = {}

        product['slogan'] = gl_item.find(class_='p-img').a.attrs['title']
        product['url'] = "https:" + gl_item.find(class_='p-img').a.attrs['href']
        product['price'] = gl_item.find(class_='p-price').strong.i.string
        pin_gou = gl_item.find(class_='price-pingou')
        if pin_gou is None:
            product['pin_price'] = ""
        else:
            product['pin_price'] = re.compile(r'\d+.\d+').search(str(pin_gou)).group(0)

        product['name'] = gl_item.find(class_='p-name').a.em.get_text()
        product['comment'] = gl_item.find(class_='p-commit').strong.a.string

        a_shop = gl_item.find(class_='p-shop').find('a')
        if a_shop is None:
            product['shop_name'] = ""
            product['shop_url'] = ""
        else:
            product['shop_name'] = a_shop.attrs['title']
            product['shop_url'] = "https:" + a_shop.attrs['href']
        products.append(product)

    return products


def main():
    # 搜索关键词
    goods = '书包'
    # 爬取深度，最大100页
    depth = 10
    # 结果列表
    products = []
    # 每页商品的起始数字
    s = 1

    odd_params = {
        'keyword': goods,
        'enc': 'utf-8',
        'wq': goods,
        'page': '1',
        'psort': '3',
        's': '1'
    }
    odd_url = 'https://search.jd.com/Search'

    even_params = {
        'keyword': goods,
        'enc': 'utf-8',
        'qrst': '1',
        'rt': '1',
        'stop': '1',
        'vt': '2',
        'psort': '3',
        'stock': '1',
        'page': '2',
        's': '31',
        'scrolling': 'y',
        'log_id': '1529759067.93124',
        'tpl': '3_M',
        'show_items': '2241345,4153317,10285134151,10503046182,13180766655,6117013,2330770,5168648,10285341597,26194798282,5471930,2804865,28561551177,1585517611,1025981997,6174172,11693982705,10540540570,5386778,11096315457,20682615258,23533395828,5248730,27306328004,3929123,5218868,12422531859,4208541,5634094,2330752'
    }
    even_url = 'https://search.jd.com/s_new.php'

    for num in range(1, 2 * depth + 1):
        # 奇数
        if num % 2 != 0:
            page_id = num
            odd_params['page'] = page_id
            odd_params['s'] = str(s)
            html = get_html(url=odd_url, params=odd_params)
            set_even_params(html, even_params)
            products.append(parse_page(html))
        # 偶数
        else:
            page_id = num
            even_params['page'] = page_id
            even_params['s'] = str(s)
            html = get_html(url=even_url, params=even_params)
            products.append(parse_page(html))
        # 每次奇偶数页面迭代，都是30个商品
        s += 30
    # 将结果保存到文件中，以json格式
    localtime = "-".join(time.asctime(time.localtime(time.time())).split(' '))
    with open("jd_" + localtime + '.json', 'w') as filename:
        filename.write(json.dumps(products, ensure_ascii=False))
    print("### 爬取完毕")


if __name__ == '__main__':
    main()