Python爬虫之小猪短租房一、xpath爬取分析二、代码三、简单分析四、问题

import requests
from lxml import etree
import pymongo

client = pymongo.MongoClient('localhost', 27017)
test = client['test']
xiaozhu = test['xiaozhu']

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}

urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)]

def get_info(url):
    html = requests.get(url,headers=headers)
    selector = etree.HTML(html.text)
    commoditys = selector.xpath('//ul[@class="pic_list clearfix"]/li')

    for commodity in commoditys:
        address = commodity.xpath('div/div/a/span[@class="result_title hiddenTxt"]/text()')[0]
        price = commodity.xpath('div/span[@class="result_price"]/i/text()')[0]
        lease_type = commodity.xpath('div/div/em/text()')[0].split('/')[0].strip()
        bed_amount = commodity.xpath('div/div/em/text()')[0].split('/')[1].strip()
        suggestion = commodity.xpath('div/div/em/text()')[0].split('/')[2].strip()
        infos = commodity.xpath('div/div/em/span/text()')[0].strip()
        comment_star = infos.split('/')[0] if '/' in infos else '无'
        comment_amount = infos.split('/')[1] if '/' in infos else infos

        content = {
            'address':address,
            'price':price,
            'lease_type':lease_type,
            'bed_amount':bed_amount,
            'suggestion':suggestion,
            'comment_star':comment_star,
            'comment_amount':comment_amount
        }
        xiaozhu.insert_one(content)

for url in urls:
    get_info(url)

mongoexport -d test -c xiaozhu --csv -f address,price,lease_type,bed_amount,suggestion,comment_star,comment_amount -o xiaozhu.csv