Python数据科学（五）- 数据处理和数据采集1.处理不同格式的数据2.网络爬虫3.小试牛刀

import requests
import pandas
from bs4 import BeautifulSoup

res = requests.get('https://news.qq.com/')  # 数据采集目标地址
soup = BeautifulSoup(res.text, 'html.parser') # 解析网页

newsary = []   # 定义空列表
for news in soup.select('.Q-tpWrap .text'):
    newsary.append({'title': news.select('a')[0].text,
                    'url':news.select('a')[0]['href']}) # 分别获取超链接中文本信息和href属性,即地址

newdf = pandas.DataFrame(newsary) # 创建一个DataFrame
newsdf.to_excel('news.xlsx')   # 输出到excel表格
print(newsary[0]) 

import requests
import pandas as pd
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua_list = UserAgent()   # 设置user-agent列表,每次请求时，随机挑选一个user-agent

my_headers = {
        'user-agent': ua_list.random
    }

# 获取所有的url
def get_url():
    num = 1
    sum_url = []
    while num < 101:
        usual_url = 'http://esf.sh.fang.com/house/i3'
        home_url = usual_url + str(num)
        print(home_url)
        res = requests.get(url=home_url, headers=my_headers)
        num+=1
        soup = BeautifulSoup(res.text, 'html.parser')
        domain = 'http://esf.sh.fang.com'
        for house in soup.select('.houseList dl'):
            try:
                # title = house.select('.title')[0].text.strip()  # 清除多余的换行
                url1 = domain + house.select('.title a')[0]['href']
                sum_url.append(url1)
            except Exception as e:
                print(e)
    print(len(sum_url))
    return sum_url


def houseary():
    houseary_url = get_url()
    houseary = []
    for url in houseary_url:
        print(url)
        content = requests.get(url=url, headers=my_headers)
        soup1 = BeautifulSoup(content.text, 'html.parser')
        try:
            info = {}
            info['标题'] = soup1.select('.title')[0].text.strip()
            info['总价'] = soup1.select('.trl-item')[0].text
            info['户型'] = soup1.select('.tt')[0].text.strip()
            info['建筑面积'] = soup1.select('.tt')[1].text
            info['单价'] = soup1.select('.tt')[2].text
            info['朝向'] = soup1.select('.tt')[3].text
            info['楼层'] = soup1.select('.tt')[4].text
            info['装修'] = soup1.select('.tt')[5].text
            info['小区'] = soup1.select('.rcont')[0].text.strip().replace('\n', '')
            info['区域'] = soup1.select('.rcont')[1].text.replace('\n', '').replace('\r', '').replace(' ', '')
            info['经纪人'] = soup1.select('.pn')[0].text
            info['联系电话'] = soup1.select('.pnum')[0].text
            houseary.append(info)
        except Exception as e:
            print(e)

    print(houseary)
    print(len(houseary))
    df = pd.DataFrame(houseary)
    df.to_excel('house.xlsx')


if __name__ == '__main__':
    houseary()