爬取knewone上的信息-- coding: utf-8 -

爬取knewone上的信息 <pre> author = 'LEE'

-- coding: utf-8 -

from bs4 import BeautifulSoup import time import requests import time import io import sys import urllib.request sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')

url = 'https://knewone.com/things/?page=' def get_page(url,data=None): wb_data = requests.get(url) soup = BeautifulSoup(wb_data.text,'lxml') imgs = soup.select('a.cover-inner > img') titles =soup.select('section.content > h4 > a') links = soup.select('section.content > h4 > a') #print(soup) if data ==None: for img,title,link in zip(imgs,titles,links): data = { 'img':img.get('src'), 'title':title.get('title'), 'link':link.get('href'), } print(data) def get_more_page(start,end): for one in range(start,end): get_page(url+str(one)) time.sleep(1)

get_more_page(1,10)

</pre>