[775]小象学院--爬虫知识点总结

pip search lxml

# mkdir ~/.pip/
# vim ~/.pip/pip.conf
[global]
index-url=https://pypi.tuna.tsinghua.edu.cn/simple

# pip install –i https://pypi.tuna.tsinghua.edu.cn/simple lxml

 Accept: text/plain
 Accept-Charset: utf-8
 Accept-Encoding: gzip, deflate
 Accept-Language: en-US
 Connection: keep-alive
 Content-Length: 348
 Content-Type: application/x-www-form-urlencoded
 Date: Tue, 15 Nov 1994 08:12:31 GMT
 Host: en.wikipedia.org:80
 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:12.0) Gecko/20100101Firefox/21.0
 Cookie: $Version=1; Skin=new;

site:www.mafengwo.cn

pip install murmurhash3 bitarray

from bitarray import bitarray
import mmh3

offset = 2147483647 // 2^31 - 1
bit_array = bitarray(4*1024*1024*1024)
bit_array.setall(0)
# mmh3 hash value 32 bit signed int
# add offset to make it unsigned int 0 ~ 2^32-1
b1 = mmh3.hash(url, 42) + offset
bit_array[b1] = 1

pip install pybloomfilter
git clone https://github.com/axiak/pybloomfiltermmap.git

>>> fruit = pybloomfilter.BloomFilter(100000, 0.1, '/tmp/words.bloom')
>>> fruit.update(('apple', 'pear', 'orange', 'apple'))
>>> len(fruit) 3
>>> 'mike' in fruit
False
>>> 'apple' in fruit
True

html = etree.HTML(html_content.lower().decode('utf-8'))
hrefs = html.xpath(u"//a")
hrefs = html.xpath(u'//a[@class="last-page"]')
hrefs = html.xpath(u'//*[@class="last-page"]')

html = lxml.html.fromstring(html_content)
elements = html.cssselect('div#page-let > a.last-page')

from hdfs import *
from hdfs.util import HdfsError
import mysql.connector
from mysql.connector import errorcode,pooling
import http.client

hdfs_client = InsecureClient('http://54.223.92.169:50070', user='ec2-user')
self.cnxpool = mysql.connector.pooling.MySQLConnectionPool(pool_name = "mypool",
                                                          pool_size = max_num_thread,
                                                          **dbconfig)

pip install hdfs

from hdfs import *
from hdfs.util import HdfsError

hdfs_client = InsecureClient(’[host]:[port]', user=’user')
with hdfs_client.write('/htmls/mfw/%s.html' % (filename)) as writer:
    writer.write(html_page)

except HdfsError as Arguments:
    print Arguments

#create an INET, STREAMing socket
s = socket.create_connection( socket.AF_INET,socket.SOCK_STREAM)

#create an INET, STREAMing socket
serversocket = socket.socket( socket.AF_INET,socket.SOCK_STREAM)
#bind the socket to a public host, and a well-known port
serversocket.bind((socket.gethostname(), 20010))
#become a server socket
serversocket.listen(5)
while True:
  #accept connections from outside
  (clientsocket, address) = serversocket.accept()
  #now do something with the clientsocket
  #in this case, we'll pretend this is a threaded
  server ct = client_thread(clientsocket)
  ct.run()

pip install networkx

g = nx.DiGraph() # 构造有向图
g.add_node(url)
g.add_edge(src, dest) # 添加边
nx.pagerank(g, 0.9) # 计算pagerank，g为有向图，0.9是PR的随机跳转概率（也称为阻尼系数）

import urllib2
url = http://jc.lo/dev/login/login.php
headers = {
'host': "jc.lo",
'connection': "keep-alive",
'cache-control': "no-cache",
'content-type': "application/x-www-form-urlencoded",
'upgrade-insecure-requests': "1",
'accept-language': "zh-CN,en-US;q=0.8,en;q=0.6",
}
data = {'name':'caca', "password":'c'}
payload = urllib.urlencode(data)
request = urllib2.Request(url, payload , headers=headers)
response = urllib2.urlopen(request)

proxy_handler = urllib2.ProxyHandler({'http': 'http://www.example.com:3128/'})
proxy_auth_handler = urllib2.ProxyBasicAuthHandler()
proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
opener = urllib2.build_opener(proxy_handler, proxy_auth_handler)
# This time, rather than install the OpenerDirector, we use it directly:
opener.open('http://www.example.com/login.html')

import cookielib
cj = cookielib.CookieJar() #创建 CookieJar 对象
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) #注册插件
request = urllib2.Request(url, payload, headers=headers)
response = opener.open(request)
print response.items() #打印headers信息
print response.read() #打印返回的网页
#打印 cookie 内容
for cookie in cj:
  print cookie.name, cookie.value, cookie.domain

# import webdriver from selenium
from selenium import webdriver
# load PhantomJS driver
driver = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true'])
# set window size, better to fit the whole page in order to
# avoid dynamically loading data
driver.set_window_size(1280, 2400) # optional
# data page content
driver.get(cur_url)
# use page_source to get html content
content = driver.page_source

--ignore-ssl-errors=[true|false] ignores SSL errors, such as expired or self-signed certificate errors (default is false). Also accepted: [yes|no].

--load-images=[true|false] load all inlined images (default is true). Also accepted: [yes|no].

--disk-cache=[true|false] enables disk cache (at desktop services cache storage location, default is false). Also accepted: [yes|no].

--cookies-file=/path/to/cookies.txt specifies the file name to store the persistent Cookies.

--debug=[true|false] prints additional warning and debug message,default is false. Also accepted: [yes|no].

--config specifies JSON-formatted configuration file (see below).

scrapy runspider quotes_spider.py -o quotes.json
scrapy startproject chdemo 创建新的工程
scrapy genspider mfw mafengwo.com 创建mfw的spider

pip install simhash

from simhash import Simhash
str0 = 'The Apache Hadoop software library is a framework that allows for
the distributed processing large data'
str1 = 'The Apache Hadoop software library is a framework that allows for
the distributed processing big data'
# 构造 SimHash 对象
sh0 = Simhash(str0)
sh1 = Simhash(str1)
# 构造特征值，关键字加权
features = [('Apache', 10),('Hadoop', 15),('framework', 3), ('distributed',
10), ('data', 6)]
# 不加权计算
sh0.distance(sh1)
# 加权计算海明距离
sh0.build_by_features(features)
sh1.build_by_features(features)
sh0.distance(sh1)

pip install pillow

from PIL import Image
import lxml

def extract_image(html):
    tree = lxml.html.fromstring(html)
    img_data = tree.cssselect('div#recaptcha img')[0].get('src')
    img_data = img_data.partition(',')[-1]
    binary_img_data = img_data.decode('base64')
    img_data = BytesIO(binary_img_data)
    img = Image.open(img_data)
    img.save('test.png')
    return img

pip install pytesseract

import pytesseract

pytesseract.image_to_string(bw)

pixdata = img.load()
colors = {}
#  统计字符颜色像素情况
for y in range(img.size[1]):
    for x in range(img.size[0]):
        if colors.has_key(pixdata[x,y]):
            colors[pixdata[x, y]] += 1
        else:
            colors[pixdata[x,y]] = 1
#  排名第一的是背景色，第二的是主要颜色
colors = sorted(colors.items(), key=lambda d:d[1], reverse=True)

significant  = colors[1][0]
for y in range(img.size[1]):
    for x in range(img.size[0]):
        if pixdata[x,y]  != significant:
            pixdata[x,y]  = (255,255,255)
        else:
            pixdata[x, y]  = (0,0,0)

word = pytesseract.image_to_string(img, lang='eng', config='ocr.conf')

driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')

<script language="javascript" type="text/javascript">if (typeof M !==
"undefined" && typeof M.loadResource === "function")
{M.loadResource("http://js.mafengwo.net/js/cv/js+pageletcommon+pageHea
dUserInfoWWWDark:js++ACnzzGaLog:js+ARecruit:js+ALazyLoad^Z11V^148
9552560.js");}</script>

from lxml.html import clean

cleaner = clean.Cleaner(style=True, scripts=True, comments=True, javascript=True,page_structure=False, safe_attrs_only=False)
content = cleaner.clean_html(content.decode('utf-8')).encode('utf-8')

reg = re.compile("<[^>]*>")
content = reg.sub('', content)