Python爬虫，批量下载小说网站上的小说

#coding=utf-8
 2 import urllib
 3 import urllib2
 4 import re
 5 import os
 6 
 7 webroot = 'http://www.xuanshu.com'
 8 
 9 for page in range(20,220):
10     print '正在下载第'+str(page)+'页小说'
11     
12     url = 'http://www.xuanshu.com/soft/sort02/index_'+str(page)+'.html'
13     headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'  }
14     try:
15         request = urllib2.Request(url,headers=headers)
16         response = urllib2.urlopen(request,timeout=180)
17         #print response.read()
18     except urllib2.URLError, e:
19         if hasattr(e,"code"):
20             print e.code
21         if hasattr(e,"reason"):
22             print e.reason
23 
24     html = response.read().decode('utf-8')
25     #print html
26     pattern = re.compile(u'<li>.*?<div class="s">.*?target="_blank">(.*?)</a><br />大小：(.*?)<br>.*?</em><br>更新：(.*?)</div>.*?<a href="(.*?)"><img.*?>(.*?)</a>.*?<div class="u">(.*?)</div>',re.S)
27     items = re.findall(pattern,html)
28     #print items
29 
30     for item in items:
31         try:
32             book_auther = item[0].encode('gbk')
33             book_size = item[1].encode('gbk')
34             book_updatetime = item[2].encode('gbk')
35             book_link = item[3].encode('gbk')
36             book_name = item[4].encode('gbk')
37             book_note = item[5].encode('gbk')
38             
39             book_full_link = webroot + book_link    # 构建书的绝对地址
40 
41             #请求地址
42             try:
43                 request = urllib2.Request(book_full_link,headers=headers)
44                 response = urllib2.urlopen(request,timeout=180)
45             except urllib2.URLError, e:
46                 if hasattr(e,"code"):
47                     print e.code
48                 if hasattr(e,"reason"):
49                     print e.reason
50             html = response.read().decode('utf-8')
51             #print html
52             pattern = re.compile('<a class="downButton.*?<a class="downButton" href=\'(.*?)\'.*?Txt.*?</a>',re.S)
53             down_link = re.findall(pattern,html)
54             print book_name
55             print down_link
56             
57             # down txt
58             try:
59                 request = urllib2.Request(down_link[0].encode('utf-8'),headers=headers)
60                 response = urllib2.urlopen(request,timeout=180)
61             except urllib2.URLError, e:
62                 if hasattr(e,"code"):
63                     print e.code
64                 if hasattr(e,"reason"):
65                     print e.reason
66             try:
67                 fp = open(book_name+'.txt','w')
68             except IOError,e:
69                 pattern = re.compile('<strong>.*?>(.*?)<.*?</strong>',re.S)
70                 book_name = re.findall(pattern,book_name)
71                 fp = open(book_name[0]+'.txt','w')
72             print 'start download'
73             fp.write(response.read())
74             print 'down finish\n'
75             fp.close()
76         except Exception,e:
77             print '该条目解析出现错误，忽略'
78             print e
79             print ''
80             fp = open('error.log','a')
81             fp.write('page:'+str(page)+'\n')
82             fp.write(item[4].encode('gbk'))
83             #fp.write(e)
84             fp.write('\nThere is an error in parsing process.\n\n')
85             fp.close()