python多线程多队列（BeautifulSoup网络爬虫）

    import Queue

    import threading

    import urllib2

    import time

    from BeautifulSoup import BeautifulSoup

hosts = ["http://yahoo.com","http://taobao.com","http://apple.com",  
         "http://ibm.com","http://www.amazon.cn"]  

queue = Queue.Queue()#存放网址的队列  
out_queue = Queue.Queue()#存放网址页面的队列  

class ThreadUrl(threading.Thread):  
    def __init__(self,queue,out_queue):  
        threading.Thread.__init__(self)  
        self.queue = queue  
        self.out_queue = out_queue  

    def run(self):  
        while True:  
            host = self.queue.get()  
            url = urllib2.urlopen(host)  
            chunk = url.read()  
            self.out_queue.put(chunk)#将hosts中的页面传给out_queue  
            self.queue.task_done()#传入一个相当于完成一个任务  

class DatamineThread(threading.Thread):  
    def __init__(self,out_queue):  
        threading.Thread.__init__(self)  
        self.out_queue = out_queue  

    def run(self):  
        while True:  
            chunk = self.out_queue.get()  
            soup = BeautifulSoup(chunk)#从源代码中搜索title标签的内容  
            print soup.findAll(['title'])  
            self.out_queue.task_done()  

start = time.time()  
def main():  
    for i in range(5):  
        t = ThreadUrl(queue,out_queue)#线程任务就是将网址的源代码存放到out_queue队列中  
        t.setDaemon(True)#设置为守护线程  
        t.start()  

    #将网址都存放到queue队列中  
    for host in hosts:  
        queue.put(host)  

    for i in range(5):  
        dt = DatamineThread(out_queue)#线程任务就是从源代码中解析出<title>标签内的内容  
        dt.setDaemon(True)  
        dt.start()  

    queue.join()#线程依次执行，主线程最后执行  
    out_queue.join()  

main()  
print "Total time :%s"%(time.time()-start)  </pre>