问刮刮式网络爬虫与数据抽取器

EN

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy import log
from krakenkrawler.items import KrakenItem

class AttractionSpider(CrawlSpider):
name = "thekraken"
allowed_domains = ["youtube.com"]
start_urls = [
    "http://www.youtube.com/?gl=GB&hl=en-GB"
]
rules = ()

def __init__(self, name=None, **kwargs):
    super(AttractionSpider, self).__init__(name, **kwargs)
    self.items_buffer = {}
    self.base_url = "http://www.youtube.com"
    from scrapy.conf import settings
    settings.overrides['DOWNLOAD_TIMEOUT'] = 360

def parse(self, response):
    print "Start scrapping Attractions...."
    try:
        hxs = HtmlXPathSelector(response)
        links = hxs.select("//h3[@class='yt-lockup-title']//a/@href")

        if not links:
            return
            log.msg("No Data to scrap")

        for link in links:
            v_url = ''.join( link.extract() )

            if not v_url:
                continue
            else:
                _url = self.base_url + v_url
                yield Request( url= _url, callback=self.parse_details )
    except Exception as e:
        log.msg("Parsing failed for URL {%s}"%format(response.request.url))
        raise 

def parse_details(self, response):
    print "Start scrapping Detailed Info...."
    try:
        hxs = HtmlXPathSelector(response)
        l_venue = KrakenItem()

        v_name = hxs.select("//*[@id='eow-title'].text").extract()
        if not v_name:
            v_name = hxs.select("//*[@id='eow-title'].text").extract()

        l_venue["name"] = v_name[0].strip()

        base = hxs.select("//*[@id='content']/div[7]")
        if base.extract()[0].strip() == "<div style=\"clear:both\"></div>":
            base = hxs.select("//*[@id='content']/div[8]")
        elif base.extract()[0].strip() == "<div style=\"padding-top:10px;margin-top:10px;border-top:1px dotted #DDD;\">\n  You must be logged in to add a tip\n  </div>":
            base = hxs.select("//*[@id='content']/div[6]")

        x_datas = base.select("div[1]/b").extract()
        v_datas = base.select("div[1]/text()").extract()
        i_d = 0;
        if x_datas:
            for x_data in x_datas:
                print "data is:" + x_data.strip()
                if x_data.strip() == "<b>Address:</b>":
                    l_venue["address"] = v_datas[i_d].strip()
                if x_data.strip() == "<b>Contact:</b>":
                    l_venue["contact"] = v_datas[i_d].strip()
                if x_data.strip() == "<b>Operating Hours:</b>":
                    l_venue["hours"] = v_datas[i_d].strip()
                if x_data.strip() == "<b>Website:</b>":
                    l_venue["website"] = (base.select("//*[@id='watch-actions-share-panel']/div/div[2]/div[2]/span[1]/input/text()").extract())[0].strip()

                i_d += 1

        v_photo = base.select("img/@src").extract()
        if v_photo:
            l_venue["photo"] = v_photo[0].strip()

        v_desc = base.select("div[3]/text()").extract()
        if v_desc:
            desc = ""
            for dsc in v_desc:
                desc += dsc
            l_venue["desc"] = desc.strip()

        v_video = hxs.select("//*[@id='content']/iframe/@src").extract()
        if v_video:
            l_venue["video"] = v_video[0].strip()


        yield l_venue
    except Exception as e:
        log.msg("Parsing failed for URL {%s}"%format(response.request.url))
        raise