问使用Scrapy/Xpath对足球(足球)评论中的ESPN进行抓取

EN

import urllib2
from scrapy.selector import HtmlXPathSelector
import re
import codecs
import timeit
start = timeit.default_timer()

class game:
    def __init__(self,game_id):
        self.game_id = game_id
        self.comms_url = 'http://espnfc.com/uk/en/gamecast/%d/gamecast.html?soccernet=true&cc=5739' % self.game_id
        self.data_text = urllib2.urlopen(self.comms_url).read()
        self.corners = []
        self.shots = []
        self.fouls = []
        self.shots_on_target = []
        self.offsides = []
        self.goals = []
        self.extractCommentaries()
        self.extractPlayers()
        self.extractTeams()
        self.findAction('^Corner', self.corners)
        self.findAction('^Foul by', self.fouls)
        self.findAction('^Attempt', self.shots)
        self.findAction('^Attempt saved', self.shots_on_target)
        self.findAction('^Offside', self.offsides)
        self.findAction('^Goal!', self.goals)
    def extractCommentaries(self):
        self.hxs = HtmlXPathSelector(text=self.data_text)
        self.match_comments = self.hxs.select("//div[@id='convo-window']/ul[@id='convo-list']/li/div[@class='comment']/p/text()")
        self.match_timestamps = self.hxs.select("//div[@id='convo-window']/ul[@id='convo-list']/li/div[@class='timestamp']/p/text()")
        self.events = zip(self.match_timestamps.extract(), self.match_comments.extract())
        self.cleanclean = []
        for time,event in self.events:
            time = re.search('\d+', time)
            if time:
                time = time.group()
                time = int(time)
                self.cleanclean.append((time,event))
        self.dumdum = range(len(self.events))
        self.clean_events = {}
        for key, event in enumerate(self.cleanclean):
            self.clean_events[key] = event
    def extractPlayers(self):
        self.player_names = self.hxs.select("//table[@class='stat-table']/tbody//a/text()")
        self.player_urls = self.hxs.select("//table[@class='stat-table']/tbody//a/@href")
        self.player_ids = []
        for url in self.player_urls.extract():
            digit = []
            for s in url:
                if s.isdigit():
                    digit.append(s)
            self.placeholder = ''.join(digit)
            self.placeholder = int(self.placeholder)
            self.player_ids.append(self.placeholder)
        self.players = dict(zip(self.player_ids, self.player_names.extract()))
    def extractTeams(self):
        self.dummy = ['home','away']
        self.zero = [0,0]
        self.team_names = self.hxs.select("//li[@class='country']/h2/text()")
        self.teams = dict(zip(self.dummy,self.team_names.extract()))
    def findAction(self, keyword, storage):
        for key, value in self.clean_events.iteritems():
            action = re.search(keyword, value[1])
            if action:
                for ha, team in self.teams.iteritems():
                    if keyword == '^Goal!':
                        split_up_action = value[1].split('.')
                        found = re.search(team, split_up_action[1])
                    else:
                        found = re.search(team, value[1])
                    if found:
                        temp_holder = [value[0], team, self.game_id]
                        storage.append(temp_holder)

a = game()
stop = timeit.default_timer()

print stop - start