如何复制内容
如何复制内容
提问于 2020-09-03 14:16:01
我试图通过对Tripadvisor中的多家酒店进行评论，并能够用150个观察数据(包括来自30家酒店的150个评论数据)对数据进行扫描。
但是，当我尝试添加hotel_name的新列并执行爬行时，酒店名称不会再次出现，观察的数量将减少到30家酒店的数量。如何将酒店名称复制到每个评审行？
这是我使用的代码：
import scrapy
from..items import ReviewItem
import re
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
for info in response.css('div.page'):
items = ReviewItem()
hotel_names = info.css('._1mTlpMC3::text').extract()
hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
reviewer_names = info.css('._1r_My98y::text').extract()
reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
review_dates = [review_date.strip() for review_date in review_dates]
review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
review_stars = [review_star.strip() for review_star in review_stars]
review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
review_texts = [review_text.strip() for review_text in review_texts]
#helpful_vote = info.css('._3kbymg8R::text').extract()
result = zip(hotel_names, reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
for hotel_name, reviewer_name, review_date, review_text, review_star, reviewer_contribution in result:
items['hotel_name'] = hotel_name
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield items回答 1
Stack Overflow用户
发布于 2020-09-03 20:06:58
您的问题是hotel_names只有一个值，但其他元素有五个值--检查：
print('hotel_names:', len(hotel_names))
print('reviewer_names:', len(reviewer_names))
print('review_dates:', len(review_dates))
print('review_stars:', len(review_stars))
print('review_texts:', len(review_texts))
print('reviewer_contributions:', len(reviewer_contributions))但是zip()使用最短列表的长度来创建项，因此它只创建一个itme。
您应该使用不带zip()的hotel_names，然后将hotel_names[0]添加到每个项中。
# without `hotel_names`
all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
hotel_name = hotel_names[0] # <-- manually get first hotel
# without `hotel_name`
for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield itemsBTW:还有其他问题- review_texts通常有超过5个项目(即。这意味着你使用了错误的方法来获取这篇文章。
当我检查CSV时，我看到它将...作为单独的评论。你必须改变它。
最小的工作代码。
您可以将所有代码放在一个文件中，然后在不创建项目- python script.py的情况下运行它。这样每个人都可以测试它。
import scrapy
#from ..items import ReviewItem
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
for info in response.css('div.page'):
hotel_names = info.css('._1mTlpMC3::text').extract()
hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
reviewer_names = info.css('._1r_My98y::text').extract()
reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
review_dates = [review_date.strip() for review_date in review_dates]
review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
review_stars = [review_star.strip() for review_star in review_stars]
review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
review_texts = [review_text.strip() for review_text in review_texts]
#helpful_vote = info.css('._3kbymg8R::text').extract()
print('hotel_names:', len(hotel_names))
print('reviewer_names:', len(reviewer_names))
print('review_dates:', len(review_dates))
print('review_stars:', len(review_stars))
print('review_texts:', len(review_texts))
print('reviewer_contributions:', len(reviewer_contributions))
print('----')
# without `hotel_names`
all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
hotel_name = hotel_names[0] # <-- manually get first hotel
# without `hotel_name`
for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield items
# --- run without project and save in `output.csv` ---
from scrapy.crawler import CrawlerProcess
c = CrawlerProcess({
'USER_AGENT': 'Mozilla/5.0',
# save in file CSV, JSON or XML
'FEED_FORMAT': 'csv', # csv, json, xml
'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()编辑：
我的版本没有zip()
首先，我搜索所有的reviews，然后使用for-loop分别处理每个评论。这样我就可以控制text并跳过... --只需在review中获得第一个text。
import scrapy
#from ..items import ReviewItem
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
hotel_name = response.css('#HEADING::text').extract_first().strip()
print('hotel_name:', hotel_name)
for review in response.xpath('.//div[@data-test-target="HR_CC_CARD"]'):
name = review.css('._1r_My98y::text').extract_first().strip()
contribution = review.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract_first().strip()
date = review.xpath('.//div[@class="_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract_first().strip().replace('wrote a review ', '')
stars = review.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract_first().strip().replace('ui_bubble_rating bubble_', '')
text = review.xpath('.//div[@class="cPQsENeY"]//span/text()').extract_first().strip()
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = name
items['reviewer_contribution'] = contribution
items['review_date'] = date
items['review_star'] = stars
items['review_text'] = text
yield items
# --- run without project and save in `output.csv` ---
from scrapy.crawler import CrawlerProcess
c = CrawlerProcess({
'USER_AGENT': 'Mozilla/5.0',
# save in file CSV, JSON or XML
'FEED_FORMAT': 'csv', # csv, json, xml
'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://stackoverflow.com/questions/63725514
复制相关文章
相似问题
腾讯云开发者
