我正在尝试删除Tripadvisor中多家酒店的评论，因此能够通过150项观测值来收集数据，其中包括30家酒店的150条评论数据。
但是，当我尝试添加新列的hotel_name并执行爬网时，该酒店名称不会再次出现，并且观测值的数量减少为30个酒店的数量。如何使每个酒店名称重复的评论行？
这是我正在使用的代码：
import scrapy
from..items import ReviewItem
import re
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
for info in response.css('div.page'):
items = ReviewItem()
hotel_names = info.css('._1mTlpMC3::text').extract()
hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
reviewer_names = info.css('._1r_My98y::text').extract()
reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
review_dates = [review_date.strip() for review_date in review_dates]
review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
review_stars = [review_star.strip() for review_star in review_stars]
review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
review_texts = [review_text.strip() for review_text in review_texts]
#helpful_vote = info.css('._3kbymg8R::text').extract()
result = zip(hotel_names, reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
for hotel_name, reviewer_name, review_date, review_text, review_star, reviewer_contribution in result:
items['hotel_name'] = hotel_name
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield items
答案 0 :(得分：0)
您的问题是hotel_names仅具有一个值，而其他元素具有五个值-检查：
print('hotel_names:', len(hotel_names))
print('reviewer_names:', len(reviewer_names))
print('review_dates:', len(review_dates))
print('review_stars:', len(review_stars))
print('review_texts:', len(review_texts))
print('reviewer_contributions:', len(reviewer_contributions))
但是zip()使用最短列表的长度来创建项目-因此它仅创建一个itme。
您应该使用zip()，而不使用hotel_names，然后将hotel_names[0]添加到每个项目中。
# without `hotel_names`
all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
hotel_name = hotel_names[0] # <-- manually get first hotel
# without `hotel_name`
for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield items
顺便说一句：还有另一个问题-review_texts通常包含超过5项（即11项），这意味着您使用错误的方法来获取此文本。
当我检查CSV时，我看到它以...的形式出现在单独的评论中。您必须更改它。
最少的工作代码。
您可以将所有代码放在一个文件中并运行它，而无需创建项目-python script.py。这样，每个人都可以对其进行测试。
import scrapy
#from ..items import ReviewItem
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
for info in response.css('div.page'):
hotel_names = info.css('._1mTlpMC3::text').extract()
hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
reviewer_names = info.css('._1r_My98y::text').extract()
reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
review_dates = [review_date.strip() for review_date in review_dates]
review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
review_stars = [review_star.strip() for review_star in review_stars]
review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
review_texts = [review_text.strip() for review_text in review_texts]
#helpful_vote = info.css('._3kbymg8R::text').extract()
print('hotel_names:', len(hotel_names))
print('reviewer_names:', len(reviewer_names))
print('review_dates:', len(review_dates))
print('review_stars:', len(review_stars))
print('review_texts:', len(review_texts))
print('reviewer_contributions:', len(reviewer_contributions))
print('----')
# without `hotel_names`
all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)
hotel_name = hotel_names[0] # <-- manually get first hotel
# without `hotel_name`
for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = reviewer_name
items['reviewer_contribution'] = reviewer_contribution
items['review_date'] = review_date
items['review_star'] = review_star
items['review_text'] = review_text
#items['helpful_vote'] = helpful_vote
yield items
# --- run without project and save in `output.csv` ---
from scrapy.crawler import CrawlerProcess
c = CrawlerProcess({
'USER_AGENT': 'Mozilla/5.0',
# save in file CSV, JSON or XML
'FEED_FORMAT': 'csv', # csv, json, xml
'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()
编辑：
我的版本没有zip()
首先，我搜索所有reviews，然后使用for循环分别处理每个评论。这样，我可以控制text并跳过...-简单地我只在text中得到第一个review。
import scrapy
#from ..items import ReviewItem
class TripAdvisorReview(scrapy.Spider):
name = "tripadvisor"
start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]
def parse(self, response):
for href in response.css("div.listing_title a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url, callback=self.parse_hotel)
def parse_hotel(self, response):
hotel_name = response.css('#HEADING::text').extract_first().strip()
print('hotel_name:', hotel_name)
for review in response.xpath('.//div[@data-test-target="HR_CC_CARD"]'):
name = review.css('._1r_My98y::text').extract_first().strip()
contribution = review.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract_first().strip()
date = review.xpath('.//div[@class="_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract_first().strip().replace('wrote a review ', '')
stars = review.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract_first().strip().replace('ui_bubble_rating bubble_', '')
text = review.xpath('.//div[@class="cPQsENeY"]//span/text()').extract_first().strip()
#items = ReviewItem()
items = dict()
items['hotel_name'] = hotel_name # <-- manually add first hotel
items['reviewer_name'] = name
items['reviewer_contribution'] = contribution
items['review_date'] = date
items['review_star'] = stars
items['review_text'] = text
yield items
# --- run without project and save in `output.csv` ---
from scrapy.crawler import CrawlerProcess
c = CrawlerProcess({
'USER_AGENT': 'Mozilla/5.0',
# save in file CSV, JSON or XML
'FEED_FORMAT': 'csv', # csv, json, xml
'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()