Question

我可以从网页上抓取所有的评论。但我没有得到完整的内容。只有半评论内容我可以刮。我需要抓取全部内容。

from bs4 import BeautifulSoup import requests import re


s = requests.Session()

def get_soup(url):

    headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0'}

    r = s.get(url, headers=headers)

    #with open('temp.html', 'wb') as f:
    #    f.write(r.content)
    #    webbrowser.open('temp.html')

    if r.status_code != 200:
        print('status code:', r.status_code)
    else:
        return BeautifulSoup(r.text, 'html.parser')
     def parse(url, response):

    if not response:
        print('no response:', url)
        return

    # get number of reviews
#    num_reviews = response.find('span', class_='reviews_header_count').text
#    num_reviews = num_reviews[1:-1] # remove `( )`
#    num_reviews = num_reviews.replace(',', '') # remove `,`
#    num_reviews = int(num_reviews)
#    print('num_reviews:', num_reviews, type(num_reviews))

    num_reviews = (20)
#    num_reviews = num_reviews[1:-1] # remove `( )`
#    num_reviews = num_reviews.replace(',', '') # remove `,`
#    num_reviews = int(num_reviews)
    print('num_reviews:', num_reviews, type(num_reviews))

    # create template for urls to pages with reviews
    url = url.replace('Hilton_New_York_Grand_Central-New_York_City_New_York.html', 'or{}-Hilton_New_York_Grand_Central-New_York_City_New_York.html')
    print('template:', url)

    # add requests to list
    for offset in range(0, num_reviews, 5):
        print('url:', url.format(offset))
        url_ = url.format(offset)
        parse_reviews(url_, get_soup(url_))
        #return # for test only - to stop after first page

def parse_reviews(url, response):
    print('review:', url)

    if not response:
        print('no response:', url)
        return

    for idx, review in enumerate(response.find_all('div', class_='review-container')):
        item = {
            'hotel_name': response.find('h1', class_='heading_title').text,
            'review_title': review.find('span', class_='noQuotes').text,
            'review_body': review.find('p', class_='partial_entry').text,
            'review_date': review.find('span', class_='relativeDate')['title'],#.text,#[idx],
#            'num_reviews_reviewer': review.find('span', class_='badgetext').text,
            'reviewer_name': review.find('span', class_='scrname').text,
            'bubble_rating': review.select_one('div.reviewItemInline span.ui_bubble_rating')['class'][1][7:],
        }
        #~ yield item

        results.append(item) 

        for key,val in item.items():
            print(key, ':', val)
        print('----')
        #return # for test only - to stop after first review
         start_urls = [
    'https://www.tripadvisor.in/Hotel_Review-g60763-d93339-Reviews-Hilton_New_York_Grand_Central-New_York_City_New_York.html',
    #'https://www.tripadvisor.com/Hotel_Review-g60795-d102542-Reviews-Courtyard_Philadelphia_Airport-Philadelphia_Pennsylvania.html',
    #'https://www.tripadvisor.com/Hotel_Review-g60795-d122332-Reviews-The_Ritz_Carlton_Philadelphia-Philadelphia_Pennsylvania.html', ]

results = [] 

for url in start_urls:
    parse(url, get_soup(url))

import pandas as pd

df = pd.DataFrame(results) # <--- convert list to DataFrame df.to_csv('output.csv')

我从csv文件中获取输出样本，例如：

I went on a family trip and it was amazing, I hope to come back soon. The room was small but what can you expect from New York. It was close to many things and the staff was perfect.I will come back again soon.More...

我只是想进一步扩展它。我需要帮助。我真的不知道这样做。请帮助。

我已经写了一个代码，但无法从下一页中提取id。代码如下所示

import re
import urllib
#import webbrowser``

s = requests.Session()

headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0'}

for i in range(0,10,5):
    url = ("https://www.tripadvisor.in/Hotel_Review-g60763-d93339-Reviews-or{}-Hilton_New_York_Grand_Central-New_York_City_New_York.html").format(i)
    print(url)
    r = s.get(url,headers=headers)
    html = BeautifulSoup(r.text, 'html.parser')
    pattern = re.compile(r"UID_(\w+)\-SRC_(\w+)")
    id = soup.find("div", id=pattern)["id"]
    uid = pattern.match(id).group(2)
    print(uid)
    url1 ="https://www.tripadvisor.in/ShowUserReviews-g60763-d93339-r"+str(uid)+"-Hilton_New_York_Grand_Central-New_York_City_New_York.html#CHECK_RATES_CONT"
    print(url1)
    url2 = ('"' + url1 + '"')`enter code here`
    print(url2)

Answer 1

该网站使用ajax扩展评论内容。在点击More链接之前，不会下载完整内容。

访问内容的一种方法是找出ajax请求格式，然后发出相同的HTTP请求。这可能很难，也许不是。

另一种更简单的方法是注意到评论标题是一个可点击的链接，可以在新页面中加载完整的评论。因此，您可以抓取每个评论的网址，并发送类似的GET请求。然后从响应中删除数据。

无法扩展更多... python

1 个答案: