Question

我的目的是从网页（http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061）获取姓名，地点，发布时间，评论标题和整个评论内容。

我的代码：

    from bs4 import BeautifulSoup
    from selenium  import webdriver
    from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

    firefox_capabilities = DesiredCapabilities.FIREFOX
    firefox_capabilities['marionette'] = True
    firefox_capabilities['binary'] = '/etc/firefox'

    driver = webdriver.Firefox(capabilities=firefox_capabilities)
    driver.get('http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061')
    soup = BeautifulSoup(driver.page_source,"lxml")
    for link in soup.select(".profile"):
        try:
           profile = link.select("p:nth-of-type(1) a")[0]
           profile1 = link.select("p:nth-of-type(2)")[0]
        except:pass      
           print(profile.text,profile1.text)
   driver = webdriver.Firefox(capabilities=firefox_capabilities)
   driver.get('http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061')
   soup1 = BeautifulSoup(driver.page_source,"lxml")
   for link in soup1.select(".col-10.review"):
      try:
        profile2 = link.select("small:nth-of-type(1)")[0]
        profile3 = link.select("span:nth-of-type(3)")[0]
        profile4 = link.select("a:nth-of-type(1)")[0]
      except:pass
        print(profile2.text,profile3.text,profile4.text)
   driver = webdriver.Firefox(capabilities=firefox_capabilities)
   driver.get('http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061')
   soup2 = BeautifulSoup(driver.page_source,"lxml")
   for link in soup2.select(".more.review"):
      try:
         containers=page_soup.findAll("div",{"class":"more reviewdata"})
         count=len(containers)
         for index in range(count):
           count1=len(containers[index].p)
           for i in range(count1):
             profile5 = link.select("p:nth-of-type(i)")[0]
      except:pass
         print(profile5.text)
   driver.quit()

我正在获取评论的名称，位置，时间和标题的输出，但我无法获得用户的完整评论。我将不胜感激，如果有人能帮助我获得相同的输出，以及我的代码的优化（即）我希望我的代码只通过加载网页一次提取所需的数据。此外，如果有人可以帮助我从网站的所有网页中提取Jio的所有客户评论，这将对我非常有帮助。

Answer 1

你可以通过几行代码和较少的痛苦来实现相同的目标。但是，我在这里定义了三个主要类别，如name，review_title，review_data以及其他可以轻松抽搐的字段。

这是您可以选择的方式：

from selenium import webdriver;import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061")
wait = WebDriverWait(driver, 10)

for item in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".review-article"))):
    link = item.find_element_by_css_selector(".reviewdata a")
    link.click()
    time.sleep(2)

    name = item.find_element_by_css_selector("p a").text
    review_title = item.find_element_by_css_selector("strong a[id^=ctl00_ctl00_ContentPlaceHolderFooter_ContentPlaceHolderBody_rptreviews]").text
    review_data = ' '.join([' '.join(items.text.split()) for items in item.find_elements_by_css_selector(".reviewdata")])
    print("Name: {}\nReview_Title: {}\nReview_Data: {}\n".format(name, review_title, review_data))

driver.quit()

或者联合使用（selenium + bs4）：

from bs4 import BeautifulSoup
from selenium import webdriver;import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("http://www.mouthshut.com/mobile-operators/Reliance-Jio-reviews-925812061")
wait = WebDriverWait(driver, 10)

for items in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".review-article"))):
    link = items.find_element_by_css_selector(".reviewdata a")
    link.click()
    time.sleep(2)

soup = BeautifulSoup(driver.page_source,"lxml")
for item in soup.select(".review-article"):
    name = item.select("p a")[0].text
    review_title = item.select("strong a[id^=ctl00_ctl00_ContentPlaceHolderFooter_ContentPlaceHolderBody_rptreviews]")[0].text
    review_data = ' '.join([' '.join(items.text.split()) for items in item.select(".reviewdata")])
    print("Name: {}\nReview_Title: {}\nReview_Data: {}\n".format(name, review_title, review_data))

driver.quit()

使用硒进行网络抓取

1 个答案: