Question

我正在尝试提取每个Twitter喜欢的信息，但它只会返回错误数量的喜欢或根本没有返回。我很确定我的代码是正确的。我认为这可能是因为Twitter试图阻止人们从其网站上抓取信息的事实。有没有办法来解决这个问题？还有没有办法看到每个人都喜欢特定的推文？

import re
import requests
import urllib
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from bs4 import BeautifulSoup
import sys
import unittest, time
import openpyxl
url = ["https://twitter.com/CocaCola?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Eauthor"]
for x in url:
   d = webdriver.Chrome()
   actions = ActionChains(d)
   d.get(x)
   res = requests.get(x)
   page = urllib.urlopen(x)
   numb = 0;
   SCROLL_PAUSE_TIME = 0.5
# Get scroll height
   last_height = d.execute_script("return document.body.scrollHeight")
   while True:
    # Scroll down to bottom
      d.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    # Wait to load page
      time.sleep(SCROLL_PAUSE_TIME)
    # Calculate new scroll height and compare with last scroll height
      new_height = d.execute_script("return document.body.scrollHeight")
      soup = BeautifulSoup(page, "html.parser")
      for posts in soup.findAll('div',{"class":"content"}):
         if(posts.find('p').text.encode('utf-8').find("Retweeted") == -1):
            print(posts.find('span',{"class": "_timestamp js-short-timestamp"}).text)
            print(posts.find('p').text.encode('utf-8'))
            retweet = posts.find('button',{"class": "ProfileTweet-actionButton js-actionButton js-actionFavorite"})
            #print(retweet.find('span',{"class":"ProfileTweet-actionCount"})["data-tweet-stat-count"])
            print(retweet)
            likes = posts.find('div',{"class":"ProfileTweet-action ProfileTweet-action--favorite js-toggleState"})
            print(likes.find('span',{"class": "ProfileTweet-actionCountForPresentation"}))
            numb = numb+1
            if new_height == last_height:
               break
            if numb > 1:
               break
      if numb > 1:
         break
      last_height = new_height
   d.close()

Answer 1

在此行中：

soup = BeautifulSoup(page, "html.parser")

，您将urllib请求看到的源传递给BeautifulSoup，而不是硒。并且由于urllib不会解析或执行在html中定义/与之相关的任何javascript和css，因此其内容与浏览器/用户使用的内容不同。

将其更改为此，因此bs可以处理呈现的html：

soup = BeautifulSoup(d.page_source, "html.parser")

获得特威特人的喜欢

1 个答案: