Question

我有以下代码，它搜索Twitter并滚动无限滚动。但“打印数据”这行并不适用于我。有什么想法吗？

# Import Selenium stuff
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException

# Import other needed packages
import sys
import unittest, time, re

# Call up Firefox, do the Twitter search, click the "All" link and start paging
class Sel(unittest.TestCase):
def setUp(self):
    self.driver = webdriver.Firefox()
    self.driver.implicitly_wait(30)
    self.base_url = "https://twitter.com"
    self.verificationErrors = []
    self.accept_next_alert = True
def test_sel(self):
    driver = self.driver
    delay = 3
    driver.get(self.base_url + "/search?q=storstrut&src=typd")
    driver.find_element_by_link_text("All").click()
    for i in range(1,100):
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(4)
    html_source = driver.page_source
    data = html_source.encode('utf-8')
    print data


if __name__ == "__main__":
    unittest.main()

Answer 1

你有很多未使用的代码和奇怪的导入，但你走在正确的轨道上。

这是一个简化版本，附有评论说明。

import time
from selenium import webdriver


# launch Firefox
driver = webdriver.Firefox()

# load Twitter page
driver.get("https://twitter.com/search?q=storstrut&src=typd")

# the following javascript scrolls down the entire page body.  Since Twitter
# uses "inifinite scrolling", more content will be added to the bottom of the
# DOM as you scroll... since it is in the loop, it will scroll down up to 100 
# times.
for _ in range(100):
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

# print all of the page source that was loaded
print driver.page_source.encode("utf-8")

# quit and close browser
driver.quit()

如何使用Selenium打印页面源

1 个答案: