Question

我正在使用Selenium / python自动向下滚动社交媒体网站并抓取帖子。我正在滚动一定次数（下面的代码）之后在一个“点击”中提取所有文本，但我想在每次滚动后仅提取新加载的文本。

例如，如果页面最初包含文本“A，B，C”，那么在第一次滚动后显示“D，E，F”，我想存储“A，B，C”，然后滚动，然后存储“D，E，F”等。

我想要提取的具体项目是帖子的日期和消息文本，可以分别使用css选择器'.message-date'和'.message-body'获取（例如{{1} }}）。

有人可以建议如何在每次滚动后只提取新加载的文字吗？

这是我当前的代码（在完成滚动之后提取所有日期/消息）：

dates = driver.find_elements_by_css_selector('.message-date')

Answer 1

您可以在变量中存储消息数量，并使用xpath和position()来获取新添加的帖子

dates = []
messages = []
num_of_posts = 1
for i in range(1, ScrollNumber):
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)
    dates.extend(driver.find_elements_by_xpath('(//div[@class="message-date"])[position()>=' + str(num_of_posts) + ']'))
    messages.extend(driver.find_elements_by_xpath('(//div[contains(@class, "message-body")])[position()>=' + str(num_of_posts) + ']'))
    num_of_posts = len(dates)

Answer 2

我在facebook帖子上遇到了同样的问题。为此，我在列表中保存帖子ID（或者帖子的唯一值，甚至哈希值），然后当您再次进行查询时，需要检查该ID是否在您的列表中

此外，您可以删除已解析的DOM，因此只存在新的DOM。

Answer 3

正如其他人所说，如果你可以通过直接点击API来做你需要做的事情，这是你最好的选择。如果您绝对必须使用Selenium，请参阅下面的解决方案。

我根据自己的需要做了类似的事情。

我正在利用CSS路径的:nth-child()方面在加载时单独查找元素。
我也使用selenium的显式等待功能（通过explicit包，pip install explicit）来有效地等待元素加载。

脚本快速退出（没有调用sleep（）），但是，网页本身在后台发生了大量垃圾，硒需要一段时间才能将控制权返回给脚本。

from __future__ import print_function

from itertools import count
import sys
import time

from explicit import waiter, CSS
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait as Wait


# The CSS selectors we will use
POSTS_BASE_CSS = 'ol.stream-list > li'              # All li elements
POST_BASE_CSS = POSTS_BASE_CSS + ":nth-child({0})"  # li child element at index {0}
POST_DATE_CSS = POST_BASE_CSS + ' div.message-date'     # li child element at {0} with div.message-date
POST_BODY_CSS = POST_BASE_CSS + ' div.message-body'     # li child element at {0} with div.message-date



class Post(object):
    def __init__(self, driver, post_index):
        self.driver = driver
        self.date_css = POST_DATE_CSS.format(post_index)
        self.text_css = POST_BODY_CSS.format(post_index)

    @property
    def date(self):
        return waiter.find_element(self.driver, self.date_css, CSS).text

    @property
    def text(self):
        return waiter.find_element(self.driver, self.text_css, CSS).text


def get_posts(driver, url, max_screen_scrolls):
    """ Post object generator """
    driver.get(url)
    screen_scroll_count = 0

    # Wait for the initial posts to load:
    waiter.find_elements(driver, POSTS_BASE_CSS, CSS)

    for index in count(1):
        # Evaluate if we need to scroll the screen, or exit the generator
        # If there is no element at this index, it means we need to scroll the screen
        if len(driver.find_elements_by_css_selector('ol.stream-list > :nth-child({0})'.format(index))) == 0:
            if screen_scroll_count >= max_screen_scrolls:
                # Break if we have already done the max scrolls
                break

            # Get count of total posts on page
            post_count = len(waiter.find_elements(driver, POSTS_BASE_CSS, CSS))

            # Scroll down
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            screen_scroll_count += 1

            def posts_load(driver):
                """ Custom explicit wait function; waits for more posts to load in """
                return len(waiter.find_elements(driver, POSTS_BASE_CSS, CSS)) > post_count

            # Wait until new posts load in
            Wait(driver, 20).until(posts_load)

        # The list elements have sponsored ads and scripts mixed in with the posts we
        # want to scrape. Check if they have a div.message-date element and continue on
        # if not
        includes_date_css = POST_DATE_CSS.format(index)
        if len(driver.find_elements_by_css_selector(includes_date_css)) == 0:
            continue

        yield Post(driver, index)


def main():
    url = "https://stocktwits.com/symbol/USDJPY?q=%24USDjpy"
    max_screen_scrolls = 4
    driver = webdriver.Chrome()
    try:
        for post_num, post in enumerate(get_posts(driver, url, max_screen_scrolls), 1):
            print("*" * 40)
            print("Post #{0}".format(post_num))
            print("\nDate: {0}".format(post.date))
            print("Text: {0}\n".format(post.text[:34]))

    finally:
        driver.quit()  # Use try/finally to make sure the driver is closed


if __name__ == "__main__":
    main()

完全披露：我是explicit包的创建者。您可以直接使用显式等待轻松地重写上述内容，但会牺牲可读性。

Answer 4

这正是您想要的。但是，我不会这样刮网站...它运行的时间越长越慢。 RAM的使用也会失控。

import time
from hashlib import md5

import selenium.webdriver.support.ui as ui
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

URL = 'https://stocktwits.com/symbol/USDJPY?q=%24USDjpy'
CSS = By.CSS_SELECTOR

driver.get(URL)


def scrape_for(max_seconds=300):
    found = set()
    end_at = time.time() + max_seconds
    wait = ui.WebDriverWait(driver, 5, 0.5)

    while True:
        # find elements
        elms = driver.find_elements(CSS, 'li.messageli')

        for li in elms:
            # get the information we need about each post
            text = li.find_element(CSS, 'div.message-content')
            key = md5(text.text.encode('ascii', 'ignore')).hexdigest()

            if key in found:
                continue

            found.add(key)

            try:
                date = li.find_element(CSS, 'div.message-date').text
            except NoSuchElementException as e:
                date = None

            yield text.text, date

        if time.time() > end_at:
            raise StopIteration

        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
        wait.until(EC.invisibility_of_element_located(
                       (CSS, 'div#more-button-loading')))

    raise StopIteration


for twit in scrape_for(60):
    print(twit)

driver.quit()

Answer 5

这不是你要求的（它不是硒解决方案），它实际上是使用页面在后台使用的API的解决方案。在我看来，使用selenium代替API是一种矫枉过正的行为。

这是一个使用API的脚本：

import re

import requests

STREAM_ID_REGEX = re.compile(r"data-stream='symbol-(?P<id>\d+)'")
CSRF_TOKEN_REGEX = re.compile(r'<meta name="csrf-token" content="(?P<csrf>[^"]+)" />')

URL = 'https://stocktwits.com/symbol/USDJPY?q=%24USDjpy'
API_URL = 'https://stocktwits.com/streams/poll?stream=symbol&substream=top&stream_id={stream_id}&item_id={stream_id}'


def get_necessary_info():
    res = requests.get(URL)

    # Extract stream_id
    match = STREAM_ID_REGEX.search(res.text)
    stream_id = match.group('id')

    # Extract CSRF token
    match = CSRF_TOKEN_REGEX.search(res.text)
    csrf_token = match.group('csrf')

    return stream_id, csrf_token


def get_messages(stream_id, csrf_token, max_messages=100):
    base_url = API_URL.format(stream_id=stream_id)

    # Required headers
    headers = {
        'x-csrf-token': csrf_token,
        'x-requested-with': 'XMLHttpRequest',
    }

    messages = []
    more = True
    max_value = None
    while more:
        # Pagination
        if max_value:
            url = '{}&max={}'.format(base_url, max_value)
        else:
            url = base_url

        # Get JSON response
        res = requests.get(url, headers=headers)
        data = res.json()

        # Add returned messages
        messages.extend(data['messages'])

        # Check if there are more messages
        more = data['more']
        if more:
            max_value = data['max']

        # Check if we have enough messages
        if len(messages) >= max_messages:
            break

    return messages


def main():
    stream_id, csrf_token = get_necessary_info()
    messages = get_messages(stream_id, csrf_token)

    for message in messages:
        print(message['created_at'], message['body'])


if __name__ == '__main__':
    main()

输出的第一行：

Tue, 03 Oct 2017 03:54:29 -0000 $USDJPY (113.170) Daily Signal remains LONG from 109.600 on 12/09. SAR point now at 112.430. website for details
Tue, 03 Oct 2017 03:33:02 -0000 JPY: Selling JPY Via Long $USDJPY Or Long $CADJPY Still Attractive  - SocGen https://www.efxnews.com/story/37129/jpy-selling-jpy-long-usdjpy-or-long-cadjpy-still-attractive-socgen#.WdMEqnCGMCc.twitter
Tue, 03 Oct 2017 01:05:06 -0000 $USDJPY buy signal on 03 OCT 2017 01:00 AM UTC by AdMACD Trading System (Timeframe=H1) http://www.cosmos4u.net/index.php/forex/usdjpy/usdjpy-buy-signal-03-oct-2017-01-00-am-utc-by-admacd-trading-system-timeframe-h1 #USDJPY #Forex
Tue, 03 Oct 2017 00:48:46 -0000 $EURUSD nice H&S to take price lower on $USDJPY just waiting 4 inner trendline to break. built up a lot of strength
Tue, 03 Oct 2017 00:17:13 -0000 $USDJPY The Instrument reached the 100% from lows at 113.25 and sold in 3 waves, still can see 114.14 area.#elliottwave $USDX
...

Answer 6

滚动后只需做睡眠，一定要像真机一样使用selenium等待页面加载新内容。我建议你在selenium中使用wait函数或者在代码中轻松添加sleep函数来加载内容。

time.sleep(5)

Selenium / python：每次滚动后从动态加载的网页中提取文本

6 个答案: