Question

我在代码下面运行，但我得到empty list。你能帮我找出问题吗？

执行：xvfb-run python dynamic_scrapy.py

import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs


class Client(QWebPage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self.on_page_load)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()
    def on_page_load(self):
        self.app.quit()

url = "https://pythonprogramming.net/parsememcparseface/"
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
print(soup)
js_test = soup.find_all('p', class_='jstest')
print(js_test)

Answer 1

您需要将QString转换为string才能将其转换为BeautifulSoup。你可以这样做：

import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs

class Client(QWebPage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self.on_page_load)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()
    def on_page_load(self):
        self.app.quit()

url = "https://pythonprogramming.net/parsememcparseface/"
client_response = Client(url)

source = client_response.mainFrame().toHtml()
source_utf = unicode(source.toUtf8(), encoding="UTF-8") # Added
soup = bs.BeautifulSoup(source_utf, 'lxml')
js_test = soup.find_all('p', class_='jstest')
print(js_test)

这将导致：

[<p class="jstest" id="yesnojs">Look at you shinin!</p>]

使用python进行动态网页抓取

1 个答案: