Question

我正在使用Qt的QWebPage来渲染一个使用javascript动态更新其内容的页面 - 因此只下载静态版本的页面（例如urllib2）的库将无效。

我的问题是，当我渲染第二页时，大约99％的时间程序崩溃了。在其他时候，它会在崩溃前工作三次。我也得到了一些段错误，但它都是随机的。

我的猜测是我用来渲染的对象没有被正确删除，所以尝试重用它可能会给自己造成一些问题。我看了一遍，似乎没有人真的遇到同样的问题。

这是我正在使用的代码。该程序从steam的社区市场下载网页，因此我可以创建所有项目的数据库。我需要多次调用getItemsFromPage函数来获取所有项目，因为它们被分解为页面（显示X量的结果1-10）。

import csv
import re
import sys
from string import replace
from bs4 import BeautifulSoup
from PyQt4.QtGui import *
from PyQt4.QtCore import *
from PyQt4.QtWebKit import *

class Item:
    __slots__ = ("name", "count", "price", "game")

    def __repr__(self):
        return self.name + "(" + str(self.count) + ")"

    def __str__(self):
        return self.name + ", " + str(self.count) + ", $" + str(self.price)

class Render(QWebPage):  
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self._loadFinished)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()

    def _loadFinished(self, result):
        self.frame = self.mainFrame()
        self.app.quit()
        self.deleteLater()

def getItemsFromPage(appid, page=1):

    r = Render("http://steamcommunity.com/market/search?q=appid:" + str(appid) + "#p" + str(page))

    soup = BeautifulSoup(str(r.frame.toHtml().toUtf8()))

    itemLst = soup.find_all("div", "market_listing_row market_recent_listing_row")

    items = []

    for k in itemLst:
        i = Item()

        i.name = k.find("span", "market_listing_item_name").string
        i.count = int(replace(k.find("span", "market_listing_num_listings_qty").string, ",", ""))
        i.price = float(re.search(r'\$([0-9]+\.[0-9]+)', str(k)).group(1))
        i.game = appid

        items.append(i)

    return items

if __name__ == "__main__":

    print "Updating market items to dota2.csv ..."

    i = 1

    with open("dota2.csv", "w") as f:
        writer = csv.writer(f)

        r = None

        while True:
            print "Page " + str(i)

            items = getItemsFromPage(570)

            if len(items) == 0:
                print "No items found, stopping..."
                break

            for k in items:
                writer.writerow((k.name, k.count, k.price, k.game))

            i += 1

    print "Done."

调用getItemsFromPage一次正常。随后的电话给我带来了问题。程序的输出通常是

Updating market items to dota2.csv ...
Page 1
Page 2

然后它崩溃了。它应该继续超过700页。

Answer 1

您的程序存在的问题是您尝试使用您获取的每个网址创建一个新的QApplication。

相反，您应该创建一个QApplication，并处理WebPage类本身内所有网页的加载和处理。关键概念是使用loadFinished信号通过在加载和处理当前网址后获取新网址来创建循环。

下面的两个演示脚本（对于PyQt4和PyQt5）是简化示例，展示了如何构建程序。希望如何使它们适合您自己的使用应该是相当明显的：

import sys
from PyQt4 import QtCore, QtGui, QtWebKit

class WebPage(QtWebKit.QWebPage):
    def __init__(self):
        super(WebPage, self).__init__()
        self.mainFrame().loadFinished.connect(self.handleLoadFinished)

    def start(self, urls):
        self._urls = iter(urls)
        self.fetchNext()

    def fetchNext(self):
        try:
            url = next(self._urls)
        except StopIteration:
            return False
        else:
            self.mainFrame().load(QtCore.QUrl(url))
        return True

    def processCurrentPage(self):
        url = self.mainFrame().url().toString()
        html = self.mainFrame().toHtml()
        # do stuff with html...
        print('loaded: [%d bytes] %s' % (self.bytesReceived(), url))

    def handleLoadFinished(self):
        self.processCurrentPage()
        if not self.fetchNext():
            QtGui.qApp.quit()

if __name__ == '__main__':

    # generate some test urls
    urls = []
    url = 'http://pyqt.sourceforge.net/Docs/PyQt4/%s.html'
    for name in dir(QtWebKit):
        if name.startswith('Q'):
            urls.append(url % name.lower())

    app = QtGui.QApplication(sys.argv)
    webpage = WebPage()
    webpage.start(urls)
    sys.exit(app.exec_())

以下是上述脚本的PyQt5 / QWebEngine版本：

import sys
from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets

class WebPage(QtWebEngineWidgets.QWebEnginePage):
    def __init__(self):
        super(WebPage, self).__init__()
        self.loadFinished.connect(self.handleLoadFinished)

    def start(self, urls):
        self._urls = iter(urls)
        self.fetchNext()

    def fetchNext(self):
        try:
            url = next(self._urls)
        except StopIteration:
            return False
        else:
            self.load(QtCore.QUrl(url))
        return True

    def processCurrentPage(self, html):
        url = self.url().toString()
        # do stuff with html...
        print('loaded: [%d chars] %s' % (len(html), url))
        if not self.fetchNext():
            QtWidgets.qApp.quit()

    def handleLoadFinished(self):
        self.toHtml(self.processCurrentPage)

if __name__ == '__main__':

    # generate some test urls
    urls = []
    url = 'http://pyqt.sourceforge.net/Docs/PyQt5/%s.html'
    for name in dir(QtWebEngineWidgets):
        if name.startswith('Q'):
            urls.append(url % name.lower())

    app = QtWidgets.QApplication(sys.argv)
    webpage = WebPage()
    webpage.start(urls)
    sys.exit(app.exec_())

使用QWebPage刮取多个URL

1 个答案: