我用CrawlerProcess编写了一个抓痒的蜘蛛，效果很好。
下一步，我想要一个使用pyqt5生成的一些输入数据的GUI对话框，并在scrapy运行时设置微调器。此任务是使用虚拟过程完成的，并且工作正常。
我知道多线程与CrawlerProcess不兼容，因此我将其更改为CrawlerRunner。无论如何，它失败了：
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
我创建了一个最小的示例来重现该错误：
from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner
class RequestRunnable(QRunnable):
def __init__(self, dialog):
super(RequestRunnable, self).__init__()
self.w = dialog
print('runnable __init__')
def run(self):
print('run')
crawler = CrawlerRunner({
'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'FEED_FORMAT': 'json',
'FEED_URI': 'output.json',
'DOWNLOAD_DELAY': 3,
'LOG_STDOUT': True,
'LOG_FILE': 'scrapy_output.txt',
'ROBOTSTXT_OBEY': False,
'RETRY_ENABLED': True,
'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
'RETRY_TIMES': 5
})
# instantiate a spider
spider = CustomSpider()
d = crawler.crawl(spider)
d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
reactor.run() #TODO gives error
QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)
class CustomSpider(Spider):
name = 'quotes_spider'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
yield {'quotes': quotes}
class DownloadDataDialog(QDialog):
def __init__(self, parent=None):
super(DownloadDataDialog, self).__init__(parent)
self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)
tabWidget = QTabWidget(self)
tabWidget.addTab(MyTab(tabWidget), "MyTab")
mainLayout = QVBoxLayout()
mainLayout.addWidget(tabWidget)
self.setLayout(mainLayout)
self.setWindowTitle("Download option chain data from web")
class MyTab(QWidget):
def __init__(self, parent=None):
super(MyTab, self).__init__(parent)
dataGroup = QGroupBox('Data')
getButton = QPushButton('Download')
getButton.clicked.connect(self.download_data)
dataLayout = QVBoxLayout()
dataLayout.addWidget(getButton)
dataGroup.setLayout(dataLayout)
mainLayout = QVBoxLayout()
mainLayout.addWidget(dataGroup)
mainLayout.addStretch(1)
self.setLayout(mainLayout)
def download_data(self):
self.parentWidget().window().spinner.start()
runnable = RequestRunnable(self)
QThreadPool.globalInstance().start(runnable)
@pyqtSlot()
def FinishedDownload(self):
self.parentWidget().window().spinner.stop()
if __name__ == '__main__':
import sys
app = QApplication(sys.argv)
tabdialog = DownloadDataDialog()
tabdialog.show()
sys.exit(app.exec_())
要执行它，您将需要waitingspinnerwidget.py：
# -*- coding: utf-8 -*-
"""
The MIT License (MIT)
Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import math
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
class QtWaitingSpinner(QWidget):
def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)
self._centerOnParent = centerOnParent
self._disableParentWhenSpinning = disableParentWhenSpinning
# WAS IN initialize()
self._color = QColor(Qt.black)
self._roundness = 100.0
self._minimumTrailOpacity = 3.14159265358979323846
self._trailFadePercentage = 80.0
self._revolutionsPerSecond = 1.57079632679489661923
self._numberOfLines = 20
self._lineLength = 10
self._lineWidth = 2
self._innerRadius = 10
self._currentCounter = 0
self._isSpinning = False
self._timer = QTimer(self)
self._timer.timeout.connect(self.rotate)
self.updateSize()
self.updateTimer()
self.hide()
# END initialize()
self.setWindowModality(modality)
self.setAttribute(Qt.WA_TranslucentBackground)
def paintEvent(self, QPaintEvent):
self.updatePosition()
painter = QPainter(self)
painter.fillRect(self.rect(), Qt.transparent)
painter.setRenderHint(QPainter.Antialiasing, True)
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
painter.setPen(Qt.NoPen)
for i in range(0, self._numberOfLines):
painter.save()
painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
rotateAngle = float(360 * i) / float(self._numberOfLines)
painter.rotate(rotateAngle)
painter.translate(self._innerRadius, 0)
distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
self._minimumTrailOpacity, self._color)
painter.setBrush(color)
painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
self._roundness, Qt.RelativeSize)
painter.restore()
def start(self):
self.updatePosition()
self._isSpinning = True
self.show()
if self.parentWidget and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(False)
if not self._timer.isActive():
self._timer.start()
self._currentCounter = 0
def stop(self):
self._isSpinning = False
self.hide()
if self.parentWidget() and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(True)
if self._timer.isActive():
self._timer.stop()
self._currentCounter = 0
def setNumberOfLines(self, lines):
self._numberOfLines = lines
self._currentCounter = 0
self.updateTimer()
def setLineLength(self, length):
self._lineLength = length
self.updateSize()
def setLineWidth(self, width):
self._lineWidth = width
self.updateSize()
def setInnerRadius(self, radius):
self._innerRadius = radius
self.updateSize()
def color(self):
return self._color
def roundness(self):
return self._roundness
def minimumTrailOpacity(self):
return self._minimumTrailOpacity
def trailFadePercentage(self):
return self._trailFadePercentage
def revolutionsPersSecond(self):
return self._revolutionsPerSecond
def numberOfLines(self):
return self._numberOfLines
def lineLength(self):
return self._lineLength
def lineWidth(self):
return self._lineWidth
def innerRadius(self):
return self._innerRadius
def isSpinning(self):
return self._isSpinning
def setRoundness(self, roundness):
self._roundness = max(0.0, min(100.0, roundness))
def setColor(self, color=Qt.black):
self._color = QColor(color)
def setRevolutionsPerSecond(self, revolutionsPerSecond):
self._revolutionsPerSecond = revolutionsPerSecond
self.updateTimer()
def setTrailFadePercentage(self, trail):
self._trailFadePercentage = trail
def setMinimumTrailOpacity(self, minimumTrailOpacity):
self._minimumTrailOpacity = minimumTrailOpacity
def rotate(self):
self._currentCounter += 1
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
self.update()
def updateSize(self):
size = (self._innerRadius + self._lineLength) * 2
self.setFixedSize(size, size)
def updateTimer(self):
self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))
def updatePosition(self):
if self.parentWidget() and self._centerOnParent:
self.move(self.parentWidget().width() / 2 - self.width() / 2,
self.parentWidget().height() / 2 - self.height() / 2)
dialogCenter = self.mapToGlobal(self.rect().center())
parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
self.move(parentWindowCenter - dialogCenter)
parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())
def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
distance = primary - current
if distance < 0:
distance += totalNrOfLines
return distance
def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
color = QColor(colorinput)
if countDistance == 0:
return color
minAlphaF = minOpacity / 100.0
distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
if countDistance > distanceThreshold:
color.setAlphaF(minAlphaF)
else:
alphaDiff = color.alphaF() - minAlphaF
gradient = alphaDiff / float(distanceThreshold + 1)
resultAlpha = color.alphaF() - gradient * countDistance
# If alpha is out of bounds, clip it.
resultAlpha = min(1.0, max(0.0, resultAlpha))
color.setAlphaF(resultAlpha)
return color
完整追溯：
Traceback (most recent call last):
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 428, in fireEvent
DeferredList(beforeResults).addCallback(self._continueFiring)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 322, in addCallback
callbackKeywords=kw)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 311, in addCallbacks
self._runCallbacks()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 441, in _continueFiring
callable(*args, **kwargs)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1256, in _reallyStartRunning
self._handleSignals()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\posixbase.py", line 295, in _handleSignals
_SignalReactorMixin._handleSignals(self)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1221, in _handleSignals
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
对正在发生的事情以及如何解决它有任何想法吗？
答案 0 :(得分：2)
不要使用线程，只需使用qt反应器进行扭曲，这样扭曲的应用程序（草皮的）和qt5应用程序都可以在同一事件循环中运行：
安装qt5reactor：
pip install qt5reactor
然后添加代码以安装反应堆并运行它：
app = QApplication(sys.argv)
import qt5reactor
qt5reactor.install()
reactor.run()