我从这里开始一个与this question链接的新主题。
我邀请您阅读背景，以获得全球性的想法。
所以我有一个下载功能，它依赖于python 3.2 API（由私人公司开发）。每个文件最多可能需要400秒。
显然，我没有一个文件可供下载，所以我几天都在尝试将每个下载进程放在一个线程池中。池中的每个线程应完全独立于GUI主线程。当其中一个完成后，它应该只向GUI发送一个信号。
我做了几次测试，但无论使用何种技术，但
我认为API提供的下载方法是一个无法进行线程化的阻止函数。
所以我的问题很简单：如何知道是否可以通过线程处理I / O方法。
2017年11月24日更新
您将在下面找到部分符合我预期的初稿（使用tandem multiprocessing.pool / map_async）。正如您将看到的，遗憾的是我不得不插入一个忙碌的等待循环＆＃34;为了获得有关正在发生的事情的QPlainTextEdit的一些信息。
任务的结果仅在全局处理结束时给出（行为map_async）。这并不是我正在寻找的。我想更多地插入实时，并在控制台上立即查看每个已完成的任务的消息。
import time
import multiprocessing
import private.library as bathy
from PyQt4 import QtCore, QtGui
import os
import sys
user = 'user'
password = 'password'
server = 'server'
basename = 'basename'
workers = multiprocessing.cpu_count()
node = bathy.NodeManager(user, password, server)
database = node.get_database(basename)
ids = (10547, 3071, 13845, 13846, 13851, 13844, 5639, 4612, 4613, 954,
961, 962, 4619, 4620, 4622, 4623, 4624, 4627, 4628, 4631,
4632, 4634, 4635, 4638, 4639, 4640, 4641, 4642, 10722, 1300,
1301, 1303, 1310, 1319, 1316, 1318, 1321, 1322, 1323, 1324,
1325, 1347, 1348, 1013, 1015, 1320, 8285, 8286, 8287, 10329,
9239, 9039, 5006, 5009, 5011, 5012, 5013, 5014, 5015, 5025,
5026, 4998, 5040, 5041, 5042, 5043, 11811, 2463, 2464, 5045,
5046, 5047, 5048, 5049, 5053, 5060, 5064, 5065, 5068, 5069,
5071, 5072, 5075, 5076, 5077, 5079, 5080, 5081, 5082, 5083,
5084, 5085, 5086, 5087, 5088, 5090, 5091, 5092, 5093)
# ---------------------------------------------------------------------------------
def download(surface_id, index):
global node
global database
t = time.time()
message = 'Surface #%d - Process started\n' % index
surface = database.get_surface(surface_id)
metadata = surface.get_metadata()
file_path = os.path.join("C:\\Users\\philippe\\Test_Download",
metadata["OBJNAM"] + ".surf")
try:
surface.download_bathymetry(file_path)
except RuntimeError as error:
message += "Error : " + str(error).split('\n')[0] + '\n'
finally:
message += ('Process ended : %.2f s\n' % (time.time() - t))
return message
# ---------------------------------------------------------------------------------
def pass_args(args):
# Method to pass multiple arguments to download (multiprocessing.Pool)
return download(*args)
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
def __init__(self):
super(self.__class__, self).__init__()
self.resize(600, 300)
self.setMinimumSize(QtCore.QSize(600, 300))
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close |
QtGui.QDialogButtonBox.Ok)
self.button_box.setObjectName("button_box")
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Ok),
QtCore.SIGNAL('clicked()'),
self.button_ok_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.pool = multiprocessing.Pool(processes=workers)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
jobs_args = [(surface_id, index) for index, surface_id in enumerate(ids)]
async = pool.map_async(pass_args, jobs_args)
pool.close()
# Busy waiting loop
while True:
# pool.map_async has a _number_left attribute, and a ready() method
if async.ready():
self.write_stream("All tasks completed\n")
pool.join()
for line in async.get():
self.write_stream(line)
break
remaining = async._number_left
self.write_stream("Waiting for %d task(s) to complete...\n" % remaining)
time.sleep(0.5)
# Other functions
# -----------------------------------------------------------------------------
def write_stream(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
app.processEvents()
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
问题
2017年11月25日更新
我尝试了apply_async：
def button_ok_clicked(self):
# Pool.apply_async - the call returns immediately instead of
# waiting for the result
for index, surface_id in enumerate(ids):
async = pool.apply_async(download,
args=(surface_id, index),
callback=self.write_stream)
pool.close()
带回调：
def write_stream(self, text):
# This is called whenever pool.apply_async(i) returns a result
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# Update the text edit
app.processEvents()
不幸的是，通过这种方式应用程序崩溃。我想我必须设置一个锁定机制，以防止所有任务同时在文本编辑中写入。
答案 0 :(得分：1)
以下是示例脚本的简化版本，其中显示了如何使用回调发布自定义事件。每个作业都通过apply_async单独处理，因此会更新一个简单的计数器，以指示所有作业何时完成。
import sys, time, random, multiprocessing
from PyQt4 import QtCore, QtGui
ids = (10547, 3071, 13845, 13846, 13851, 13844, 5639, 4612, 4613, 954,
961, 962, 4619, 4620, 4622, 4623, 4624, 4627, 4628, 4631,
4632, 4634, 4635, 4638, 4639, 4640, 4641, 4642, 10722, 1300,
1301, 1303, 1310, 1319, 1316, 1318, 1321, 1322, 1323, 1324,
1325, 1347, 1348, 1013, 1015, 1320, 8285, 8286, 8287, 10329,
9239, 9039, 5006, 5009, 5011, 5012, 5013, 5014, 5015, 5025,
5026, 4998, 5040, 5041, 5042, 5043, 11811, 2463, 2464, 5045,
5046, 5047, 5048, 5049, 5053, 5060, 5064, 5065, 5068, 5069,
5071, 5072, 5075, 5076, 5077, 5079, 5080, 5081, 5082, 5083,
5084, 5085, 5086, 5087, 5088, 5090, 5091, 5092, 5093)
def download(surface_id, index):
t = time.time()
message = 'Surface #%s (%s) - Process started\n' % (index, surface_id)
time.sleep(random.random())
message += 'Process ended : %.2f s\n' % (time.time() - t)
return message
def pass_args(args):
return download(*args)
class CustomEvent(QtCore.QEvent):
DownloadComplete = QtCore.QEvent.registerEventType()
def __init__(self, typeid, *args):
super().__init__(typeid)
self.data = args
class Console(QtGui.QDialog):
def __init__(self):
super().__init__()
self.resize(600, 300)
self.setMinimumSize(QtCore.QSize(600, 300))
self.setWindowTitle("Console")
self.verticalLayout = QtGui.QVBoxLayout(self)
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(
QtGui.QDialogButtonBox.Close | QtGui.QDialogButtonBox.Ok)
self.button_box.setObjectName("button_box")
self.verticalLayout.addWidget(self.button_box)
self.button_box.button(QtGui.QDialogButtonBox.Close
).clicked.connect(self.button_cancel_clicked)
self.button_box.button(QtGui.QDialogButtonBox.Ok
).clicked.connect(self.button_ok_clicked)
self.pool = multiprocessing.Pool(None)
def event(self, event):
if event.type() == CustomEvent.DownloadComplete:
message, complete = event.data
self.write_stream(message)
if complete:
self.write_stream('Downloads complete!')
return super().event(event)
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
total = len(ids)
def callback(message):
nonlocal total
total -= 1
QtGui.qApp.postEvent(self, CustomEvent(
CustomEvent.DownloadComplete, message, not total))
for index, surface_id in enumerate(ids):
self.pool.apply_async(
pass_args, [(surface_id, index)], callback=callback)
def write_stream(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()