我正在使用多处理来调试问题 我有以下孩子:
class Target(multiprocessing.Process):
def __init__(self, work_queue, result_queue, suite_name, test_settings, html_log_dir, output_file, ip_address, fit_dir):
multiprocessing.Process.__init__(self)
# initialize other variables
def run(self):
print multiprocessing.current_process()
suite_start_time = time.clock()
while not self.kill_received:
# get a task
try:
job = self.work_queue.get(True, 2)
except Queue.Empty:
self._log('Work queue empty, creating XML result file')
self.create_xml_result_file(suite_start_time)
break
# the actual processing, run the test.
fitnesse_common.log_output("\n(PID " + str(self.pid) + "): End of process")
def create_xml_result_file(self, start_time):
# generate result
父进程基本上只启动了几个(12)目标并等待它们全部加入。
问题是由于某种原因,子进程正在运行到run函数的末尾(我看到进程打印结束),然后由于某种原因没有终止,这阻止了父进程的继续。
编辑 - 并非所有衍生进程都挂起,只有几个。在12个衍生过程中,通常只有2-4个过程在完成其运行功能后挂起。
我考虑在run函数结束时调用terminate,但Python文档表明这是一个坏主意。
我已经看过有关Python多处理的Stack Overflow的几篇不同文章,其中大部分与父进程的问题有关。
非常感谢任何想法或帮助。
更新:这是一个很容易复制问题的脚本:
import multiprocessing, Queue
import subprocess
import time
import sys
class Target(multiprocessing.Process):
def __init__(self, work_queue, results_queue, delay_length):
# base class initialization
multiprocessing.Process.__init__(self)
# job management stuff
self.work_queue = work_queue
self.results_queue = results_queue
self.delay_length = delay_length
self.kill_received = False
def run(self):
while not self.kill_received:
# get a task
try:
job = self.work_queue.get(True, 2)
except Queue.Empty:
self._log('Work queue empty, prepare to terminate')
break
time.sleep(self.delay_length)
self._log("Sleeping done")
results = self._run_an_application(job)
self.results_queue.put(results)
self._log("Have put results on queue " + str(job) + "-" + results)
self._log("\n(PID " + str(self.pid) + "): End of process")
def _log(self, text):
print ('PID ' + str(self.pid) + ' => ' + text)
sys.stdout.flush()
def _run_an_application(self, app):
try:
test_output = subprocess.check_output(app)
except subprocess.CalledProcessError, e:
log_output('### Process check_output threw exception CalledProcessError')
test_output = e.output
return test_output
if __name__ == "__main__":
test_jobs = []
started_targets = []
# run
# load up work queue
for i in range(500):
test_jobs.append('spewage')
work_queue = multiprocessing.Queue()
for job in test_jobs:
work_queue.put(job)
# create a queue to pass to targets to store the results
result_queue = multiprocessing.Queue()
# spawn targets
for i in range(12):
started_targets.append(Target(work_queue, result_queue, i))
# start all targets
for i in range(len(started_targets)):
started_targets[i].start()
print "starting process no %s with id: %s" % (i, started_targets[i].pid)
print 'Waiting for all processes to join'
# wait for all targets to finish
for i in range(len(started_targets)):
started_targets[i].join()
print 'All processes have joined'
# collect the results off the queue
while not result_queue.empty():
target_result = result_queue.get()
print "Test job - " + target_result
print ('All Tests completed')
这是“spewage”应用程序的源代码(它是C ++)。
#include <iostream>
#include <windows.h>
using namespace std;
int main()
{
for (int i = 0; i < 500; i++)
{
cout << "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" << endl;
Sleep(20);
}
return 0;
}
由于它似乎与推送到stdout的数量有关,因此C ++程序很容易被另一个打印很多东西的脚本所取代。
答案 0 :(得分:1)
我设法解决了这个问题。它似乎与子流程中的输出量有关。在进程'run()函数结束时,我需要放置self.results_queue.cancel_join_thread()
我仍然很好奇为什么它在没有很多stdout的情况下工作,但是当有时,进程会挂起。根据Python文档,我使用result_queue的方式应该一直锁定,即使它没有。