在question之后，我使用下面的代码将数据从多处理任务写入文件。
#!/usr/bin/env python3.5
# coding: utf-8
"""Python multiprocessing safely writing to a file"""
# https://stackoverflow.com/questions/13446445/python-multiprocessing-safely-writing-to-a-file
import random
import multiprocessing as mp
import time
fn = '/tmp/tmp.txt'
def worker(arg, q):
"""stupidly simulates long running process"""
idw = random.randint(0, 200)
print("[%s] start worker" % idw)
start = time.clock()
s = 'this is a test\n'
txt = s
for i in range(2000000):
txt += s
done = time.clock() - start
with open(fn, 'rb') as f:
size = len(f.read())
res = 'Process worker ' + str(idw) + ' :: ' + str(arg), str(size), done
print("[%s] worker finish job" % idw)
q.put(res)
return res
def listener(q):
"""listens for messages on the q, writes to file. """
print("start listener")
f = open(fn, 'wb')
while 1:
m = q.get()
print("listener get message: {}".format(m))
if m == 'kill':
print("listener get kill message")
f.write('killed')
break
f.write(str(m) + '\n')
f.flush()
f.close()
def main():
print("start main")
# must use Manager queue here, or will not work
manager = mp.Manager()
q = manager.Queue()
pool = mp.Pool(mp.cpu_count() + 2)
# put listener to work first
watcher = pool.apply_async(listener, (q,))
# fire off workers
jobs = []
for i in range(20):
job = pool.apply_async(worker, (i, q))
jobs.append(job)
# collect results from the workers through the pool result queue
for job in jobs:
job.get()
# now we are done, kill the listener
q.put('kill')
pool.close()
pool.join()
if __name__ == "__main__":
main()
适用于Python 2.7：
start main
start listener
[139] start worker
[115] start worker
[195] start worker
[86] start worker
[124] start worker
[195] worker finish job
[139] worker finish job
listener get message: ('Process worker 195 :: 2', '0', 0.126242)
listener get message: ('Process worker 139 :: 0', '0', 0.125833)
[188] start worker
[123] start worker
[115] worker finish job
listener get message: ('Process worker 115 :: 1', '86', 0.129909)
[196] start worker
[86] worker finish job
listener get message: ('Process worker 86 :: 3', '130', 0.127318)
[124] worker finish job
listener get message: ('Process worker 124 :: 4', '174', 0.124823)
[88] start worker
[148] start worker
[123] worker finish job
listener get message: ('Process worker 123 :: 6', '219', 0.138156)
[196] worker finish job
listener get message: ('Process worker 196 :: 7', '264', 0.13563200000000003)
[188] worker finish job
listener get message: ('Process worker 188 :: 5', '320', 0.136552)
[88] worker finish job
listener get message: ('Process worker 88 :: 8', '365', 0.13255199999999998)
[148] worker finish job
listener get message: ('Process worker 148 :: 9', '420', 0.12930799999999998)
listener get message: kill
listener get kill message
但是使用Python 3.5或3.6，每个工作程序都已启动，但只有第一个工作人员将消息发送给监听器。
start main
start listener
[163] start worker
[138] start worker
[128] start worker
[199] start worker
[190] start worker
[128] worker finish job
[163] worker finish job
listener get message: ('Process worker 128 :: 3', '0', 0.129444)
[33] start worker
[78] start worker
[138] worker finish job
[170] start worker
[42] start worker
[199] worker finish job
[170] worker finish job
[153] start worker
[78] worker finish job
[190] worker finish job
[33] worker finish job
[42] worker finish job
[153] worker finish job
你有解决这个问题的线索吗？
感谢。