Question

我有一个脚本来读取一个文件，这个文件可以是10次演出，我希望使用多处理来处理它。

这是一个压缩算法，我希望用户定义一个缓冲区，然后启动3个进程，一个从文件中读取缓冲区的行数，将这些行传递给处理过程，然后将处理过的行传递给将行写入新文件的进程。我希望所有这些能够同时发生，并且每个进程都要等待下一行。

我已经有了脚本，但是当我运行它时，它并没有结束。我觉得这些过程有问题。我认为它与我的阅读功能中的islice有关，但我不知道如何更好地编写它。

import multiprocessing as mp
import time
from itertools import islice

def read(from_filename, buffer, process_queue):
  file = open(from_filename, 'r')
  slice = islice(file, buffer)
  while slice:
    to_process = []
    for line in slice:
      to_process.append(line)
    process_queue.put(to_process)
  process_queue.put('kill')

def write(to_filename, write_queue):
  to_file = open(to_filename, 'a+')
  while 1:
    to_write = write_queue.get()
    if to_write == 'kill':
      break
    to_file.write(to_write + '\n')

def compress(process_queue, write_queue):
  while 1:
    to_process = process_queue.get()
    if to_process == 'kill':
      write_queue.put('kill')
      break
    # process, put output in to_write
    write_queue.put(to_write)

def decompress(process_queue, write_queue):
  while 1:
    to_process = process_queue.get()
    if to_process == 'kill':
      write_queue.put('kill')
      break
    # process, put output in to_write
    write_queue.put(to_write)

def main():
  option = raw_input("C for Compress OR D for Decompress: ")
  from_file = raw_input("Enter input filename: ")
  buf = int(raw_input("Enter line buffer: "))
  to_file = raw_input("Enter output filename: ")
  start = time.time()
  write_queue = mp.Queue()
  process_queue = mp.Queue()
  reader = mp.Process(target=read, args=(from_file, buf, process_queue))
  writer = mp.Process(target=write, args=(to_file, write_queue))
  if option == 'c' or option == 'C':
    processor = mp.Process(target=compress, args=(process_queue, write_queue))
  elif option == 'd' or option == 'D':
    processor = mp.Process(target=decompress, args=(process_queue, write_queue))
  else:
    print "Invalid Options..."
  writer.start()
  processor.start()
  reader.start()
  reader.join()
  processor.join()
  writer.join()
  end = time.time()
  elapsed = (end - start)
  print "\n\nTotal Time Elapsed: " + str(elapsed) + " secs"

if __name__=='__main__':
  main()

这是我第一次尝试多处理。当我运行它时，它不会结束。我认为某个过程会被困在某个地方。

Answer 1

这段代码错了：

def read(from_filename, buffer, process_queue):
  file = open(from_filename, 'r')
  slice = islice(file, buffer)
  while slice:
    to_process = []
    for line in slice:
      to_process.append(line)
    process_queue.put(to_process)
  process_queue.put('kill')

由于slice是islice对象，因此条件while slice将始终为真，因此就像在那里while True一样。您应该每次都重新创建切片对象。

def read(from_filename, buffer, process_queue):
  file = open(from_filename, 'r')

  while True:
    slice = islice(file, buffer)
    to_process = []
    for line in slice:
      to_process.append(line)
    process_queue.put(to_process)
    if not to_process:
        # input ended
        break
  process_queue.put('kill')

或者你可以这样做：

def read_chunk(file, buffer):
    return [file.readline() for _ in xrange(buffer)]
    # or, "more" equivalent to using islice
    #return [line for i,line in itertools.izip(xrange(buffer), file)]

def read(from_filename, buffer, process_queue):
  file = open(from_filename, 'r')

  for to_process in iter(lambda: read_chunk(file, buffer), []):
    process_queue.put(to_process)
  process_queue.put('kill')

请注意，如果必须构建列表，则使用itertools.islice没有意义。

进程不是以python结尾

1 个答案: