Question

嘿，我尝试创建一个多处理复制过程。但是我有几个问题。我总是收到错误OSError：[Errno 24]打开的文件太多。我认为我的函数创建了太多同时打开的进程。我是多处理领域的新手。我有一个多线程版本，但我想通过多处理来实现。函数copyFileList的输入是一个具有两个列的数据帧，分别带有文件源和文件目标。那么我在这里做错了吗？最好的问候！

from multiprocessing import Queue, Process, Value, Lock, cpu_count 

class CTError(Exception):
    def __init__(self, errors):
        self.errors = errors

try:
    O_BINARY = os.O_BINARY
except:
    O_BINARY = 0

READ_FLAGS = os.O_RDONLY | O_BINARY
WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
BUFFER_SIZE = 128*1024

def copyfile2(src, dst):

    try:
        fin = os.open(src, READ_FLAGS)
        stat = os.fstat(fin)
        fout = os.open(dst, WRITE_FLAGS, stat.st_mode)
        for x in iter(lambda: os.read(fin, BUFFER_SIZE), b""):
            os.write(fout, x)
    finally:
        try: os.close(fin)
        except: pass
        try: os.close(fout)
        except: pass

def addToQueueFile(src,dst, queue):
    queue.put(copyfile2(src,dst))

    queue.get()

def copyFileList(srcDst,multiproc=2):
    from multiprocessing import Queue, Process, cpu_count
    if multiproc == 2:
        if __name__ == '__main__':

            queue = Queue()
            processes = []

            for count,row in enumerate(srcDst.itertuples()):

                processes.append(Process(target = addToQueueFile,
                                         args = (row[1],row[2],queue,)))
                processes[-1].start()

Python：多处理复制

0 个答案: