我的目标是遍历目录并计算其中所有文件的MD5。我使用代码来解决类似的问题
Parallel file matching, Python
import os
import re
import sys
import time
import md5
from stat import S_ISREG
import multiprocessing
global queue
size_limit = 500000
target = sys.argv[1]
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
############Analysis and Multiprocessing####################
def walk_files(topdir):
"""yield up full pathname for each file in tree under topdir"""
for dirpath, dirnames, filenames in os.walk(topdir):
for fname in filenames:
pathname = os.path.join(dirpath, fname)
yield pathname
def files_to_search(topdir):
"""yield up full pathname for only files we want to search"""
for fname in walk_files(topdir):
try:
# if it is a regular file and big enough, we want to search it
sr = os.stat(fname)
if S_ISREG(sr.st_mode) and sr.st_size <= size_limit:
yield fname
except OSError:
pass
def worker_search_fn(fname):
fp = open(fname, 'rt')
# read one line at a time from file
contents = fp.read()
hash = md5.md5(contents)
global queue
print "enqueue"
queue.put(fname+'-'+hash.hexdigest())
################MAIN MAIN MAIN#######################
################MAIN MAIN MAIN#######################
################MAIN MAIN MAIN#######################
################MAIN MAIN MAIN#######################
################MAIN MAIN MAIN#######################
#kick of processes to md5 the files and wait till completeion
queue = multiprocessing.Queue()
pool = multiprocessing.Pool()
pool.map(worker_search_fn, files_to_search(target))
pool.close()
pool.join()
#Should be done, now lets send do our analysis
while not queue.empty():
print queue.get()
我添加了“print enqueue”语句作为调试目的,我注意到在递归大目录树时代码确实锁定了。我不确定两个进程是否同时尝试访问队列,从而导致死锁。
也许有更好的方法可以做到这一点?该结构不必是队列,但必须是无锁的,以便充分利用多处理。我想要一个recurse和md5并行一个目录,一旦完成,就可以对整个列表做一些事情。为了调试我只是打印完成的队列。有什么建议吗?
答案 0 :(得分:3)
目前还不清楚你的程序是I / O还是CPU绑定,即,如果任务受I / O限制,单个进程可以比几个进程执行得更好,例如,通过最小化磁盘搜索次数。您可以通过指定不同的nprocesses
值(下方)来检查这一点,并查看在您的案例中提供更好结果的内容。
在这种情况下你不需要队列:
#!/usr/bin/env python
import os
import sys
from hashlib import md5
from multiprocessing import Pool, freeze_support
from stat import S_ISREG
def walk_files(topdir):
"""yield up full pathname for each file in tree under topdir"""
for dirpath, dirnames, filenames in os.walk(topdir):
for fname in filenames:
pathname = os.path.join(dirpath, fname)
yield pathname
def files_to_process(topdir, size_limit):
"""yield up full pathname for only files we want to process"""
for fname in walk_files(topdir):
try: sr = os.stat(fname)
except OSError: pass
else:
# if it is a regular file and small enough, we want to process it
if S_ISREG(sr.st_mode) and sr.st_size <= size_limit:
yield fname
def md5sum(fname):
with open(fname, 'rb') as fp:
# read all file at once
contents = fp.read()
hash = md5(contents)
return fname, hash.hexdigest()
def main(argv=None):
if argv is None:
argv = sys.argv
topdir = argv[1]
size_limit = 500000
nprocesses = 1
pool = Pool(processes=nprocesses)
files = files_to_process(topdir, size_limit)
for fname, hexdigest in pool.imap_unordered(md5sum, files):
print("%s\t%s" % (fname, hexdigest))
if __name__=="__main__":
freeze_support()
main()
$ python md5sum.py .
./md5sum.py 9db44d3117673790f1061d4b8f00e8ce
答案 1 :(得分:1)
因为大目录需要很长时间才能执行walk_files()
它不是死锁
和...
删除pool.join()
multiprocessing.Pool().map()
阻止,直到结果准备就绪,因此您不需要pool.join()