为什么在某些情况下并行计算使用python的pp模块会更慢？
查看代码。当genMotifs的参数设置为n_seq = 5000且n_pos = 10时，并行版本getPairedSeqNames3和getPairedSeqNames1要慢得多。但是当n_seq = 50且n_pos = 2000时，并行版本的性能会更好。不幸的是，我正在处理的数据更像是n_seq = 5000和n_pos = 10。谁能告诉我为什么会这样呢？当n_seq = 5000和n_pos = 10时，有没有办法让并行版本表现更好？
以下是代码：
#! /usr/bin/env python
import pp, sys, random, time
def getMotif_SeqName(Motifs):
return dict([(uid, set(Motifs[uid].keys())) for uid in Motifs.keys()])
def getPairedList(uids):
return [(id1, id2) for i, id1 in enumerate(uids) for id2 in uids[i:] if id1 != id2]
def is_overlap(pos_pair):
(posA, posB) = pos_pair
if max(posA) < min(posB) or min(posA) > max(posB):
return False
else:
return True
def caclDist(pos_pair):
(posA, posB) = pos_pair
d1 = min(posB) - max(posA)
d2 = min(posA) - max(posB)
return {True: d1, False: -d2}[d1 > d2]
def getDist(posA, posB, low, high):
comb = [(i, j) for i in posA for j in posB]
not_overlap = [e for e in comb if not is_overlap(e)]
distances = map(caclDist, not_overlap)
CoDist = {}
for i, d in enumerate(distances):
if abs(d) >= low and abs(d) <= high:
CoDist[not_overlap[i]] = d
return CoDist
def getDist2(uidA, uidB, seqname, posA, posB, low, high):
comb = [(i, j) for i in posA for j in posB]
not_overlap = [e for e in comb if not is_overlap(e)]
distances = map(caclDist, not_overlap)
CoDist = {}
for i, d in enumerate(distances):
if abs(d) >= low and abs(d) <= high:
CoDist[not_overlap[i]] = d
return (uidA, uidB, seqname, CoDist)
def ppCacl(job_server, inputs, equation, funs, packages, Progress=True):
num_inputs = len(inputs) / 100 + 1
jobs = [job_server.submit(equation, pars, funs, packages) for pars in inputs]
return [job() for job in jobs]
def ssCacl(inputs, equation):
ps = []
for i, (X, n, m, N) in enumerate(inputs):
ps.append(equation(X, n, m, N))
return ps
def getPairedSeqNames1(Motifs):
SeqNames = getMotif_SeqName(Motifs)
MotifPairs = set(getPairedList(Motifs.keys()))
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = {}
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions = [(PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect]
distances = ppCacl(job_server, positions, getDist, (is_overlap, caclDist), (), False)
distances = dict([(intersect[i], d) for i, d in enumerate(distances) if d])
if distances:
PairedMotifs[(uidA, uidB)] = distances
return PairedMotifs
def getPairedSeqNames2(Motifs):
SeqNames = getMotif_SeqName(Motifs)
MotifPairs = set(getPairedList(Motifs.keys()))
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = {}
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions = [(PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect]
distances = ssCacl(positions, getDist)
distances = dict([(intersect[i], d) for i, d in enumerate(distances) if d])
if distances:
PairedMotifs[(uidA, uidB)] = distances
return PairedMotifs
def getPairedSeqNames3(Motifs):
SeqNames = getMotif_SeqName(Motifs)
MotifPairs = set(getPairedList(Motifs.keys()))
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = {}
positions = []
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions.extend([(uidA, uidB, seqname, PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect])
distances = ppCacl(job_server, positions, getDist2, (is_overlap, caclDist), (), False)
for (uidA, uidB, seqname, CoDist) in distances:
if CoDist:
if not PairedMotifs.has_key((uidA, uidB)):
PairedMotifs[(uidA, uidB)] = {}
PairedMotifs[(uidA, uidB)][seqname] = CoDist
return PairedMotifs
def genMotifs(n_seq=5000, n_pos=10):
digits = range(1, 60000)
Motifs = {}
uids = random.sample(digits, 50)
for uid in uids:
seqnames = random.sample(digits, random.randint(0, n_seq))
Motifs[uid] = {}
for seqname in seqnames:
Motifs[uid][seqname] = genPos(random.randint(0, n_pos))
return Motifs
def genPos(n):
return [(random.randint(0, 3000),random.randint(0, 3000)) for i in xrange(0,n)]
job_server = pp.Server()
Motifs = genMotifs()
timestamp = time.time()
getPairedSeqNames1(Motifs)
print time.time() - timestamp
timestamp = time.time()
getPairedSeqNames2(Motifs)
print time.time() - timestamp
timestamp = time.time()
getPairedSeqNames3(Motifs)
print time.time() - timestamp
Motifs = genMotifs(50, 2000)
timestamp = time.time()
getPairedSeqNames1(Motifs)
print time.time() - timestamp
timestamp = time.time()
getPairedSeqNames2(Motifs)
print time.time() - timestamp
timestamp = time.time()
getPairedSeqNames3(Motifs)
print time.time() - timestamp
我的电脑上的结果：
1225 pairs to go
57.377081871 16666 20431
1225 pairs to go
15.1005380154 16666 20431
1225 pairs to go
59.9019329548 16666 20431
1225 pairs to go
43.1178700924 11721 46015
1225 pairs to go
77.7199709415 11721 46015
1225 pairs to go
10.1687381268 11721 46015
getPairedSeqNames3的cProfile n_seq = 5000 n_pos = 10
getPairedSeqNames3的cProfile n_seq = 10 n_pos = 5000
getPairedSeqNames3的cProfile n_seq = 20 n_pos = 2500
1 个答案:
答案 0 :(得分：0)
我改变了你的代码以使用更好的python习语：
#! /usr/bin/env python
import pp
import sys
import random
import time
from collections import defaultdict
job_server = pp.Server()
def getMotif_SeqName(Motifs):
return {uid: set(d.keys()) for uid, d in Motifs.items()}
def getPairedList(uids):
return [(id1, id2) for i, id1 in enumerate(uids) for id2 in uids[i:] if id1 != id2]
def is_overlap(pos_pair):
(posA, posB) = pos_pair
return not (max(posA) < min(posB) or min(posA) > max(posB))
def caclDist(pos_pair):
(posA, posB) = pos_pair
d1 = min(posB) - max(posA)
d2 = min(posA) - max(posB)
return d1 if d1 > d2 else -d2
def getDist(posA, posB, low, high):
comb = ((i, j) for i in posA for j in posB)
not_overlap = [e for e in comb if not is_overlap(e)]
distances = map(caclDist, not_overlap)
return {
not_over: d
for not_over, d in zip(not_overlap, distances)
if low <= abs(d) <= high
}
def getDist2(uidA, uidB, seqname, posA, posB, low, high):
return (uidA, uidB, seqname, getDist(posA, posB, low, high))
def ppCacl(job_server, inputs, equation, funs, packages, Progress=True):
jobs = (job_server.submit(equation, pars, funs, packages) for pars in inputs)
return [job() for job in jobs]
def ssCacl(inputs, equation):
return [equation(X, n, m, N) for (X, n, m, N) in inputs]
def getPairedSeqNames1(Motifs, SeqNames, MotifPairs):
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = {}
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions = [(PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect]
distances = ppCacl(job_server, positions, getDist, (is_overlap, caclDist), (), False)
distances = {index: d for index, d in zip(intersect, distances) if d}
if distances:
PairedMotifs[(uidA, uidB)] = distances
return PairedMotifs
def getPairedSeqNames2(Motifs, SeqNames, MotifPairs):
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = {}
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions = ((PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect)
distances = ssCacl(positions, getDist)
distances = {index: d for index, d in zip(intersect, distances) if d}
if distances:
PairedMotifs[(uidA, uidB)] = distances
return PairedMotifs
def getPairedSeqNames3(Motifs, SeqNames, MotifPairs):
num_MotifPairs = len(MotifPairs)
print "%s pairs to go" % num_MotifPairs
num_MotifPairs = num_MotifPairs / 100 + 1
PairedMotifs = defaultdict(dict)
positions = []
for i, (uidA, uidB) in enumerate(MotifPairs):
intersect = list(SeqNames[uidA] & SeqNames[uidB])
if intersect:
PosA = Motifs[uidA]
PosB = Motifs[uidB]
sys.stderr.write("Progress:%d%%\t%s\t%s\r" % (i / num_MotifPairs, uidA, uidB))
positions.extend([(uidA, uidB, seqname, PosA[seqname], PosB[seqname], 10, 250) for seqname in intersect])
distances = ppCacl(job_server, positions, getDist2, (is_overlap, caclDist), (), False)
for (uidA, uidB, seqname, CoDist) in distances:
if CoDist:
PairedMotifs[(uidA, uidB)][seqname] = CoDist
return PairedMotifs
def genMotifs(n_seq, n_pos):
digits = range(1, 60000)
uids = random.sample(digits, 50)
return {
uid: {
seqname: genPos(random.randint(0, n_pos))
for seqname in random.sample(digits, random.randint(0, n_seq))
}
for uid in uids
}
def genPos(n):
return [(random.randint(0, 3000), random.randint(0, 3000)) for _ in xrange(n)]
def driver(Motifs):
SeqNames = getMotif_SeqName(Motifs)
MotifPairs = set(getPairedList(Motifs.keys()))
for fn in (getPairedSeqNames1, getPairedSeqNames2, getPairedSeqNames3):
timestamp = time.time()
fn(Motifs, SeqNames, MotifPairs)
print time.time() - timestamp
if __name__ == '__main__':
for x, y in ((5000, 10), (50, 2000)):
print '=' * 30
driver(genMotifs(x, y))
我不能保证你会更快。如果您希望优化代码，我可以使用cProfile或使用numpy或cython来查看分析。
相关问题
最新问题
- 我写了这段代码，但我无法理解我的错误
- 我无法从一个代码实例的列表中删除 None 值，但我可以在另一个实例中。为什么它适用于一个细分市场而不适用于另一个细分市场？
- 是否有可能使 loadstring 不可能等于打印？卢阿
- java中的random.expovariate()
- Appscript 通过会议在 Google 日历中发送电子邮件和创建活动
- 为什么我的 Onclick 箭头功能在 React 中不起作用？
- 在此代码中是否有使用“this”的替代方法？
- 在 SQL Server 和 PostgreSQL 上查询，我如何从第一个表获得第二个表的可视化
- 每千个数字得到
- 更新了城市边界 KML 文件的来源？