Python Pandas Multiprocessing Apply

Question

我想知道是否有办法并行执行pandas数据帧应用功能。我环顾四周，找不到任何东西。至少从理论上讲，我认为实施它应该相当简单，但却没有看到任何东西。这实际上毕竟是平行的教科书定义..有没有其他人尝试过这种方式或知道一种方法？如果没有人有任何想法，我想我可能会尝试自己编写。

我正在使用的代码如下。很抱歉缺少import语句。它们与许多其他东西混在一起。

def apply_extract_entities(row):
     names=[]
     counter=0
     print row
     for sent in nltk.sent_tokenize(open(row['file_name'], "r+b").read()):
         for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
             if hasattr(chunk, 'node'):
                 names+= [chunk.node, ' '.join(c[0] for c in chunk.leaves())]
                 counter+=1
                 print counter
     return names

data9_2['proper_nouns']=data9_2.apply(apply_extract_entities, axis=1) 

编辑：

所以这就是我的尝试。我尝试使用我的iterable的前五个元素来运行它，并且它比连续运行它所花费的时间更长所以我认为它不起作用。

os.chdir(str(home))
data9_2=pd.read_csv('edgarsdc3.csv')
os.chdir(str(home)+str('//defmtest'))

#import stuff
from nltk import pos_tag, ne_chunk
from nltk.tokenize import SpaceTokenizer

#define apply function and apply it
os.chdir(str(home)+str('//defmtest'))

####

#this is our apply function
def apply_extract_entities(row):
    names=[]
    counter=0
    print row
    for sent in nltk.sent_tokenize(open(row['file_name'], "r+b").read()):
        for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
            if hasattr(chunk, 'node'):
            names+= [chunk.node, ' '.join(c[0] for c in chunk.leaves())]
            counter+=1
            print counter
    return names


#need something that populates a list of sections of a dataframe
def dataframe_splitter(df):
     df_list=range(len(df))
     for i in xrange(len(df)):
         sliced=df.ix[i]
         df_list[i]=sliced
     return df_list

df_list=dataframe_splitter(data9_2)
#df_list=range(len(data9_2))
print df_list

#the multiprocessing section
import multiprocessing

def worker(arg):
    print arg
    (arg)['proper_nouns']=arg.apply(apply_extract_entities, axis=1)
    return arg

pool = multiprocessing.Pool(processes=10)

# get list of pieces
res = pool.imap_unordered(worker, df_list[:5])
res2= list(itertools.chain(*res))
pool.close()
pool.join()

# re-assemble pieces into the final output
output = data9_2.head(1).concatenate(res)
print output.head()

Answer 1

通过多处理，最好生成几个大块数据，然后重新组装它们以产生最终输出。

源

import multiprocessing

def worker(arg):
    return arg*2

pool = multiprocessing.Pool()

# get list of pieces
res = pool.map(worker, [1,2,3])
pool.close()
pool.join()

# re-assemble pieces into the final output
output = sum(res)
print 'got:',output

输出

got: 12