我正在开发涉及大量数据的程序。我正在使用python pandas模块来查找数据中的错误。这通常非常快。然而，我写的这段代码似乎比应该的速度慢，我正在寻找一种方法来加速它。
为了让你们正确测试它，我上传了一大堆代码。你应该能够按原样运行它。代码中的注释应该解释我在这里要做的事情。任何帮助将不胜感激。
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
# Filling dataframe with data
# Just ignore this part for now, real data comes from csv files, this is an example of how it looks
TimeOfDay_options = ['Day','Evening','Night']
TypeOfCargo_options = ['Goods','Passengers']
np.random.seed(1234)
n = 10000
df = pd.DataFrame()
df['ID_number'] = np.random.randint(3, size=n)
df['TimeOfDay'] = np.random.choice(TimeOfDay_options, size=n)
df['TypeOfCargo'] = np.random.choice(TypeOfCargo_options, size=n)
df['TrackStart'] = np.random.randint(400, size=n) * 900
df['SectionStart'] = np.nan
df['SectionStop'] = np.nan
grouped_df = df.groupby(['ID_number','TimeOfDay','TypeOfCargo','TrackStart'])
for index, group in grouped_df:
if len(group) == 1:
df.loc[group.index,['SectionStart']] = group['TrackStart']
df.loc[group.index,['SectionStop']] = group['TrackStart'] + 899
if len(group) > 1:
track_start = group.loc[group.index[0],'TrackStart']
track_end = track_start + 899
section_stops = np.random.randint(track_start, track_end, size=len(group))
section_stops[-1] = track_end
section_stops = np.sort(section_stops)
section_starts = np.insert(section_stops, 0, track_start)
for i,start,stop in zip(group.index,section_starts,section_stops):
df.loc[i,['SectionStart']] = start
df.loc[i,['SectionStop']] = stop
#%% This is what a random group looks like without errors
#Note that each section neatly starts where the previous section ended
#There are no gaps (The whole track is defined)
grouped_df.get_group((2, 'Night', 'Passengers', 323100))
#%% Introducing errors to the data
df.loc[2640,'SectionStart'] += 100
df.loc[5390,'SectionStart'] += 7
#%% This is what the same group looks like after introducing errors
#Note that the 'SectionStop' of row 1525 is no longer similar to the 'SectionStart' of row 2640
#This track now has a gap of 100, it is not completely defined from start to end
grouped_df.get_group((2, 'Night', 'Passengers', 323100))
#%% Try to locate the errors
#This is the part of the code I need to speed up
def Full_coverage(group):
if len(group) > 1:
#Sort the grouped data by column 'SectionStart' from low to high
#Updated for newer pandas version
#group.sort('SectionStart', ascending=True, inplace=True)
group.sort_values('SectionStart', ascending=True, inplace=True)
#Some initial values, overwritten at the end of each loop
#These variables correspond to the first row of the group
start_km = group.iloc[0,4]
end_km = group.iloc[0,5]
end_km_index = group.index[0]
#Loop through all the rows in the group
#index is the index of the row
#i is the 'SectionStart' of the row
#j is the 'SectionStop' of the row
#The loop starts from the 2nd row in the group
for index, (i, j) in group.iloc[1:,[4,5]].iterrows():
#The start of the next row must be equal to the end of the previous row in the group
if i != end_km:
#Add the faulty data to the error list
incomplete_coverage.append(('Expected startpoint: '+str(end_km)+' (row '+str(end_km_index)+')', \
'Found startpoint: '+str(i)+' (row '+str(index)+')'))
#Overwrite these values for the next loop
start_km = i
end_km = j
end_km_index = index
return group
#Check if the complete track is completely defined (from start to end) for each combination of:
#'ID_number','TimeOfDay','TypeOfCargo','TrackStart'
incomplete_coverage = [] #Create empty list for storing the error messages
df_grouped = df.groupby(['ID_number','TimeOfDay','TypeOfCargo','TrackStart']).apply(lambda x: Full_coverage(x))
#Print the error list
print('\nFound incomplete coverage in the following rows:')
for i,j in incomplete_coverage:
print(i)
print(j)
print()
#%%Time the procedure -- It is very slow, taking about 6.6 seconds on my pc
%timeit df.groupby(['ID_number','TimeOfDay','TypeOfCargo','TrackStart']).apply(lambda x: Full_coverage(x))
答案 0 :(得分：11)
我认为，问题在于您的数据有5300个不同的组。因此，您的功能中任何缓慢的东西都会被放大。您可以在函数中使用向量化操作而不是for循环来节省时间，但更简单的方法是缩短几秒钟，而不是return 0而不是return group。当您return group时，pandas实际上会创建一个新的数据对象，将您排序的组合在一起，而您似乎并未使用这些组。当你return 0时，pandas会将5300个零组合起来，这要快得多。
例如：
cols = ['ID_number','TimeOfDay','TypeOfCargo','TrackStart']
groups = df.groupby(cols)
print(len(groups))
# 5353
%timeit df.groupby(cols).apply(lambda group: group)
# 1 loops, best of 3: 2.41 s per loop
%timeit df.groupby(cols).apply(lambda group: 0)
# 10 loops, best of 3: 64.3 ms per loop
将您不使用的结果组合起来大约需要2.4秒;剩下的时间是循环中的实际计算，你应该尝试向量化。
编辑：
在for循环之前快速进行额外的矢量化检查并返回0而不是group，我将时间缩短到约2秒，这基本上是每个排序的成本组。试试这个功能：
def Full_coverage(group):
if len(group) > 1:
group = group.sort('SectionStart', ascending=True)
# this condition is sufficient to find when the loop
# will add to the list
if np.any(group.values[1:, 4] != group.values[:-1, 5]):
start_km = group.iloc[0,4]
end_km = group.iloc[0,5]
end_km_index = group.index[0]
for index, (i, j) in group.iloc[1:,[4,5]].iterrows():
if i != end_km:
incomplete_coverage.append(('Expected startpoint: '+str(end_km)+' (row '+str(end_km_index)+')', \
'Found startpoint: '+str(i)+' (row '+str(index)+')'))
start_km = i
end_km = j
end_km_index = index
return 0
cols = ['ID_number','TimeOfDay','TypeOfCargo','TrackStart']
%timeit df.groupby(cols).apply(Full_coverage)
# 1 loops, best of 3: 1.74 s per loop
编辑2：这里有一个例子，其中包含我的建议，以便在groupby之外移动排序并删除不必要的循环。对于给定的示例，删除循环的速度并不快，但如果存在大量不完整的情况，则会更快：
def Full_coverage_new(group):
if len(group) > 1:
mask = group.values[1:, 4] != group.values[:-1, 5]
if np.any(mask):
err = ('Expected startpoint: {0} (row {1}) '
'Found startpoint: {2} (row {3})')
incomplete_coverage.extend([err.format(group.iloc[i, 5],
group.index[i],
group.iloc[i + 1, 4],
group.index[i + 1])
for i in np.where(mask)[0]])
return 0
incomplete_coverage = []
cols = ['ID_number','TimeOfDay','TypeOfCargo','TrackStart']
df_s = df.sort_values(['SectionStart','SectionStop'])
df_s.groupby(cols).apply(Full_coverage_nosort)
答案 1 :(得分：0)
我发现pandas locate命令（.loc或.iloc）也在减慢进度。通过将循环移出循环并在函数开始时将数据转换为numpy数组，我获得了更快的结果。我知道数据不再是数据帧，但列表中返回的索引可用于查找原始df中的数据。
如果有任何方法可以进一步加快这一过程，我将不胜感激。到目前为止我所拥有的：
def Full_coverage(group):
if len(group) > 1:
group_index = group.index.values
group = group.values
# this condition is sufficient to find when the loop will add to the list
if np.any(group[1:, 4] != group[:-1, 5]):
start_km = group[0,4]
end_km = group[0,5]
end_km_index = group_index[0]
for index, (i, j) in zip(group_index, group[1:,[4,5]]):
if i != end_km:
incomplete_coverage.append(('Expected startpoint: '+str(end_km)+' (row '+str(end_km_index)+')', \
'Found startpoint: '+str(i)+' (row '+str(index)+')'))
start_km = i
end_km = j
end_km_index = index
return 0
incomplete_coverage = []
df.sort(['SectionStart','SectionStop'], ascending=True, inplace=True)
cols = ['ID_number','TimeOfDay','TypeOfCargo','TrackStart']
%timeit df.groupby(cols).apply(Full_coverage)
# 1 loops, best of 3: 272 ms per loop