Question

我有一个如下所示的DataFrame：

import numpy as np
raw_data = {'Series_Date':['2017-03-10','2017-03-13','2017-03-14','2017-03-15'],'SP':[35.6,56.7,41,41],'1M':[-7.8,56,56,-3.4],'3M':[24,-31,53,5]}
import pandas as pd
df = pd.DataFrame(raw_data,columns=['Series_Date','SP','1M','3M'])
print df

我想对此DataFrame中的某些列运行测试，此列表中的所有列名称都是：

check = {'1M','SP'}
print check

对于这些列，我想知道这些列中任何一列中的值何时与前一天的值相同。因此输出数据帧应返回系列日期和注释，例如（在本例中为示例:)

output_data = {'Series_Date':['2017-03-14','2017-03-15'],'Comment':["Value for 1M data is same as previous day","Value for SP data is same as previous day"]}
output_data_df = pd.DataFrame(output_data,columns = ['Series_Date','Comment'])
print output_data_df

请您提供一些帮助以解决这个问题？

Answer 1

我不确定这是最干净的方法。但是，它的工作原理

check = {'1M', 'SP'}
prev_dict = {c: None for c in check}

def check_prev_value(row):
    global prev_dict
    msg = ""
    # MAYBE add clause to check if both are equal
    for column in check:
        if row[column] == prev_dict[column]:
            msg = 'Value for %s data is same as previous day' % column
        prev_dict[column] = row[column]
    return msg

df['comment'] = df.apply(check_prev_value, axis=1)

output_data_df = df[df['comment'] != ""]
output_data_df = output_data_df[["Series_Date", "comment"]].reset_index(drop=True)

您的意见：

  Series_Date    SP    1M  3M
0  2017-03-10  35.6  -7.8  24
1  2017-03-13  56.7  56.0 -31
2  2017-03-14  41.0  56.0  53
3  2017-03-15  41.0  -3.4   5

输出结果为：

  Series_Date                                    comment
0  2017-03-14  Value for 1M data is same as previous day
1  2017-03-15  Value for SP data is same as previous day

Answer 2

以下内容或多或少地符合您的要求。列item_ok将添加到原始数据框中，指定该值是否与前一天相同：

from datetime import timedelta
df['Date_diff'] = pd.to_datetime(df['Series_Date']).diff()
for item in check:
    df[item+'_ok'] = (df[item].diff() == 0) & (df['Date_diff'] == timedelta(1))
df_output = df.loc[(df[[item + '_ok' for item in check]]).any(axis=1)]

Answer 3

参考：this answer

cols = ['1M','SP']
for col in cols:
    df[col + '_dup'] = df[col].groupby((df[col] != df[col].shift()).cumsum()).cumcount()

当找到重复时，输出列将具有大于零的整数。

df:

  Series_Date    SP    1M  3M  1M_dup  SP_dup
0  2017-03-10  35.6  -7.8  24       0       0
1  2017-03-13  56.7  56.0 -31       0       0
2  2017-03-14  41.0  56.0  53       1       0
3  2017-03-15  41.0  -3.4   5       0       1

切片查找重复：

col = 'SP'
dup_df = df[df[col + '_dup'] > 0][['Series_Date', col + '_dup']]

dup_df:

  Series_Date  SP_dup
3  2017-03-15       1

以上是上述功能版本（增加了处理多个列的功能）：

import pandas as pd
import numpy as np

def find_repeats(df, col_list, date_col='Series_Date'):
    dummy_df = df[[date_col, *col_list]].copy()
    dates = dummy_df[date_col]
    date_series = []
    code_series = []
    if len(col_list) > 1:
        for col in col_list:
            these_repeats = df[col].groupby((df[col] != df[col].shift()).cumsum()).cumcount().values
            repeat_idx = list(np.where(these_repeats > 0)[0])
            date_arr = dates.iloc[repeat_idx]
            code_arr = [col] * len(date_arr)
            date_series.extend(list(date_arr))
            code_series.extend(code_arr)
        return pd.DataFrame({date_col: date_series, 'col_dup': code_series}).sort_values(date_col).reset_index(drop=True)
    else:
        col = col_list[0]
        dummy_df[col + '_dup'] = df[col].groupby((df[col] != df[col].shift()).cumsum()).cumcount()
        return dummy_df[dummy_df[col + '_dup'] > 0].reset_index(drop=True)

find_repeats(df, ['1M'])

  Series_Date    1M  1M_dup
0  2017-03-14  56.0       1

find_repeats(df, ['1M', 'SP'])

  Series_Date col_dup
0  2017-03-14      1M
1  2017-03-15      SP

这是另一种使用pandas diff的方式：

def find_repeats(df, col_list, date_col='Series_Date'):
    code_list = []
    dates = list()

    for col in col_list:
        these_dates = df[date_col].iloc[np.where(df[col].diff().values == 0)[0]].values
        code_arr = [col] * len(these_dates)
        dates.extend(list(these_dates))
        code_list.extend(code_arr)
    return pd.DataFrame({date_col: dates, 'val_repeat': code_list}).sort_values(date_col).reset_index(drop=True)

检查特定列上的Pandas Dataframe中的填充数据

3 个答案: