Question

我需要对多个文件执行pandas df操作，

df1 = pd.read_csv("~/pathtofile/sample1.csv")
some_df=pd.read_csv("~/pathtofile/metainfo.csv")
df1.sort_values('col2')
df1 = df1[df1.col5 != 'N']
df1['new_col'] = df1['col3'] - df1['col2'] + 1
f = lambda row: '{col1}:{col2}-{col3}({col4})'.format(**row)
df1.astype(str).apply(f,1)
df4 = df1.assign(Unique=df1.astype(str).apply(f,1))
# print(df4)
##merge columns 
df44 = df4.merge(some_df, left_on='genes', right_on='name', suffixes=('','_1'))
df44 = df44.rename(columns={'id':'id_new'}).drop(['name_1'], axis=1)
# print(df44)
df44['some_col'] = df44['some_col'] + ':E' + 
df44.groupby('some_col').cumcount().add(1).astype(str).str.zfill(3)
print(df44)
##drop unwanted columns adapted from http://stackoverflow.com/questions/13411544/delete-column-from-pandas-dataframe
df4 = df44
df4.drop(df4.columns[[3,7,9,11,12,13]], axis=1, inplace=True)

df4 = df4[['col0', 'col1', 'col2', 'col4', 'col5', 'col6', 'col8']]
df4
df4.to_csv('foo.csv', index=False)

以上代码仅适用于一个文件，几个问题 1）我有~15个文件我需要执行这组命令如何在所有15个文件上使用它 2）并写入15个不同的csv 3）合并来自所有15个df的某些列并制作矩阵（例如仅合并3个dfs）

sample1 = pd.DataFrame.set_index(df4,['col1'])["col4"]
sample2 = pd.DataFrame.set_index(df5,['col1'])["col4"]
sample3 = pd.DataFrame.set_index(df6, ['col1'])["col4"]
concat =  pd.concat([sample1,sample2,sample3], axis=1).fillna(0)
# print(concat)
concat.reset_index(level=0, inplace=True)
concat.columns = ["newcol0", "col1", "col2", "col3"]
concat.to_csv('bar.csv', index=False)

有没有更好的方法来复制粘贴15次？

Answer 1

好吧，我只是快速将它们放在一起用于上述代码。我建议学习如何编写脚本和概括事物。我没有清理代码或删除冗余，我会把它留给你。如果您发布的代码有效，这应该可以在命令行中使用。

import sys
import pandas as pd

def load_df(input_file):
    df = pd.DataFrame(pd.read_csv(input_file))
    return df

def perform_operations(df):
    df.sort_values('col2')
    df = df[df.col5 != 'N']
    df['new_col'] = df['col3'] - df['col2'] + 1
    f = lambda row: '{col1}:{col2}-{col3}({col4})'.format(**row)
    df.astype(str).apply(f,1)
    df4 = df.assign(Unique=df.astype(str).apply(f,1))
    return df4

def merge_stuff(df, df1):
    df44 = df.merge(df1, left_on='genes', right_on='name', suffixes=('','_1'))
    df44 = df44.rename(columns={'id':'id_new'}).drop(['name_1'], axis=1)
    return df44


def group_and_drop(df):
    df['some_col'] = df['some_col'] + ':E' + 
    df.groupby('some_col').cumcount().add(1).astype(str).str.zfill(3)
    df4 = df
    df4.drop(df4.columns[[3,7,9,11,12,13]], axis=1, inplace=True)
    return df4

def write_out_csv(df):
    df = df[['col0', 'col1', 'col2', 'col4', 'col5', 'col6', 'col8']]
    df.to_csv('foo.csv', index=False)


def main():
    file_1 = sys.argv[1]
    file_2 = sys.argv[2]
    df = load_df(file_1)
    df1 = load_df(file_2)
    df4 = perform_operations(df)
    df44 = merge_stuff(df4, df1)
    grouped = group_and_drop(df44)
    write_out_csv(grouped)

if __name__ == '__main__':
    main()

pandas对几个文件进行操作并合并

1 个答案: