Question

我在Python中存在循环问题。我的文件夹如下：

|folder_initial
       |--data_loop
                   |--example1
                   |--example2
                   |--example3
       |--python_jupyter_notebook

我想遍历data_loop中的所有文件，打开它们，运行一个简单的操作，用另一个名称保存它们，然后对后续文件执行相同的操作。我创建了以下代码：

import pandas as pd
import numpy as np
import os

def scan_folder(parent):
# iterate over all the files in directory 'parent'
for file_name in os.listdir(parent):
    if file_name.endswith(".csv"):

       print(file_name)
       df = pd.read_csv("RMB_IT.csv", low_memory=False, header=None,     names=['column1','column2','column3','column4']

        df = df[['column2','column4']
        #Substitute ND with missing data
        df = df.replace('ND,1',np.nan)
        df = df.replace('ND,2',np.nan)
        df = df.replace('ND,3',np.nan)
        df = df.replace('ND,4',np.nan)
        df = df.replace('ND,5',np.nan)
        df = df.replace('ND,6',np.nan)

    else:
        current_path = "".join((parent, "/", file_name))
        if os.path.isdir(current_path):
            # if we're checking a sub-directory, recall this method
            scan_folder(current_path)

scan_folder("./data_loop")  # Insert parent direcotry's path

我得到了错误：

FileNotFoundError 
FileNotFoundError: File b'example2.csv' does not exist

此外，我想运行代码而无需在文件夹folder_initial中放置Jupyter笔记本，但是我想要这样的东西：

|scripts
        |--Jupiter Notebook
|data
     |---csv files
                  |--example1.csv
                  |--example2.csv

有什么主意吗？

-编辑：我根据用户建议创建了类似的内容

import os                                                                   
import glob                                                                 
os.chdir('C:/Users/bedinan/Documents/python_scripts_v02/data_loop')         
for file in list(glob.glob('*.csv')):                                       
df = pd.read_csv(file, low_memory=False, header=None, names=[

df = df[[

#Substitute ND with missing data
df = df.replace('ND,1',np.nan)
df = df.replace('ND,2',np.nan)
df = df.replace('ND,3',np.nan)
df = df.replace('ND,4',np.nan)
df = df.replace('ND,5',np.nan)
df = df.replace('ND,6',np.nan)   

df.to_pickle(file+"_v02"+".pkl")

f = pd.read_pickle('folder\\data_loop\\RMB_PT.csv_v02.pkl')

但是结果文件的名称组成不正确，因为文件名中带有扩展名-csv

Answer 1

您可以使用this answer遍历所有子文件夹：

import os
import shutil
import pathlib
import pandas as pd

def scan_folder(root):
    for path, subdirs, files in os.walk(root):
        for name in files:
            if name.endswith('.csv'):
                src = pathlib.PurePath(path, name)
                dst = pathlib.PurePath(path, 'new_' + name)
                shutil.copyfile(src, dst)
                df = pd.read_csv(dst)
                # do something with DF
                df.to_csv()

scan_folder(r'C:\User\Desktop\so\55648849')

Answer 2

这是一个仅使用pathlib的解决方案，我非常喜欢。我将您的DataFrame操作提取到了自己的函数中，您可以对其进行重命名和重写以实际执行您想要的操作。

import pandas as pd
import numpy as np

from pathlib import Path

# rename the function to something more relevant
def df_operation(csv_path):
    df = pd.read_csv(
        csv_path.absolute(),
        low_memory=False,
        header=None,
        names=['column1','column2','column3','column4']
    )
    # do some stuff with the dataframe

def scan_folder(parent):

    p = Path(parent)

    # Probably want a check here to make sure the provided 
    # parent is a directory, not a file
    assert p.is_dir()

    [df_operation(f) for f in p.rglob('*') if f.suffix == '.csv']

print(scan_folder("./example/dir"))

在Python中的其他文件夹中循环浏览文件

2 个答案: