Question

我在一个文件夹中有44496个csv文件。

如果我想在一个文件夹中阅读所有这些csv文件，我可以这样做：

files = list.files(pattern="*.csv")
library(data.table)
DT = do.call(rbind, lapply(files, fread)

每个文件的名称为wrX_Y.csv。我有5562个X值和8个Y值。例如，我为X的每个值都有8个csv文件。

  wr1_258, wr1_260, wr1_265, wr1_280, wr1_290, wr1_300, wr1_310,wr1_320 
  wr2_258, wr2_260, wr2_265, wr2_280, wr2_290, wr2_300, wr2_310,wr2_320
  .
  .
  .
  .
  wr5562_258, wr5562_260, wr5562_265, wr5562_280, wr5562_290, wr5562_300, wr5562_310,wr5562_320

我想要合并属于给定X的所有文件。例如，

  wr1_258, wr1_260, wr1_265, wr1_280, wr1_290, wr1_300, wr1_310,wr1_320 into a single csv
  wr2_258, wr2_260, wr2_265, wr2_280, wr2_290, wr2_300, wr2_310,wr2_320 into a single csv and so on

让我们说names.list是一个包含所有X值的向量。如何读取属于单个X的所有csv，合并它们并将它们写出来

  for(i in names.list){
   files <- list.files(pattern = "*.csv", full.names = T)
   DT = do.call(rbind, lapply(files, fread) # one read those csv files which belong to i
   fwrite(DT,paste0(i,"alldata.csv"))
  }

Answer 1

这似乎更像regex而不是data.table qn。您对以下list.files函数的模式输入进行了改进：

  for(i in names.list) {
     files <- list.files(pattern=paste0("wr", i, "_(.*).csv"), full.names=TRUE)
     DT <- rbindlist(lapply(files, fread))
     fwrite(DT, paste0(i,"alldata.csv))
  }

Answer 2

这应该有用。

## list all files in the folder, enter the path of folder containing .csv files
list_files <- list.files(path = 'path') 

## number of iterations 
len <- 5562
ffiles <- vector('list', length = len)

## create a dictionary
## this groups the name of files based on X values (1,2...5562)
for(i in seq(list_files))
{
    file_name <- list_files[i]
    string <- unlist(strsplit(file_name, split = '_'))[1]
    string <- gsub(pattern = '[a-z]', replacement = '', x = string)
    print(string)
    if (string %in% names(ffiles))
    {
        ffiles[[string]] <- append(ffiles[[string]], file_name)
    } else {
        ffiles[[string]] <- file_name
    }

}

## this will be used in next step
full_path <- list.files(path = 'folder_path', full.names = T)

## rbind and write all files
for (i in names(ffiles))
{
    files_path <- append(files_path, sapply(ffiles[i], function(x) list.files(path = full_path, pattern = x, full.names = T)))
    assign(paste0('df',i), do.call(rbind, lapply(files_path, fread)))
    fwrite(get(paste0('df',i), .GlobalEnv), paste0('df_',i,'.csv'))
}

Answer 3

您可以使用rbindlist()函数来绑定list data.tables和idcol参数，以添加一个额外的列，指示每行的原始文件：

library(data.table)

# Load all files into a named list of data.tables:
files <- list.files(pattern="*.csv")
dt_list <- lapply(files, fread)
names(dt_list) <- files # for the idcol argument in rbindlist

# Concatenate all data.tables into a single data.table, with a column
# indicating each row's file of origin.
# add use.names = TRUE if you columns are not in the same order in 
# each file.
# add fill = TRUE if some columns are not present in all files
dt <- rbindlist(dt_list, idcol = "file")

# Convert file column to a column of X and Y
dt[, fileX := gsub("_.*", "", file)]
dt[, fileX := gsub("^wr", "", fileX)]
dt[, fileY := gsub(".*_", "", file)]
dt[, fileY := gsub(".csv", "", fileY)]

# For each X, output the corresponding data.table:
for (xtype in unique(dt$fileX)) {
   # subset dt and drop file identifier columns
   xdt <- dt[fileX == xtype]
   xdt[, file := NULL]
   xdt[, fileX := NULL]
   xdt[, fileY := NULL]

   # write table:
   fwrite(xdt, file=paste0("wr", xtype, ".csv"))
}

读取单个组的文件夹中的多个csv文件

3 个答案: