Question

我正在研究一个需要读取并合并大量数据表的模型。虽然数据表将具有相同的列和相同的标题，但每个数据表将具有不同数量的行。我目前执行此操作的方法效率低下，并且取决于要合并的数据表的数量，可能需要几个小时。

我目前的方法是使用rbind组合数据表；下面是一个具有较小数据集的可重现示例：

library(data.table)

old.way <- function() {
wildfire_data <- data.table()

for(tile in 1:3) {
# Normally this data would be read in from an external file, but we'll make some dummy data for this example
new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

wildfire_data <- rbind(wildfire_data,new_wildfire_data)
}
return(wildfire_data)
}

再来看其他问题，这似乎是一种效率低下的方法（Growing a data.frame in a memory-efficient manner），而我应该预先分配大小并在for循环内使用数据表的“ set”函数，以填充空数据表。我也尝试过：

new.way <- function() {
num.needed.rows <- 3000

# Create a data table of a pre-allocated size    
wildfire_data <- data.table(x = integer(num.needed.rows), y = integer(num.needed.rows), total_PM10 = integer(num.needed.rows), total_PM2.5 = integer(num.needed.rows), total_CH4 = integer(num.needed.rows), total_CO = integer(num.needed.rows), total_CO2 = integer(num.needed.rows), total_NOx = integer(num.needed.rows), total_SO2 = integer(num.needed.rows), total_VOC = integer(num.needed.rows), total_char = integer(num.needed.rows))

start.row <- as.integer(0)

for(tile in 1:3) {
# Again, this data would normally be read in from an external file
new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

for(raw.data.row.i in 1:nrow(new_wildfire_data)) {
set(wildfire_data,start.row + raw.data.row.i,"x", new_wildfire_data[raw.data.row.i,x])
set(wildfire_data,start.row + raw.data.row.i,"y", new_wildfire_data[raw.data.row.i,y])
set(wildfire_data,start.row + raw.data.row.i,"total_PM10", new_wildfire_data[raw.data.row.i,total_PM10])
set(wildfire_data,start.row + raw.data.row.i,"total_PM2.5", new_wildfire_data[raw.data.row.i,total_PM2.5])
set(wildfire_data,start.row + raw.data.row.i,"total_PM2.5", new_wildfire_data[raw.data.row.i,total_PM2.5])
set(wildfire_data,start.row + raw.data.row.i,"total_CH4", new_wildfire_data[raw.data.row.i,total_CH4])
set(wildfire_data,start.row + raw.data.row.i,"total_CO", new_wildfire_data[raw.data.row.i,total_CO])
set(wildfire_data,start.row + raw.data.row.i,"total_CO2", new_wildfire_data[raw.data.row.i,total_CO2])
set(wildfire_data,start.row + raw.data.row.i,"total_NOx", new_wildfire_data[raw.data.row.i,total_NOx])
set(wildfire_data,start.row + raw.data.row.i,"total_SO2", new_wildfire_data[raw.data.row.i,total_SO2])
set(wildfire_data,start.row + raw.data.row.i,"total_VOC", new_wildfire_data[raw.data.row.i,total_VOC])
set(wildfire_data,start.row + raw.data.row.i,"total_char", new_wildfire_data[raw.data.row.i,total_char])
}
start.row <- start.row + nrow(new_wildfire_data)
}
return(wildfire_data)
}

但是新方法要慢得多。这是我的基准测试结果：

library(microbenchmark)
microbenchmark(old.way(),new.way(),times=2

Unit: milliseconds
      expr         min          lq        mean      median          uq         max neval
 old.way()    24.29792    24.29792    25.06512    25.06512    25.83233    25.83233     2
 new.way() 12961.41358 12961.41358 13070.96187 13070.96187 13180.51016 13180.51016     2

是否存在使用“集合”的适当方法，该方法将比使用“ rbind”带来更高的效率？

Answer 1

set通常是:=的替代方法，用于快速分配给data.table的元素。 This是通常用法的一个例子。

chinsoon12指出，rbindlist(lapply(filepaths, fread))应该是一个更快的解决方案。就给定的示例而言，一种选择是定义正确尺寸的列表并使用rbindlist：

list.way <- function() {
wildfire_data_list <- vector("list", length = 3)
for(tile in 1:3) {
    # Normally this data would be read in from an external file, but we'll make some dummy data for this example
    new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
                                    total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

    wildfire_data_list[[tile]] <- new_wildfire_data
}
wildfire_data <- rbindlist(wildfire_data_list)
return(wildfire_data)
}

为什么在增长数据表时，“ rbind”的工作比“ set”的工作更快？

1 个答案: