Question

我有一个大的data.table（+ 12M行），我需要这样转换：
将具有相同第一列值（将其称为BookId）的每一行折叠为一行，并将其他列合并为一个大的＆＃34;数据＆＃34;领域。此表包含2.7M独特的BookId＆＃39>

即：

BookId    Col1      Col2     ...      ColN
B001      Author    Bob      ...      ...
B002      Author    Marc     ...      ...
B002      Editor    Bob Inc  ...      ...
B001      Editor    MyBooks  ...      ...

成绩结果：

BookId    data
B001      Bob,MyBooks, ...
B002      Marc,Bob Inc, ...

目前，我已经能够使用子集重现此结构，但这非常慢，构建一行需要300毫秒，这意味着实现该过程最多需要9天。

所以我决定使用并行的foreach循环来加速这个过程我的第一个问题是循环使用bookId List，但它只会将全局总时间除以不满足的核心数（8核意味着+1天）。此外，这意味着每个进程都会自动导出大量数据，因为它们都需要整个data.table对象。

我发现了另一种通过将主数据.table分割为基于bookId列表的独立子集来改进过程的方法，然后使每个集群在该子集上工作（较少的行意味着更快的子集生成）。不幸的是，我无法将我的子集导出到群集，因为他们有一个＆＃34;动态＆＃34;名称。我试过＆＃34; .export＆＃34; param，但我想它并不知道当前的＆＃34; i＆＃34;评估时的价值。我怎样才能做到这一点？它甚至可能吗？

我是R的新手，我被告知总有很多方法可以实现同样的目标，我是否选择了实现这一目标的最佳方法？

这是我的代码：

# Create cluster based on available cores
cores = detectCores()
cl <- makeCluster(cores)
registerDoParallel(cl)

# Load datas and generate BookId lists
books <- fread("books.tab")
bookId.unique.list <- unique(books$BookId)
bookId.list <- books$BookId

# Split datatable into "equals" subsets
subset.length = ceiling(length(book.unique.list)/cores)
for (i in 1:(cores)) {
    start = (i-1)*subset.length
    end = (i)*subset.length
    list = book.unique.list[start:end]
    assign(paste("books",i,sep=""), books[books$BookId %in% list])
    assign(paste("book.list",i,sep=""), list )
}

# Prepare resulting DT
res = data.table(BookId = character(0), data = character(0))

# Parallel loop
res  <- foreach(i = 1:cores, .combine = rbind, .export = paste0("book", i),  .packages = c("data.table")) %dopar% {

    #Try to get the named subset corresponding to the current iteration (i)
    # IE : Books1, Books2...
    BookSubset = get(paste0("book", i))
    Book.list.subset = unique(BookSubset$BookId)

    temp = data.table(BookId = character(0), data = character(0))

    for (i in 1:length(Book.list.subset)) {
        bookId = Book.list.subset[i]

        subset <- BookSubset[which(Book.list.subset ==bookId)]
        output = capture.output(write.table(subset, stdout()quote=FALSE, row.names=FALSE,col.names=FALSE)

      temp <- rbind(hist, data.table(zkf_BOOK = c(bookId), data = c(output)))
    }
    temp
}

以下是dput[head(books))的结果：

structure(list(BookId = c("BOOKXXXX774051532082", "BOOKXXXX776514515608", 
    "BOOKXXXX776287821289", "BOOKXXXX776514515608", "BOOKXXXX774051532082", 
    "BOOKXXXX774051532082"), V2 = c("ZUSRXXXX842901236553", 
    "ZUSRXXXX371255229634", 
     "ZUSRXXXX656080986411", "ZUSRXXXX371255229634", "ZUSRXXXX842901236553", 
    "ZUSRXXXX842901236553"), V3 = c("BOOKEVTX776757835463", 
    "BOOKEVTX776762775464", 
    "BOOKEVTX776772854465", "BOOKEVTX776773643466", "", "BOOKEVTX776995487467"
    ), V4 = c("ZACTIONX215229995154", "ZACTIONX533300043134", 
    "ZACTIONX533300043134", 
    "ZACTIONX533300043134", "", "ZACTIONX215229995154"), V5 = c("", 
    "", "", "", "", ""), V6 = c("", "", "", "", "MAILOUTX776774376684", 
    ""), V7 = c("", "", "", "", "", ""), V8 = c("", "", "", "", "", 
    ""), V9 = c("", "", "", "", "", ""), V10 = c("", "", "", "", 
    "", ""), V11 = c("", "", "", "", "", "")), .Names = c("zkf_BOOK", 
    "V2", "V3", "V4", "V5", "V6", "V7", "V8", "V9", "V10", "V11"), class = 
    c("data.table", 
    "data.frame"), row.names = c(NA, -6L))

以下是我＆＃34;真实＆＃34;的示例。数据输入：

BOOKXXXX774051532082    ZUSRXXXX842901236553    BOOKEVTX776757835463    ZACTIONX215229995154                            
BOOKXXXX776514515608    ZUSRXXXX371255229634    BOOKEVTX776762775464    ZACTIONX533300043134                            
BOOKXXXX776287821289    ZUSRXXXX656080986411    BOOKEVTX776772854465    ZACTIONX533300043134                            
BOOKXXXX776514515608    ZUSRXXXX371255229634    BOOKEVTX776773643466    ZACTIONX533300043134                            
BOOKXXXX774051532082    ZUSRXXXX842901236553                MAILOUTX776774376684                    
BOOKXXXX774051532082    ZUSRXXXX842901236553    BOOKEVTX776995487467    ZACTIONX215229995154                            
BOOKXXXX776287821289    ZUSRXXXX656080986411    BOOKEVTX777107387468    ZACTIONX533300043134

和预期的输出

BOOKXXXX774051532082    ZUSRXXXX842901236553|BOOKEVTX776757835463|ZACTIONX215229995154|||||||;ZUSRXXXX842901236553||||MAILOUTX776774376684|||||;ZUSRXXXX842901236553|BOOKEVTX776995487467|ZACTIONX215229995154|||||||
BOOKXXXX776514515608    ZUSRXXXX371255229634|BOOKEVTX776762775464|ZACTIONX533300043134|||||||;ZUSRXXXX371255229634|BOOKEVTX776773643466|ZACTIONX533300043134|||||||
BOOKXXXX776287821289    ZUSRXXXX656080986411|BOOKEVTX776772854465|ZACTIONX533300043134|||||||;ZUSRXXXX656080986411|BOOKEVTX777107387468|ZACTIONX533300043134|||||||

Answer 1

OP要求两次崩溃操作：

对于每一行，将所有列（ID列zkf_BOOK除外）折叠到由|分隔的一个数据字段中。
对于每个zkf_BOOK组，请展开由;

通过调用Reduce()完成列内折叠，而跨行折叠则使用paste()分组完成。使用data.table时，by =参数中的列不会包含在.SD的操作中。

library(data.table)
setDT(books)[, paste(Reduce(function(x, y) paste(x, y, sep = "|"), .SD), collapse = ";"), 
             by = zkf_BOOK]

               zkf_BOOK
1: BOOKXXXX774051532082
2: BOOKXXXX776514515608
3: BOOKXXXX776287821289
                                                                                                                                                                                              V1
1: ZUSRXXXX842901236553|BOOKEVTX776757835463|ZACTIONX215229995154|||||||;ZUSRXXXX842901236553||||MAILOUTX776774376684|||||;ZUSRXXXX842901236553|BOOKEVTX776995487467|ZACTIONX215229995154|||||||
2:                                                   ZUSRXXXX371255229634|BOOKEVTX776762775464|ZACTIONX533300043134|||||||;ZUSRXXXX371255229634|BOOKEVTX776773643466|ZACTIONX533300043134|||||||
3:                                                                                                                         ZUSRXXXX656080986411|BOOKEVTX776772854465|ZACTIONX533300043134|||||||

请注意，与预期结果的差异是由于dput[head(books))仅返回6行，而打印数据输入和预期输出基于7行（或更多）。

在foreach循环中导出命名变量

1 个答案: