我使用R和我在Linux机器上的7个内核上运行并行代码。
使用小型数据集时，我的代码大约需要2个小时，工作正常。 当使用6x更大的数据集时，代码需要更长的时间（可能因为它需要交换），但随后将随机结束，有时为10％，有时为18％，20％，30％左右。看起来完全随意。 RAM使用率通常为90％左右，SWAP使用率为<50％。
我在foreach后端使用doSNOW。
这是错误代码：
Error in unserialize(socklist[[n]]) : error reading from connection
Calls: %dopar% ... tryCatch -> tryCatchList -> tryCatchOne -> <Anonymous>
Execution halted
*** caught bus error ***
address 0x7f829d2adbd0, cause 'non-existent physical address'
An irrecoverable exception occurred. R is aborting now ...
这是通过在outfile="outfile.out"电话中设置makeCluster获得的SNOW输出文件：
starting worker for localhost:11567
Type: EXEC
Loading required package: MASS
Loading required package: survival
Loading required package: sp
Attaching package: 'raster'
The following objects are masked from 'package:MASS':
area, select
Attaching package: 'data.table'
The following object is masked from 'package:raster':
shift
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
Type: EXEC
*** caught bus error ***
address 0x7ffada863636, cause 'non-existent physical address'
Traceback:
1: .Call("Rsx_nc4_get_vara_double", as.integer(ncid), as.integer(varid), as.integer(c.start), as.integer(c.count), fixmiss, imvstate, as.double(passed_missval), PACKAGE = "ncdf4")
2: ncvar_get_inner(ncid2use, varid2use, nc$var[[li]]$missval, addOffset, scaleFact, start = start, count = count, verbose = verbose, signedbyte = signedbyte, collapse_degen = collapse_degen)
3: getfun(nc, varid = zvar, start = start, count = count)
4: .readBrickCellsNetCDF(x, cells, layer, nl)
5: .cellValues(x, i)
6: .doExtract(x, i, drop = drop)
7: por[i]
8: por[i]
9: as.vector(por[i])
10: mainF(as.vector(por[i]))
11: eval(expr, envir, enclos)
12: eval(.doSnowGlobals$expr, envir = .doSnowGlobals$exportenv)
13: doTryCatch(return(expr), name, parentenv, handler)
14: tryCatchOne(expr, names, parentenv, handlers[[1L]])
15: tryCatchList(expr, classes, parentenv, handlers)
16: tryCatch(eval(.doSnowGlobals$expr, envir = .doSnowGlobals$exportenv), error = function(e) e)
17: fun(quote(list(i = 23339L)))
18: do.call("fun", lapply(args, enquote))
19: docall(msg$data$fun, msg$data$args)
20: doTryCatch(return(expr), name, parentenv, handler)
21: tryCatchOne(expr, names, parentenv, handlers[[1L]])
22: tryCatchList(expr, classes, parentenv, handlers)
23: tryCatch(docall(msg$data$fun, msg$data$args), error = handler)
24: doTryCatch(return(expr), name, parentenv, handler)
25: tryCatchOne(expr, names, parentenv, handlers[[1L]])
26: tryCatchList(expr, classes, parentenv, handlers)
27: tryCatch({ msg <- recvData(master) cat(paste("Type:", msg$type, "\n")) if (msg$type == "DONE") { closeNode(master) break } else if (msg$type == "EXEC") { success <- TRUE handler <- function(e) { success <<- FALSE structure(conditionMessage(e), class = c("snow-try-error", "try-error")) } t1 <- proc.time() value <- tryCatch(docall(msg$data$fun, msg$data$args), error = handler) t2 <- proc.time() value <- list(type = "VALUE", value = value, success = success, time = t2 - t1, tag = msg$data$tag) sendData(master, value) }}, interrupt = function(e) NULL)
28: slaveLoop(makeSOCKmaster(master, port))
29: eval(expr, envir, enclos)
30: eval(quote({ master <- "localhost" port <- "" snowlib <- Sys.getenv("R_SNOW_LIB") outfile <- Sys.getenv("R_SNOW_OUTFILE") args <- commandArgs() pos <- match("--args", args) args <- args[-(1:pos)] for (a in args) { pos <- regexpr("=", a) name <- substr(a, 1, pos - 1) value <- substr(a, pos + 1, nchar(a)) switch(name, MASTER = master <- value, PORT = port <- value, SNOWLIB = snowlib <- value, OUT = outfile <- value) } if (!(snowlib %in% .libPaths())) .libPaths(c(snowlib, .libPaths())) library(methods) library(snow) if (port == "") port <- getClusterOption("port") sinkWorkerOutput(outfile) cat("starting worker for", paste(master, port, sep = ":"), "\n") slaveLoop(makeSOCKmaster(master, port))}), new.env())
31: eval(expr, envir, enclos)
32: eval(expr, p)
33: eval.parent(substitute(eval(quote(expr), envir)))
34: local({ master <- "localhost" port <- "" snowlib <- Sys.getenv("R_SNOW_LIB") outfile <- Sys.getenv("R_SNOW_OUTFILE") args <- commandArgs() pos <- match("--args", args) args <- args[-(1:pos)] for (a in args) { pos <- regexpr("=", a) name <- substr(a, 1, pos - 1) value <- substr(a, pos + 1, nchar(a)) switch(name, MASTER = master <- value, PORT = port <- value, SNOWLIB = snowlib <- value, OUT = outfile <- value) } if (!(snowlib %in% .libPaths())) .libPaths(c(snowlib, .libPaths())) library(methods) library(snow) if (port == "") port <- getClusterOption("port") sinkWorkerOutput(outfile) cat("starting worker for", paste(master, port, sep = ":"), "\n") slaveLoop(makeSOCKmaster(master, port))})
An irrecoverable exception occurred. R is aborting now ...
代码使用8个本地计算机中的7个线程。 foreach调用是这样的：
#Packages
packageVec <- c("RcppRoll", "FAdist", "fitdistrplus", "minpack.lm", "raster", "foreach", "data.table")
#Register cluster
cl <- makeCluster(nThreads, outfile=paste0(dischargefile, ".out"))
registerDoSNOW(cl)
#Create progress bar
pb <- txtProgressBar(max = ncells, style = 3)
progress <- function(n) setTxtProgressBar(pb, n)
SNOWopts <- list(progress = progress)
#Compute
a <-
foreach(i=cells, .packages=packageVec, .combine='rbind', .options.snow = SNOWopts) %dopar% {
mainF(as.vector(por[i]))
}
stopCluster(cl)
很难为此功能生成MRE，因为它很复杂。代码是here，我将在下面描述该函数的功能。我想强调的是，所有这些都在一个小数据集上运行得非常好，但在大型数据集上失败，即使它包含多次复制的小数据集的相同数据。
mainF()是一个函数，其输入por[i]是125000个元素向量，其输出为244个元素向量。此函数基本上对输入向量执行240滚动方式（使用RcppRoll）并为每个均值获取每个时间片（通常为15）的最大值（使用data.table），使数据适合给定的分布（使用FAdist，raster和fitdistrplus）执行另一个循环来计算另外240个值，并将这些值拟合（使用minpack.lm）函数以获取更多参数。返回240个元素，返回4个拟合参数（每个拟合2个）。
如何修复上述错误？这是什么意思？