Question

我正在尝试使用parlapply从多个网址中抓取数据。我收到此错误

sites <- c("https://forums.vwvortex.com/showthread.php?5494121-euro-spec-parts", "https://forums.vwvortex.com/showthread.php?5489376-Is-this-normal", "https://forums.vwvortex.com/showthread.php?5490376-rear-hatch-light")

我要遍历约5000个网址。

nCores <- detectCores(logical = FALSE)
cat(nCores, " cores detected.")

# detect threads with parallel()
nThreads<- detectCores(logical = TRUE)
cat(nThreads, " threads detected.")

# Create doSNOW compute cluster (try 64)
# One can increase up to 128 nodes
# Each node requires 44 Mbyte RAM under WINDOWS.
cluster <- makeCluster(nThreads, type = "SOCK")
class(cluster);

# register the cluster
registerDoSNOW(cluster)

#get info
getDoParWorkers(); getDoParName();

strt <- Sys.time()

results <- parLapply(cluster, sites, function(i) {
  library(dplyr)
  library(xml2)
  library(magrittr)
  library(rvest)
  review <- read_html(url(i))  
  threads<- cbind(review %>% html_nodes("blockquote.postcontent.restore") %>% html_text())
  datethreads <- cbind(review %>% html_nodes("span.date") %>% html_text())
  userinfo <- cbind(review %>% html_nodes("div.username_container") %>% html_text())
  title <- cbind(review %>% html_nodes("li.navbit.lastnavbit") %>%  html_text())
  urls <- cbind(review %>% html_nodes("span.threadtitle") %>% html_nodes("a") %>%   html_attr("href") %>%  paste0("https://forums.vwvortex.com/", .) )
  links <- sub("&.*","", urls)
  library(rowr)
  x <- data.frame(rowr::cbind.fill(threads, datethreads, userinfo, title, links, fill = NA), stringsAsFactors = FALSE)
  return(x)

}) 
print(Sys.time()-strt)

Error in checkForRemoteErrors(val) : 
  one node produced an error: cannot open the connection

由于错误不明确。我想使用trycatch（）进行错误处理。

我不确定，但我认为错误一定是在read_html（url（i）），我尝试了以下代码：

results <- parLapply(cluster, sites, function(i) {
  library(dplyr)
  library(xml2)
  library(magrittr)
  library(rvest)
  tryCatch(
  review <- read_html(url(i)),
error = function(e) {

message("Here's the original error message:")
message(e)
return(NULL)  
})
  threads<- cbind(review %>% html_nodes("blockquote.postcontent.restore") %>% html_text())
  datethreads <- cbind(review %>% html_nodes("span.date") %>% html_text())
  userinfo <- cbind(review %>% html_nodes("div.username_container") %>% html_text())
  title <- cbind(review %>% html_nodes("li.navbit.lastnavbit") %>%  html_text())
  urls <- cbind(review %>% html_nodes("span.threadtitle") %>% html_nodes("a") %>%   html_attr("href") %>%  paste0("https://forums.vwvortex.com/", .) )
  links <- sub("&.*","", urls)
  library(rowr)
  x <- data.frame(rowr::cbind.fill(threads, datethreads, userinfo, title, links, fill = NA), stringsAsFactors = FALSE)
  return(x)

})

不确定如何处理。关于可能导致无法打开连接错误或如何为上述代码编写trycatch（）的任何帮助？

提前谢谢！

如何使用trycatch（）进行parLapply的错误：无法打开连接？

0 个答案: