Question

我正面临着虫子，这让我真的很紧张。所以，从一开始就开始：我编写了一个代码，该代码从IMF DOT下载数据（国家/地区导出广告导入数据）。有时此代码有效（下载所有数据）。有时。其他时间，在下载过程中我得到错误：

No encoding supplied: defaulting to UTF-8.
Error: lexical error: invalid char in json text.
                                       <!DOCTYPE HTML PUBLIC "-//W3C//
                     (right here) ------^

有什么好笑的 - 有时出现错误，在下载的开始，有时在中间。基本上，它是随机的。所以调试它基本上是与阴影搏斗。也许有人也面临这个问题，可以帮忙吗？

Code: 

   rm(list = ls())
#Code downloads data from DOT (IMF). 
#DOT(date).csv

# Libraries
suppressPackageStartupMessages(library(plyr))
suppressPackageStartupMessages(library(dplyr))
suppressPackageStartupMessages(library(tidyr))
suppressPackageStartupMessages(library(reshape2))
suppressPackageStartupMessages(library(stringr))
suppressPackageStartupMessages(library(IMFData))
suppressPackageStartupMessages(library(TTR))
suppressPackageStartupMessages(library(readxl))

#Parameters of download -------------------------------------------------------

databaseID <- 'DOT'
startdate <- '1977-01-01'
enddate <- format(Sys.Date(),"%Y-%m-%d")
checkquery = FALSE

  # Frequency
download.freq <- c("A")

  # Area
available.codes <- DataStructureMethod('DOT')
cn <- available.codes$CL_AREA_DOT


# Download data -----------------------------------------------------------

print("Downloading")
datalist <- list(); queryfilter<- list()
for(i in 1:length(cn[,"CodeValue"])) queryfilter[[i]] <- list(CL_FREA=download.freq, CL_AREA_DOT=cn[,"CodeValue"][i],  CL_INDICATOR_DOT = "TXG_FOB_USD"
)

datalist<- plyr::llply(queryfilter, function(x) {
  Sys.sleep(runif(1,2,5))
  Dot.downloader(databaseID,x, startdate, enddate) 
}, .progress = "text")

#WHERE ERRORS HAPPENS

data <- do.call(rbind.data.frame, datalist) #..............................................................................................................................................................................................................................................

哦和Dot.downloader函数看起来像这样（它来自IMFdata库，只是稍微适应了这种情况）：

    Dot.downloader <- function(databaseID, queryfilter=NULL,
                           startdate='1977-01-01', enddate='2016-12-31'){

  queryfilterstr <- ''
  if (length(queryfilter) > 0){
    queryfilterstr <- paste0(
      unlist(plyr::llply(queryfilter,
                         function(x)(paste0(x, collapse="+")))), collapse=".")
  }

  APIstr <- paste0('http://dataservices.imf.org/REST/SDMX_JSON.svc/CompactData/',
                   databaseID,'/',queryfilterstr,
                   '?startPeriod=',startdate,'&endPeriod=',enddate)
  r <- httr::GET(APIstr)

  if(httr::http_status(r)$reason != "OK"){
    stop(paste(unlist(httr::http_status(r))))
    return(list())
  }
  r.parsed <- jsonlite::fromJSON(httr::content(r, "text"))

  if(is.null(r.parsed$CompactData$DataSet$Series)){
    warning("No data available")
    return(NULL)
  }

  if(class(r.parsed$CompactData$DataSet$Series) == "data.frame"){
    r.parsed$CompactData$DataSet$Series <- r.parsed$CompactData$DataSet$Series[!plyr::laply(r.parsed$CompactData$DataSet$Series$Obs, is.null),]
    if(nrow(r.parsed$CompactData$DataSet$Series) ==0){
      warning("No data available")
      return(NULL)
    }
  }

  if(class(r.parsed$CompactData$DataSet$Series) == "list"){
    if(is.null(r.parsed$CompactData$DataSet$Series$Obs)){
      warning("No data available")
      return(NULL)
    }
    ret.df <- as.data.frame(r.parsed$CompactData$DataSet$Series[1:(length(r.parsed$CompactData$DataSet$Series)-1)])
    ret.df$Obs <- list(r.parsed$CompactData$DataSet$Series$Obs)
    names(ret.df) <- names(r.parsed$CompactData$DataSet$Series)
    r.parsed$CompactData$DataSet$Series <- ret.df
  }

  return(r.parsed$CompactData$DataSet$Series)
}

IMF数据下载错误

0 个答案: