闪亮文件上传内存的问题

时间:2016-02-17 21:48:09

标签: r memory memory-management shiny reactive-programming

我一直在尝试上传10个文件,每个文件至少70,000 KB大,将它们组合在一起,然后在其中一个列上运行renderPrint()函数。当我这样做时,它不起作用,因为存在内存分配问题。真正的问题是文件上传,因为即使我将400万行的合并文件压缩到100万行,仍然存在错误。主要问题:如何将大量文件上传到Shiny?

以下是代码:

UI

date1<-read.csv('E:/Shiny_App_First/data/Company_20160201.csv')
date2<-read.csv('E:/Shiny_App_First/data/Company_20160202.csv')
date3<-read.csv('E:/Shiny_App_First/data/Company_20160203.csv')
date4<-read.csv('E:/Shiny_App_First/data/Company_20160204.csv')
date5<-read.csv('E:/Shiny_App_First/data/Company_20160205.csv')
date6<-read.csv('E:/Shiny_App_First/data/Company_20160206.csv')
date7<-read.csv('E:/Shiny_App_First/data/Company_20160207.csv')
date8<-read.csv('E:/Shiny_App_First/data/Company_20160208.csv')
date9<-read.csv('E:/Shiny_App_First/data/Company_20160209.csv')
date10<-read.csv('E:/Shiny_App_First/data/Company_20160210.csv')

one<-rbind(date3,date4)
two<-rbind(one,date5)
three<-rbind(two,date6)
four<-rbind(three,date7)
five<-rbind(four,date8)
six<-rbind(five,date9)
seven<-rbind(six,date10)

library(dplyr)
library(tidyr)
res <- cbind(seven %>% select(c(Supply.Source.ID,Zone.ID,ABC.ID,Company.Domain.ID,Company2.Domain.ID)), # preserve the year column as-is
         seven %>% select(-c(Supply.Source.ID,Zone.ID,ABC.ID,Company.Domain.ID,Company2.Domain.ID, Supply.Source.Label, Zone.Label, ABC.Label,Company.Domain.Label,Company2.Domain.Label)) %>% mutate_each(funs(extract_numeric))
)

res3<-aggregate(cbind(Category1,Category2,Category3,Category4,Category5,Category6,Category7,Category8,Category9,Category10,Category11,Category12,Category13,Category14,Category15,Category16,Category17)~ Supply.Source.ID+Company.Domain.ID,data=res,mean, na.action=na.pass, na.rm=TRUE)
res4<-aggregate(cbind(CategoryA,CategoryB,CategoryC)~Supply.Source.ID+Company.Domain.ID,data=res,sum, na.action=na.pass,na.rm=TRUE)

final <- merge(res3, res4,
           by=c("Supply.Source.ID","Company.Domain.ID"),all.y=TRUE)

final$Category1<-final$Category1/100
final$Category2<-final$Cateogry2/100
final$Category3<-final$Category3/100
final$Category4<-final$Category4/100
final$Category5<-final$Category5/100
final$Category6<-final$Category6/100
final$Category7<-final$Category7/100
final$Category8<-final$Category8/100
final$Category9<-final$Category9/100
final$Category10<-final$Category10/100
final$Category11<-final$Category11/100
final$Category12<-final$Category12/100
final$Category13<-final$Category13/100
final$Category14<-final$Category14/100
final$Category15<-final$Category15/100
final$Category16<-final$Category16/100

final[is.na(final)] <- " " 

library(shiny)

library(shiny)

shinyUI(fluidPage(
selectInput('in6', 'Category', final$Category, multiple=TRUE, selectize=TRUE)
))

服务器

options(shiny.maxRequestSize=110*1024^2) 

shinyServer(function(input, output, session) {
  output$out6 <- renderPrint(input$in6)
  })

文件大小:

date1 = 559763 obs. of 30 variables
date2 = 493755 obs. of 30 variables
date3 = 555774 obs. of 30 variables
date4 = 626387 obs. of 30 variables
date5 = 615375 obs. of 30 variables
date6 = 535136 obs. of 30 variables
date7 = 529833 obs. of 30 variables
date8 = 665174 obs. of 30 variables
date9 = 643329 obs. of 30 variables
date10 = 609979 obs. of 30 variables

seven = 4780987 obs. of 30 variables

当我这样做时,它不起作用。关于我做错了什么想法?

Loading required package: shiny

Listening on http://127.0.0.1:3640
Warning in type.convert(data[[i]], as.is = as.is[i], dec = dec, numerals = numerals,  :

  Reached total allocation of 8107Mb: see help(memory.size)
Warning in as.vector(levels(x)[x], mode) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in as.vector(levels(x)[x], mode) :
  Reached total allocation of 8107Mb: see help(memory.size)
 Warning in `[<-.factor`(`*tmp*`, ri, value = c("default", "vpaidtag", "default",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("default", "vpaidtag", "default",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("VAST_2089", "b23d794s8w",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("VAST_2089", "b23d794s8w",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("VAST_2089", "b23d794s8w",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("VAST_2089", "b23d794s8w",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "s685.photobucket.com",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "s685.photobucket.com",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "s685.photobucket.com",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net",  "s685.photobucket.com",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
 Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
 Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("wealthyman.net", "Other",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in rbind(deparse.level, ...) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in as.vector(levels(x)[x], mode) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in as.vector(levels(x)[x], mode) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in match(value, lx) :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Warning in `[<-.factor`(`*tmp*`, ri, value = c("", "", "", "", "", "", "",  :
  Reached total allocation of 8107Mb: see help(memory.size)
Error : cannot allocate vector of size 15.9 Mb

0 个答案:

没有答案