purrr :: map()-分组读取文件

时间:2018-11-12 20:26:34

标签: r purrr

我需要从分组列表中读取一堆文件,然后根据组进行合并(来自同一组的文件将具有相同的列,因此可以使用bind_rows()进行缩小。

当我通过purrr::map()函数移动时,由于似乎不断收到关于不能在原子向量上使用$的警告,我似乎无法掌握数据的变化方式。

我要做的第一件事是按组划分,以便获得要在每个组中读取的文件列表的列表。然后,我使用map浏览该列表中的每个项目,并使用第二张map浏览每个子列表中的行以读取文件。但是,在该级别上发生了一些事情,不再像对待顶级团队那样处理数据了。

(缺少能够调试和查看map函数内部环境的问题,这实际上是理解机制的问题。)

require(tidyverse)
#> Loading required package: tidyverse

x <- structure(list(survey = c("adm2014", "adm2015", "adm2016", "eap2008", 
                          "eap2009", "eap2011", "eap2012", "eap2013", "eap2014", "eap2015", 
                          "eap2016", "ef2008a", "ef2008b", "ef2008c", "ef2008cp", "ef2008d", 
                          "ef2009a", "ef2009b", "ef2009c", "ef2009d", "ef2010a", "ef2010b", 
                          "ef2010c", "ef2010cp", "ef2010d", "ef2011a", "ef2011b", "ef2011c", 
                          "ef2011d", "ef2012a", "ef2012b", "ef2012c", "ef2012cp", "ef2012d", 
                          "ef2013a", "ef2013b", "ef2013c", "ef2013d", "ef2014a", "ef2014b", 
                          "ef2014c", "ef2014cp", "ef2014d", "ef2015a", "ef2015b", "ef2015c", 
                          "ef2015d", "ef2016a", "ef2016b", "ef2016c", "ef2016cp", "ef2016d", 
                          "efest2008", "efest2009", "effy2008", "effy2009", "effy2010", 
                          "effy2011", "effy2012", "effy2013", "effy2014", "effy2015", "effy2016", 
                          "effy2017", "efia2008", "efia2009", "efia2011", "efia2012", "efia2013", 
                          "efia2014", "efia2015", "efia2016", "efia2017", "f0708_f1a", 
                          "f0708_f2", "f0708_f3", "f0809_f1a", "f0809_f2", "f0809_f3", 
                          "f0910_f1a", "f0910_f2", "f0910_f3", "f1011_f1a", "f1011_f2", 
                          "f1011_f3", "f1112_f1a", "f1112_f2", "f1112_f3", "f1213_f1a", 
                          "f1213_f2", "f1213_f3", "f1314_f1a", "f1314_f2", "f1314_f3", 
                          "f1415_f1a", "f1415_f2", "f1415_f3", "f1516_f1a", "f1516_f2", 
                          "f1516_f3", "gr2008", "gr2008_l2", "gr2009", "gr2009_l2", "gr200_08", 
                          "gr200_09", "gr200_10", "gr200_11", "gr200_12", "gr200_13", "gr200_14", 
                          "gr200_15", "gr200_16", "gr2010", "gr2010_l2", "gr2011", "gr2011_l2", 
                          "gr2012", "gr2012_l2", "gr2013", "gr2013_l2", "gr2014", "gr2014_l2", 
                          "gr2015", "gr2015_l2", "gr2016", "gr2016_l2", "hd2008", "hd2009", 
                          "hd2010", "hd2011", "hd2012", "hd2013", "hd2014", "hd2015", "hd2017", 
                          "ic2008", "ic2008_ay", "ic2008_py", "ic2009", "ic2009_ay", "ic2009_py", 
                          "ic2010", "ic2010_ay", "ic2010_py", "ic2011", "ic2011_ay", "ic2011_py", 
                          "ic2012", "ic2012_ay", "ic2012_py", "ic2013", "ic2013_ay", "ic2013_py", 
                          "ic2014", "ic2014_ay", "ic2014_py", "ic2015", "ic2015_ay", "ic2015_py", 
                          "ic2016", "ic2016_ay", "ic2016_py", "ic2017", "ic2017_ay", "ic2017_py", 
                          "s2008_abd", "s2008_cn", "s2008_f", "s2008_g", "s2009_abd", "s2009_cn", 
                          "s2009_f", "s2009_g", "s2010_abd", "s2010_cn", "s2010_f", "s2010_g", 
                          "s2011_abd", "s2011_cn", "s2011_f", "s2011_g", "sal2008_a", "sal2008_a_lt9", 
                          "sal2008_b", "sal2008_faculty", "sal2009_a", "sal2009_a_lt9", 
                          "sal2009_b", "sal2009_faculty", "sal2010_a", "sal2010_a_lt9", 
                          "sal2010_b", "sal2010_faculty", "sal2011_a", "sal2011_a_lt9", 
                          "sal2011_faculty"), survgroup = c("adm", "adm", "adm", "eap", 
                                                            "eap", "eap", "eap", "eap", "eap", "eap", "eap", "efa", "efb", 
                                                            "efc", "efcp", "efd", "efa", "efb", "efc", "efd", "efa", "efb", 
                                                            "efc", "efcp", "efd", "efa", "efb", "efc", "efd", "efa", "efb", 
                                                            "efc", "efcp", "efd", "efa", "efb", "efc", "efd", "efa", "efb", 
                                                            "efc", "efcp", "efd", "efa", "efb", "efc", "efd", "efa", "efb", 
                                                            "efc", "efcp", "efd", "efest", "efest", "effy", "effy", "effy", 
                                                            "effy", "effy", "effy", "effy", "effy", "effy", "effy", "efia", 
                                                            "efia", "efia", "efia", "efia", "efia", "efia", "efia", "efia", 
                                                            "f_f1a", "f_f2", "f_f3", "f_f1a", "f_f2", "f_f3", "f_f1a", "f_f2", 
                                                            "f_f3", "f_f1a", "f_f2", "f_f3", "f_f1a", "f_f2", "f_f3", "f_f1a", 
                                                            "f_f2", "f_f3", "f_f1a", "f_f2", "f_f3", "f_f1a", "f_f2", "f_f3", 
                                                            "f_f1a", "f_f2", "f_f3", "gr", "gr_l2", "gr", "gr_l2", "gr_08", 
                                                            "gr_09", "gr_10", "gr_11", "gr_12", "gr_13", "gr_14", "gr_15", 
                                                            "gr_16", "gr", "gr_l2", "gr", "gr_l2", "gr", "gr_l2", "gr", "gr_l2", 
                                                            "gr", "gr_l2", "gr", "gr_l2", "gr", "gr_l2", "hd", "hd", "hd", 
                                                            "hd", "hd", "hd", "hd", "hd", "hd", "ic", "ic_ay", "ic_py", "ic", 
                                                            "ic_ay", "ic_py", "ic", "ic_ay", "ic_py", "ic", "ic_ay", "ic_py", 
                                                            "ic", "ic_ay", "ic_py", "ic", "ic_ay", "ic_py", "ic", "ic_ay", 
                                                            "ic_py", "ic", "ic_ay", "ic_py", "ic", "ic_ay", "ic_py", "ic", 
                                                            "ic_ay", "ic_py", "s_abd", "s_cn", "s_f", "s_g", "s_abd", "s_cn", 
                                                            "s_f", "s_g", "s_abd", "s_cn", "s_f", "s_g", "s_abd", "s_cn", 
                                                            "s_f", "s_g", "sal_a", "sal_a_lt9", "sal_b", "sal_faculty", "sal_a", 
                                                            "sal_a_lt9", "sal_b", "sal_faculty", "sal_a", "sal_a_lt9", "sal_b", 
                                                            "sal_faculty", "sal_a", "sal_a_lt9", "sal_faculty")), class = c("tbl_df", 
                                                                                                                            "tbl", "data.frame"), row.names = c(NA, -197L))

x %>% 
  split(.$survgroup) %>% 
  map(function(currentgroup) {
    #currentgroup should now be a tibble of each group.
    currentgroup %>% 
      map(function(singlesurvey) { #singlesurvey should be each row in the group
        x <- read_csv(path_expand(paste0("~data/IPEDS/API Pulls/datadownloaded/", singlesurvey$survey, ".csv")))
    }) %>% bind_rows()

  })
#> Error in path_expand(paste0("~data/IPEDS/API Pulls/datadownloaded/", singlesurvey$survey, : could not find function "path_expand"

reprex package(v0.2.1)于2018-11-12创建

2 个答案:

答案 0 :(得分:2)

问题在于我们需要遍历列中的各个文件,而不是遍历数据集中的列。在OP的帖子中,第二个mapdata.frame中循环显示一列。在这里,基本单位是带有一列的data.frame。如果将列提取为vector,则该单位将变为vector,并循环遍历vector

的每个元素
x %>% 
   split(.$survgroup) %>%
        map(~ .x %>%
                 pull(survey) %>%
                 map(~ .x %>% 
                    paste0("~data/IPEDS/API Pulls/datadownloaded/", ., '.csv') %>% 
                    path.expand %>% 
                    read_csv)))

答案 1 :(得分:1)

另一种解决方案是使用列表列将数据帧读入列,然后进行拆分。

x %>% 
  mutate(data = map(survey, ~ read_csv(path.expand(paste0("~data/IPEDS/API Pulls/datadownloaded/", .x, ".csv"))))) %>%
  unnest() %>%
  split(.$survgroup)