Question

我在固定宽度文件中有时间序列数据，其中观察行（n根据样本大小而变化）发生在＆＃34;标题＆＃34;包含重要元数据的行（即样本号，日期等）。两种类型的行都包含字母数字字符。它看起来像这样（字符串缩短以便于阅读：

4  64001416230519844TP blahblah  
5416001130  1 F   492273
5416001140  3 F   492274
5416001145  1 F   492275
5416001150 19 F   492276
5416001155 21 F   492277
5416001160 21 F   492278
5416001165 13 F   492279
5416001170  3 F   492280
5416001180  1 F   492281
4  64001544250619844RA blahblah
5544001125  1 F   492291
5544001130  3 F   492292
5544001135  4 F   492293
5544001140 11 F   492294
5544001145 13 F   492295
4  64002544250619844RA blahblah
etc.

标题行由字符串== 4中的第一个字符区分，并且有89个字符。观察行== 5并且有24个字符。

我想要的是将标题行粘贴到每个后续观察行（数据的子集），以便稍后我可以使用read_fwf解析字符串，并确保我可以通过标题中包含的信息对每个观察进行排序行。我不在乎原始标题行是否被删除。像这样：

5416001130  1 F   492273 4  64001416230519844TP blahblah  
5416001140  3 F   492274 4  64001416230519844TP blahblah  
5416001145  1 F   492275 4  64001416230519844TP blahblah  
5416001150 19 F   492276 4  64001416230519844TP blahblah  
5416001155 21 F   492277 4  64001416230519844TP blahblah  
5416001160 21 F   492278 4  64001416230519844TP blahblah  
5416001165 13 F   492279 4  64001416230519844TP blahblah  
5416001170  3 F   492280 4  64001416230519844TP blahblah  
5416001180  1 F   492281 4  64001416230519844TP blahblah  
5544001125  1 F   492291 4  64001544250619844RA blahblah
5544001130  3 F   492292 4  64001544250619844RA blahblah
5544001135  4 F   492293 4  64001544250619844RA blahblah
5544001140 11 F   492294 4  64001544250619844RA blahblah
5544001145 13 F   492295 4  64001544250619844RA blahblah
etc...

我找到的最接近的解决方案是fwf file with headers every 5th row, headers were characters and observations numeric

提供的解决方案是一个循环，迭代地滚动行并测试它们是字符还是数字并相应地粘贴在一起。

    text <- readLines('/path/to/file')                   # read in the file
split_text <- strsplit(text, "\\s+")                 # split each line on whitespace

for (line in split_text) {                           # iterate through lines
  numeric_line <- suppressWarnings(as.numeric(line)) # try to convert the current line into a vector of numbers
  if (is.na(numeric_line[[1]])) {                    # if it fails, we know we're on a header line
    header <- line
  } else {
    for (i in seq(1, length(line), 2)) {             # otherwise, we're on a data line, so take two numbers at once
      print(c(header, line[[i]], line[[i+1]]))       # and output the latest header with each pair of values
    }
  }
}

我尝试通过首先使用read.fwf（）或read_fwf（）读取fwf并将第一个字符定义为列来区分标题和观察值，从而使其适应我的数据：

    packages = c('tidyverse','rgdal','car','audio','beepr','xlsx','magrittr','lubridate','RColorBrewer','haven')
invisible(lapply(packages, function(x) {if (!require(x, character.only = T)) {install.packages(x);require(x)}}))
DF <- read.fwf("directory/.dat",  widths = c(1, 88), header = FALSE)

我的改编：

newdf <- for (i in DF) {                           # iterate through lines

  if (DF$V1 == 4) {          # if true, we know we're on a header row
    header <- i
  } else {
    for (i in seq(1, length(DF$V2), 1)) { # otherwise = observation row 
      print(c(header, DF$V2[[i]], DF$V2[[i+1]]))  # and output the latest header with each observation until you hit another header
    }
  }
}
#this is very slow and/or does not work
# I get the following error message
#Warning messages:
1: In if (DF$V1 == 4) { :
  the condition has length > 1 and only the first element will be used

我也试过通过nchar（）听众= 89和观察= 24来指定标题与观察行。我意识到这里的循环解决方案可能是使用ifelse，但另一个问题出现了。数据集长约39700行，我始终获得新数据。循环需要很长时间......

我想用data.table或dplyr语法来做这件事。

我已根据这些帖子尝试使用dplyr :: lag：dplyr example 1 和dplyr example 2并接近我想要的东西：

newdf<-DF %>% 
  mutate(new = replace(lag(V2), V1 != '5', NA))

但是正如您所看到的那样，新列仅粘贴上一行的信息......正如lag（）所做的那样。

非常感谢任何帮助，谢谢你提前。

作为旁注。这些数据以前是在SAS中处理的，但是因为我不在那里做SAS。如果有帮助，我确实有SAS代码：

DATA A1;
FILENAME FREQLONG 'dir/FL.DAT';
INFILE FREQLONG;
INPUT
       TYPE   1   @ ;
        IF TYPE EQ 4 THEN LINK LIGNE4;
        IF TYPE EQ 5 THEN DELETE;
        RETURN;


LIGNE4:
INPUT             var1     $   6 -  8
                  var2     $   9 - 11
                  var3     12 - 13
                  var4     14 - 15
                  var5     18 - 19
                  var6     $  20 - 22
                  var7     $  44 - 46
                  var8     $    78;


DATA A2;
FILENAME FREQLONG 'dir/FL.DAT';
INFILE FREQLONG;
INPUT
       TYPE   1   @ ;
        IF TYPE EQ 4 THEN DELETE;
        IF TYPE EQ 5 THEN LINK LIGNE5;
        RETURN;


LIGNE5:
INPUT             var1     $    5 - 7
                  var2     $    2 - 4
                  varz     8 - 10
                  vara     11 - 13
                  varb     $     15;


DATA A3;
SET A1;
PROC SORT;
     BY var1 var2;
     RUN;

DATA A4;
SET A2;
PROC SORT;
     BY var1 var2;
     RUN;

DATA A5;
MERGE A4 A3;
      BY var1 var2;
      RUN;

如您所见，它会拆分文件，对变量进行排序，合并它们。然而，这是逐年完成的，我希望与所有年份的一个文件一起工作。

Answer 1

以下是使用tidyverse的解决方案。

它创建一个只包含标题行的新列，然后填充没有带上标头的标题的行。最后，如果需要，您可以paste列。

 x <- read.table(text = "4  64001416230519844TP blahblah  
       5416001130  1 F   492273
       5416001140  3 F   492274
       5416001145  1 F   492275
       5416001150 19 F   492276
       5416001155 21 F   492277
       5416001160 21 F   492278
       5416001165 13 F   492279
       5416001170  3 F   492280
       5416001180  1 F   492281
       4  64001544250619844RA blahblah
       5544001125  1 F   492291
       5544001130  3 F   492292
       5544001135  4 F   492293
       5544001140 11 F   492294
       5544001145 13 F   492295", header = FALSE, sep = "\t")

library("tidyverse")
 x %>% 
   rename(body = V1) %>% 
   mutate(
     body = trimws(body),
     head = if_else(grepl("^4", body), body, NA_character_),
     body = if_else(is.na(head), body, NA_character_)
   ) %>% 
   fill(head, .direction  = "down") %>% 
   filter(!is.na(body))

输出

                       body                            head
1  5416001130  1 F   492273 4  64001416230519844TP blahblah
2  5416001140  3 F   492274 4  64001416230519844TP blahblah
3  5416001145  1 F   492275 4  64001416230519844TP blahblah
4  5416001150 19 F   492276 4  64001416230519844TP blahblah
5  5416001155 21 F   492277 4  64001416230519844TP blahblah
6  5416001160 21 F   492278 4  64001416230519844TP blahblah
7  5416001165 13 F   492279 4  64001416230519844TP blahblah
8  5416001170  3 F   492280 4  64001416230519844TP blahblah
9  5416001180  1 F   492281 4  64001416230519844TP blahblah
10 5544001125  1 F   492291 4  64001544250619844RA blahblah
11 5544001130  3 F   492292 4  64001544250619844RA blahblah
12 5544001135  4 F   492293 4  64001544250619844RA blahblah
13 5544001140 11 F   492294 4  64001544250619844RA blahblah
14 5544001145 13 F   492295 4  64001544250619844RA blahblah

Answer 2

另一种可能的解决方案（无tidyverse）是每行读入文件，查找标题行并将这些行粘贴到没有标题的行的末尾。之后，这些行被拆分并放入data.frame。

lines <- readLines("asd.dat")

# last index + 1 for iteration
headers <- c(which(grepl("^4 ", lines)), length(lines) + 1) 

pastedLines <- c()
for(i in 1:(length(headers) - 1)) {
  pastedLines <- c(pastedLines, 
                   paste(lines[(headers[i] + 1) : (headers[i + 1] - 1)], lines[headers[i]]))
}

DF <- as.data.frame(matrix(unlist(strsplit(pastedLines, "\\s+")), nrow =  length(pastedLines), byrow=T))

输出：

           V1 V2 V3     V4 V5                  V6       V7
1  5416001130  1  F 492273  4 64001416230519844TP blahblah
2  5416001140  3  F 492274  4 64001416230519844TP blahblah
3  5416001145  1  F 492275  4 64001416230519844TP blahblah
4  5416001150 19  F 492276  4 64001416230519844TP blahblah
5  5416001155 21  F 492277  4 64001416230519844TP blahblah
6  5416001160 21  F 492278  4 64001416230519844TP blahblah
7  5416001165 13  F 492279  4 64001416230519844TP blahblah
8  5416001170  3  F 492280  4 64001416230519844TP blahblah
9  5416001180  1  F 492281  4 64001416230519844TP blahblah
10 5544001125  1  F 492291  4 64001544250619844RA blahblah
11 5544001130  3  F 492292  4 64001544250619844RA blahblah
12 5544001135  4  F 492293  4 64001544250619844RA blahblah
13 5544001140 11  F 492294  4 64001544250619844RA blahblah
14 5544001145 13  F 492295  4 64001544250619844RA blahblah

Answer 3

基础R的两个选项。两者都使用readLines来读取原始文本数据（参见本答案的结尾）。

选项1：

i <- grepl(pattern = '^4 ', x)
x1 <- strsplit(x[!i], '\\s+')
x2 <- strsplit(x[i], '\\s+')

d1 <- do.call(rbind.data.frame, x1)
d2 <- do.call(rbind.data.frame, x2)

d <- cbind(d1, d2[cumsum(i)[-which(i)],])
names(d) <- paste0('V',1:ncol(d))

给出：

> d
            V1 V2 V3     V4 V5                  V6       V7
1   5416001130  1  F 492273  4 64001416230519844TP blahblah
1.1 5416001140  3  F 492274  4 64001416230519844TP blahblah
1.2 5416001145  1  F 492275  4 64001416230519844TP blahblah
1.3 5416001150 19  F 492276  4 64001416230519844TP blahblah
1.4 5416001155 21  F 492277  4 64001416230519844TP blahblah
1.5 5416001160 21  F 492278  4 64001416230519844TP blahblah
1.6 5416001165 13  F 492279  4 64001416230519844TP blahblah
1.7 5416001170  3  F 492280  4 64001416230519844TP blahblah
1.8 5416001180  1  F 492281  4 64001416230519844TP blahblah
2   5544001125  1  F 492291  4 64001544250619844RA blahblah
2.1 5544001130  3  F 492292  4 64001544250619844RA blahblah
2.2 5544001135  4  F 492293  4 64001544250619844RA blahblah
2.3 5544001140 11  F 492294  4 64001544250619844RA blahblah
2.4 5544001145 13  F 492295  4 64001544250619844RA blahblah

选项2：

rawlist <- split(x, cumsum(grepl(pattern = '^4 ', x)))

l1 <- lapply(rawlist, function(x) read.table(text = x, skip = 1, header = FALSE))
l2 <- lapply(rawlist, function(x) read.table(text = x, nrows = 1, header = FALSE))
reps <- sapply(l1, nrow)

d1 <- do.call(rbind, l1)
d2 <- do.call(rbind, l2)[rep(1:length(l2), reps),]

d <- cbind(d1, d2)
names(d) <- paste0('V',1:ncol(d))

给出：

> d
            V1 V2    V3     V4 V5                  V6       V7
1.1 5416001130  1 FALSE 492273  4 64001416230519844TP blahblah
1.2 5416001140  3 FALSE 492274  4 64001416230519844TP blahblah
1.3 5416001145  1 FALSE 492275  4 64001416230519844TP blahblah
1.4 5416001150 19 FALSE 492276  4 64001416230519844TP blahblah
1.5 5416001155 21 FALSE 492277  4 64001416230519844TP blahblah
1.6 5416001160 21 FALSE 492278  4 64001416230519844TP blahblah
1.7 5416001165 13 FALSE 492279  4 64001416230519844TP blahblah
1.8 5416001170  3 FALSE 492280  4 64001416230519844TP blahblah
1.9 5416001180  1 FALSE 492281  4 64001416230519844TP blahblah
2.1 5544001125  1 FALSE 492291  4 64001544250619844RA blahblah
2.2 5544001130  3 FALSE 492292  4 64001544250619844RA blahblah
2.3 5544001135  4 FALSE 492293  4 64001544250619844RA blahblah
2.4 5544001140 11 FALSE 492294  4 64001544250619844RA blahblah
2.5 5544001145 13 FALSE 492295  4 64001544250619844RA blahblah

使用过的数据：

x <- readLines(textConnection('4  64001416230519844TP blahblah  
5416001130  1 F   492273
5416001140  3 F   492274
5416001145  1 F   492275
5416001150 19 F   492276
5416001155 21 F   492277
5416001160 21 F   492278
5416001165 13 F   492279
5416001170  3 F   492280
5416001180  1 F   492281
4  64001544250619844RA blahblah
5544001125  1 F   492291
5544001130  3 F   492292
5544001135  4 F   492293
5544001140 11 F   492294
5544001145 13 F   492295'))

要阅读您的实际数据，您可以使用以下内容：

x <- readLine('name-of-datafile.txt')

Answer 4

这是一个可能的基本R解决方案试图更高效的内存：

rawtext <- "4  64001416230519844TP blahblah  
5416001130  1 F   492273
5416001140  3 F   492274
5416001145  1 F   492275
5416001150 19 F   492276
5416001155 21 F   492277
5416001160 21 F   492278
5416001165 13 F   492279
5416001170  3 F   492280
5416001180  1 F   492281
4  64001544250619844RA blahblah
5544001125  1 F   492291
5544001130  3 F   492292
5544001135  4 F   492293
5544001140 11 F   492294
5544001145 13 F   492295"

首先读取数据一次，然后获取标题行号。请注意，这可以通过命令行实用程序完成，例如...... grep，在R：

之外

text <- readLines(textConnection(rawtext))
header_rows <- grep("^4", text)
lengths <- diff(c(header_rows, length(text) + 1)) - 1
rm(text)

然后实际上重新读取每一块，但只有必要的行数：

do.call(rbind, mapply(
  function(skip, nrows, ...) data.frame(
    read.table(skip = skip, nrows = nrows, ...),
    read.table(skip = skip - 1, nrows = 1, ...)
  ),
  MoreArgs = list(text = rawtext),
  header_rows,
  lengths,
  SIMPLIFY = FALSE
))

#            V1 V2    V3     V4 V1.1                V2.1     V3.1
# 1  5416001130  1 FALSE 492273    4 64001416230519844TP blahblah
# 2  5416001140  3 FALSE 492274    4 64001416230519844TP blahblah
# 3  5416001145  1 FALSE 492275    4 64001416230519844TP blahblah
# 4  5416001150 19 FALSE 492276    4 64001416230519844TP blahblah
# 5  5416001155 21 FALSE 492277    4 64001416230519844TP blahblah
# 6  5416001160 21 FALSE 492278    4 64001416230519844TP blahblah
# 7  5416001165 13 FALSE 492279    4 64001416230519844TP blahblah
# 8  5416001170  3 FALSE 492280    4 64001416230519844TP blahblah
# 9  5416001180  1 FALSE 492281    4 64001416230519844TP blahblah
# 10 5544001125  1 FALSE 492291    4 64001544250619844RA blahblah
# 11 5544001130  3 FALSE 492292    4 64001544250619844RA blahblah
# 12 5544001135  4 FALSE 492293    4 64001544250619844RA blahblah
# 13 5544001140 11 FALSE 492294    4 64001544250619844RA blahblah
# 14 5544001145 13 FALSE 492295    4 64001544250619844RA blahblah

每隔n（不同）行如何整理带有标题的固定宽度文件？

4 个答案: