Question

我有一个大约有2000万行的数据集，格式如下：

Userid attributid timeid
1      -1         0
1      -2         0
1      -3         0
1      -4         0
1      -5         0
...

和另一个将attributeid与四种属性类型之一匹配的索引：

attributeid attributetype
-1          A
-2          B
-3          C
-4          D
-5          B

我想通过将数据集转换为以下格式将数据集批量导入neo4j：

UserID A     B      C     D     timeid
1      -1    -2,-5  -3    -4    0

我通过使用userid命令数据帧并扫描它来尝试R.但它太慢了。我想知道什么是最有效的方法呢？或者我可以做任何事情来优化我的代码？这是我的代码：

names(node_attri)[1] = 'UserID'
names(node_attri)[2] = 'AttriID'
names(node_attri)[3] = 'TimeID'
names(attri_type)[1] = 'AttriID'
names(attri_type)[2] = 'AttriType'
#attri_type <- attri_type[order(attri_type),]
#node_attri <- node_attri[order(node_attri),]

N = length(unique(node_attri$TimeID))*length(unique(node_attri$UserID))
new_nodes = data.frame(UserID=rep(NA,N), employer=rep(NA,N), major=rep(NA,N), 
                  places_lived=rep(NA,N), school=rep(NA,N), TimeID=rep(NA,N))
row = 0
start = 1
end = 1
M =nrow(node_attri)  
while(start <= M) {
  row = row + 1
  em = ''
  ma = ''
  pl = ''
  sc = ''
  while(node_attri[start,1] == node_attri[end,1]) {
    if (attri_type[abs(node_attri[end,2]),2] == 'employer') 
       em = paste(em, node_attri[end,2], sep=',')
    else if (attri_type[abs(node_attri[end,2]),2] == 'major')
       ma = paste(ma, node_attri[end,2], sep=',')
    else if (attri_type[abs(node_attri[end,2]),2] == 'places_lived')
       pl = paste(pl, node_attri[end,2], sep=',')
    else if (attri_type[abs(node_attri[end,2]),2] == 'school')
      sc = paste(sc, node_attri[end,2], sep=',')
    end = end + 1
    if (end > M) break
  }
  new_nodes[row,] = list(UserID=node_attri[start,1], employer=substring(em,2), 
                       major=substring(ma,2), places_lived=substring(pl,2), 
                       school=substring(sc,2), TimeID=node_attri[start,3])
  start = end
  end = start
}
new_nodes = new_nodes[1:row,]

Answer 1

您需要合并，聚合然后重塑。假设您的数据框分别为DF和DF2：

x <- merge(DF, DF2)
y <- aggregate(attributeid~., data=x, FUN=function(x)paste(x, collapse=","))
z <- reshape(y, direction="wide", idvar=c("Userid","timeid"), timevar="attributetype")

结果：

> z
  Userid timeid attributeid.A attributeid.B attributeid.C attributeid.D
1      1      0            -1         -5,-2            -3            -4

重命名和重新排列列很简单。

Answer 2

以下是使用reshape2包和match的解决方案。

library(reshape2)

##Create some sample data
dat1 <- data.frame(Userid=rep(1:4,each=5),attributeid=rep(-1:-5,4),timeid=rep(0:3,each=5))
index <- data.frame(attruibuteid=-1:-5,attributetype=c("A","B","C","D","B"))

##Merge the two using match
dat1$attributetype = index$attributetype[match(dat1$attributeid,index$attruibuteid)]

##Reformat using aggregate and dcast 
dat2 <- aggregate(attributeid~attributetype+timeid+Userid,function(x){paste(x,collapse=",")},data=dat1)
dat3 <- dcast(formula=Userid+timeid~attributetype,value.var="attributeid",data=dat2)

> dat3
  Userid timeid  A     B  C  D
1      1      0 -1 -2,-5 -3 -4
2      2      1 -1 -2,-5 -3 -4
3      3      2 -1 -2,-5 -3 -4
4      4      3 -1 -2,-5 -3 -4

合并数据集中的多行

2 个答案: