从路径/节点名称混乱的XML文件中提取数据

时间:2019-02-14 09:53:10

标签: r xml xml2

我正在尝试使用R从机构XML文件中提取值。这是此类文件的示例:

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<missions xmlns="http://sensored">
    <mission missiontype="2" year="2012" platform="4135" missionnumber="1" missiontypename="Referanseflåten-Hav" callsignal="LJBD" platformname="Nybo">
        <startdate>30/01/2012</startdate>
        <stopdate>28/11/2012</stopdate>
        <purpose></purpose>
        <fishstation serialno="86001">
            <nation>58</nation>
            <platform>4135</platform>
            <station>1</station>
            <startdate>30/01/2012</startdate>
            <starttime>18:56:00</starttime>
            <stopdate>30/01/2012</stopdate>
            <stoptime>23:19:00</stoptime>
            <latitudestart>65.13833</latitudestart>
            <longitudestart>-11.04833</longitudestart>
            <system>2</system>
            <area>59</area>
            <location>07</location>
            <bottomdepthstart>653.0</bottomdepthstart>
            <bottomdepthstop>653.0</bottomdepthstop>
            <fishingdepthmax>85.0</fishingdepthmax>
            <fishingdepthmin>40.0</fishingdepthmin>
            <gear>3711</gear>
            <gearcount>1</gearcount>
            <gearcondition>4</gearcondition>
            <trawlquality>7</trawlquality>
            <dataquality>2</dataquality>
            <catchsample species="161722.G03" samplenumber="1" noname="sild'G03" aphia="126417">
                <sampletype>20</sampletype>
                <conservation>1</conservation>
                <producttype>1</producttype>
                <weight>10.0</weight>
                <count>46</count>
                <sampleproducttype>1</sampleproducttype>
                <lengthmeasurement>E</lengthmeasurement>
                <lengthsampleweight>0.863</lengthsampleweight>
                <lengthsamplecount>4</lengthsamplecount>
                <specimensamplecount>4</specimensamplecount>
                <individual specimenno="1">
                    <producttype>1</producttype>
                    <weight>0.222</weight>
                    <lengthunit>2</lengthunit>
                    <length>0.28</length>
                </individual>
                <individual specimenno="2">
                    <producttype>1</producttype>
                    <weight>0.127</weight>
                    <lengthunit>2</lengthunit>
                    <length>0.245</length>
                </individual>
                <individual specimenno="3">
                    <producttype>1</producttype>
                    <weight>0.172</weight>
                    <lengthunit>2</lengthunit>
                    <length>0.26</length>
                </individual>
                <individual specimenno="4">
                    <producttype>1</producttype>
                    <weight>0.342</weight>
                    <lengthunit>2</lengthunit>
                    <length>0.325</length>
                </individual>
            </catchsample>
        </fishstation>
    </mission>
</missions>

我设法毫无问题地解析文件。尽管名称在解析列表中显示正确,但是我无法使用XPath语言正确解析它们:

library(xml2)
library(xmltools)
doc <- xml2::read_xml("test.xml")
doc %>% xmltools::xml_view_trees()
#└── mission
#  ├── startdate
#  ├── stopdate
#  ├── purpose
#  └── fishstation
#    ├── nation
#    ├── platform
#    ├── station
#    ├── startdate
#    ├── starttime
#    ├── stopdate
#    ├── stoptime
#    ├── latitudestart
#    ├── longitudestart
#    ├── system
#    ├── area
#    ├── location
#    ├── bottomdepthstart
#    ├── bottomdepthstop
#    ├── fishingdepthmax
#    ├── fishingdepthmin
#    ├── gear
#    ├── gearcount
#    ├── gearcondition
#    ├── trawlquality
#    ├── dataquality
#    └── catchsample
#      ├── sampletype
#      ├── conservation
#      ├── producttype
#      ├── weight
#      ├── count
#      ├── sampleproducttype
#      ├── lengthmeasurement
#      ├── lengthsampleweight
#      ├── lengthsamplecount
#      ├── specimensamplecount
#      ├── individual
#        ├── producttype
#        ├── weight
#        ├── lengthunit
#        └── length
#      ├── individual
#        ├── producttype
#        ├── weight
#        ├── lengthunit
#        └── length
#      ├── individual
#        ├── producttype
#        ├── weight
#        ├── lengthunit
#        └── length
#      └── individual
#        ├── producttype
#        ├── weight
#        ├── lengthunit
#        └── length
doc %>% xmltools::xml_get_paths()
#[[1]]
# [1] "/*/*"         "/*/*/*"       "/*/*/*"       "/*/*/*"       #"/*/*/*"       "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     #"/*/*/*/*"     "/*/*/*/*"    
# [12] "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     #"/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     #"/*/*/*/*"     "/*/*/*/*"    
# [23] "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     "/*/*/*/*"     #"/*/*/*/*"     "/*/*/*/*/*"   "/*/*/*/*/*"   "/*/*/*/*/*"   "/*/*/*/*/*"   #"/*/*/*/*/*"   "/*/*/*/*/*"  
# [34] "/*/*/*/*/*"   "/*/*/*/*/*"   "/*/*/*/*/*"   "/*/*/*/*/*"   #"/*/*/*/*/*"   "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*/*" #"/*/*/*/*/*"   "/*/*/*/*/*/*"
# [45] "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*"   #"/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*/*" "/*/*/*/*/*"   #"/*/*/*/*/*/*" "/*/*/*/*/*/*"
# [56] "/*/*/*/*/*/*" "/*/*/*/*/*/*"

说我想提取每个fishstation的经度和纬度,以及每个采样鱼的物种代码和个体重量。通过将XML文档转换为列表并使用标准R代码,我可以使用内存效率不高的方式来做到这一点:

library(XML)

doc <- xmlInternalTreeParse("test.xml")

getNodeSet(doc, "//fishstation") # Does not work
getNodeSet(doc, "/*/*/*/*") # Works

tmp <- xmlToList(doc)[[1]]
tmp <- tmp[names(tmp) == "fishstation"]

lapply(tmp, function(k) {

  sp <- k$catchsample$.attrs
  inds <- k$catchsample
  inds <- inds[names(inds) == "individual"]

  data.frame(lon = k$longitudestart, lat = k$latitudestart, 
sp = unname(sp[names(sp) == "species"]), 
weight = unname(sapply(inds, function(j) j$weight)))
})

# $fishstation
#        lon             lat         sp   weight
# 1 -11.04833        65.13833 161722.G03  0.222
# 2 -11.04833        65.13833 161722.G03  0.127
# 3 -11.04833        65.13833 161722.G03  0.172
# 4 -11.04833        65.13833 161722.G03  0.342

但是,能够非常方便地使用XPath语言,因为这样可以使处理过程更快(我的文件可以是几GB)。到目前为止,我还没有调整提取程序的运气,因此使用XPath语言可以处理这些文件。我很清楚,这可能是由于基本用户错误引起的。我以前没有使用过XML文件。

1)如何将示例文件读到R,以保留节点名?

2)如何以更节省内存的方式提取上面示例中的值?

1 个答案:

答案 0 :(得分:1)

这个文件比较棘手的是xml文本中的名称空间。 xml2程序包具有删除命名空间的功能,这使过程更容易。
完成后,只需查找个体,然后向上移动树以找到父节点,然后提取请求的信息即可。

library(xml2)
#read the file in as xml
page<-read_xml("fish.xml")
#strip the name space out
page<-xml_ns_strip(page)

#find all of the individuals
individuals<-xml_find_all(page, ".//individual")
#find the parent catchsample nodes for the individuals
catchnodes<-xml_find_first( individuals, ".//parent::catchsample")
#find the parents for the catchsample node the fishstation
fishstations<-xml_find_first( catchnodes, ".//parent::fishstation ")

#individuals, catchnodes and fishstations should all have the same length

#extract the desired information from each set of nodes
weights<-xml_text(xml_find_first(individuals, ".//weight"))
lat<-xml_text(xml_find_first(fishstations, ".//latitudestart"))
long<-xml_text(xml_find_first(fishstations, ".//longitudestart"))
species<-xml_attr(catchnodes, "species")  #information is an attribute

#package together
data.frame(lat, long, species, weights)
#        lat      long    species weights
# 1 65.13833 -11.04833 161722.G03   0.222
# 2 65.13833 -11.04833 161722.G03   0.127
# 3 65.13833 -11.04833 161722.G03   0.172
# 4 65.13833 -11.04833 161722.G03   0.342

这里是相同的代码,但没有剥离名称空间:

page<-read_xml("fish.xml")
ns <- xml_ns(page)

#find all of the individuals
individuals<-xml_find_all(page, ".//d1:individual", ns)

catchnodes<-xml_find_first(individuals, ".//parent::d1:catchsample", ns)
fishstations<-xml_find_first(catchnodes, ".//parent::d1:fishstation", ns)

weights<-xml_text(xml_find_first(individuals, ".//d1:weight", ns))
lat<-xml_text(xml_find_first(fishstations, ".//d1:latitudestart", ns))
long<-xml_text(xml_find_first(fishstations, ".//d1:longitudestart", ns))
species<-xml_attr(catchnodes, "species")