Question

我必须通过对页面进行分页来刮取多个数据。我能够从一个页面跳到另一个页面，但我不能从其他页面中删除必要的数据。它只返回空字段。

部分

item['path'] = response.xpath('//[@id="articleTop"]/table/tbody/tr[1]/td[2]/text()').extract()

不会返回任何内容。完整代码如下。

import scrapy
import re
from scrapy.item import Item, Field
from provat.items import ProvatItem
from scrapy.selector import HtmlXPathSelector
from urlparse import urljoin
import urlparse
import urllib
import os
from os import path
from scrapy.http import Request


class RegebaSpider(scrapy.Spider):
 name = "regeba"
 start_urls = [
    'https://www.eba.europa.eu/regulation-and-policy/single-rulebook/interactive-single-rulebook/-/interactive-single-rulebook/toc/2',
]

 def parse(self, response):
  sites = response.xpath('//*[@class="yui3-widget aui-component aui-tree-  data aui-tree-node"]/div')
  item = ProvatItem()
  for elements in sites:
   item['url'] = elements.xpath('./span/a/@href').extract()
   for moreurls in item['url']:
    request = scrapy.Request(moreurls,callback=self.save_article_data)
    request.meta['item'] = item
    yield request

   yield item

 def save_article_data(self,response):
  item=response.meta['item']
  item['path'] = response.xpath('//*[@id="articleTop"]/table/tbody/tr[1]/td[2]/text()').extract()
  yield item

使用Scrapy搜索数据

0 个答案: