Question

所以我在这里很困惑。当我使用scrapy shell并输入xpath时，会返回正确的数据，但是当我将相同的xpath设置为等于脚本中的变量时，它会输出一个空白。我真的不确定发生了什么。

import scrapy

类FestivalSpider（scrapy.Spider）： name =＆＃39; basketball＆＃39;

custom_settings = {
    "DOWNLOAD_DELAY": 3,
    "CONCURRENT_REQUESTS_PER_DOMAIN": 3,
    "HTTPCACHE_ENABLED": True
}

start_urls = [
    'http://www.basketball-reference.com/leagues/NBA_2017_per_game.html'
]

def parse(self, response):
    start_urls = [
    'http://www.basketball-reference.com/leagues/NBA_2017_per_game.html']
    for href in start_urls:
        yield scrapy.Request(
            url=href,
            callback=self.parse_festival,
            meta={'url': href}
        )


def parse_festival(self, response):

    url = response.request.meta['url']

    name = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "player"]/a/text()').extract()

    pos = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "pos"]/text()').extract()

    age = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "age"]/text()').extract()

    #team = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "team_id"]/text()').extract()

    games = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "g"]/text()').extract()

    games_s = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "gs"]/text()').extract()

    fg_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_per_mp"]/text()').extract()

    #fga_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fga_per_mp"]/text()').extract()

    #fg_pct = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_pct"]/text()').extract()

    #fg3_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3_per_mp"]/text()').extract()

    #fg3a_per_mp = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3a_per_mp"]/text()').extract()

    #fg3_pct = response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg3_pct"]/text()').extract()
    #location = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[3])

    #dates = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[5])

    #tickets = (
    #    response.xpath('//div[@id="festival-basics"]/text()').extract()[7])

    #website = (
    #    response.xpath(
    #        '//div[@id="festival-basics"]/a/@href').extract()[0]
    #)

    #logo = (
    #    response.xpath(
    #        '//div[@id="festival-basics"]/img/@src').extract()[0]
    #)

    #lineup = (
    #    response.xpath(
    #        '//div[@class="lineupguide"]/ul/li/text()').extract() +
    #    response.xpath(
    #        '//div[@class="lineupguide"]/ul/li/a/text()').extract()
    #)
    print(response.xpath('//tr[@class = "full_table"]/td[@data-stat = "fg_per_mp"]/text()').extract())

    yield {
        'url': url,
        'name': name,
        'pos': pos,
        'age' : age,
        'games' : games,
        'games_s': games_s, 
        'fg_per_mp': fg_per_mp
        #'fga_per_mp': fga_per_mp,
        #'fg_pct': fg_pct,
        #'fg3_per_mp': fg3_per_mp,
        #'fg3a_per_mp': fg3a_per_mp
        #'team' : team
        #'location': location,
        #'dates': dates,
        #'tickets': tickets,
        #'website': website,
        #'logo': logo,
        #'lineup': lineup
        }

当我使用response.xpath时，有问题的项目是fg_per_mp（＆＃39; // tr [@class =＆＃34; full_table＆＃34;] / td [@ data-stat =＆＃34 ; fg_per_mp＆＃34;] / text（）＆＃39;）。extract（）在它工作的shell中，但脚本中的同一行返回一个空列表。

我做错了什么？

Scrapy Shell具有正确的输出，但脚本没有

0 个答案: