Question

我想抓住英国航空公司的机票并将其存放在mongodb。我能够通过搜索表单，但我无法获取给定的数据。

我的蜘蛛：

from scrapy import Spider
from scrapy.selector import Selector
from scrapy.http import FormRequest
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
import time
from flight.items import FlightItem

class BASpider(Spider):
name = "BA"
allowed_domains = ["britishairways.com"]
start_urls = [
    "http://www.britishairways.com/travel/home/public/en_za?DM1_Channel=PPC&DM1_Mkt=ZA&DM1_Campaign=APMEA_ZA_EN_PUREBRAND_MASTERBRAND&Brand=Y&gclid=CLvt24zsqMgCFUGg2wodds4Prw",
]

def __init__(self):
    self.driver = webdriver.Firefox()

def parse(self, response):
    self.driver.get(response.url)
    WebDriverWait(self.driver, 10).until(lambda s: s.find_element_by_xpath('//select[@id="depCountry"]').is_displayed())

    departCountry_form = self.driver.find_element_by_id('depCountry')
    departCity_form = self.driver.find_element_by_id('from')
    oneWay = self.driver.find_element_by_id('journeyTypeOW')
    oneWay.click()
    dest_form = self.driver.find_element_by_id('planTripFlightDestination')
    date_form = self.driver.find_element_by_id('depDate')
    butt = self.driver.find_element_by_class_name('button')
    departCountry_form.send_keys("South Africa")
    departCity_form.send_keys("Johannesburg")
    dest_form.send_keys("London")
    date_form.clear()
    date_form.send_keys("05/10/15")

    actions = ActionChains(self.driver)
    actions.click(butt)
    actions.perform()
    time.sleep(35)



def parse_post(self, response): 
    flightList = Selector(response).xpath('//table[@class="flightList directFlightsTable connectflights"]/tbody/tr')

    for flight in flightList:
        item = FlightItem()
        item['dTime'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" departure"]/div/div/span[1]/text()').extract()[0]
        item['aTime'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" arrival"]/span[1]/text()').extract()[0]
        item['flightNr'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" operator"]/div/div/span[2]/href').extract()[0]
        item['price_economy'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-M ch3 col1"]/span/span[2]/label/text()').extract()[0]
        item['price_premium'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-W ch3 col2"]/span/span[2]/label/text()').extract()[0]
        item['price_business'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-C ch3 col3"]/span/span[2]/label/text()').extract()[0]
        yield item  

    self.driver.close()

我没有收到任何错误，只是没有抓错。

Answer 1

你什么都得不到的原因是永远不会调用parse_post()方法。

您可能直接在Selector回调中直接从self.driver.page_source实例化parse()：

selector = Selector(text=self.driver.page_source)
flightList = selector.xpath('//table[@class="flightList directFlightsTable connectflights"]/tbody/tr')

for flight in flightList:
    # ...

Python，Selenium和Scrapy不能一起工作？

1 个答案: