使用scrapy

Question

我正在写一只蜘蛛。其中我试图通过登录该网站使用抓取来抓取网站。我写了一个蜘蛛，但在登录网站时仍然遇到问题。我编写了整个蜘蛛，但无法解决登录问题。请查看我的代码。

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import FormRequest
from scrapy.utils.response import open_in_browser

class ScotlandSpider(scrapy.Spider):
    name = 'scotland'
    allowed_domains = ['www.whoownsscotland.org.uk']
    login_url = r'http://www.whoownsscotland.org.uk/login.php?p=%2Fsearch.php'
    start_urls = ['http://www.whoownsscotland.org.uk/search.php']

    def login(self , response):
        data = {
            'name' : 'USERNAME',
            'pass' : 'PASSWORD',
            'previous' : r'%2Fsearch.php',
            'login' : 'login'
        }
        yield FormRequest(url=self.login_url, formdata=data ,callback=self.parse)

    def parse(self, response):
        open_in_browser(response)
        links = response.xpath('//p/a/@href').extract()
        for link in links:
            absoulute_url = response.urljoin(link)
            yield scrapy.Request(absoulute_url , callback=self.parse_links)

    def parse_links(self , response):
        cities = response.xpath('//*[@id="layout-right"]/table/tr/td/p/a/@href').extract()
        for city in cities:
            absoulute_url_new = response.urljoin(city)
            yield scrapy.Request(absoulute_url_new , callback=self.parse_cities)

    def parse_cities(self , response):
        record = response.xpath('//*[@id="layout-left"]/table/tr')
        estate =  record[0].xpath('.//td/text()').extract()
        courty =  record[1].xpath('.//td/text()').extract()
        grid_ref =  record[2].xpath('.//td/text()').extract()
        acreage =  record[3].xpath('.//td/text()').extract()
        os_15 =  record[4].xpath('.//td/text()').extract()
        owner  = record[5].xpath('.//td/text()').extract()
        owner_address = record[6].xpath('.//td/text()').extract()
        property_address = record[7].xpath('.//td/text()').extract()
        website  = record[8].xpath('.//td/text()').extract()
        further_info = record[9].xpath('.//td//text()').extract()
        contacts = record[10].xpath('.//td//text()').extract()
        regsiters_sheet = record[11].xpath('.//td//text()').extract()
        regsiters_certificate = record[12].xpath('.//td//text()').extract()
        currency_of_data = record[13].xpath('.//td//text()').extract()

        yield {
            "Estate" : estate,
            "County" : courty,
            "Grid Reference" : grid_ref,
            "Acreage" : acreage,
            "OS 1:50k Sheet" : os_15,
            "Owner" : owner,
            "Owner Address" : owner_address,
            "Property Address" : property_address,
            "Website" : website,
            "Further Information" : further_info,
            "Contacts" : contacts,
            "Registers of Scotland Sasines Search Sheet No" : regsiters_sheet,
            "Registers of Scotland Land Certificate No" : regsiters_certificate ,
            "Currency of Data" : currency_of_data
        }

Answer 1

问题很简单：你已经创建了login()方法，但是你从来没有打过它。

解决此问题的最简单方法是将该方法重命名为start_requests() 然后scrapy将调用此方法来生成初始请求，而不是从start_urls生成它们。