Question

我试图抓住以下网站：https://www.climatempo.com.br/climatologia/558/saopaulo-sp。它有一个两个下拉菜单，第二个取决于第一个菜单，因此我选择使用scrapy并通过scrapy-splash进行喷溅。

我需要首先选择州，然后选择城市来自动更改位置。我尝试过SplashFormRequest，但我无法更改城市列表。我的蜘蛛是（打印用于调试）：

import scrapy
from scrapy_splash import SplashRequest, SplashFormRequest


class ExampleSpider(scrapy.Spider):
    name = 'climatologia'

    def start_requests(self):
        urls = ['https://www.climatempo.com.br/climatologia/558/saopaulo-sp']
        for url in urls:
            yield SplashRequest(url=url, callback=self.parse,
                                endpoint='render.html',
                                args={'wait': 0.5},)

    def parse(self, response):
        print(response.url)
        state = response.css("select.slt-geo")[0].css("option::attr(value)").extract()
        print(state)

        return SplashFormRequest(response.url, method='POST',
                                 formdata={'sel-state-geo': 'SP'},
                                 callback=self.state_selected,
                                 args={'wait': 0.5})

    def state_selected(self, response):
        print('\t:+)\t:+)\t:+)\t:+)\t:+)\t:+)')
        print(response.css("select.slt-geo")[0].css("option::text").extract())
        print(response.css("select.slt-geo")[1].css("option::text").extract())

Answer 1

如果您绝对必须使用网站菜单，这是我建议Selenium的工作。脚本Splash的唯一方法是通过LUA脚本。您必须发送到执行端点并创建LUA脚本。我找到了您尝试选择的选项，但没有找到提交表单的位置或它在网站上的运行方式。我确实要翻译成英文。

我的建议是在浏览器检查器中查找端点，例如这是一些看起来特别有趣的端点之一： https://www.climatempo.com.br/json/busca-estados

此端点为json提供如下

{"success":true,"message":"Resultados encontrados","time":"2017-11-30 16:05:20","totalRows":null,"totalPages":null,"page":null,"data":[{"idlocale":338,"idstate":31,"uf":"AC","state":"Acre","region":"N","latitude":null,"longitude":null},{"idlocale":339,"idstate":49,"uf":"AL","state":"Alagoas","region":"NE","latitude":null,"longitude":null},{"idlocale":340,"idstate":41,"uf":"AM","state":"Amazonas","region":"N","latitude":null,"longitude":null},{"idlocale":341,"idstate":30,"uf":"AP","state":"Amap\u00e1","region":"N","latitude":null,"longitude":null},{"idlocale":342,"idstate":56,"uf":"BA","state":"Bahia","region":"NE","latitude":null,"longitude":null},{"idlocale":343,"idstate":44,"uf":"CE","state":"Cear\u00e1","region":"NE","latitude":null,"longitude":null},{"idlocale":344,"idstate":47,"uf":"DF","state":"Distrito Federal","region":"CO","latitude":null,"longitude":null},{"idlocale":345,"idstate":45,"uf":"ES","state":"Esp\u00edrito Santo","region":"SE","latitude":null,"longitude":null},{"idlocale":346,"idstate":54,"uf":"GO","state":"Goi\u00e1s","region":"CO","latitude":null,"longitude":null},{"idlocale":347,"idstate":52,"uf":"MA","state":"Maranh\u00e3o","region":"NE","latitude":null,"longitude":null},{"idlocale":348,"idstate":53,"uf":"MG","state":"Minas Gerais","region":"SE","latitude":null,"longitude":null},{"idlocale":349,"idstate":39,"uf":"MS","state":"Mato Grosso do Sul","region":"CO","latitude":null,"longitude":null},{"idlocale":350,"idstate":40,"uf":"MT","state":"Mato Grosso","region":"CO","latitude":null,"longitude":null},{"idlocale":351,"idstate":50,"uf":"ND","state":"N\u00e3o Aplic\u00e1vel","region":"ND","latitude":null,"longitude":null},{"idlocale":352,"idstate":55,"uf":"PA","state":"Par\u00e1","region":"N","latitude":null,"longitude":null},{"idlocale":353,"idstate":37,"uf":"PB","state":"Para\u00edba","region":"NE","latitude":null,"longitude":null},{"idlocale":354,"idstate":29,"uf":"PE","state":"Pernambuco","region":"NE","latitude":null,"longitude":null},{"idlocale":355,"idstate":33,"uf":"PI","state":"Piau\u00ed","region":"NE","latitude":null,"longitude":null},{"idlocale":356,"idstate":32,"uf":"PR","state":"Paran\u00e1","region":"S","latitude":null,"longitude":null},{"idlocale":357,"idstate":46,"uf":"RJ","state":"Rio de Janeiro","region":"SE","latitude":null,"longitude":null},{"idlocale":358,"idstate":35,"uf":"RN","state":"Rio Grande do Norte","region":"NE","latitude":null,"longitude":null},{"idlocale":359,"idstate":38,"uf":"RO","state":"Rond\u00f4nia","region":"N","latitude":null,"longitude":null},{"idlocale":360,"idstate":43,"uf":"RR","state":"Roraima","region":"N","latitude":null,"longitude":null},{"idlocale":361,"idstate":48,"uf":"RS","state":"Rio Grande do Sul","region":"S","latitude":null,"longitude":null},{"idlocale":362,"idstate":36,"uf":"SC","state":"Santa Catarina","region":"S","latitude":null,"longitude":null},{"idlocale":363,"idstate":51,"uf":"SE","state":"Sergipe","region":"NE","latitude":null,"longitude":null},{"idlocale":364,"idstate":34,"uf":"SP","state":"S\u00e3o Paulo","region":"SE","latitude":null,"longitude":null},{"idlocale":365,"idstate":42,"uf":"TO","state":"Tocantins","region":"N","latitude":null,"longitude":null}]}

希望这是获取您正在寻找的数据的另一种方式吗？

然后您可以使用普通请求来获取数据。你只需要形成相同的请求。通常添加一个accept，useragent和requested with header足以通过。

使用scrapy-splash

1 个答案: