将scrapy + splash脚本转换为.exe

时间:2019-10-02 17:42:42

标签: python web-scraping scrapy

我制作了一个抓痒的脚本(也使用了飞溅)。尝试使用pyinstaller和py2exe制作.exe,但是没有运气。 Pyinstaller能够制作一个.exe文件,但没有运行(由于模块错误)。关于如何做到这一点的任何建议?下面是脚本的代码。

# -*- coding: utf-8 -*-
import scrapy

from scrapy_splash import SplashRequest
from scrapy.selector import Selector


class DarazsSpider(scrapy.Spider):
    name = 'darazs'
    script = '''
    function main(splash, args)
  assert(splash:go(args.url))
  assert(splash:wait(0.5))
  treat=require('treat')
  result = {}
  for i=2,68,1
  do
    assert(splash:runjs('document.querySelector(".ant-pagination-next .ant-pagination-item-link").click()'))
    assert(splash:wait(5.0))
    result[i]=splash:html()


  end  
  return treat.as_array(result)
end
    '''

    def start_requests(self):
        url = 'https://www.daraz.pk/smartphones/'
        yield SplashRequest(url=url, callback=self.parse, endpoint='render.html', args={'wait': 0.5})
        yield SplashRequest(url=url, callback=self.parse_other_pages, endpoint='execute',
                            args={'wait': 0.5, 'lua_source': self.script, 'timeout': 3600}, dont_filter=True)

    def parse(self, response):
        for phone in response.xpath('//div[@class="c2prKC"]'):
            yield {
                'Name': phone.xpath('.//div[@class="c16H9d"]/a/text()').extract(),
                'Price': phone.xpath('.//span[@class="c13VH6"]/text()').extract(),
                'old_price': phone.xpath('.//del[@class="c13VH6"]/text()').extract(),

            }

    def parse_other_pages(self, response):
        for page in response.data:
            sel = Selector(text=page)
            for phone in sel.xpath('//div[@class="c2prKC"]'):
                yield {
                    'Name': phone.xpath('.//div[@class="c16H9d"]/a/text()').extract(),
                    'Price': phone.xpath('.//span[@class="c13VH6"]/text()').extract(),
                    'old_price': phone.xpath('.//del[@class="c13VH6"]/text()').extract(),

                }

0 个答案:

没有答案