我已经在setting.py文件中设置了downloadmiddleware
class JsScrollMiddleware(object):
@classmethod
def process_request(cls, request, spider):
if spider.job_cfg.scroll:
scroll_to_the_end_get_data(spider)
html=spider.driver.page_source
spider.job_cfg.scroll=False
return HtmlResponse(url=spider.driver.current_url, encoding='utf-8', body=html, request=request)