我正在废弃一个网站,其中进行了多个ajax调用并显示了内容。首先,我通过单击主页中的选项导航到页面。从生成的HTML中,我必须再次单击并导航到另一个page.如何使用scrapy-splash进行此操作?
这是我的代码
import json
import scrapy
from scrapy_splash import SplashRequest
class SA(scrapy.Spider):
name = "sa"
start_urls = ["http://ted.europa.eu/TED/browse/browseByBO.do"]
def start_requests(self):
script = """
function main(splash)
local url = splash.args.url
assert(splash:go(url)) #goes to the home url
assert(splash:wait(1))
assert(splash:runjs("$('#link-kUK').click()")) #clicks an option in the page
assert(splash:runjs("$('#filterTree > ul > li:nth-child(1) > a').click()")) #this click has to be done from the resulting html(i am struck with this part) and this click open a new page from where i have to scrap details
return {
html = splash:html()
}
end
"""
for url in self.start_urls:
yield scrapy.Request(url, self.parse_result, meta={
'splash': {
'args': {'lua_source': script},
'endpoint': 'execute',
}
})
def parse_result(self, response):
print("response")