Question

以下用于返回医疗信息的scrapy代码确实会返回第一组返回的结果，但不会返回链接。在stackoverflow上学习代码并检查类似的结果，但是集成它们不起作用。没错，我正在学习。任何指针将不胜感激。

import urlparse

from scrapy.spider import BaseSpider
from scrapy.selector import Selector
from scrapy.http import Request
import w3lib.url

from yelp.items import YelpItem


class YelpSpider(BaseSpider):
    name = "yelp"
    download_delay = 10
    concurrent_requests = 1
    concurrent_requests_per_domain = 1
    allowed_domains = ["yelp.com"]
    start_urls = ["http://www.yelp.com/search?find_desc=cancer+treatment&find_loc=manhattan%2Cny&start=0",
"http://www.yelp.com/search?find_desc=cancer+treatment&find_loc=manhattan%2Cny&start=20",
"http://www.yelp.com/search?find_desc=cancer+treatment&find_loc=manhattan%2Cny&start=30"]

    def parse(self, response):
        selector = Selector(response)
        for title in selector.css("span.indexed-biz-name"):
            page_url = urlparse.urljoin(response.url,
                                        title.xpath("a/@href").extract()[0])
            self.log("page URL: %s" % page_url)
            #continue
            yield Request(page_url,
                          callback=self.parse_page)

        for next_page in selector.css(u'ul > li > a.prev-next:contains(\u2192)'):
            next_url = urlparse.urljoin(response.url,
                                        next_page.xpath('@href').extract()[0])
            self.log("next URL: %s" % next_url)
            #continue
            yield Request(next_url,
                          callback=self.parse)

    def parse_page(self, response):
        selector = Selector(response)
        item = YelpItem()
        item["name"] = selector.xpath('.//h1[@itemprop="name"]/text()').extract()[0].strip()
        item["addresslocality"] = u"\n".join(
            selector.xpath('.//address[@itemprop="address"]//text()').extract()).strip()
        item["link"] = response.url
        website = selector.css('div.biz-website a')
        if website:
            website_url = website.xpath('@href').extract()[0]
            item["website"] = w3lib.url.url_query_parameter(website_url, "url")
        return item

Answer 1

您的下一个网址提取和选择逻辑不正确。定位具有next和pagination-links_anchor类的链接元素。以下适用于我：

next_url = response.css('a.pagination-links_anchor.next::attr(href)').extract_first()
if next_url:
    next_url = urlparse.urljoin(response.url, next_url)
    self.log("next URL: %s" % next_url)
    yield Request(next_url, callback=self.parse)

scrapy没有关注链接

1 个答案: