Question

我抓取了一个网站以及该网站的图片。我对图像使用自定义管道。我想在URL列表中创建一个链接（例如“ www.urls.com/image”），并添加相应图像的名称。

我的spider.py：

import json
import scrapy
import re
import pkgutil

from scrapy.loader import ItemLoader
from auctions_results.items import AuctionItem
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem

from datetime import datetime

class Spider(scrapy.Spider):

    name = 'results'

    def __init__(self, *args, **kwargs):
        data_file = pkgutil.get_data(
            "auctions_results", "json/input/scrape_demo_db.json")
        self.data = json.loads(data_file)

    def start_requests(self):
        for item in self.data:
            request = scrapy.Request(item['gm_url'], callback=self.parse)
            request.meta['item'] = item
            yield request

    def parse(self, response):
        item = response.meta['item']
        item['results'] = []

        for caritem in response.css("div.car-item-border"):
            data = AuctionItem()

            data["marque"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[1]

            data["auction_house"] = caritem.css("div.auctionHouse::text").extract_first().split("-", 1)[0].strip()
            data["auction_country"] = caritem.css("div.auctionHouse::text").extract_first().rsplit(",", 1)[1].strip()
            data["auction_date"] = caritem.css("div.date::text").extract_first().replace(",", "").strip()

            data["image_urls"] = caritem.css("div.view-auction a img::attr(src)").extract_first()
            data["image_path"] = "urls.com/" + NAME OF THE SCRAPED IMAGE

            item['results'].append(data)

        yield item

我的图片管道：

import scrapy

from scrapy.pipelines.images import ImagesPipeline

class DownloadImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for result in item['results']:
            image_url = result['image_urls']
            if image_url is not None:
                request = scrapy.Request(url=image_url)
                yield request

输出示例：

[{
"gm_url": "url",
"results": [{
    "marque": "ferrari",
    "auction_house": "auction",
    "auction_country": "japan",
    "auction_date": "2019",
    "image_urls": "imgurl"
},
{
    "marque": "porsche",
    "auction_house": "auction2",
    "auction_country": "gb",
    "auction_date": "2018",
    "image_urls": "imgurl2"
}],
"images": [
{
    "url": "imgurl",
    "path": "full/0c2d1678d25dfed305d9a1f64366f005f2c3a138.jpg",
    "checksum": "e33f726186f72e2e9810555615964fb1"
},
{
    "url": "imgurl2",
    "path": "full/758825eeb421e6ef49e73bd6af2e0eeb9090cfe1.jpg",
    "checksum": "c172e40ab961108bdd5be1874d688f62"
}]

现在，我想在每个结果中为我的URL添加一个键值“ image_path”（在示例中），并添加图像名称以创建另一个URL。可以在Python中做到吗？

Answer 1

我现在无法对其进行测试，但是您可以在parse()中创建自己的名称，然后在Pipeline中使用它

在Spider中：

在filename中创建url和parse()：

    data["filename"] = '... create your filename ...'

    data["image_path"] = "urls.com/" + data["filename"]

    item['results'].append(data)

在DownloadImagesPipeline中：

get_media_requests

使用filename中meta=中的Request()

    if image_url is not None:

        filename = result['filename']

        request = scrapy.Request(url=image_url, meta={'filename:', filename} )

        yield request

创建file_path来生成下载文件的名称并使用filename下载的文件

def file_path(self, request, response=None, info=None):
    return request.meta['filename']

或多或少的所有代码

def parse(self, response):
        item = response.meta['item']
        item['results'] = []

        for caritem in response.css("div.car-item-border"):
            data = AuctionItem()

            data["marque"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[1]

            data["auction_house"] = caritem.css("div.auctionHouse::text").extract_first().split("-", 1)[0].strip()
            data["auction_country"] = caritem.css("div.auctionHouse::text").extract_first().rsplit(",", 1)[1].strip()
            data["auction_date"] = caritem.css("div.date::text").extract_first().replace(",", "").strip()

            data["image_urls"] = caritem.css("div.view-auction a img::attr(src)").extract_first()

            data["filename"] = '... your filename ...'

            data["image_path"] = "urls.com/" + data["filename"]

            item['results'].append(data)


class DownloadImagesPipeline(ImagesPipeline):

    def get_media_requests(self, item, info):
        for result in item['results']:
            image_url = result['image_urls']
            if image_url is not None:

                filename = result['filename']

                request = scrapy.Request(url=image_url, meta={'filename:', filename} )
                yield request

    def file_path(self, request, response=None, info=None):
        return request.meta['filename']

创建具有刮取图像名称的链接

1 个答案: