我设置了Scrapy Spider。我抓取了一些网址,然后按网址存储了多个结果。我在每个项目中都有嵌套数组。 我想使用这些项目中包含的图像URL来下载图像,但是我不能这样做。
import * as moment from 'moment';
我的JSON输出如下:
def init(self, *args, **kwargs):
data_file = pkgutil.get_data(
"auctions_results", "json/input/demo_db_urls_glenmarch.json")
self.data = json.loads(data_file)
def start_requests(self):
for item in self.data:
request = scrapy.Request(item['gm_url'], callback=self.parse)
request.meta['item'] = item
yield request
def parse(self, response):
item = response.meta['item']
item['results'] = []
for caritem in response.css("div.car-item-border"):
data = AuctionItem()
data["marque"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[1]
data["model"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[2]
data["model_year"] = caritem.css("div.make::text").extract_first().strip().split(" ", 1)[0]
data["price_str"] = caritem.css("div.price::text").extract_first().strip().replace(",", " ")
if caritem.css("div.price::text").extract_first().find("Estimate"):
data["price_int"] = re.sub("\D", "", data["price_str"])
data["price_int"] = int(data["price_int"])
data["price_currency"] = re.sub(
"[0-9]", "", data["price_str"]).replace(" ", "")
data["sold"] = True
else:
data["price_int"] = None
data["price_currency"] = None
data["sold"] = False
data["auction_house"] = caritem.css("div.auctionHouse::text").extract_first().split("-", 1)[0].strip()
data["auction_country"] = caritem.css("div.auctionHouse::text").extract_first().rsplit(",", 1)[1].strip()
data["auction_date"] = caritem.css("div.date::text").extract_first().replace(",", "").strip()
if " - " in data["auction_date"]:
auctiondate = re.sub(r".*-", "", data["auction_date"]).strip()
data["auction_datetime"] = datetime.strptime(auctiondate, '%d %B %Y').date()
else:
data["auction_datetime"] = datetime.strptime(data["auction_date"], '%d %B %Y').date()
auctionurl = caritem.css("div.view-auction a::attr(href)").extract_first()
if auctionurl != None and "/auction-cars/show-backup-image" not in auctionurl:
data["auction_url"] = caritem.css("div.view-auction a::attr(href)").extract_first()
else :
data["auction_url"] = None
data["image_urls"] = caritem.css("div.view-auction a img::attr(src)").extract_first()
item['results'].append(data)
yield item`
如何从“ images_urls”下载图像?
答案 0 :(得分:0)
ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
答案 1 :(得分:0)
图像管道期望根项上的image_urls
字段。
虽然您可以override the field name used,但由于用例不足,因为您想处理每个项目的多个字段。
因此,根据文档:
如果您需要更复杂的东西并想覆盖自定义管道行为,请参见Extending the Media Pipelines。