我在从亚马逊网站提取图片网址时遇到了一些麻烦。
我设法获得了包含所有URL的脚本，但是，当提取工作开始进行时，事情就无法正常进行。
例如，让我们来看看这个亚马逊产品：
使用xpath
//script[contains(., "ImageBlockATF")]/text()
您可以获取源代码中包含URL的部分。我一直在通过做这个脚本
s = response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first()
然后，为了获得单个URL，类似这样的
m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
我遇到的问题是，要使re.search工作，我需要s作为字符串，但这没有发生。当代码到达data = m.groups()[0]时，将发生错误，其中m由于s不是字符串，因此无法应用.groups方法。
我不知道发生了什么，因为s实际上是一个字符串，/text()返回了一个字符串，我检查了类型，结果为str（也尝试做str(s)，但也无济于事。
此外，如果我将字符串强行插入s，则类似：
s = "P.when('A').register("ImageBlockATF", function(A){
var data = {
'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
'colorToAsin': {'initial': {}},
'holderRatio': 1.2,
'holderMaxHeight': 700,
'heroImage': {'initial': []},
'heroVideo': {'initial': []},
'spin360ColorData': {'initial': {}},
'spin360ColorEnabled': {'initial': 0},
'spin360ConfigEnabled': false,
'spin360LazyLoadEnabled': false,
'playVideoInImmersiveView':'false',
'tabbedImmersiveViewTreatment':'C',
'totalVideoCount':'0',
'videoIngressATFSlateThumbURL':'',
'mediaTypeCount':'0',
'atfEnhancedHoverOverlay' : true,
'winningAsin': 'B072596K2C',
'weblabs' : {},
'aibExp3Layout' : 1,
'aibRuleName' : 'frank-powered',
'acEnabled' : false
};
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});"
它很好用。
错误回溯：
Traceback (most recent call last):
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 59, in parse
data = m.group()
AttributeError: 'NoneType' object has no attribute 'group'
有什么想法吗？
答案 0 :(得分：0)
使用另一种方法修复了该问题。
imagenesString = str(response.xpath('//*[contains(@type, "text/javascript") and contains(.,"ImageBlockATF") and not(contains(.,"jQuery"))]').extract())
imagenes = re.findall('\"large\":\"(https.*?\.jpg)\"', imagenesString)
现在imagenes是包含所有URL的列表