Question

我正在尝试从Amazon产品（例如this link）中抓取图像URL。

在页面源代码中，有一个部分包含了不同大小（大，中，hirez等）图像的所有url。我可以通过抓取来完成脚本的那一部分

imagesString = (response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first())

哪个给了我一个看起来像这样的字符串，

P.when('A').register("ImageBlockATF", function(A){
    var data = {
                'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31HoKqtljqL._SS40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31HoKqtljqL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX355_.jpg":[308,355],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX450_.jpg":[390,450],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX425_.jpg":[369,425],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX466_.jpg":[404,466],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX522_.jpg":[453,522],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX569_.jpg":[494,569],"https://images-na.ssl-images-amazon.com/images/I/81FED1p-sTL._SX679_.jpg":[589,679]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31Y%2B8oE5DtL._SS40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31Y%2B8oE5DtL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX355_.jpg":[308,355],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX450_.jpg":[390,450],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX425_.jpg":[369,425],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX466_.jpg":[404,466],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX522_.jpg":[453,522],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX569_.jpg":[494,569],"https://images-na.ssl-images-amazon.com/images/I/81e8905DlhL._SX679_.jpg":[589,679]},"variant":"PT01","lowRes":null},{"hiRes":null,"thumb":"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL._SS40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL._SX355_.jpg":[236,355],"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL._SX450_.jpg":[300,450],"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL._SX425_.jpg":[283,425],"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL._SX466_.jpg":[310,466],"https://images-na.ssl-images-amazon.com/images/I/51rORrvh0hL.jpg":[333,500]},"variant":"PT02","lowRes":null},{"hiRes":null,"thumb":"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL._SS40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL._SX355_.jpg":[236,355],"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL._SX450_.jpg":[300,450],"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL._SX425_.jpg":[283,425],"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL._SX466_.jpg":[310,466],"https://images-na.ssl-images-amazon.com/images/I/41L2OU5rPyL.jpg":[333,500]},"variant":"PT03","lowRes":null},{"hiRes":null,"thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL._SS40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL._SX355_.jpg":[236,355],"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL._SX450_.jpg":[300,450],"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL._SX425_.jpg":[283,425],"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL._SX466_.jpg":[310,466],"https://images-na.ssl-images-amazon.com/images/I/51%2BsCYjx6OL.jpg":[333,500]},"variant":"PT04","lowRes":null}]},
                'colorToAsin': {'initial': {}},
                'holderRatio': 1.0,
                'holderMaxHeight': 700,
                'heroImage': {'initial': []},
                'heroVideo': {'initial': []},
                'spin360ColorData': {'initial': {}},
                'spin360ColorEnabled': {'initial': 0},
                'spin360ConfigEnabled': false,
                'spin360LazyLoadEnabled': false,
                'playVideoInImmersiveView':'false',
                'tabbedImmersiveViewTreatment':'T2',
                'totalVideoCount':'0',
                'videoIngressATFSlateThumbURL':'',
                'mediaTypeCount':'0',
                'atfEnhancedHoverOverlay' : true,
                'winningAsin': 'B00XLSS79Y',
                'weblabs' : {},
                'aibExp3Layout' : 1,
                'aibRuleName' : 'frank-powered',
                'acEnabled' : false
                };
    A.trigger('P.AboveTheFold'); // trigger ATF event.
    return data;
});

我的目标是将colorImages中的数据放入Json词典中，这样我就可以轻松获取每个URL。

我试图做这样的事情：

m = re.search(r'^var data = ({.*};)', imagesString , re.S | re.M)
data = m.group()
jsonObj = json.loads(data[:-1].replace("'", '"'))

但是似乎imagesString与re.search不能很好地工作，关于imagesString实际上不是字符串，我不断收到错误消息。

我使用re.findall从亚马逊页面上获得了类似的数据，类似这样的内容（script是我从页面上获取的一大块文本）。

variationValues = re.findall(r'variationValues\" : ({.*?})', ' '.join(script))[0]

然后

variationValuesDict = json.loads(variationValues)

但是我对正则表达式的了解不是很好。

从我上面粘贴的字符串中，我删除了开始和结束，因此只保留了数据，所以我只剩下了这个：

https://jsoneditoronline.org/?id=9ea92643044f4ac88bcc3e76d98425fc

我不知道如何使用colorImages（或json编辑器中的数据）来获取re.findall()，因此我可以将其加载到Json中并像字典一样使用，关于如何做到这一点？

Answer 1

您只需要最初将var data转换为正确的标记json。这很容易））））只需将所有字符'替换为"并删除 SPACES 。然后您将获得json对象：（这是您正确的json）

从亚马逊产品页面获取图片网址

1 个答案: