使用python从网页获取Json数据

时间：2019-10-15 15:56:22

标签： python-3.x 

import requests
from bs4 import BeautifulSoup
import json

# import webpage
r= requests.get('https://www.zooplus.fi/shop/koirat/kuivaruoka/royal_canin_size')
#find required data-set
#results = soup.find_all('main', attrs={'class':'row'})
src=r.content
#get web data structured
soup= BeautifulSoup(src, 'html.parser')
scrap = soup.find('div',id='productListDataLayer')
print (scrap)

结果：

<div id="productListDataLayer" style="display:none;">[{"product":{"product_id":375148,"product_name":"Royal Canin Size -bonuspakkaus: 15 + 3 kg!","average_prod_review":5,"amount_prod_reviews":55},"variants":[{"bestPriceArticle":false,"variant_id":"375148.5","variant_name":"Medium Puppy","product_variant_name":"Royal Canin Size -bonuspakkaus: 15 + 3 kg! Medium Puppy","variant_price":58.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"375148.13","variant_name":"Giant Adult","product_variant_name":"Royal Canin Size -bonuspakkaus: 15 + 3 kg! Giant Adult","variant_price":57.99,"out_stock":false}]},{"product":{"product_id":764539,"product_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle!","average_prod_review":5,"amount_prod_reviews":3},"variants":[{"bestPriceArticle":false,"variant_id":"764539.1","variant_name":"Maxi Adult (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Adult (15 kg)","variant_price":54.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.3","variant_name":"Maxi Puppy Active (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Puppy Active (15 kg)","variant_price":64.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.4","variant_name":"Maxi Puppy (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Puppy (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.2","variant_name":"Maxi Adult 5+ (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Adult 5+ (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.0","variant_name":"Maxi Ageing 8+ (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Ageing 8+ (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.6","variant_name":"Maxi Starter (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Maxi Starter (15 kg)","variant_price":64.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.9","variant_name":"Medium Adult (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Medium Adult (15 kg)","variant_price":53.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.11","variant_name":"Medium Puppy (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Medium Puppy (15 kg)","variant_price":58.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.18","variant_name":"Medium Starter (12 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Medium Starter (12 kg)","variant_price":57.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.13","variant_name":"Medium Ageing 10+ (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Medium Ageing 10+ (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.14","variant_name":"Medium Adult 7+ (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Medium Adult 7+ (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.28","variant_name":"Giant Adult (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Giant Adult (15 kg)","variant_price":57.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.29","variant_name":"Giant Junior Active (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Giant Junior Active (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.30","variant_name":"Giant Junior (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Giant Junior (15 kg)","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.31","variant_name":"Giant Puppy (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Giant Puppy (15 kg)","variant_price":62.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.32","variant_name":"Giant Starter (15 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Giant Starter (15 kg)","variant_price":62.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.23","variant_name":"Mini Adult 8+ (8 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Mini Adult 8+ (8 kg)","variant_price":43.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.17","variant_name":"Mini Adult (8 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Mini Adult (8 kg)","variant_price":36.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.22","variant_name":"Mini Puppy (8 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Mini Puppy (8 kg)","variant_price":44.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"764539.25","variant_name":"Mini Starter Mother &amp; Babydog (8,5 kg)","product_variant_name":"Royal Canin Size -suurpakkaus + kuivatut kaninkorvat kaupan päälle! Mini Starter Mother &amp; Babydog (8,5 kg)","variant_price":46.99,"out_stock":false}]},{"product":{"product_id":749406,"product_name":"Juhlapainos Royal Canin zooplus-syntymäpäiväpakkaus pienille koirille","average_prod_review":0,"amount_prod_reviews":0},"variants":[{"bestPriceArticle":false,"variant_id":"749406.0","variant_name":"1 kpl","product_variant_name":"Juhlapainos Royal Canin zooplus-syntymäpäiväpakkaus pienille koirille 1 kpl","variant_price":29.99,"out_stock":false}]},{"product":{"product_id":140795,"product_name":"Royal Canin Maxi Adult","average_prod_review":5,"amount_prod_reviews":9},"variants":[{"bestPriceArticle":false,"variant_id":"140795.11","variant_name":"4 kg","product_variant_name":"Royal Canin Maxi Adult 4 kg","variant_price":21.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140795.8","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Adult 15 kg","variant_price":54.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140795.9","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Adult säästöpakkaus: 2 x 15 kg","variant_price":104.99,"out_stock":false}]},{"product":{"product_id":140794,"product_name":"Royal Canin Maxi Puppy / Junior","average_prod_review":4,"amount_prod_reviews":6},"variants":[{"bestPriceArticle":false,"variant_id":"140794.6","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Puppy / Junior 15 kg","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140794.7","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Puppy / Junior säästöpakkaus: 2 x 15 kg","variant_price":114.99,"out_stock":false}]},{"product":{"product_id":317741,"product_name":"Royal Canin Maxi Ageing 8+","average_prod_review":0,"amount_prod_reviews":0},"variants":[{"bestPriceArticle":false,"variant_id":"317741.6","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Ageing 8+ 15 kg","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"317741.7","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Ageing 8+ säästöpakkaus: 2 x 15 kg","variant_price":112.99,"out_stock":false}]},{"product":{"product_id":140797,"product_name":"Royal Canin Maxi Adult 5+","average_prod_review":5,"amount_prod_reviews":2},"variants":[{"bestPriceArticle":false,"variant_id":"140797.11","variant_name":"4 kg","product_variant_name":"Royal Canin Maxi Adult 5+ 4 kg","variant_price":21.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140797.9","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Adult 5+ 15 kg","variant_price":59.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140797.10","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Adult 5+ säästöpakkaus: 2 x 15 kg","variant_price":117.99,"out_stock":false}]},{"product":{"product_id":214805,"product_name":"Royal Canin Maxi Starter Mother &amp; Babydog","average_prod_review":0,"amount_prod_reviews":0},"variants":[{"bestPriceArticle":false,"variant_id":"214805.3","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Starter Mother &amp; Babydog 15 kg","variant_price":64.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"214805.4","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Starter Mother &amp; Babydog säästöpakkaus: 2 x 15 kg","variant_price":124.99,"out_stock":false}]},{"product":{"product_id":214906,"product_name":"Royal Canin Maxi Puppy / Junior Active","average_prod_review":0,"amount_prod_reviews":0},"variants":[{"bestPriceArticle":false,"variant_id":"214906.7","variant_name":"15 kg","product_variant_name":"Royal Canin Maxi Puppy / Junior Active 15 kg","variant_price":64.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"214906.8","variant_name":"säästöpakkaus: 2 x 15 kg","product_variant_name":"Royal Canin Maxi Puppy / Junior Active säästöpakkaus: 2 x 15 kg","variant_price":114.99,"out_stock":false}]},{"product":{"product_id":140789,"product_name":"Royal Canin Medium Adult","average_prod_review":4,"amount_prod_reviews":15},"variants":[{"bestPriceArticle":false,"variant_id":"140789.12","variant_name":"4 kg","product_variant_name":"Royal Canin Medium Adult 4 kg","variant_price":21.99,"out_stock":false},{"bestPriceArticle":false,"variant_id":"140789.9","variant_name":"15 k…</div>

问题： 1.我的结果带有“ divs”，我只需要“ divs”之间的内容 2.由于“ divs”中的数据看起来像json，因此我也想将其转换为可读格式。（例如csv）

1 个答案:

答案 0 :(得分：0)

您只需要将div的内容加载为json：

...
import json
print(json.loads(scrap.text))

将数据打印为Python对象：

[{'product': {'product_id': 375148, 'product_name': 'Royal Canin Size -bonuspakkaus: 15 + 3 kg!', 'average_prod_review': 5, 'amount_prod_reviews': 55}, 'variants': [{'bestPriceArticle': False, ...

由于数据是嵌套的（每个产品有多个变体），因此可能很难将其转换为平面CSV结构。

相关问题
最新问题