如何使用BeautifulSoup从网站获取var脚本中的json数据？

时间：2019-07-06 09:42:41

标签： python json python-3.x web-scraping beautifulsoup

我正在尝试获取在检查页面源中的特定URL时看到的JSON数据。这是网址：https://www.uniqlo.com/de/de/product/ut-bedrucktes-t-shirt-kaws-summer-421771.html

我的目标是为“ ats”：“ xxx”获得“定价”和“可用性”

这是源代码的一部分：

<template>
  <section class="pre-footer">
    <div class="wrapper">
      <div class="watermark">
        <SVGIcon class="watermark-left" icon="LearnLongShort" />
        <SVGIcon class="watermark-right" icon="FreeEducation" />
      </div>
      <div class="content">
        <div class="content-left content-wrapper">
          <div>
            <h3 class="title">Evidence base</h3>
            <div class="body">
              <p>The Academy of Medical Cannabis Evidence Base is an advanced referencing tool for CBMP research. Containing the history of research to date and all emerging findings, this cutting-edge engine underpins our learning content.</p>
            </div>
            <WhiteButton text="View database" />
          </div>
        </div>
        <div class="content-right content-wrapper">
          <div>
            <h3 class="title">White Papers</h3>
            <div class="body">
              <p>Alongside our learning platform, The Academy publishes a series of authoritative papers discussing the core topics and themes related to CBMPs. Register to view and download the archive.</p>
            </div>
            <WhiteButton text="Archive" />
          </div>
        </div>
      </div>
    </div>
  </section>
</template>

我已经用json.loads（）尝试过了，但这会给我这个错误：

“从无引发JSONDecodeError（”期望值“，s，err.value） json.decoder.JSONDecodeError：期望值：第1行第1列（字符0）“

这是到目前为止我的代码片段的样子：

</div>
<div class="product-detail">

<script>
var pdpVariationsJSON = 

{"color-COL08|size-SMA002":{"id":"421771COL08SMA002000","attributes":{"color":"Dunkelgrau","size":"XS"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"203","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA003":{"id":"421771COL08SMA003000","attributes":{"color":"Dunkelgrau","size":"S"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"106","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA004":{"id":"421771COL08SMA004000","attributes":{"color":"Dunkelgrau","size":"M"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"218","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA005":{"id":"421771COL08SMA005000","attributes":{"color":"Dunkelgrau","size":"L"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"7","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA006":{"id":"421771COL08SMA006000","attributes":{"color":"Dunkelgrau","size":"XL"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"83","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA007":{"id":"421771COL08SMA007000","attributes":{"color":"Dunkelgrau","size":"XXL"},"availability":{"status":"NOT_AVAILABLE","statusQuantity":"0","inStock":false,"ats":"0","inStockDate":"","availableForSale":false,"purchaseLevel":"","levels":{"IN_STOCK":0,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":1},"isAvailable":false,"inStockMsg":"0 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL08|size-SMA008":{"id":"421771COL08SMA008000","attributes":{"color":"Dunkelgrau","size":"3XL"},"availability":{"status":"NOT_AVAILABLE","statusQuantity":"0","inStock":false,"ats":"0","inStockDate":"","availableForSale":false,"purchaseLevel":"","levels":{"IN_STOCK":0,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":1},"isAvailable":false,"inStockMsg":"0 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA002":{"id":"421771COL68SMA002000","attributes":{"color":"Blau","size":"XS"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"159","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA003":{"id":"421771COL68SMA003000","attributes":{"color":"Blau","size":"S"},"availability":{"status":"NOT_AVAILABLE","statusQuantity":"0","inStock":false,"ats":"0","inStockDate":"","availableForSale":false,"purchaseLevel":"","levels":{"IN_STOCK":0,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":1},"isAvailable":false,"inStockMsg":"0 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA004":{"id":"421771COL68SMA004000","attributes":{"color":"Blau","size":"M"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"63","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA005":{"id":"421771COL68SMA005000","attributes":{"color":"Blau","size":"L"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"9","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA006":{"id":"421771COL68SMA006000","attributes":{"color":"Blau","size":"XL"},"availability":{"status":"IN_STOCK","statusQuantity":"0","inStock":true,"ats":"136","inStockDate":"","availableForSale":true,"purchaseLevel":"","levels":{"IN_STOCK":1,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":0},"isAvailable":true,"inStockMsg":"1 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA007":{"id":"421771COL68SMA007000","attributes":{"color":"Blau","size":"XXL"},"availability":{"status":"NOT_AVAILABLE","statusQuantity":"0","inStock":false,"ats":"0","inStockDate":"","availableForSale":false,"purchaseLevel":"","levels":{"IN_STOCK":0,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":1},"isAvailable":false,"inStockMsg":"0 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}},"color-COL68|size-SMA008":{"id":"421771COL68SMA008000","attributes":{"color":"Blau","size":"3XL"},"availability":{"status":"NOT_AVAILABLE","statusQuantity":"0","inStock":false,"ats":"0","inStockDate":"","availableForSale":false,"purchaseLevel":"","levels":{"IN_STOCK":0,"PREORDER":0,"BACKORDER":0,"NOT_AVAILABLE":1},"isAvailable":false,"inStockMsg":"0 Produkt(e) verfügbar","preOrderMsg":"0 Produkt(e) sind zur Vorbestellung verfügbar.","backOrderMsg":"Back Order 0 item(s)"},"pricing":{"showStandardPrice":false,"isPromoPrice":false,"standard":14.9,"formattedStandard":"14,90 €","sale":14.9,"formattedSale":"14,90 €","salePriceMoney":{},"standardPriceMoney":{},"quantities":[{"unit":"","value":0}]}}}
;
</script>

<div id="product-content">

这是我现在在打印时得到的结果：

import requests

from bs4 import BeautifulSoup

import json

url = "https://www.uniqlo.com/de/de/product/ut-bedrucktes-t-shirt-kaws-summer-421771.html"

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}

response = requests.get(url, headers=headers)

soup = BeautifulSoup(response.text, "lxml")



div = soup.find("div", class_="product-detail")

scr = str(div.find("script"))

print(scr)

1 个答案:

答案 0 :(得分：0)

您可以使用re从<script>标记中提取重要部分，然后使用json.loads对其进行解析：

import re
import json
import requests
from bs4 import BeautifulSoup

url = "https://www.uniqlo.com/de/de/product/ut-bedrucktes-t-shirt-kaws-summer-421771.html"

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}

response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, "lxml")

div = soup.find("div", class_="product-detail")
scr = str(div.find("script"))

data = json.loads(re.findall(r'pdpVariationsJSON[^{]+(.*);', str(scr), flags=re.DOTALL)[0])

print(json.dumps(data, indent=4))

打印：

{
    "color-COL08|size-SMA002": {
        "id": "421771COL08SMA002000",
        "attributes": {
            "color": "Dunkelgrau",
            "size": "XS"
        },
        "availability": {
            "status": "IN_STOCK",
            "statusQuantity": "0",
            "inStock": true,
            "ats": "203",
            "inStockDate": "",
            "availableForSale": true,
            "purchaseLevel": "",
            "levels": {
                "IN_STOCK": 1,
                "PREORDER": 0,
                "BACKORDER": 0,
                "NOT_AVAILABLE": 0
            },
            "isAvailable": true,
            "inStockMsg": "1 Produkt(e) verf\u00fcgbar",
            "preOrderMsg": "0 Produkt(e) sind zur Vorbestellung verf\u00fcgbar.",
            "backOrderMsg": "Back Order 0 item(s)"
        },
        "pricing": {
            "showStandardPrice": false,
            "isPromoPrice": false,
            "standard": 14.9,
            "formattedStandard": "14,90 \u20ac",
            "sale": 14.9,
            "formattedSale": "14,90 \u20ac",
            "salePriceMoney": {},
            "standardPriceMoney": {},
            "quantities": [
                {
                    "unit": "",
                    "value": 0
                }
            ]
        }
    },
    "color-COL08|size-SMA003": {
        "id": "421771COL08SMA003000",
        "attributes": {
            "color": "Dunkelgrau",
            "size": "S"
        },
        "availability": {
            "status": "IN_STOCK",
            "statusQuantity": "0",
            "inStock": true,
            "ats": "106",
            "inStockDate": "",
            "availableForSale": true,
            "purchaseLevel": "",
            "levels": {
                "IN_STOCK": 1,
                "PREORDER": 0,
                "BACKORDER": 0,
                "NOT_AVAILABLE": 0
            },
            "isAvailable": true,
            "inStockMsg": "1 Produkt(e) verf\u00fcgbar",
            "preOrderMsg": "0 Produkt(e) sind zur Vorbestellung verf\u00fcgbar.",
            "backOrderMsg": "Back Order 0 item(s)"
        },
        "pricing": {
            "showStandardPrice": false,
            "isPromoPrice": false,
            "standard": 14.9,
            "formattedStandard": "14,90 \u20ac",
            "sale": 14.9,
            "formattedSale": "14,90 \u20ac",
            "salePriceMoney": {},
            "standardPriceMoney": {},
            "quantities": [
                {
                    "unit": "",
                    "value": 0
                }
            ]
        }
    },

... and so on.
相关问题
最新问题