Question

因此，我有一个小小的问题，可以从标记中检索数据，然后将其写入csv。现在，我需要修改xpath，以便它可以读取如下所示的var。从“ var digitalData”中，我将需要来自“ product”的数据。我也会在下面发布我的代码。

var digitalData = {
        "page" : {
            "pageInfo" : {
                "siteCode" : siteCode,
                "siteSection": "", 
                "pageName" : "", 
                "pageURL" : pageURL,
                "pageTrack" : "" 
                },
            "pathIndicator" : { 
                "depth_2" : "mobile",
                "depth_3" : "mobile",
                "depth_4" : "smartphones",
                "depth_5" : "galaxy-s9"
                }
        },
        "user" : {
            "loginStatus" : ""
            },
        "product" : {
            "category" : "", 
            "model_code" : "SM-G960FZPDBTU",
            "model_name" : "SM-G960F/DS",
            "displayName" : "Galaxy S9 Hybrid Sim 64GB",
            "pvi_type_code" : "", 
            "pvi_type_name" : "Mobile", 
            "pvi_subtype_code" : "", 
            "pvi_subtype_name" : "Smartphone"
            }
};

这是我的代码：

import scrapy
import json
import csv
import re

class QuotesSpider(scrapy.Spider):
    name = "quotes1"

    def start_requests(self):
        with open('so_52069753.csv','r') as csvf:
            urlreader = csv.reader(csvf, delimiter=',',quotechar='"')
            for url in urlreader:
                if url[0]=="y":
                    yield scrapy.Request(url[1])
        with open('so_52069753_out.csv', 'w') as csvfile:
            fieldnames = ['Category', 'Type', 'Model', 'SK']
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()

    def parse(self, response):

        regex = re.compile(r'"product"\s*:\s*(.+?\})', re.DOTALL)
        source_json = response.xpath("//script[contains(., 'var digitalData')]/text()").re_first(regex)

        if source_json:
            source_json = re.sub(r'//[^\n]+', "", source_json)
            product = json.loads(source_json)
            product_category = product["pvi_type_name"]
            product_type = product["pvi_subtype_name"]
            product_model = product["displayName"]
            product_name = product["model_name"]

        if source_json:
            source = source_json[0]
            #yield ({'Category': get_values("pvi_subtype_name", source), 'Type': get_values("pathIndicator.depth_5", source), 'SK': get_values("model_name", source)})
            with open('so_52069753_out.csv', 'a') as csvfile:
                fieldnames = ['Category', 'Type', 'Model', 'SK']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                writer.writerow({'Category': product_category, 'Type': product_type, 'Model': product_model, 'SK': product_name})

如何修改我的xpath以读取“ var DigitalData”？预先谢谢你！

Answer 1

由于

，我们无法处理（使用json.loads()）整个digitalData变量

“ siteCode”：siteCode，

和

“ pageURL”：pageURL，

所以我尝试仅获取产品部分：

def parse(self, response):

    regex = re.compile(r'"product"\s*:\s*(.+?\})', re.DOTALL)        
    source_json = response.xpath("//script[contains(., 'var digitalData')]/text()").re_first(regex)

    if source_json:

        # Now we need to remove comments from the JSON:
        # "category" : "", // pathIndicator depth정보 이용하여 설정

        # source_json = re.sub(r'//.+$', "", source_json, re.MULTILINE) # this regex doesn't work for me
        source_json = re.sub(r'//[^\n]+', "", source_json)

        product = json.loads(source_json)

        product_category = product["category"]

如何从我的代码中修改xpath以检索所需的数据

1 个答案: