Question

这是收集和导出异步数据的模块：scraper.js

const express = require('express')
const cheerio = require('cheerio')
const request = require("tinyreq")
const fs = require('fs')
const _ = require('lodash')
const uuid = require('uuid/v4')
const async = require('async')


const mental_models = {
    url: 'https://www.farnamstreetblog.com/mental-models/',
    data: {}
}
const decision_making = {
    url: 'https://www.farnamstreetblog.com/smart-decisions/',
    data: {}
}
const cognitive_bias = {
    url: 'https://betterhumans.coach.me/cognitive-bias-cheat-sheet-55a472476b18',
    data: {}
}
const DATA_URLS = [mental_models, decision_making, cognitive_bias]

const filterScrape = async (source, params) => {

    let filtered_data = {
        topics: [],
        content: [],
        additional_content: []
    }

    let response = await scrape(source)

    try {
        let $ = cheerio.load(response)

        params.forEach((elem) => {
            let headers = ['h1', 'h2', 'h3']
            if ($(elem) && headers.includes(elem)) {

                let topic = {}
                let content = {}
                let id = uuid()
                topic.id = id
                topic.text = $(elem).text()

                if ($(elem).closest('p')) {
                    content.text = $(elem).closest('p').text()
                    content.id = id
                }
                filtered_data.topics.push(topic)
                filtered_data.content.push(content)
            } else if ($(elem) && !headers.includes(elem)) {

                let content = {}
                let id = uuid()
                content.text = $(elem).text()
                content.id = id
                filtered_data.additional_content.push(content)

            } else {

            }
        })

    }
    catch (err) {
        console.log(err)
    }

    return filtered_data

}

const scrape = (source) => {
    return new Promise((resolve, reject) => {
        request(source.url, function (err, body) {
            if (err) {
                reject(err)
                return
            }
            resolve(body)
        })
    })
}

const DATA = _.map(DATA_URLS, async (source) => {

    let params = ['h1', 'h2', 'h3', 'p']
    let new_data = await filterScrape(source, params)

    try {
        source.data = new_data
     }
     catch (err) {
         console.log(err)
     }

})
module.exports = DATA

这是导入数据的模块：neural.js

const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const DATA = require('./scraper')

console.log(DATA)

显然没有太多进展，我删除了代码，因为变量无法解析。记录时，它会记录一个承诺，但承诺无法解决。但是，在导入的模块中，将记录promise，然后解析。是什么赋予了？我应该导入一个解析数据的函数吗？

Answer 1

当然最好导入该功能，但不会改变代码中的问题：

const DATA = _.map(DATA_URLS, async (source) => {

Lodash不支持异步迭代 - 所以你需要有一些其他方法，一个是使用最新的nodejs版本（10.x）并使用异步迭代 - 但这不会使用全部功能异步代码。

您还可以使用scramjet - 我公司支持的框架。上面的代码将采用以下形式：

const {DataStream} = require("scramjet");
const DATA_URLS = [mental_models, decision_making, cognitive_bias];

module.exports = async () => DataStream.fromArray(DATA_URLS)
    .setOptions({maxParallel: 2})   // if you need to limit that at all.
    .map(async ({url}) => {
        let params = ['h1', 'h2', 'h3', 'p']
        let data = await filterScrape(source, params);

        return { url, data };
    })
    .toArray();

另一个文件采用以下形式：

const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const scraper = require('./scraper')

(async (){
    const DATA = await scraper();
    console.log(DATA);   // or do whatever else you were expecting...
})();

从Node中的另一个模块导入的异步数据无法解析，我该如何解决？

1 个答案: