Question

我在Node中玩Cheerio。我有一个刮刀进入文章列表，抓取所有文章URL，然后转到每篇文章并抓取标题和网址。一切正常，除非我试图将结果追加到我的Mongodb，我得到了未定义。

我假设它在定义值之前尝试upsert ...但即使使用Request-Response我也无法使其工作。任何帮助将不胜感激！由于代码不是太长，我只需粘贴整个内容，这样就可以更容易地看到我正在尝试做什么。同样，主要问题是让upsertArticle实际上插入变量。

const request = require('request');
const cheerio = require('cheerio');
const rp = require('request-promise');
const mongoose = require('mongoose');
const Article = require('./models/article');

var urls = [];
//get the list of articles to scrape
rp('https://www.somesite.com/', function(error, response, html) {
    if (!error && response.statusCode == 200) {
        var $ = cheerio.load(html);
        $('.c-entry-box--compact__title').each(function(i, element) {
            var a = $(this);
            urls.push(a.children().attr('href'));
        });     }  })
    //scrape over each article individually
    .then(function(getStuff) {
        var arrayLength = urls.length;
        //get the list of articles to scrape and upsert each one
        for (var i = 0; i < arrayLength; i++) {
            const result = rp(urls[i], function(error, response, html) {
                if (!error && response.statusCode == 200) {
                    var $ = cheerio.load(html);
                    var parsedResults = [];
                    $('.l-main-content').each(function(n, element) {
                        var a = $(this);
                        var title = a.find('.c-page-title').text();
                        var url = response.request.uri.href;
                        //I also tried upserting the variables right here, that didn't work
                        return { title, url };
                    });
                } else {console.log(error);}
            }).then(function(upsertStuff) {
                    //also tried returning and upserting stuff here... but nothing gets upserted
                    upsertArticle({
                        title: result.title,
                        source: result.url,
                        dateCrawled: new Date()
                    });
                    console.log('Upserted ' + result.title);
                }).catch(function(err) {console.log(err);   });     }
    })  .catch(function(err) {console.log(err); });

function upsertArticle(userObj) {
    const DB_URL = 'mongodb://localhost/articles';
    if (mongoose.connection.readyState == 0) {
        mongoose.connect(DB_URL, {
            useMongoClient: true
        });
    }
    let conditions = {
        title: userObj.title
    };
    let options = {
        upsert: true,
        new: true,
        setDefaultsOnInsert: true
    };
    Article.findOneAndUpdate(conditions, userObj, options, (err, result) => {
        if (err) throw err;
    });
}

Answer 1

我对提供的代码做了一些更改。也就是说，我使用promises而不是回调来保持逻辑的一致性，并确保一切都在运行时。

对于for循环，我将upsertArticle({...})移回each函数内部，以便title和url定义时它运行。

最后，我使用Bluebird＆＃39; Promise.all（request-promise已经依赖Bluebird）来表示所有链接都已被插入。此更改是可选的，但我认为在完成所有操作后获得反馈非常有用：

尝试一下：

const request = require('request');
const cheerio = require('cheerio');
const rp = require('request-promise');
const mongoose = require('mongoose');
const Article = require('./models/article');
const Promise = require("bluebird");

var urls = [];

rp({uri: 'https://www.somesite.com',  resolveWithFullResponse: true}).then(function(response) {

    if(response.statusCode != 200) throw "Response: " + response.statusCode;

    var html = response.body;

    var $ = cheerio.load(html);

    $('.c-entry-box--compact__title').each(function(i, element) {
        var a = $(this);
        urls.push(a.children().attr('href'));
    });

}).then(function(getStuff) {

    var arrayLength = urls.length;
    var promiseArray = [];

    for(var i = 0; i < arrayLength; i++) {

        const p = rp({uri: urls[i],  resolveWithFullResponse: true}).then(function(response) {

            if(response.statusCode != 200) throw "Response: " + response.statusCode;

            var html = response.body;

            var $ = cheerio.load(html);
            var parsedResults = [];

            $('.l-main-content').each(function(n, element) {

                var a = $(this);
                var title = a.find('.c-page-title').text();
                var url = response.request.uri.href;

                upsertArticle({
                    title: title,
                    source: url,
                    dateCrawled: new Date()
                });

                console.log('Upserted ' + title);
            });

        });

        promiseArray.push(p);
    }

    return Promise.all(promiseArray);

}).then(function() {
    console.log("Done upserting!");
})
.catch(function(err) {
    console.log(err); 
});

function upsertArticle(userObj) {
    const DB_URL = 'mongodb://localhost/articles';
    if (mongoose.connection.readyState == 0) {
        mongoose.connect(DB_URL, {
            useMongoClient: true
        });
    }
    let conditions = {
        title: userObj.title
    };
    let options = {
        upsert: true,
        new: true,
        setDefaultsOnInsert: true
    };
    Article.findOneAndUpdate(conditions, userObj, options, (err, result) => {
        if (err) throw err;
    });
}

我无法在不知道https://www.somesite.com的真实价值的情况下测试代码，因此请告诉我代码是否会给您带来任何新错误。

将节点请求 - 响应响应上载到MongoDB

1 个答案: