Question

所以我正在构建一个空间地图并将关系转换为mongodb。在这个场景中，我有地区，星座和太阳系。其中1个区域可以有许多星座，每个星座可以有许多太阳系。我有一个api允许抓取其中一些，但它需要每个项目api调用。提出我正在制作的问题~6000 api电话。

继承第三方的api标准：

一般费率限制：每秒150个请求
爆裂尺寸：400
并发连接：20

这是我的db模型：

地区模型：

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var regionSchema = new Schema({
    _id: Number,
    name: String,
    description: String,
    href: String,
    constellations: [{
        type: Schema.ObjectId,
        ref: 'Constellation'
    }]//Reference
});

//Expose (export) the model
module.exports = mongoose.model('Region', regionSchema);

星座模型：

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var constellationSchema = new Schema({
    _id: Number,
    name: String,
    href: String,
    solarSystems: [{
        type: Schema.ObjectId,
        ref: 'SolarSystem'
    }]
});

//Expose (export) the model
module.exports = mongoose.model('Constellation', constellationSchema);

太阳系模型：

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var solarSystemSchema = new Schema({
    _id: Number,
    name: String,
    imgUrl: String
});

//Expose (export) the model
module.exports = mongoose.model('SolarSystem', solarSystemSchema);

我也试图以正确的顺序保存它们，以便为关系填充引用。

这是我的代码：

function getAllRegions(req, res){
   getAllRegionsHrefs().then(function (hrefs){
       var newRegions = [];
       for(var href in hrefs){
           var options = {
               uri: hrefs[href],
               json: true
           };
           RequestPromise(options).then(function (responseItem){
               var constellationObjects = [];
               for(var item in responseItem.constellations){
                   var newConstellation = constellationModel({
                       _id: responseItem.constellations[item].id,
                       href: 'https://getspaceInfoHere.com/constellations/'+responseItem.constellations[item].id+'/'
                   });
                   newConstellation.save();
                   constellationObjects.push(newConstellation);
               }
               var newRegion = regionModel({
                   _id: responseItem.id,
                   name: responseItem.name,
                   description: responseItem.description,
                   href: 'https://getspaceInfoHere.com/regions/'+responseItem.id+'/',
                   constellations: constellationObjects
               });
               newRegion.save();
               newRegions.push(newRegion);
               console.log(newRegion);
           });
       }
   });
}

function getAllRegionsHrefs(){
    var options = {
        uri: universeInfoEndpoint,
        json: true
    };
   return RequestPromise(options).then(function (responseItems){
        var regionHrefs = [];
        for(var item in responseItems.items){
                regionHrefs.push(responseItems.items[item].href);
        }
        return regionHrefs;
    });
}

现在我甚至没有尝试获取提供系统信息的详细星座信息（然后提供了一个href来获取详细的系统信息）并且我正在进入我的最大值。限制这个的最佳方法是什么，以便我可以保持在参数范围内？

更新

function getAllRegions(req, res){
    getAllRegionsHrefs().then(function (hrefs){
        var chunks = _.chunk(hrefs, 25);
        return Promise.map(chunks, function(chunk) {
            return Promise.map(chunk, getRegion).then(function (getRegionResults){
                for(var item in getRegionResults) {
                    Promise.map(getRegionResults[item].constellations, getConstellationInfo).then(function (constellationInfo) {
                        var chunks = _.chunk(constellationInfo, 150);
                        return Promise.map(chunks, function (chunk) {
                            return Promise.map(chunk, getSystem).delay(20000);
                        })
                    }).delay(20000);
                }
            }).delay(200000);
        });
    });
}

function getSystem(systems){
    for(var updateSystem in systems){
        var options = {
            uri: systems[updateSystem].href,
            json: true
        };
         RequestPromise(options).then(function (responseItem){
             //Grab the system in the db and update it with its info
            systemModel.findOne({ _id: systems[updateSystem]._id }, function (err, doc){
                doc.name = responseItem.name;
                doc.save();
            });

        });
    }
}

function getConstellationInfo(constellation) {
    var options = {
        uri: constellation.href,
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var arrayOfSystems = [];
        for(var system in responseItem.systems){
            var newSystem = new systemModel({
                _id: responseItem.systems[system].id,
                href: responseItem.systems[system].href
            });
            newSystem.save();
            arrayOfSystems.push(newSystem);
        }
        //find the constellation and update it with its info
        constellationModel.findOne({ _id: constellation._id }, function (err, doc){
            doc.name = responseItem.name;
            doc.solarSystems = arrayOfSystems;
            doc.save();
        });
        return arrayOfSystems;
    });
}


function getRegion(href) {
    var options = {
        uri: href,
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var constellationObjects = [];
        for(var item in responseItem.constellations){
            var newConstellation = constellationModel({
                _id: responseItem.constellations[item].id,
                href: eveConstellationCrestEndpoint + responseItem.constellations[item].id+'/'
            });
            newConstellation.save();
            constellationObjects.push(newConstellation);
        }
        var newRegion = regionModel({
            _id: responseItem.id,
            name: responseItem.name,
            description: responseItem.description,
            href: universeEndpoint + responseItem.id+'/',
            constellations: constellationObjects
        });
        newRegion.save();
        return newRegion;
    });
}

function getAllRegionsHrefs(){
    var options = {
        uri: universeEndpoint,
        json: true
    };
   return RequestPromise(options).then(function (responseItems){
        var regionHrefs = [];
        for(var item in responseItems.items){
                regionHrefs.push(responseItems.items[item].href);
        }
        return regionHrefs;
    });
}

现在这适用于整个链（获取区域，星座和系统信息），但超时工作正在开始拒绝系统级别的连接。有什么建议？

Answer 1

你可以通过将你的href分成20组并在每个块之后设置延迟来实现这一点，你可能想要使用这些参数：

使用lodash＆＃39; _.chunk和Bluebird＆＃39; Promise.delay与Promise.map：

function getAllRegions(req, res){
   getAllRegionsHrefs().then(function (hrefs){
       var chunks = _.chunk(hrefs, 20);
       return Promise.map(chunks, function(chunk) {
         // tune the delay to what you need it to be
         // it will wait the delay (in ms) before starting the next chunk of requests
         return Promise.map(chunk, getRegion).delay(150000);
       });
   });
}

function getRegion(href) {
    var options = {
        uri: hrefs[href],
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var constellationObjects = [];
        for(var item in responseItem.constellations){
            var newConstellation = constellationModel({
                _id: responseItem.constellations[item].id,
                href: 'https://getspaceInfoHere.com/constellations/'+responseItem.constellations[item].id+'/'
            });
            newConstellation.save();
            constellationObjects.push(newConstellation);
        }
        var newRegion = regionModel({
            _id: responseItem.id,
            name: responseItem.name,
            description: responseItem.description,
            href: 'https://getspaceInfoHere.com/regions/'+responseItem.id+'/',
            constellations: constellationObjects
        });
        newRegion.save();
        console.log(newRegion);
        return newRegion;
    });
}

使用Node限制对第3部分Apis的请求

更新

1 个答案: