The below code dumps all the followers from TWITTER_USERNAME in a mongodb collection. When it's running, the memory utilization of my MongoDB database slowly increases until it crashes. I have to pause the script and restart mongo every few hours to avoid this. Is there a way for the script to get Mongo to release this memory?
var TwitterUserId = require(PATH + "/models/twitterUserId").Model;
var TwitterConnection = require(PATH + "/models/twitterConnection").Model;
util.promiseWhile = Promise.method(function(condition, action, lastValue) {
if (!condition()) return lastValue;
return action().then(promiseWhile.bind(null, condition, action));
});
var go = function(cursor) {
if(!cursor) cursor = -1;
var pollTwitter = function(cnn) {
return new Promise(function(resolve,reject) {
var parms = {
screen_name: TWITTER_USERNAME,
stringify_ids:true
cursor: cursor
};
cnn.get('followers/ids', parms,function(err,res) {
if(err) return reject(err);
cursor = res.next_cursor_str;
resolve(res.ids);
});
});
};
mongoose.connect(config.db.uri,config.db.options)
.then(function() {
return TwitterConnection.findOne({});
})
.then(function(twitterCnnCOnfig) {
return new twitterObject({
consumer_key: twitterCnnCOnfig.consumerKey
,consumer_secret: twitterCnnCOnfig.consumerSecret
,access_token_key: twitterCnnCOnfig.accessTokenKey
,access_token_secret: twitterCnnCOnfig.accessTokenSecret
});
})
.then(function(twitterCnn) {
util.promiseWhile(
function() {
return cursor;
},
function () {
return new Promise(function(outerResolve,outerReject) {
twitter.parms.cursor = cursor;
pollTwitter(twitterCnn)
.each(function(twitterId) {
return new Promise(function(resolve,reject) {
return TwitterUserId.create({twitterId: twitterId})
.then(resolve,reject);
});
})
.then(function() {
setTimeout(function() {
outerResolve();
},50 * 1000);
})
});
}
);
})
};