nodejs lodash map synchronous requests

Question

This is the current situation, i have list of payload JSON entry (it could be anything , if you want to try) in the enteries map, i iterate through the map using lodash , and post the requests, in async manner as below

_.map(enteries, async (entry) => {
let response_1 = await postToAPI('apiurl', entry);
}

Problem :- I tried, with 100 enteries , api can't accept more than 1 request in 1/10 of a second (its a limitation, and can't change :( ), therefore as a workaround, what i thought is two options 1. i would need to introduce a pause of 100ms , between each requests 2. process them sequentially.

Can someone please assist or advise, how to achieve option 1 or 2 . Any help is greatly appreciated

Updated standalone code :-

var _ = require('lodash');

// function for the delay
var delay = function (delayMs) {
    return new Promise((resolve) => {
        setTimeout(resolve, delayMs);
    });
};

// Create some sample entries
var entries = _.map(_.range(0,20), (i) => {
    return {id: i, name: 'entry #' + i}
});

// Create sample post request returning a promise
async function postToAPI(apiurl, entry) {
    console.log('postToAPI - '+apiurl+ ' request sent = '+ JSON.stringify(entry));
    return new Promise((resolve) => {
        resolve(entry);
    }); 
};

async function postEntries(entries, allDoneCallback) {
    var responses = [];
    while(entries.length > 0) {
        var entry = entries.shift();
        try
        {
            var response = await postToAPI('apiurl', entry);
            console.log('postToAPI - reply '+ JSON.stringify(entry));
            responses.push(response);
        }
        catch (err) {
            console.log("An error occurred: " + err);
        }
        await delay(300);
    };

    console.log('postEntries (after posting all the enteries) --> '+responses.length);
    allDoneCallback(responses);
};

postEntries(entries, (res) => { console.log('All done ', res.length);});

Answer 1

I'm making the assumption that postToAPI returns a promise. Posting the entries sequentially will only help if the API takes more than 100 ms to process an an entry. Many APIs will be much quicker than this.

I've revised this method again! Here goes, and I think this really does do what you want!!

var delay = function (delayMs) {
     return new Promise((resolve,reject) => {
          setTimeout(resolve, delayMs);
     });
};


async function postEntries(entries, allDoneCallback) {
    var responses = [];
    while(entries.length > 0) {
        var entry = entries.shift();
        try
        {
            var response = await postToAPI('apiurl', entry);
            responses.push(response);
        }
        catch (err) {
            console.log("An error occurred: " + err);
        }
        await delay(100);
    };

    console.log(responses);
    allDoneCallback(responses);
};

// Create some sample entries
var entries = _.map(_.range(0,10), (i) => {
   return {id: i, name: 'entry #' + i}
});

postEntries(entries, (res) => { console.log('All done', res);});

You could modify the response array to include any errors that occur, you'll note I wrapped the postToAPI in a try/catch block in case any call chokes.

Suggested structure for postToAPI:

var postToAPI = function(apiURL, entry) {
    console.log(new Date().getTime(), ' - postToAPI');
        return postHTTP(apiURL, entry).then ((result) => {
            return saveToDB(result);
    });
}

Once we have postToAPI in this format, the await will work correctly. I've assumed that postHTTP and saveToDB return promises.