Question

我的代码有问题。我需要通过API捕获大量数据。首先，我遇到内存问题，现在我看到：连接ETIMEDOUT错误

我的代码：

var fetch = require('node-fetch');
const async = require('async');
const request = require('request');
var jsonfile = require('jsonfile');

var file = '/temp/data.json'
let urls = [];

for (var i = 1; i <= 608469; i++) {
  let url = "https://api.demo.com/v2.1/people?user_key=auth_key&page="+i+"&sort_order=created_at%20DESC";
  urls.push(url);
}

function httpGet(url, callback) {
  const options = {
    url :  url,
    json : true
  };
  request(options,
    function(err, res, body) {
      callback(err, body);
    }
  );
}

async.map(urls, httpGet, function (err, res){
  if (err) return console.log(err);
  console.log(typeof(res));
  jsonfile.writeFileSync(file, res);
});

Answer 1

只需用map替换异步eachLimit方法即可。这里的关键是limit。它仍将为每个链接运行http请求，但不超过limit个并行线程。这是一个文档：https://caolan.github.io/async/docs.html#eachLimit

很多HTTP请求并保存为json：connect ETIMEDOUT

1 个答案: