Question

我正在尝试抓取网站的数据，我正在使用node.js来执行此操作，因为使用javascript来处理遍历网页的DOM似乎是一个好主意。

我遇到的问题是我不想因为显而易见的原因而过快地点击Web服务器，所以我使用递归来延迟时间。但是在大约10,000条记录之后，我达到了node.js的内存限制。目前共有17,000条记录。

var i = 0;                     //  set your counter to 1

function myLoop (n) {           //  create a loop function
   setTimeout(function () {    //  call a 1s setTimeout when the loop is called
     request(url, function(error, response, html){
         if(!error){
     }

     }) ;
      i++;                     //  increment the counter
      if (i < ids.length) {    //  if the counter < ids.length, call the loop function
         myLoop();             //  ..  again which will trigger another
      }                        //  ..  setTimeout()
   }, 2000)
}

myLoop();                      //  start the loop

上面是代码中使用的递归循环。我使用了递归，因为请求是异步的，超时不会停止for循环。

我目前使用的解决方案是运行两次并将输出复制并粘贴在一起。必须有更好的方法来做到这一点。

以下是代码的完整版本（匿名）

var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var json2csv = require('json2csv');

console.log('Starting');

ids = []

count = -1;

fs.readFile('data.csv', 'utf-8', function (err,data) {
  if (err) {
    return console.log(err);
  }
  console.log('### CSV data incoming ###');
  console.log();
  console.log(data);
  console.log('### End of CSV data ###');
  ids = data.split('\n');
});

var jsonFinal = new Array();

var i = 0;                     //  set your counter to 1

function myLoop (n) {           //  create a loop function
   setTimeout(function () {    //  call a 1s setTimeout when the loop is called
     console.log('for loop ' + i);

     url = 'http://www.example.org.uk/location/' + ids[i];

     request(url, function(error, response, html){
         if(!error){
         var $ = cheerio.load(html);

         var example1, example2, example3, example4, example5;
         var json = {uid : "", name : "", example1 : "", example2 : "", example3 : "", example4 : "", example5 : ""};

         //Traverse DOM here

         jsonFinal.push(json);
         count += 1;
         console.log('Pushed JSON array ' + count)
         if (count+1 == ids.length){
           // To write to the system we will use the built in 'fs' library.
           // In this example we will pass 3 parameters to the writeFile function
           // Parameter 1 :  output.json - this is what the created filename will be called
           // Parameter 2 :  JSON.stringify(json, null, 4) - the data to write, here we do an extra step by calling JSON.stringify to make our JSON easier to read
           // Parameter 3 :  callback function - a callback function to let us know the status of our function

           //fields for the csv
           var fields = ['uid', 'name', 'example1', 'example2', 'example3', 'example4', 'example5'];

           fs.writeFile('output.json', JSON.stringify(jsonFinal, null, 4), function(err){

               console.log('File successfully written! - Check your project directory for the output.json file');

           })
           json2csv({ data: jsonFinal, fields: fields }, function(err, csv) {
             if (err) console.log(err);
             fs.writeFile('output.csv', csv, function(err) {
               if (err) throw err;
               console.log('File successfully written! - Check your project directory for the output.csv file');
             });
           });
         }
     }

     }) ;
      i++;                     //  increment the counter
      if (i < ids.length) {    //  if the counter < ids.length, call the loop function
         myLoop();             //  ..  again which will trigger another
      }                        //  ..  setTimeout()
   }, 2000)
}

myLoop();                      //  start the loop

Answer 1

只需将递归调用移动到请求的回调，就不会损坏堆栈

i++;                     //  increment the counter

request(url, function(error, response, html){

    if (i < ids.length) {    //  if the counter < ids.length, call the loop function
        myLoop();             //  ..  again which will trigger another
    }
    // ...
    if(!error){
        // ...
    }
}) ;

Node.js递归和内存限制

1 个答案: