Question

此问题与 node.js 中的抓取工具有关。在他抓取网址时会给出start_url，并且＆＃34;推送＆＃34;他们到。 json -file（output.json）。目前，他仅使用start_url运行请求功能，并将收集的网址保存在output.json中。我希望他使用保存的网址，将start_url替换为第一个收集的网址，然后再次收集链接......依此类推......

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');

var start_url = ["http://stackoverflow.com/"]

var req = function(url){
    request(url, function(error, response, html){
        var $ = cheerio.load(html);

        var data = [];

        $("a").each(function() {
            var link = $(this);
                var exurls = {exurl: new Array(link.attr("href"))}

                data.push(exurls);

                // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
                // save to "output.json" from time to time, so you can stop it anytime
        });

        fs.writeFile("output.json", JSON.stringify(data, null, 4), function(err){
            if(err){
                console.log(err);
            } else {
                console.log("File successfully written!");
            }
        });
    });
}
for (var i = 0; i < start_url.length; i++){
    req(start_url[i]);
}

Answer 1

所以你可以做的是递归地调用函数。以下示例应该有效：

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');

var start_url = ["http://stackoverflow.com/"]

var req = function(url){
  var count = 0;
  
  request(url, function(error, response, html){
    var $ = cheerio.load(html);

    $("a").each(function() {
      var link = $(this);
      var exurls = {exurl: new Array(link.attr("href"))}

      start_url.push(exurls);

      // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
      // save to "output.json" from time to time, so you can stop it anytime
    });

    try {
      fs.writeFileSync("output.json");
      console.log("File successfully written!");
     }catch(err){
       console.log(err);
     }
      
      ++count;
      
      if(start_url.length > count) {
        req(start_url[count]);
      }
  });
}

return req(start_url[0]);

这个问题是你每次都完全重写文件。如果这种情况持续一段时间，你将会耗尽内存。另一种选择是创建写入流

var fs = require('fs');
    var request = require('request');
    var cheerio = require('cheerio');

    var start_url = ["http://stackoverflow.com/"]
    
    var wstream = fs.createWriteStream("output.json");

    var req = function(url){
      
      request(url, function(error, response, html){
        var $ = cheerio.load(html);

        $("a").each(function() {
          var link = $(this);
          var exurls = {exurl: new Array(link.attr("href"))}

          start_url.push(exurls);

          // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
          // save to "output.json" from time to time, so you can stop it anytime
          wstream.write('"'+ exurls + '",');
        });
          
        start_url.shift();
        if(start_url.length > 0) {
          return req(start_url[0]);
        }
          
          wstream.end();
      });
    }

    req(start_url[0]);

编辑：切换到基本队列，以解决内存问题

迭代node.js请求函数

1 个答案: