Question

我试图让我的PhantomJS使用while循环抓取多个页面，但是我发现它是异步的，并且它只返回最后一页的结果。我有什么想法可以让它返回每个页面的结果吗？

以下是我的代码：

var  _output = {'cookies':[],'resources':{'js':[]}};
var fs = require('fs');
var file_h = fs.open('file.csv', 'r');
var line = file_h.readLine();

while(line)
{

  var page = require('webpage').create(),
      system = require('system'),
      address;

  console.log(line);

  // open web page
  phantom.cookiesEnabled = true;
  address = line;
  page.open(address, function (status) {
      console.log("status " + status);
      if(status=='success'){

        _output.cookies = phantom.cookies; // record cookies
        _output.cookies.forEach(function(cookie){
          console.log("cookie " + cookie);
        });
      }else{
        console.log('Unable to open provided URL: '+address);
        phantom.exit(-2); // -2: unable to open provided URL
      }
  });

  // to avoid errors detected while parsing the page (eg. Syntax Error, Type Error, etc.)
  // getting into stdout, so breaking the JSON decoding of returned output.
  page.onError = function (msg, trace) {

  }
  line = file_h.readLine(); 
}
file_h.close();

Answer 1

将Promise用于异步任务。

var page = require('webpage').create(),
    system = require('system');
new Promise(function(resolve, reject) {
  var  _output = {'cookies':[],'resources':{'js':[]}};
  var fs = require('fs');
  var file_h = fs.open('file.csv', 'r');
  var line = file_h.readLine();
  var promises = [];
  while(line)
  {
    promises.push(process_line(line));
  }
  file_h.close();
  Promise.all(promises).then(resolve, reject);
  function process_line(line) {
    return Promise((resolve, reject) => {
      var address;

      console.log(line);

      // open web page
      phantom.cookiesEnabled = true;
      address = line;
      page.open(address, function (status) {
        console.log("status " + status);
        if(status=='success'){

          _output.cookies = phantom.cookies; // record cookies
          _output.cookies.forEach(function(cookie){
            console.log("cookie " + cookie);
          });
          resolve(status)
        }else{
          reject('Unable to open provided URL: '+address);
        }
      });

      // to avoid errors detected while parsing the page (eg. Syntax Error, Type Error, etc.)
      // getting into stdout, so breaking the JSON decoding of returned output.
      page.onError = function (msg, trace) {
        reject(msg);
      }
      line = file_h.readLine(); 
    });
  }
})
  .then((statuses) => {
    // use the results
  })
  .catch((err) => {
    // report an error
    phantom.exit(-2); // -2: unable to open provided URL
  });

PhantomJS抓取多个页面

1 个答案: