NodeJS / Meteor如何通过request.get和future从外部服务器下载多个文件?

时间:2013-12-05 10:45:13

标签: node.js meteor web-scraping future

我在尝试通过外部请求下载许多图片时遇到问题。如下所示,下面是代码

var future = new Future();
var images, nome, blob;

.each(imoveis, function(dadosImovel, numeroImovel){

   images = dadosImovel.images;

   _.each(imagens, function(value, key){

      // the name of the image to a permalink format, this function is working
      nome = Meteor.createPermalinkFromString(value[3]);

      // the link pointing to the image
      blob = Meteor.getImage(value[0]);

      Meteor.saveImage(blob, nome, '.jpeg');
   });
});

// Get a image through a url
Meteor.getImage = function(url){

    var options = {
        url : url,
        encoding : null
    };

    // get raw image
    request.get(options, function(error, result, body) {

        if (error) {
            return console.error(error);
        }

        //  pause until binaries are fully loaded
        future['return'](body);

    });

    return future.wait()
};

// save a image in a server folder
Meteor.saveImage = function(name, blob, ext, encoding) {

    var projectPath = basepath.resolve('.').split('.meteor')[0],
        chroot  = Meteor.chroot || projectPath + 'public', // (process.env['PWD'] +'/public') ;
        path    = chroot + (path ? '/' + path + '/' : '/'),
        name    = Meteor.cleanName(name || 'file'),
        encoding= encoding || 'binary';


    // TODO Add file existance checks, etc...
    fs.writeFile(path + name + ext, blob, encoding, function(err) {
        if (err) {
            console.log(err);
            throw (new Meteor.Error(500, 'Failed to save file.', err));
        } else {
            console.log('The file ' + name + ' (' + encoding + ') was saved to ' + path);
        }
    });

    return true;
}

现在发生了什么:在循环的第一次迭代中,我接收图像就好了,下一次迭代出现问题。

如果有的话,10个图像全部以10个不同的名称保存(并且它们还有10个指向正确图像链接的链接)但是当您可视化图像时,它是列表中的第一个图像,除了所有其他图像9,就像第一张图像加载流星时不要等待下一个二进制/图像代码到达之前保存。

有关如何解决这个问题的想法吗?

2 个答案:

答案 0 :(得分:0)

更改

var Future = new Future();

从outter范围到函数getImage内部修复了图像的问题,但又创建了另一个。

我目前的代码是

        _.each(imoveis, function(dadosImovel, numeroImovel){

            var imagens = dadosImovel.imagens;

            _.each(imagens, function(value, key){

                var nome = Meteor.createPermalinkFromString(value[3]);

                var blob = Meteor.getImage(value[0]);

                Meteor.saveImage(blob, nome, '.jpeg');

            });
        });

现在我有一个无限的外循环,一次又一次地保存来自循环第一项的图像集。

答案 1 :(得分:0)

对于referente(如果将来可能会帮助某人),这是我用来使其工作的代码:

// imovel[0] = link, imovel[1] = width, imovel[2] = height, imovel[3] = title, imovel[4] =               codigo
    var futures = _.map(links, function(imovel) {

    var future = new Future();

    var onComplete = future.resolver();

    var options = {
            url : imovel[0],
            encoding : null
        };

    // get raw image
    request.get(options, function(error, result, body) {

        if (error) {
             return console.error(error);
        }

        var nome = Meteor.createPermalinkFromString(imovel[3]);
        var data = {'codigo' : imovel[4], 'blob' : body, 'nome' : nome};

        onComplete(error, data);
    });

    return future;
});

// wait for all futures to finish
Future.wait(futures);

// and grab the results out.
links    = _.invoke(futures, 'get');