Question

我正在尝试使用nodejs将google staorage中的Avro文件上传到Google大查询

var gcloud = require('gcloud')({
  keyFilename: '../config/keyfile.json',
  projectId: 'my-project-id'
});

var request = require('request');

var bigquery = gcloud.bigquery();


var dataset = bigquery.dataset('my-data-set');
var table = dataset.table('my-table');



  var metadata = {

   sourceFormat: 'AVRO'
  };

   var url = 'https://storage.googleapis.com/' + 'my-bucket' + '/' + 'yob1900.avro';

    request.get(url)
      .pipe(table.createWriteStream(metadata))
      .on('complete', function (job) {
        // Wait up to 60 seconds for job to complete
        pollJobUntilDone(job, 60000, 0, function (err, metadata) {
          if (err) {

            return callback(err);
          }
          console.log('job completed', metadata);

        });
      });

function pollJobUntilDone (job, timeout, timeWaited, callback) {
  job.getMetadata(function (err, metadata) {
    if (err) {
      return callback(err);
    }
    if (timeWaited > timeout) {
      return callback(new Error('Timed out waiting for job to complete'));
    }
    if (metadata.status && (metadata.status.state === 'RUNNING' ||
      metadata.status.state === 'PENDING')) {
      setTimeout(function () {
        console.log('working...');
        pollJobUntilDone(job, timeout, timeWaited + 5000, callback);
      }, 5000);
    } else {
      callback(null, metadata);
    }
  });
}

我收到此错误

'The Apache Avro library failed to parse file file-00000000.'

（'yob1900'文件是从bigquery文档https://cloud.google.com/bigquery/loading-data#loading_json_files下载的）当我尝试通过webUI这样做时，我成功了。有什么建议吗？

Answer 1

Avro格式并没有被gcloud嘲笑。问题已得到修复＆＃34; gcloud＆＃34;：＆＃34; ^ 0.36.0＆＃34;

Apache Avro库无法解析文件nodejs

1 个答案: