Question

使用appscript将大型（16GB）Biq查询表导出到GCS时，无法进行压缩工作。我将压缩类型设置为GZIP并将目标格式设置为NEWLINE_DELIMITED_JSON，但它不压缩文件，只输出116个文件？

我已尝试extract.compression =和extract.setcompression =，但它没有任何区别。我该如何解决这个问题？

function extractBigQueryToCloudStorage(compressionType,csFileUri, datasetId, tableId,projectId) { 

  //
  var compressionType='GZIP';
  var csFileUri='gs://xxxxxxxxxx/bq_extracts/xxxxxxxxxx.*.JSON';
  var datasetId='xxxxxxxxxx';
  var tableId='xxxxxxxxxx';
  var projectId='xxxxxxxxxx';
  var bqTable = checkBigQueryTable(projectId, datasetId, tableId);

  var fnStart = new Date();

    try {
        var tableReference = BigQuery.newTableReference();
        tableReference.setProjectId(projectId);
        tableReference.setDatasetId(datasetId);
        tableReference.setTableId(tableId);

        var extract = BigQuery.newJobConfigurationExtract()
        extract.setDestinationFormat('NEWLINE_DELIMITED_JSON');
        extract.compression=(compressionType);
        extract.setDestinationUris([csFileUri]);
        extract.setSourceTable(tableReference);

        var configuration = BigQuery.newJobConfiguration();
        configuration.setExtract(extract);
        var newJob = BigQuery.newJob();
        newJob.setConfiguration(configuration);

        var job = BigQuery.Jobs.insert(newJob, projectId);
        var jobId = job.getJobReference().getJobId();    
        var status = job.getStatus();

        while (status.getState() != 'DONE'){
            Logger.log(status.getState());
            if(status.getState() == 'PENDING'){
                Utilities.sleep(100);
            }      
            if (status.getErrorResult() == true){     
                Logger.log('BigQuery file upload error: %s', status.getErrors());
            }       
            status = BigQuery.Jobs.get(projectId, jobId).getStatus();
        }  
    } catch(err) { 
        Logger.log('BigQuery file upload error: %s', err);  
        return err;   
    } 
    var fnEnd = new Date();
    Logger.log(status.getState());
    Logger.log('Function loadCloudStorageFileToBigQuery elapsed time: %sms', fnEnd - fnStart);
    Logger.log(status.errorResult);   // check for notification of extract too big (e.g. > 1 Gb)
    return status.getState();


    // Function to determine if a BigQuery table exists. Returns boolean
function checkBigQueryTable(projectId, datasetId, tableId) {  
    try { 
        var job = BigQuery.Tables.get(projectId, datasetId, tableId);
        return true;
    } catch(err) { 
        Logger.log('Table %s does not exist' , tableId);    
        return false;   
    } 

}   
}

Answer 1

你所做的对我来说似乎是正确的。你确定产生的输出不是GZIP吗？即使将压缩设置为GZIP，BigQuery仍将以GZIP格式输出116个文件，每个分片一个（使用＆＃34; .JSON和＃34;作为输出URI中指示的扩展名）。

我尝试使用setCompression使用以下代码，它适用于我：

var tableReference = BigQuery.newTableReference();
tableReference.setProjectId(projectId);
tableReference.setDatasetId(datasetId);
tableReference.setTableId(tableId);

var extract = BigQuery.newJobConfigurationExtract()
extract.setDestinationFormat('NEWLINE_DELIMITED_JSON');
extract.setCompression('GZIP');
extract.setDestinationUris(['gs://xxxxx/output.*.JSON']);
extract.setSourceTable(tableReference);

var configuration = BigQuery.newJobConfiguration();
configuration.setExtract(extract);
var newJob = BigQuery.newJob();
newJob.setConfiguration(configuration);

var job = BigQuery.Jobs.insert(newJob, projectId);
Logger.log("JobId is " + projectId + ":" + jobId);

您可以使用bq command-line client：

检查您的setCompression是否生效

bq --format=prettyjson show -j <datasetId>:<jobId>

当它工作时，您应该看到以下这些行：

...
"extract": {
  "compression": "GZIP", 
  "destinationFormat": "NEWLINE_DELIMITED_JSON", 
...

使用Google Apps脚本设置压缩查询导出

1 个答案: