谷歌应用程序脚本中的谷歌Bigquery API抛出错误(响应代码413:'响应太大')加入相对较小的表

时间:2018-02-04 04:52:58

标签: join google-apps-script google-bigquery

我使用Google App Script从BigQuery API面临413错误(响应太大)。这些表格相对较小,可以解决这种错误。任何帮助将受到高度赞赏。它背靠背运行两个查询。几个星期前,它运行得非常好。

查询1.我正在加入两个表。两个表每天都会累积数据。一个表(revRaw)很大,> 11000行。其他表,cmpnMeta较小。它是所有广告系列的元信息,其效果信息在我们的广告服务器的表格“revRaw”中捕获。因此,加入将包含广告系列的效果和元信息。

查询2.在连接中,根据特定字段和adType的值,我正在更新另一个字段的值。

**请注意,实际查询会带来比我的代码显示更多的字段。我在这里删掉了那些无关紧要的东西。

错误引发第48行:

queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
function saveQueryToTable() {
 // destination dataset and table id
  const datasetId = "destination_dataset_ID";
  const tableId = "destination_table_ID";


  const query = "SELECT revRaw.date AS revRawdate, revRaw.adType AS adType, cmpnMeta.slaStartDate AS slaStartDate, cmpnMeta.slaEndDate AS slaEndDate, cmpnMeta.cmpnStartDate AS cmpnStartDate, cmpnMeta.slaRevenue AS slaRevenue, (cmpnMeta.slaRevenue / cmpnMeta.slaImpression) * revRaw.impressionDelivered AS directRevenue FROM [dataset1.revRaw] as revRaw LEFT JOIN [dataset1.cmpnMeta] AS cmpnMeta ON revRaw.campaignName = cmpnMeta.campaignName GROUP BY ...... : 


const updateQuery = "UPDATE `" + datasetId + "." + tableId +"` SET directRevenue = 0 WHERE adType = 'Direct' AND (slaRevenue <= 1 OR slaRevenue IS NULL)";


  resolveQuery(query, updateQuery, datasetId, tableId);
}

function resolveQuery(query, updateQuery, datasetId, tableId){
  const projectId = 'possible-stock-172109';

  var destinationTable = BigQuery.newTableReference();
  destinationTable.datasetId = datasetId;
  destinationTable.projectId = projectId;
  destinationTable.tableId = tableId;

  var myJobQueryConfig = BigQuery.newJobConfigurationQuery();
  myJobQueryConfig.allowLargeResults = true;
  myJobQueryConfig.query = query;
  myJobQueryConfig.writeDisposition = "WRITE_TRUNCATE";
  myJobQueryConfig.destinationTable = destinationTable; 

  var myJobConfig = BigQuery.newJobConfiguration();
  myJobConfig.query = myJobQueryConfig;

  var myJob = BigQuery.newJob();
  myJob.configuration = myJobConfig;

  var jobResult = BigQuery.Jobs.insert(myJob, projectId);
  Logger.log(jobResult);

  var jobId = jobResult.jobReference.jobId;
  Logger.log(jobId);

  // Check on status of the Query Job.
  var sleepTimeMs = 500;
  var queryResults= "a";
    while (!queryResults.jobComplete) {
    Utilities.sleep(sleepTimeMs);
    sleepTimeMs *= 2;
    queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);



    //send notification/write to sheet or nothing
  }

  // Update impressionDelivered to 1 where adType = 'Direct' AND (slaRevenue <= 1 OR IS NULL)
  if(queryResults.jobComplete){
    var updateQueryConfig = BigQuery.newJobConfigurationQuery();
    updateQueryConfig.allowLargeResults = true;
    updateQueryConfig.useLegacySql = false;
    updateQueryConfig.query = updateQuery;

    var updateJobConfig = BigQuery.newJobConfiguration();
    updateJobConfig.query = updateQueryConfig;

    var updateJob = BigQuery.newJob();
    updateJob.configuration = updateJobConfig;

    var updateResult = BigQuery.Jobs.insert(updateJob, projectId);

    var updateJobId = updateResult.jobReference.jobId;
    Logger.log(updateJobId);

    // Check on status of the Query Job.
    var sleepTimeMs = 500;
    var updateQueryResults= "a";
    while (!updateQueryResults.jobComplete) {
      Utilities.sleep(sleepTimeMs);
      sleepTimeMs *= 2;
      updateQueryResults = BigQuery.Jobs.getQueryResults(projectId, updateJobId);
    }

    Logger.log(updateQueryResults);
    Logger.log("---------------------------------------");
  }

   Logger.log(queryResults);
}

0 个答案:

没有答案