GAS:BigQuery上载到现有表

时间:2018-07-09 22:46:51

标签: json rest google-apps-script google-bigquery ringcentral

我有超过350,000个RingCentral呼叫日志,我想上传到BigQuery,以便可以使用SQL查询来提取和摘要报告。它们当前存储为23个.csv文件,以使每个文件均低于BigQuery API规定的10MB限制。我想使用Google Apps脚本上传CSV数据,因此所有350K条目都在一个表中。这是我到目前为止的代码:

function uploadCSVtoBigQuery() {
  
  try {
    
  var CSVFolder = "Drive ID to Folder A";
  var ProcessedFolder = "Drive ID to Folder B";
  
  var projectId = 'ring-central-call-logs';
  var datasetId = 'RingCentral';
  var tableId   = 'Calls';
  
  //CSVFolder = getDriveFolder(CSVFolder);
  var CSVFolder = DriveApp.getFolderById(CSVFolder);
  //ProcessedFolder = getDriveFolder(ProcessedFolder);
  var ProcessedFolder = DriveApp.getFolderById(ProcessedFolder); 
  
  if (CSVFolder && ProcessedFolder) {
    Logger.log("Folders Appear Valid");
    var data, job, file, files = CSVFolder.getFiles();
    
    while (files.hasNext()) {
      
      file = files.next();
      
      if (file.getMimeType() === "text/csv") {
        
        data = file.getBlob().setContentType('application/octet-stream');
        
        job = {
          configuration: {
            load: {
              destinationTable: {
                projectId: projectId,
                datasetId: datasetId,
                tableId: tableId
              },
              skipLeadingRows: 1
            }
          }
        };

        job = BigQuery.Jobs.insert(job, projectId, data);
        file.makeCopy(file.getName(), ProcessedFolder);
        file.setTrashed(true);

        Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
      } else{Logger.log(file.getMimeType()+" Is not a valid file type.");}
    }
    
  } else{Logger.log('One of your folders is not valid');}
    
   Logger.log('Finished'); 
  } catch(e) {
    Logger.log(e.toString());
  }
}

// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {
  
  var results, folders = name.split("\\");
  var folder = DriveApp.getRootFolder();
  
  for (var i=0; i<folders.length; i++) {
    if (folders[i] === "") continue;
    results = folder.getFoldersByName(folders[i]);
    if (results.hasNext()) {
      folder = results.next(); 
    } else {
      folder = folder.createFolder(folders[i]);
    }
  }
  
  return folder;
}

运行该函数时,我将收到如下JSON响应:

 GoogleJsonResponseException: Using table ring-central-call-logs:RingCentral.Calls is not allowed for this operation because of its type. Try using a different table that is of type TABLE.

为检查我的大脑,我尝试在API Explorer中运行REST API,如下所示:

POST https://www.googleapis.com/bigquery/v2/projects/ring-central-call-logs/datasets/RingCentral/tables/Calls/insertAll?key={YOUR_API_KEY}
 
{


 "skipInvalidRows": true,
 "rows": [
  {
   "json": {
    "User": "Nathaniel",
    "CallStartTime": "2018-07-09 15:45:10",
    "From": "123456789",
    "To": "9876543211",
    "DurationSeconds": "15",
    "CallDirection": "out",
    "CallResult": "Failed",
    "QueueName": "Test Queue"
   }
  }
 ],
 "kind": "bigquery#tableDataInsertAllRequest"
}
 

并得到了类似的错误:

400
 
- Show headers -
  
{
 "error": {
  "code": 400,
  "message": "Cannot add rows to a table of type EXTERNAL.",
  "errors": [
   {
    "message": "Cannot add rows to a table of type EXTERNAL.",
    "domain": "global",
    "reason": "invalid"
   }
  ],
  "status": "INVALID_ARGUMENT"
 }
}

这里的Schema屏幕快照看起来与我输入的内容匹配: enter image description here  我想通过阅读可以解决权限问题,但是我可以在GAS中运行查询功能。我可以得到协助以弄清楚为什么这行不通吗?谢谢!

0 个答案:

没有答案