我有超过350,000个RingCentral呼叫日志,我想上传到BigQuery,以便可以使用SQL查询来提取和摘要报告。它们当前存储为23个.csv文件,以使每个文件均低于BigQuery API规定的10MB限制。我想使用Google Apps脚本上传CSV数据,因此所有350K条目都在一个表中。这是我到目前为止的代码:
function uploadCSVtoBigQuery() {
try {
var CSVFolder = "Drive ID to Folder A";
var ProcessedFolder = "Drive ID to Folder B";
var projectId = 'ring-central-call-logs';
var datasetId = 'RingCentral';
var tableId = 'Calls';
//CSVFolder = getDriveFolder(CSVFolder);
var CSVFolder = DriveApp.getFolderById(CSVFolder);
//ProcessedFolder = getDriveFolder(ProcessedFolder);
var ProcessedFolder = DriveApp.getFolderById(ProcessedFolder);
if (CSVFolder && ProcessedFolder) {
Logger.log("Folders Appear Valid");
var data, job, file, files = CSVFolder.getFiles();
while (files.hasNext()) {
file = files.next();
if (file.getMimeType() === "text/csv") {
data = file.getBlob().setContentType('application/octet-stream');
job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1
}
}
};
job = BigQuery.Jobs.insert(job, projectId, data);
file.makeCopy(file.getName(), ProcessedFolder);
file.setTrashed(true);
Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
} else{Logger.log(file.getMimeType()+" Is not a valid file type.");}
}
} else{Logger.log('One of your folders is not valid');}
Logger.log('Finished');
} catch(e) {
Logger.log(e.toString());
}
}
// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {
var results, folders = name.split("\\");
var folder = DriveApp.getRootFolder();
for (var i=0; i<folders.length; i++) {
if (folders[i] === "") continue;
results = folder.getFoldersByName(folders[i]);
if (results.hasNext()) {
folder = results.next();
} else {
folder = folder.createFolder(folders[i]);
}
}
return folder;
}
运行该函数时,我将收到如下JSON响应:
GoogleJsonResponseException: Using table ring-central-call-logs:RingCentral.Calls is not allowed for this operation because of its type. Try using a different table that is of type TABLE.
为检查我的大脑,我尝试在API Explorer中运行REST API,如下所示:
POST https://www.googleapis.com/bigquery/v2/projects/ring-central-call-logs/datasets/RingCentral/tables/Calls/insertAll?key={YOUR_API_KEY}
{
"skipInvalidRows": true,
"rows": [
{
"json": {
"User": "Nathaniel",
"CallStartTime": "2018-07-09 15:45:10",
"From": "123456789",
"To": "9876543211",
"DurationSeconds": "15",
"CallDirection": "out",
"CallResult": "Failed",
"QueueName": "Test Queue"
}
}
],
"kind": "bigquery#tableDataInsertAllRequest"
}
并得到了类似的错误:
400
- Show headers -
{
"error": {
"code": 400,
"message": "Cannot add rows to a table of type EXTERNAL.",
"errors": [
{
"message": "Cannot add rows to a table of type EXTERNAL.",
"domain": "global",
"reason": "invalid"
}
],
"status": "INVALID_ARGUMENT"
}
}
这里的Schema屏幕快照看起来与我输入的内容匹配: 我想通过阅读可以解决权限问题,但是我可以在GAS中运行查询功能。我可以得到协助以弄清楚为什么这行不通吗?谢谢!