我正在从多个表中读取数据并创建7天的分区,试图将数据写入相应的日分区中以调整该特定日期的更改。
我正在使用WRITE.TRUNCATE
方法来避免重复。但是,当写入表时,我仅获取最后一天分区的数据,而其他所有分区均未填充。原来的工作在Legacy SQL中运行,并且工作正常,但是我正尝试迁移到Standard SQL,此后此问题一直存在。附加以下代码:
for (int i = 1; i <= PARTITION_CREATE_DAYS; ++i) {
LocalDate partDate = params.referenceDate.minusDays(PARTITION_CREATE_DAYS - i);
String query = String.format(sql, partDate.toString(), params.referenceDate.toString(),params.TableA, params.TableB, params.TableC, params.TableD);
jobs.add(createJob(params, bq, query, partDate));
}
boolean allSucceeded = waitForJobs(jobs.listIterator());
if (allSucceeded) System.out.println("Job finished successfully!");
else System.exit(1);
}
private static Job createJob(Params params, BigQuery bq, String query, LocalDate partDate) {
String partition = "$" + partDate.format(BQ_PARTITION_FORMAT);
TableId tableId = TableId.of(params.outProj, params.outDataset, params.outTable + partition);
Job job = bq.create(JobInfo.of(QueryJobConfiguration.newBuilder(query)
.setDestinationTable(tableId)
//.setAllowLargeResults(true)
.setUseLegacySql(false)
.setUseQueryCache(true)
.setPriority(QueryJobConfiguration.Priority.values()[params.priority == 'i' ? 0 : 1])
.setWriteDisposition(JobInfo.WriteDisposition.WRITE_TRUNCATE)
.build()));
System.out.println("Submitted BQ job " + job.getJobId().getJob() + " for partition " + partition);
return job;
}
}
PARTITION_CREATE_DAYS = 8
referenceDate = current date