我在名为review_date的字段上分区了一个表，并尝试在其上运行MERGE查询。
MERGE操作一直有效，直到我填满大约5000个分区为止……然后它一直失败。出现此错误：
An internal error occurred and the request could not be completed. Error: 7367027
这是我正在使用的MERGE查询。
MERGE `ps-erep.data.product_review` a
USING (
SELECT * EXCEPT(row_number)
, CASE WHEN review_date < '1995-01-01' THEN '1995-01-01' ELSE review_date END norm_review_date
FROM (
SELECT *,
ROW_NUMBER() OVER (PARTITION BY product_review_id ORDER BY date_loaded desc) row_number
from `ps-erep.raw_data.product_review_log`
WHERE date_loaded > '2019-01-01'
and review_date >= '2005-01-01'
and review_date < '2010-01-01'
)
where row_number = 1
) b
ON a.product_review_id = b.product_review_id
AND a.date_loaded = b.date_loaded
WHEN NOT MATCHED THEN
INSERT (
product_review_id
, date_loaded
, seller_id
, expert_review_source_id
, product_id
, unique_review_id
, title
, body
, review_date
, product_review_type_id
, reviewer_name
, reviewer_location
, reviewer_duration_of_product_ownership
, reviewer_length_of_experience
, reviewer_gender
, ratio
, review_rating_points
, review_rating_points_possible
, percentage_of_users_found_review_useful
, users_found_review_useful_count
, users_found_review_useful_sample_count
, review_detail_link
, date_entered
, date_modified
, record_status
, parsed_vendor_id
, review_page_number
)
VALUES (
product_review_id
, date_loaded
, seller_id
, expert_review_source_id
, product_id
, unique_review_id
, title
, body
, norm_review_date
, product_review_type_id
, reviewer_name
, reviewer_location
, reviewer_duration_of_product_ownership
, reviewer_length_of_experience
, reviewer_gender
, ratio
, review_rating_points
, review_rating_points_possible
, percentage_of_users_found_review_useful
, users_found_review_useful_count
, users_found_review_useful_sample_count
, review_detail_link
, date_entered
, date_modified
, record_status
, parsed_vendor_id
, review_page_number
)
这是我看到的输出：
{
"kind": "bigquery#job",
"etag": "\"s6cRwJMpZRqrw_B9JD5ZljlYNOs/j4hRRg2Zf2Etacsz6QsiwxXB_N4\"",
"id": "ps-erep:US.bquxjob_6379ee69_168572ec216",
"selfLink": "https://www.googleapis.com/bigquery/v2internal/projects/ps-erep/jobs/bquxjob_6379ee69_168572ec216?location=US",
"jobReference": {
"projectId": "ps-erep",
"jobId": "bquxjob_6379ee69_168572ec216",
"location": "US"
},
"configuration": {
"jobType": "QUERY",
"query": {
"query": "MERGE `ps-erep.data.product_review` a\nUSING (\n SELECT * EXCEPT(row_number)\n , CASE WHEN review_date \u003c '1995-01-01' THEN '1995-01-01' ELSE review_date END norm_review_date\n FROM (\n SELECT *, \n ROW_NUMBER() OVER (PARTITION BY product_review_id ORDER BY date_loaded desc) row_number\n from `ps-erep.raw_data.product_review_log`\n WHERE date_loaded \u003e '2019-01-01'\n and review_date \u003e= '2005-01-01'\n and review_date \u003c '2010-01-01'\n )\n where row_number = 1\n) b\nON a.product_review_id = b.product_review_id\nAND a.date_loaded = b.date_loaded\nWHEN NOT MATCHED THEN\n INSERT (\n product_review_id\n , date_loaded\n , seller_id \n , expert_review_source_id \n , product_id \n , unique_review_id\n , title\n , body\n , review_date\n , product_review_type_id\n , reviewer_name\n , reviewer_location\n , reviewer_duration_of_product_ownership\n , reviewer_length_of_experience\n , reviewer_gender\n , ratio\n , review_rating_points\n , review_rating_points_possible\n , percentage_of_users_found_review_useful\n , users_found_review_useful_count\n , users_found_review_useful_sample_count\n , review_detail_link\n , date_entered\n , date_modified\n , record_status\n , parsed_vendor_id\n , review_page_number\n )\n VALUES (\n product_review_id\n , date_loaded\n , seller_id \n , expert_review_source_id \n , product_id \n , unique_review_id\n , title\n , body\n , norm_review_date\n , product_review_type_id\n , reviewer_name\n , reviewer_location\n , reviewer_duration_of_product_ownership\n , reviewer_length_of_experience\n , reviewer_gender\n , ratio\n , review_rating_points\n , review_rating_points_possible\n , percentage_of_users_found_review_useful\n , users_found_review_useful_count\n , users_found_review_useful_sample_count\n , review_detail_link\n , date_entered\n , date_modified\n , record_status\n , parsed_vendor_id\n , review_page_number\n )\n",
"destinationTable": {
"projectId": "ps-erep",
"datasetId": "data",
"tableId": "product_review"
},
"priority": "INTERACTIVE",
"allowLargeResults": false,
"useQueryCache": true,
"useLegacySql": false
}
},
"status": {
"state": "DONE",
"errorResult": {
"reason": "internalError",
"message": "An internal error occurred and the request could not be completed. Error: 7367027"
},
"errors": [
{
"reason": "internalError",
"message": "An internal error occurred and the request could not be completed. Error: 7367027"
}
]
},
"statistics": {
"creationTime": "1547653418070",
"startTime": "1547653418435",
"endTime": "1547653491857",
"query": {
"queryPlan": [
{
"name": "S00: Input",
"id": "0",
"waitMsAvg": "0",
"waitMsMax": "0",
"readMsAvg": "1057",
"readMsMax": "8303",
"computeMsAvg": "659",
"computeMsMax": "185366",
"writeMsAvg": "33",
"writeMsMax": "1439",
"shuffleOutputBytes": "0",
"shuffleOutputBytesSpilled": "0",
"recordsRead": "0",
"recordsWritten": "0",
"parallelInputs": "281",
"completedParallelInputs": "281",
"status": "COMPLETE",
"steps": [
{
"kind": "READ",
"substeps": [
"$10:product_review_id, $11:date_loaded, $12:seller_id, $13:expert_review_source_id, $14:product_id, $15:unique_review_id, $16:title, $17:body, $18:review_date, $19:product_review_type_id, $20:reviewer_name, $21:reviewer_location, $22:reviewer_duration_of_product_ownership, $23:reviewer_length_of_experience, $24:reviewer_gender, $25:ratio",
"FROM ps-erep.raw_data.product_review_log",
"WHERE and(greater($11, 1546300800.000000000), greater_or_equal($18, 1104537600.000000000), less($18, 1262304000.000000000))"
]
},
{
"kind": "WRITE",
"substeps": [
"$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25",
"TO __stage00_output",
"BY HASH($10)"
]
}
]
},
{
"name": "S01: Input",
"id": "1",
"waitMsAvg": "0",
"waitMsMax": "0",
"readMsAvg": "51",
"readMsMax": "286",
"computeMsAvg": "29",
"computeMsMax": "96253",
"writeMsAvg": "256",
"writeMsMax": "1828",
"shuffleOutputBytes": "0",
"shuffleOutputBytesSpilled": "0",
"recordsRead": "0",
"recordsWritten": "0",
"parallelInputs": "3299",
"completedParallelInputs": "3299",
"status": "COMPLETE",
"steps": [
{
"kind": "READ",
"substeps": [
"$1:product_review_id, $2:date_loaded",
"FROM ps-erep.data.product_review"
]
},
{
"kind": "COMPUTE",
"substeps": [
"$90 := 1"
]
},
{
"kind": "WRITE",
"substeps": [
"$1, $2, $90",
"TO __stage01_output",
"BY HASH($1, $2)"
]
}
]
},
{
"name": "S02: Sort+",
"id": "2",
"inputStages": [
"0"
],
"waitMsAvg": "0",
"waitMsMax": "0",
"readMsAvg": "0",
"readMsMax": "0",
"computeMsAvg": "636",
"computeMsMax": "178969",
"writeMsAvg": "128",
"writeMsMax": "460",
"shuffleOutputBytes": "0",
"shuffleOutputBytesSpilled": "0",
"recordsRead": "0",
"recordsWritten": "0",
"parallelInputs": "281",
"completedParallelInputs": "281",
"status": "COMPLETE",
"steps": [
{
"kind": "READ",
"substeps": [
"$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25",
"FROM __stage00_output"
]
},
{
"kind": "FILTER",
"substeps": [
"equal($100, 1)"
]
},
{
"kind": "ANALYTIC_FUNCTION",
"substeps": [
"$100 := ROW_NUMBER() OVER (PARTITION BY $110 ORDER BY $111 DESC)"
]
},
{
"kind": "SORT",
"substeps": [
"$10 ASC, $11 DESC"
]
},
{
"kind": "WRITE",
"substeps": [
"$140, $141, $142, $143, $144, $145, $146, $147, $148, $149, $150, $151, $152, $153, $154, $155",
"TO __stage02_output",
"BY HASH($140, $141)"
]
}
]
},
{
"name": "S03: Join+",
"id": "3",
"inputStages": [
"2",
"1"
],
"waitMsAvg": "0",
"waitMsMax": "0",
"readMsAvg": "0",
"readMsMax": "0",
"computeMsAvg": "2977",
"computeMsMax": "297774",
"writeMsAvg": "181",
"writeMsMax": "667",
"shuffleOutputBytes": "0",
"shuffleOutputBytesSpilled": "0",
"recordsRead": "0",
"recordsWritten": "0",
"parallelInputs": "100",
"completedParallelInputs": "100",
"status": "COMPLETE",
"steps": [
{
"kind": "READ",
"substeps": [
"$1, $2, $90",
"FROM __stage01_output"
]
},
{
"kind": "READ",
"substeps": [
"$140, $141, $142, $143, $144, $145, $146, $147, $148, $149, $150, $151, $152, $153, $154, $155",
"FROM __stage02_output"
]
},
{
"kind": "COMPUTE",
"substeps": [
"$40 := COMPUTE_PARTITION_ID($58, PROTO\u003c...\u003e)"
]
},
{
"kind": "COMPUTE",
"substeps": [
"$50 := if(equal(1, $80), if(is_null($170), $171, NULL), NULL)",
"$51 := if(equal(1, $80), if(is_null($170), $172, NULL), NULL)",
"$52 := if(equal(1, $80), if(is_null($170), $173, NULL), NULL)",
"$53 := if(equal(1, $80), if(is_null($170), $174, NULL), NULL)",
"$54 := if(equal(1, $80), if(is_null($170), $175, NULL), NULL)",
"$55 := if(equal(1, $80), if(is_null($170), $176, NULL), NULL)",
"$56 := if(equal(1, $80), if(is_null($170), $177, NULL), NULL)",
"$57 := if(equal(1, $80), if(is_null($170), $178, NULL), NULL)",
"$58 := if(equal(1, $80), if(is_null($170), case_no_value(less($179, 788918400.000000000), 788918400.000000000, $179), NULL), NULL)",
"$59 := if(equal(1, $80), if(is_null($170), $180, NULL), NULL)",
"$60 := if(equal(1, $80), if(is_null($170), $181, NULL), NULL)",
"$61 := if(equal(1, $80), if(is_null($170), $182, NULL), NULL)",
"$62 := if(equal(1, $80), if(is_null($170), $183, NULL), NULL)",
"$63 := if(equal(1, $80), if(is_null($170), $184, NULL), NULL)",
"$64 := if(equal(1, $80), if(is_null($170), $185, NULL), NULL)",
"$65 := if(equal(1, $80), if(is_null($170), $186, NULL), NULL)"
]
},
{
"kind": "COMPUTE",
"substeps": [
"$80 := if(is_null($170), 1, -1)"
]
},
{
"kind": "FILTER",
"substeps": [
"not(equal(if(is_null($170), 10001, 0), 0))"
]
},
{
"kind": "JOIN",
"substeps": [
"RIGHT OUTER HASH JOIN EACH WITH EACH ON $1 = $140, $2 = $141"
]
},
{
"kind": "WRITE",
"substeps": [
"$40, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64",
"TO __stage03_output"
]
}
]
},
{
"name": "S04: Output",
"id": "4",
"inputStages": [
"3"
],
"waitMsAvg": "0",
"waitMsMax": "0",
"readMsAvg": "0",
"readMsMax": "0",
"computeMsAvg": "466",
"computeMsMax": "851717",
"writeMsAvg": "1253",
"writeMsMax": "1761",
"shuffleOutputBytes": "0",
"shuffleOutputBytesSpilled": "0",
"recordsRead": "0",
"recordsWritten": "0",
"parallelInputs": "1826",
"completedParallelInputs": "1826",
"status": "COMPLETE",
"steps": [
{
"kind": "READ",
"substeps": [
"$40, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64",
"FROM __stage03_output"
]
},
{
"kind": "WRITE",
"substeps": [
"$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65",
"TO __stage04_output"
]
}
]
}
],
"estimatedBytesProcessed": "62225303406",
"timeline": [
{
"elapsedMs": "3806",
"totalSlotMs": "0",
"pendingUnits": "281",
"completedUnits": "0",
"activeUnits": "24"
},
{
"elapsedMs": "4820",
"totalSlotMs": "49",
"pendingUnits": "280",
"completedUnits": "1",
"activeUnits": "70"
},
{
"elapsedMs": "5836",
"totalSlotMs": "23917",
"pendingUnits": "71",
"completedUnits": "210",
"activeUnits": "88"
},
{
"elapsedMs": "6939",
"totalSlotMs": "69413",
"pendingUnits": "55",
"completedUnits": "226",
"activeUnits": "66"
},
{
"elapsedMs": "7973",
"totalSlotMs": "148772",
"pendingUnits": "33",
"completedUnits": "248",
"activeUnits": "42"
},
{
"elapsedMs": "9018",
"totalSlotMs": "157110",
"pendingUnits": "31",
"completedUnits": "250",
"activeUnits": "32"
},
{
"elapsedMs": "10350",
"totalSlotMs": "161760",
"pendingUnits": "30",
"completedUnits": "251",
"activeUnits": "30"
},
{
"elapsedMs": "11432",
"totalSlotMs": "173937",
"pendingUnits": "28",
"completedUnits": "253",
"activeUnits": "28"
},
{
"elapsedMs": "13416",
"totalSlotMs": "281185",
"pendingUnits": "14",
"completedUnits": "267",
"activeUnits": "28"
},
{
"elapsedMs": "14135",
"totalSlotMs": "335221",
"pendingUnits": "8",
"completedUnits": "273",
"activeUnits": "16"
},
{
"elapsedMs": "16223",
"totalSlotMs": "408813",
"pendingUnits": "1",
"completedUnits": "280",
"activeUnits": "12"
},
{
"elapsedMs": "17696",
"totalSlotMs": "685570",
"pendingUnits": "1363",
"completedUnits": "2498",
"activeUnits": "1367"
},
{
"elapsedMs": "18698",
"totalSlotMs": "1116960",
"pendingUnits": "766",
"completedUnits": "3095",
"activeUnits": "988"
},
{
"elapsedMs": "19711",
"totalSlotMs": "1687356",
"pendingUnits": "0",
"completedUnits": "3861",
"activeUnits": "12"
},
{
"elapsedMs": "21328",
"totalSlotMs": "1687356",
"pendingUnits": "100",
"completedUnits": "3861",
"activeUnits": "100"
},
{
"elapsedMs": "23114",
"totalSlotMs": "1863046",
"pendingUnits": "9",
"completedUnits": "3952",
"activeUnits": "86"
},
{
"elapsedMs": "24218",
"totalSlotMs": "1884966",
"pendingUnits": "1826",
"completedUnits": "3961",
"activeUnits": "1826"
},
{
"elapsedMs": "26193",
"totalSlotMs": "4529289",
"pendingUnits": "88",
"completedUnits": "5699",
"activeUnits": "1448"
},
{
"elapsedMs": "26974",
"totalSlotMs": "4700893",
"pendingUnits": "0",
"completedUnits": "5787",
"activeUnits": "58"
}
],
"statementType": "MERGE"
}
},
"user_email": ""
}
除了“发生内部错误并且无法完成请求。错误7367027”之外，该错误中似乎没有其他任何详细信息。
BigQuery小组的任何帮助将不胜感激
答案 0 :(得分：0)
您阅读BigQuery限制吗？每个表有4000个分区的限制。 https://cloud.google.com/bigquery/quotas#partitioned_tables
可能是原因吗？