Google BigQuery发生内部错误,无法完成请求。错误:7367027

时间:2019-01-16 16:04:57

标签: google-bigquery

我在名为review_date的字段上分区了一个表,并尝试在其上运行MERGE查询。

MERGE操作一直有效,直到我填满大约5000个分区为止……然后它一直失败。出现此错误:

An internal error occurred and the request could not be completed. Error: 7367027

这是我正在使用的MERGE查询。

MERGE `ps-erep.data.product_review` a
USING (
  SELECT * EXCEPT(row_number)
  , CASE WHEN review_date < '1995-01-01' THEN '1995-01-01' ELSE review_date END norm_review_date
  FROM (
    SELECT *, 
    ROW_NUMBER() OVER (PARTITION BY product_review_id ORDER BY date_loaded desc) row_number
    from `ps-erep.raw_data.product_review_log`
    WHERE date_loaded > '2019-01-01'
    and review_date >= '2005-01-01'
    and review_date < '2010-01-01'
  )
  where row_number = 1
) b
ON a.product_review_id = b.product_review_id
AND a.date_loaded = b.date_loaded
WHEN NOT MATCHED THEN
  INSERT (
    product_review_id
    , date_loaded
    , seller_id 
    , expert_review_source_id 
    , product_id 
    , unique_review_id
    , title
    , body
    , review_date
    , product_review_type_id
    , reviewer_name
    , reviewer_location
    , reviewer_duration_of_product_ownership
    , reviewer_length_of_experience
    , reviewer_gender
    , ratio
    , review_rating_points
    , review_rating_points_possible
    , percentage_of_users_found_review_useful
    , users_found_review_useful_count
    , users_found_review_useful_sample_count
    , review_detail_link
    , date_entered
    , date_modified
    , record_status
    , parsed_vendor_id
    , review_page_number
  )
  VALUES (
    product_review_id
    , date_loaded
    , seller_id 
    , expert_review_source_id 
    , product_id 
    , unique_review_id
    , title
    , body
    , norm_review_date
    , product_review_type_id
    , reviewer_name
    , reviewer_location
    , reviewer_duration_of_product_ownership
    , reviewer_length_of_experience
    , reviewer_gender
    , ratio
    , review_rating_points
    , review_rating_points_possible
    , percentage_of_users_found_review_useful
    , users_found_review_useful_count
    , users_found_review_useful_sample_count
    , review_detail_link
    , date_entered
    , date_modified
    , record_status
    , parsed_vendor_id
    , review_page_number
  )


这是我看到的输出:

{
 "kind": "bigquery#job",
 "etag": "\"s6cRwJMpZRqrw_B9JD5ZljlYNOs/j4hRRg2Zf2Etacsz6QsiwxXB_N4\"",
 "id": "ps-erep:US.bquxjob_6379ee69_168572ec216",
 "selfLink": "https://www.googleapis.com/bigquery/v2internal/projects/ps-erep/jobs/bquxjob_6379ee69_168572ec216?location=US",
 "jobReference": {
  "projectId": "ps-erep",
  "jobId": "bquxjob_6379ee69_168572ec216",
  "location": "US"
 },
 "configuration": {
  "jobType": "QUERY",
  "query": {
   "query": "MERGE `ps-erep.data.product_review` a\nUSING (\n  SELECT * EXCEPT(row_number)\n  , CASE WHEN review_date \u003c '1995-01-01' THEN '1995-01-01' ELSE review_date END norm_review_date\n  FROM (\n    SELECT *, \n    ROW_NUMBER() OVER (PARTITION BY product_review_id ORDER BY date_loaded desc) row_number\n    from `ps-erep.raw_data.product_review_log`\n    WHERE date_loaded \u003e '2019-01-01'\n    and review_date \u003e= '2005-01-01'\n    and review_date \u003c '2010-01-01'\n  )\n  where row_number = 1\n) b\nON a.product_review_id = b.product_review_id\nAND a.date_loaded = b.date_loaded\nWHEN NOT MATCHED THEN\n  INSERT (\n    product_review_id\n    , date_loaded\n    , seller_id \n    , expert_review_source_id \n    , product_id \n    , unique_review_id\n    , title\n    , body\n    , review_date\n    , product_review_type_id\n    , reviewer_name\n    , reviewer_location\n    , reviewer_duration_of_product_ownership\n    , reviewer_length_of_experience\n    , reviewer_gender\n    , ratio\n    , review_rating_points\n    , review_rating_points_possible\n    , percentage_of_users_found_review_useful\n    , users_found_review_useful_count\n    , users_found_review_useful_sample_count\n    , review_detail_link\n    , date_entered\n    , date_modified\n    , record_status\n    , parsed_vendor_id\n    , review_page_number\n  )\n  VALUES (\n    product_review_id\n    , date_loaded\n    , seller_id \n    , expert_review_source_id \n    , product_id \n    , unique_review_id\n    , title\n    , body\n    , norm_review_date\n    , product_review_type_id\n    , reviewer_name\n    , reviewer_location\n    , reviewer_duration_of_product_ownership\n    , reviewer_length_of_experience\n    , reviewer_gender\n    , ratio\n    , review_rating_points\n    , review_rating_points_possible\n    , percentage_of_users_found_review_useful\n    , users_found_review_useful_count\n    , users_found_review_useful_sample_count\n    , review_detail_link\n    , date_entered\n    , date_modified\n    , record_status\n    , parsed_vendor_id\n    , review_page_number\n  )\n",
   "destinationTable": {
    "projectId": "ps-erep",
    "datasetId": "data",
    "tableId": "product_review"
   },
   "priority": "INTERACTIVE",
   "allowLargeResults": false,
   "useQueryCache": true,
   "useLegacySql": false
  }
 },
 "status": {
  "state": "DONE",
  "errorResult": {
   "reason": "internalError",
   "message": "An internal error occurred and the request could not be completed. Error: 7367027"
  },
  "errors": [
   {
    "reason": "internalError",
    "message": "An internal error occurred and the request could not be completed. Error: 7367027"
   }
  ]
 },
 "statistics": {
  "creationTime": "1547653418070",
  "startTime": "1547653418435",
  "endTime": "1547653491857",
  "query": {
   "queryPlan": [
    {
     "name": "S00: Input",
     "id": "0",
     "waitMsAvg": "0",
     "waitMsMax": "0",
     "readMsAvg": "1057",
     "readMsMax": "8303",
     "computeMsAvg": "659",
     "computeMsMax": "185366",
     "writeMsAvg": "33",
     "writeMsMax": "1439",
     "shuffleOutputBytes": "0",
     "shuffleOutputBytesSpilled": "0",
     "recordsRead": "0",
     "recordsWritten": "0",
     "parallelInputs": "281",
     "completedParallelInputs": "281",
     "status": "COMPLETE",
     "steps": [
      {
       "kind": "READ",
       "substeps": [
        "$10:product_review_id, $11:date_loaded, $12:seller_id, $13:expert_review_source_id, $14:product_id, $15:unique_review_id, $16:title, $17:body, $18:review_date, $19:product_review_type_id, $20:reviewer_name, $21:reviewer_location, $22:reviewer_duration_of_product_ownership, $23:reviewer_length_of_experience, $24:reviewer_gender, $25:ratio",
        "FROM ps-erep.raw_data.product_review_log",
        "WHERE and(greater($11, 1546300800.000000000), greater_or_equal($18, 1104537600.000000000), less($18, 1262304000.000000000))"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25",
        "TO __stage00_output",
        "BY HASH($10)"
       ]
      }
     ]
    },
    {
     "name": "S01: Input",
     "id": "1",
     "waitMsAvg": "0",
     "waitMsMax": "0",
     "readMsAvg": "51",
     "readMsMax": "286",
     "computeMsAvg": "29",
     "computeMsMax": "96253",
     "writeMsAvg": "256",
     "writeMsMax": "1828",
     "shuffleOutputBytes": "0",
     "shuffleOutputBytesSpilled": "0",
     "recordsRead": "0",
     "recordsWritten": "0",
     "parallelInputs": "3299",
     "completedParallelInputs": "3299",
     "status": "COMPLETE",
     "steps": [
      {
       "kind": "READ",
       "substeps": [
        "$1:product_review_id, $2:date_loaded",
        "FROM ps-erep.data.product_review"
       ]
      },
      {
       "kind": "COMPUTE",
       "substeps": [
        "$90 := 1"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$1, $2, $90",
        "TO __stage01_output",
        "BY HASH($1, $2)"
       ]
      }
     ]
    },
    {
     "name": "S02: Sort+",
     "id": "2",
     "inputStages": [
      "0"
     ],
     "waitMsAvg": "0",
     "waitMsMax": "0",
     "readMsAvg": "0",
     "readMsMax": "0",
     "computeMsAvg": "636",
     "computeMsMax": "178969",
     "writeMsAvg": "128",
     "writeMsMax": "460",
     "shuffleOutputBytes": "0",
     "shuffleOutputBytesSpilled": "0",
     "recordsRead": "0",
     "recordsWritten": "0",
     "parallelInputs": "281",
     "completedParallelInputs": "281",
     "status": "COMPLETE",
     "steps": [
      {
       "kind": "READ",
       "substeps": [
        "$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25",
        "FROM __stage00_output"
       ]
      },
      {
       "kind": "FILTER",
       "substeps": [
        "equal($100, 1)"
       ]
      },
      {
       "kind": "ANALYTIC_FUNCTION",
       "substeps": [
        "$100 := ROW_NUMBER() OVER (PARTITION BY $110 ORDER BY $111 DESC)"
       ]
      },
      {
       "kind": "SORT",
       "substeps": [
        "$10 ASC, $11 DESC"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$140, $141, $142, $143, $144, $145, $146, $147, $148, $149, $150, $151, $152, $153, $154, $155",
        "TO __stage02_output",
        "BY HASH($140, $141)"
       ]
      }
     ]
    },
    {
     "name": "S03: Join+",
     "id": "3",
     "inputStages": [
      "2",
      "1"
     ],
     "waitMsAvg": "0",
     "waitMsMax": "0",
     "readMsAvg": "0",
     "readMsMax": "0",
     "computeMsAvg": "2977",
     "computeMsMax": "297774",
     "writeMsAvg": "181",
     "writeMsMax": "667",
     "shuffleOutputBytes": "0",
     "shuffleOutputBytesSpilled": "0",
     "recordsRead": "0",
     "recordsWritten": "0",
     "parallelInputs": "100",
     "completedParallelInputs": "100",
     "status": "COMPLETE",
     "steps": [
      {
       "kind": "READ",
       "substeps": [
        "$1, $2, $90",
        "FROM __stage01_output"
       ]
      },
      {
       "kind": "READ",
       "substeps": [
        "$140, $141, $142, $143, $144, $145, $146, $147, $148, $149, $150, $151, $152, $153, $154, $155",
        "FROM __stage02_output"
       ]
      },
      {
       "kind": "COMPUTE",
       "substeps": [
        "$40 := COMPUTE_PARTITION_ID($58, PROTO\u003c...\u003e)"
       ]
      },
      {
       "kind": "COMPUTE",
       "substeps": [
        "$50 := if(equal(1, $80), if(is_null($170), $171, NULL), NULL)",
        "$51 := if(equal(1, $80), if(is_null($170), $172, NULL), NULL)",
        "$52 := if(equal(1, $80), if(is_null($170), $173, NULL), NULL)",
        "$53 := if(equal(1, $80), if(is_null($170), $174, NULL), NULL)",
        "$54 := if(equal(1, $80), if(is_null($170), $175, NULL), NULL)",
        "$55 := if(equal(1, $80), if(is_null($170), $176, NULL), NULL)",
        "$56 := if(equal(1, $80), if(is_null($170), $177, NULL), NULL)",
        "$57 := if(equal(1, $80), if(is_null($170), $178, NULL), NULL)",
        "$58 := if(equal(1, $80), if(is_null($170), case_no_value(less($179, 788918400.000000000), 788918400.000000000, $179), NULL), NULL)",
        "$59 := if(equal(1, $80), if(is_null($170), $180, NULL), NULL)",
        "$60 := if(equal(1, $80), if(is_null($170), $181, NULL), NULL)",
        "$61 := if(equal(1, $80), if(is_null($170), $182, NULL), NULL)",
        "$62 := if(equal(1, $80), if(is_null($170), $183, NULL), NULL)",
        "$63 := if(equal(1, $80), if(is_null($170), $184, NULL), NULL)",
        "$64 := if(equal(1, $80), if(is_null($170), $185, NULL), NULL)",
        "$65 := if(equal(1, $80), if(is_null($170), $186, NULL), NULL)"
       ]
      },
      {
       "kind": "COMPUTE",
       "substeps": [
        "$80 := if(is_null($170), 1, -1)"
       ]
      },
      {
       "kind": "FILTER",
       "substeps": [
        "not(equal(if(is_null($170), 10001, 0), 0))"
       ]
      },
      {
       "kind": "JOIN",
       "substeps": [
        "RIGHT OUTER HASH JOIN EACH  WITH EACH  ON $1 = $140, $2 = $141"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$40, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64",
        "TO __stage03_output"
       ]
      }
     ]
    },
    {
     "name": "S04: Output",
     "id": "4",
     "inputStages": [
      "3"
     ],
     "waitMsAvg": "0",
     "waitMsMax": "0",
     "readMsAvg": "0",
     "readMsMax": "0",
     "computeMsAvg": "466",
     "computeMsMax": "851717",
     "writeMsAvg": "1253",
     "writeMsMax": "1761",
     "shuffleOutputBytes": "0",
     "shuffleOutputBytesSpilled": "0",
     "recordsRead": "0",
     "recordsWritten": "0",
     "parallelInputs": "1826",
     "completedParallelInputs": "1826",
     "status": "COMPLETE",
     "steps": [
      {
       "kind": "READ",
       "substeps": [
        "$40, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64",
        "FROM __stage03_output"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65",
        "TO __stage04_output"
       ]
      }
     ]
    }
   ],
   "estimatedBytesProcessed": "62225303406",
   "timeline": [
    {
     "elapsedMs": "3806",
     "totalSlotMs": "0",
     "pendingUnits": "281",
     "completedUnits": "0",
     "activeUnits": "24"
    },
    {
     "elapsedMs": "4820",
     "totalSlotMs": "49",
     "pendingUnits": "280",
     "completedUnits": "1",
     "activeUnits": "70"
    },
    {
     "elapsedMs": "5836",
     "totalSlotMs": "23917",
     "pendingUnits": "71",
     "completedUnits": "210",
     "activeUnits": "88"
    },
    {
     "elapsedMs": "6939",
     "totalSlotMs": "69413",
     "pendingUnits": "55",
     "completedUnits": "226",
     "activeUnits": "66"
    },
    {
     "elapsedMs": "7973",
     "totalSlotMs": "148772",
     "pendingUnits": "33",
     "completedUnits": "248",
     "activeUnits": "42"
    },
    {
     "elapsedMs": "9018",
     "totalSlotMs": "157110",
     "pendingUnits": "31",
     "completedUnits": "250",
     "activeUnits": "32"
    },
    {
     "elapsedMs": "10350",
     "totalSlotMs": "161760",
     "pendingUnits": "30",
     "completedUnits": "251",
     "activeUnits": "30"
    },
    {
     "elapsedMs": "11432",
     "totalSlotMs": "173937",
     "pendingUnits": "28",
     "completedUnits": "253",
     "activeUnits": "28"
    },
    {
     "elapsedMs": "13416",
     "totalSlotMs": "281185",
     "pendingUnits": "14",
     "completedUnits": "267",
     "activeUnits": "28"
    },
    {
     "elapsedMs": "14135",
     "totalSlotMs": "335221",
     "pendingUnits": "8",
     "completedUnits": "273",
     "activeUnits": "16"
    },
    {
     "elapsedMs": "16223",
     "totalSlotMs": "408813",
     "pendingUnits": "1",
     "completedUnits": "280",
     "activeUnits": "12"
    },
    {
     "elapsedMs": "17696",
     "totalSlotMs": "685570",
     "pendingUnits": "1363",
     "completedUnits": "2498",
     "activeUnits": "1367"
    },
    {
     "elapsedMs": "18698",
     "totalSlotMs": "1116960",
     "pendingUnits": "766",
     "completedUnits": "3095",
     "activeUnits": "988"
    },
    {
     "elapsedMs": "19711",
     "totalSlotMs": "1687356",
     "pendingUnits": "0",
     "completedUnits": "3861",
     "activeUnits": "12"
    },
    {
     "elapsedMs": "21328",
     "totalSlotMs": "1687356",
     "pendingUnits": "100",
     "completedUnits": "3861",
     "activeUnits": "100"
    },
    {
     "elapsedMs": "23114",
     "totalSlotMs": "1863046",
     "pendingUnits": "9",
     "completedUnits": "3952",
     "activeUnits": "86"
    },
    {
     "elapsedMs": "24218",
     "totalSlotMs": "1884966",
     "pendingUnits": "1826",
     "completedUnits": "3961",
     "activeUnits": "1826"
    },
    {
     "elapsedMs": "26193",
     "totalSlotMs": "4529289",
     "pendingUnits": "88",
     "completedUnits": "5699",
     "activeUnits": "1448"
    },
    {
     "elapsedMs": "26974",
     "totalSlotMs": "4700893",
     "pendingUnits": "0",
     "completedUnits": "5787",
     "activeUnits": "58"
    }
   ],
   "statementType": "MERGE"
  }
 },
 "user_email": ""
}

除了“发生内部错误并且无法完成请求。错误7367027”之外,该错误中似乎没有其他任何详细信息。

BigQuery小组的任何帮助将不胜感激

1 个答案:

答案 0 :(得分:0)

您阅读BigQuery限制吗?每个表有4000个分区的限制。 https://cloud.google.com/bigquery/quotas#partitioned_tables

可能是原因吗?