我正在尝试使用Hive 2.3.0在Amazon EMR 5.8上执行以下hive查询
CREATE EXTERNAL TABLE IF NOT EXISTS base_session (
property_num string,
visitId string,
fullVisitorId string,
adate string,
sessionStartTime timestamp,
totalVisits int,
totalHits int,
totalPageViews int,
totalTimeOnSite int,
totalBounces int,
totalNewVisits int,
totalScreenViews INT,
totalUniqueScreenViews int,
totalTimeOnScreen int,
browser string,
browserVersion string,
os string,
osVer String,
screenColors string,
screenResolution string,
deviceCategory string,
geoNetworkCountry string,
geoNetworkRegion string,
orgId int,
userType string,
landingScreen string,
exitScreen string,
org_session_num int,
geoNetworkMetro string,
deviceMobileBranding string,
start_business_context string,
end_business_context string,
totalViews int,
hits_appinfo_id STRING,
hits_appinfo_name STRING,
hits_appinfo_version STRING
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'
LOCATION '${BUCKET}/base/session/';
CREATE EXTERNAL TABLE IF NOT EXISTS bt_session (
property_num string,
visitId string,
fullVisitorId string,
adate string,
sessionStartTime int,
totalVisits int,
totalHits int,
totalPageViews int,
totalTimeOnSite int,
totalBounces int,
totalNewVisits int,
totalScreenViews INT,
totalUniqueScreenViews int,
totalTimeOnScreen int,
browser string,
browserVersion string,
os string,
osVer String,
screenColors string,
screenResolution string,
deviceCategory string,
geoNetworkCountry string,
geoNetworkRegion string,
geoNetworkMetro string,
deviceMobileBranding string,
totalViews int,
hits_appinfo_id STRING,
hits_appinfo_name STRING,
hits_appinfo_version STRING
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'
LOCATION '${BUCKET}/temp/bt/session/';
CREATE TABLE temp_session_orgid as
SELECT
sorgid.property_num, sorgid.visitid, sorgid.fullvisitorid, sorgid.adate, sorgid.hits_customvariables_customvarvalue as orgid
FROM
(
SELECT
*,
row_number() over (partition by property_num, visitid, fullvisitorid, adate order by hitsid) as rn
FROM bt_hits_custom_vars
WHERE hits_customvariables_customvarname = 'orgId'
) sorgid
WHERE
sorgid.rn = 1
;
CREATE TABLE temp_session_userType as
SELECT
sut.property_num, sut.visitid, sut.fullvisitorid, sut.adate, sut.hits_customvariables_customvarvalue as userType
FROM
(
SELECT
*,
row_number() over (partition by property_num, visitid, fullvisitorid, adate order by hitsid) as rn
FROM bt_hits_custom_vars
WHERE hits_customvariables_customvarname = 'userType'
) sut
WHERE
sut.rn = 1
;
CREATE TABLE temp_session_hitdata as
SELECT DISTINCT
sh.property_num, sh.visitid, sh.fullvisitorid, sh.adate,
sh2.business_context as start_business_context,
sh2.appinfo_exitscreenname as exitscreen,
sh2.appinfo_landingscreenname as landingscreen,
sh3.business_context as end_business_context
FROM
(
SELECT property_num, visitid, fullvisitorid, adate,
min(hitid) over (partition by property_num, visitid, fullvisitorid, adate ) as first_id,
max(hitid) over (partition by property_num, visitid, fullvisitorid, adate ) as last_id
FROM bt_hits
) sh
left join bt_hits as sh2
on (sh.property_num = sh2.property_num
and sh.visitid = sh2.visitid
and sh.fullvisitorid = sh2.fullvisitorid
and sh.adate = sh2.adate
and sh.first_id = sh2.hitid)
left join bt_hits as sh3
on (sh.property_num = sh3.property_num
and sh.visitid = sh3.visitid
and sh.fullvisitorid = sh3.fullvisitorid
and sh.adate = sh3.adate
and sh.last_id = sh3.hitid)
;
create table bt_session2 as
select
s.*, O.orgid, U.userType, sh.landingscreen, sh.exitscreen, sh.start_business_context, sh.end_business_context
from
bt_session s
left join temp_session_orgid O
on (O.property_num = s.property_num and O.visitid = s.visitid and O.fullvisitorid = s.fullvisitorid and O.adate = s.adate)
left join temp_session_userType U
on (U.property_num = s.property_num and U.visitid = s.visitid and U.fullvisitorid = s.fullvisitorid and U.adate = s.adate)
left join temp_session_hitdata sh
on (sh.property_num = s.property_num and sh.visitid = s.visitid and sh.fullvisitorid = s.fullvisitorid and sh.adate = s.adate)
;
INSERT INTO TABLE base_session
select
property_num,
visitId,
fullVisitorId,
adate,
from_unixtime(sessionStartTime) as sessionStartTime,
totalVisits,
totalHits,
totalPageViews,
totalTimeOnSite,
totalBounces,
totalNewVisits,
totalScreenViews,
totalUniqueScreenViews,
totalTimeOnScreen,
browser,
browserVersion,
os,
osVer,
screenColors,
screenResolution,
deviceCategory,
geoNetworkCountry,
geoNetworkRegion,
orgId,
userType,
landingScreen,
exitScreen,
if( orgid is not null,
row_number() over (partition by property_num, visitid, adate, orgid order by sessionstarttime),
null) as org_session_num,
geoNetworkMetro,
deviceMobileBranding,
start_business_context,
end_business_context,
totalViews,
hits_appinfo_id,
hits_appinfo_name,
hits_appinfo_version
from
bt_session2
;
当我执行此查询时，我遇到以下异常：
Vertex failed, vertexName=Reducer 2, vertexId=vertex_1504998200702_0026_1_01, diagnostics=[Task failed, taskId=task_1504998200702_0026_1_01_000023, diagnostics=[TaskAttempt 0 failed, info=[Container container_1504998200702_0026_01_000061 finished with diagnostics set to [Container failed, exitCode=-104. Container [pid=17003,containerID=container_1504998200702_0026_01_000061] is running beyond physical memory limits. Current usage: 1.5 GB of 1.5 GB physical memory used; 3.2 GB of 7.5 GB virtual memory used. Killing container.
Dump of the process-tree for container_1504998200702_0026_01_000061 :
|- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE
|- 17003 17000 17003 17003 (bash) 0 0 115810304 679 /bin/bash -c /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061 -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx1229m -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061 -Dtez.root.logger=INFO,CLA -Djava.io.tmpdir=/mnt/yarn/usercache/hadoop/appcache/application_1504998200702_0026/container_1504998200702_0026_01_000061/tmp org.apache.tez.runtime.task.TezChild ip-10-212-196-172.johndeerecloud.com 41390 container_1504998200702_0026_01_000061 application_1504998200702_0026 1 1>/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061/stdout 2>/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061/stderr
|- 17016 17003 17003 17003 (java) 6237 246 3331600384 393016 /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061 -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN -Xmx1229m -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1504998200702_0026/container_1504998200702_0026_01_000061 -Dtez.root.logger=INFO,CLA -Djava.io.tmpdir=/mnt/yarn/usercache/hadoop/appcache/application_1504998200702_0026/container_1504998200702_0026_01_000061/tmp org.apache.tez.runtime.task.TezChild ip-10-212-196-172.johndeerecloud.com 41390 container_1504998200702_0026_01_000061 application_1504998200702_0026 1
Container killed on request. Exit code is 143
Container exited with a non-zero exit code 143
]], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_1504998200702_0026_1_01_000023_1:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row (tag=0) {"key":{"reducesinkkey0":"89634781","reducesinkkey1":"1412865647","reducesinkkey2":"4585400128498998895","reducesinkkey3":"20141009","reducesinkkey4":"1"},"value":{"_col1":"5268"}}
Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row (tag=0) {"key":{"reducesinkkey0":"89634781","reducesinkkey1":"1498837079","reducesinkkey2":"4333289744492811332","reducesinkkey3":"20170630","reducesinkkey4":"1"},"value":{"_col1":"CUSTOMER"}}
at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:297)
at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row (tag=0) {"key":{"reducesinkkey0":"89634781","reducesinkkey1":"1498837079","reducesinkkey2":"4333289744492811332","reducesinkkey3":"20170630","reducesinkkey4":"1"},"value":{"_col1":"CUSTOMER"}}
at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:287)
... 16 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.nio.channels.ClosedChannelException
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:796)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at org.apache.hadoop.hive.ql.exec.FilterOperator.process(FilterOperator.java:126)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at org.apache.hadoop.hive.ql.exec.PTFOperator$PTFInvocation.handleOutputRows(PTFOperator.java:337)
at org.apache.hadoop.hive.ql.exec.PTFOperator$PTFInvocation.processRow(PTFOperator.java:325)
at org.apache.hadoop.hive.ql.exec.PTFOperator.process(PTFOperator.java:139)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
Caused by: java.nio.channels.ClosedChannelException
at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1546)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:104)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:60)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat$1.write(HiveIgnoreKeyTextOutputFormat.java:87)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
我甚至尝试为hive shell设置此配置： 蜂房＆GT; set hive.vectorized.execution.enabled = false; 蜂房＆GT; set hive.vectorized.execution.reduce.enabled = false;
但仍然失败了。 有时这个查询成功运行，而有时它由于上述原因而失败。是否还有其他解决方法可以解决这个问题？