更新3/11
我遇到了一个基于我的SparkSQL查询从Spark收到的错误。我正在运行spark 1.2.1版本。我尝试检查我的查询对我在堆栈溢出时发现的几个答案，但我无法确切地确定问题在这里。
错误：
Application Failed...java.lang.RuntimeException: [1.226] failure: identifier expected
SELECT A_HOSTNAME, A_MODEL FROM (SELECT A_HOSTNAME, A_MODEL, COUNT(*) FROM (SELECT A_HOSTNAME, A_IF_DESC, A_MODEL FROM TOPOLOGY WHERE IS_PROD > 0 AND A_TYPE = 'GWR' AND Z_TYPE = 'LCR' GROUP BY A_HOSTNAME, A_IF_DESC, A_MODEL) GROUP BY A_HOSTNAME, A_MODEL HAVING COUNT(*) > 1) GROUP BY A_HOSTNAME, A_MODEL
代码：modelTopology.java
public class modelTopology implements Serializable {
//Variables are here + getters and setters
}
代码：从RDD创建JavaSchemaRDD作为modelTOpologySchema
JavaRDD<modelTopology> MODEL_TOPOLOGYRDD = TopologyRDD.map(
new Function<Object[], modelTopology>() {
public modelTopology call(Object[] line) throws Exception {
modelTopology toporow = new modelTopology();
toporow.setA_TYPE(line[0].toString().trim());
toporow.setZ_TYPE(line[1].toString().trim());
toporow.setA_CLLI(line[2].toString().trim());
toporow.setZ_CLLI(line[3].toString().trim());
toporow.setA_HOSTNAME(line[4].toString().trim());
toporow.setZ_HOSTNAME(line[5].toString().trim());
toporow.setA_LOCATION(line[6].toString().trim());
toporow.setA_LOC_TYPE(line[7].toString().trim());
toporow.setZ_LOCATION(line[8].toString().trim());
toporow.setZ_LOC_TYPE(line[9].toString().trim());
toporow.setA_SHELF(line[10].toString().trim());
toporow.setA_SLOT(line[11].toString().trim());
toporow.setA_CARD(line[12].toString().trim());
toporow.setA_PORT(line[13].toString().trim());
toporow.setA_INTERFACE(line[14].toString().trim());
toporow.setA_IF_DESC(line[15].toString().trim());
toporow.setZ_SHELF(line[16].toString().trim());
toporow.setZ_SLOT(line[17].toString().trim());
toporow.setZ_CARD(line[18].toString().trim());
toporow.setZ_PORT(line[19].toString().trim());
toporow.setZ_INTERFACE(line[20].toString().trim());
toporow.setZ_IF_DESC(line[21].toString().trim());
toporow.setA_CARD_NAME(line[22].toString().trim());
toporow.setZ_CARD_NAME(line[23].toString().trim());
toporow.setPHY_CIRCUIT_ID(line[24].toString().trim());
toporow.setLAG_CIRCUIT_ID(line[25].toString().trim());
toporow.setPHY_CIRCUIT_ALIAS(line[26].toString().trim());
toporow.setA_VENDOR(line[27].toString().trim());
toporow.setA_MODEL(line[28].toString().trim());
toporow.setA_TECHNOLOGY(line[29].toString().trim());
toporow.setZ_VENDOR(line[30].toString().trim());
toporow.setZ_MODEL(line[31].toString().trim());
toporow.setZ_TECHNOLOGY(line[32].toString().trim());
toporow.setA_EH_ELEMENT_ID(line[33].toString().trim());
toporow.setA_EH_MACHINE_ID(line[34].toString().trim());
toporow.setZ_EH_ELEMENT_ID(line[35].toString().trim());
toporow.setZ_EH_MACHINE_ID(line[36].toString().trim());
toporow.setA_EH_SPEED(line[37].toString().trim());
toporow.setZ_EH_SPEED(line[38].toString().trim());
toporow.setA_EH_SPEED1(line[39].toString().trim());
toporow.setZ_EH_SPEED1(line[40].toString().trim());
toporow.setA_EH_EHEALTH_DOMAIN(line[41].toString().trim());
toporow.setZ_EH_EHEALTH_DOMAIN(line[42].toString().trim());
toporow.setA_MRTG_HOSTID(line[43].toString().trim());
toporow.setA_MRTG_GRPID(line[44].toString().trim());
toporow.setA_MRTG_IFID(line[45].toString().trim());
toporow.setZ_MRTG_HOSTID(line[46].toString().trim());
toporow.setZ_MRTG_GRPID(line[47].toString().trim());
toporow.setZ_MRTG_IFID(line[48].toString().trim());
toporow.setA_MGMT_IP(line[49].toString().trim());
toporow.setZ_MGMT_IP(line[50].toString().trim());
toporow.setA_IF_INDEX(line[51].toString().trim());
toporow.setZ_IF_INDEX(line[52].toString().trim());
toporow.setIS_PROD(line[53].toString().trim());
toporow.setTOPOLOGY_KEY(line[54].toString().trim());
toporow.setCOMMIT_TS(line[55].toString().trim());
return toporow;
}
});
JavaSchemaRDD schemaTopology = sqlContext.applySchema(MODEL_TOPOLOGYRDD, modelTopology.class);
schemaTopology.registerAsTable("TOPOLOGY");
JavaSchemaRDD FILTERED_TOPOLOGY = sqlContext.sql("SELECT A_HOSTNAME, A_MODEL FROM (SELECT A_HOSTNAME, A_MODEL, COUNT(*) FROM (SELECT A_HOSTNAME, A_IF_DESC, A_MODEL FROM TOPOLOGY WHERE IS_PROD > 0 AND A_TYPE = 'GWR' AND Z_TYPE = 'LCR' GROUP BY A_HOSTNAME, A_IF_DESC, A_MODEL) GROUP BY A_HOSTNAME, A_MODEL HAVING COUNT(*) > 1) GROUP BY A_HOSTNAME, A_MODEL").cache();
JavaSchemaRDD布局
root
|-- COMMIT_TS: string (nullable = true)
|-- IS_PROD: string (nullable = true)
|-- LAG_CIRCUIT_ID: string (nullable = true)
|-- PHY_CIRCUIT_ALIAS: string (nullable = true)
|-- PHY_CIRCUIT_ID: string (nullable = true)
|-- TOPOLOGY_KEY: string (nullable = true)
|-- a_CARD: string (nullable = true)
|-- a_CARD_NAME: string (nullable = true)
|-- a_CLLI: string (nullable = true)
|-- a_EH_EHEALTH_DOMAIN: string (nullable = true)
|-- a_EH_ELEMENT_ID: string (nullable = true)
|-- a_EH_MACHINE_ID: string (nullable = true)
|-- a_EH_SPEED: string (nullable = true)
|-- a_EH_SPEED1: string (nullable = true)
|-- a_HOSTNAME: string (nullable = true)
|-- a_IF_DESC: string (nullable = true)
|-- a_IF_INDEX: string (nullable = true)
|-- a_INTERFACE: string (nullable = true)
|-- a_LOCATION: string (nullable = true)
|-- a_LOC_TYPE: string (nullable = true)
|-- a_MGMT_IP: string (nullable = true)
|-- a_MODEL: string (nullable = true)
|-- a_MRTG_GRPID: string (nullable = true)
|-- a_MRTG_HOSTID: string (nullable = true)
|-- a_MRTG_IFID: string (nullable = true)
|-- a_PORT: string (nullable = true)
|-- a_SHELF: string (nullable = true)
|-- a_SLOT: string (nullable = true)
|-- a_TECHNOLOGY: string (nullable = true)
|-- a_TYPE: string (nullable = true)
|-- a_VENDOR: string (nullable = true)
|-- z_CARD: string (nullable = true)
|-- z_CARD_NAME: string (nullable = true)
|-- z_CLLI: string (nullable = true)
|-- z_EH_EHEALTH_DOMAIN: string (nullable = true)
|-- z_EH_ELEMENT_ID: string (nullable = true)
|-- z_EH_MACHINE_ID: string (nullable = true)
|-- z_EH_SPEED: string (nullable = true)
|-- z_EH_SPEED1: string (nullable = true)
|-- z_HOSTNAME: string (nullable = true)
|-- z_IF_DESC: string (nullable = true)
|-- z_IF_INDEX: string (nullable = true)
|-- z_INTERFACE: string (nullable = true)
|-- z_LOCATION: string (nullable = true)
|-- z_LOC_TYPE: string (nullable = true)
|-- z_MGMT_IP: string (nullable = true)
|-- z_MODEL: string (nullable = true)
|-- z_MRTG_GRPID: string (nullable = true)
|-- z_MRTG_HOSTID: string (nullable = true)
|-- z_MRTG_IFID: string (nullable = true)
|-- z_PORT: string (nullable = true)
|-- z_SHELF: string (nullable = true)
|-- z_SLOT: string (nullable = true)
|-- z_TECHNOLOGY: string (nullable = true)
|-- z_TYPE: string (nullable = true)
|-- z_VENDOR: string (nullable = true)
编辑2016年3月11日 根据下面收到的每个子查询反馈编辑了我的sql语句。
JavaSchemaRDD FILTERED_TOPOLOGY = sqlContext.sql("SELECT t2.A_HOSTNAME, t2.A_MODEL FROM " +
"(SELECT t1.A_HOSTNAME, t1.A_MODEL, COUNT(*) FROM " +
"(SELECT A_HOSTNAME, A_IF_DESC, A_MODEL " +
"FROM TOPOLOGY WHERE IS_PROD > 0 AND A_TYPE = 'GWR' AND Z_TYPE = 'LCR' " +
"GROUP BY A_HOSTNAME, A_IF_DESC, A_MODEL) t1 " +
"GROUP BY A_HOSTNAME, A_MODEL HAVING COUNT(*) > 1) t2 " +
"GROUP BY t2.A_HOSTNAME, t2.A_MODEL").cache();
错误：
Application Failed...org.apache.spark.sql.catalyst.errors.package$TreeNodeException: Unresolved attributes: 't2.A_HOSTNAME.,'t2.A_MODEL.,'t2.A_HOSTNAME.,'t2.A_MODEL., tree:
'Aggregate ['t2.A_HOSTNAME.,'t2.A_MODEL.], ['t2.A_HOSTNAME.,'t2.A_MODEL.]
'Subquery t2
'Filter (COUNT(1) > CAST(1, LongType))
'Aggregate ['A_HOSTNAME,'A_MODEL], ['t1.A_HOSTNAME.,'t1.A_MODEL.,COUNT(1) AS c2#59L]
'Subquery t1
'Aggregate ['A_HOSTNAME,'A_IF_DESC,'A_MODEL], ['A_HOSTNAME,'A_IF_DESC,'A_MODEL]
'Filter (((CAST(IS_PROD#1, DoubleType) > CAST(0, DoubleType)) && ('A_TYPE = GWR)) && ('Z_TYPE = LCR))
Subquery TOPOLOGY
LogicalRDD [COMMIT_TS#0,IS_PROD#1,LAG_CIRCUIT_ID#2,PHY_CIRCUIT_ALIAS#3,PHY_CIRCUIT_ID#4,TOPOLOGY_KEY#5,a_CARD#6,a_CARD_NAME#7,a_CLLI#8,a_EH_EHEALTH_DOMAIN#9,a_EH_ELEMENT_ID#10,a_EH_MACHINE_ID#11,a_EH_SPEED#12,a_EH_SPEED1#13,a_HOSTNAME#14,a_IF_DESC#15,a_IF_INDEX#16,a_INTERFACE#17,a_LOCATION#18,a_LOC_TYPE#19,a_MGMT_IP#20,a_MODEL#21,a_MRTG_GRPID#22,a_MRTG_HOSTID#23,a_MRTG_IFID#24,a_PORT#25,a_SHELF#26,a_SLOT#27,a_TECHNOLOGY#28,a_TYPE#29,a_VENDOR#30,z_CARD#31,z_CARD_NAME#32,z_CLLI#33,z_EH_EHEALTH_DOMAIN#34,z_EH_ELEMENT_ID#35,z_EH_MACHINE_ID#36,z_EH_SPEED#37,z_EH_SPEED1#38,z_HOSTNAME#39,z_IF_DESC#40,z_IF_INDEX#41,z_INTERFACE#42,z_LOCATION#43,z_LOC_TYPE#44,z_MGMT_IP#45,z_MODEL#46,z_MRTG_GRPID#47,z_MRTG_HOSTID#48,z_MRTG_IFID#49,z_PORT#50,z_SHELF#51,z_SLOT#52,z_TECHNOLOGY#53,z_TYPE#54,z_VENDOR#55], MapPartitionsRDD[2] at mapPartitions at JavaSQLContext.scala:102
答案 0 :(得分：0)
您的SQL不必要地复杂，并且缺少子查询的别名。
这是一个简化的查询（为了便于阅读，在多行上）：
SELECT A_HOSTNAME, A_MODEL FROM
(SELECT A_HOSTNAME, A_IF_DESC, A_MODEL
FROM TOPOLOGY
WHERE IS_PROD > 0 AND A_TYPE = 'GWR' AND Z_TYPE = 'LCR') D
GROUP BY A_HOSTNAME, A_MODEL
HAVING COUNT(*) > 1;
这会返回具有多个界面的设备的主机名和型号，这是我理解你想要做的。
答案 1 :(得分：0)
我不明白你的查询，因为它看起来太复杂了。 以下是相同的：
SELECT A_HOSTNAME, A_MODEL FROM TOPOLOGY
WHERE IS_PROD > 0 AND A_TYPE = 'GWR' AND Z_TYPE = 'LCR'
GROUP BY A_HOSTNAME, A_MODEL
HAVING COUNT(*) > 1
正如zero323所说，你为什么要运行一个非常旧版本的Spark，你能尝试至少1.5.x版本（或新的1.6.1版本）
此致
卢瓦克