Question

输入JSON文件：

Exception in thread "main" java.lang.IllegalArgumentException: Error while instantiating 'org.apache.spark.sql.hive.HiveSessionStateBuilder':
        at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$instantiateSessionState(SparkSession.scala:1075)
        at org.apache.spark.sql.SparkSession$$anonfun$sessionState$2.apply(SparkSession.scala:142)
        at org.apache.spark.sql.SparkSession$$anonfun$sessionState$2.apply(SparkSession.scala:141)
        at scala.Option.getOrElse(Option.scala:121)
        at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:141)
        at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:138)
        at org.apache.spark.sql.SparkSession.conf$lzycompute(SparkSession.scala:167)
        at org.apache.spark.sql.SparkSession.conf(SparkSession.scala:167)
        at com.ibm.ccs.spark.FTR.FTRHiveReader.main(FTRHiveReader.java:52)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.ClassNotFoundException: java.lang.NoClassDefFoundError: org/apache/tez/dag/api/SessionNotRunning when creating Hive client using classpath: file:/home/sifsuser/test/FTR-HiveReader.jar, <Some more jars for hive and hadoop>
Please make sure that jars for your version of hive and hadoop are included in the paths passed to spark.sql.hive.metastore.jars.
        at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:274)
        at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:362)
        at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:266)
        at org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)
        at org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)
        at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)
        at org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)
        at org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:105)
        at org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:93)
        at org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)
        at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)
        at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)
        at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:35)
        at org.apache.spark.sql.internal.BaseSessionStateBuilder.build(BaseSessionStateBuilder.scala:289)
        at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$instantiateSessionState(SparkSession.scala:1072)
        ... 17 more
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:268)
        ... 34 more
Caused by: java.lang.NoClassDefFoundError: org/apache/tez/dag/api/SessionNotRunning
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:576)
        at org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:191)
        ... 39 more
Caused by: java.lang.ClassNotFoundException: org.apache.tez.dag.api.SessionNotRunning
        at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1.doLoadClass(IsolatedClientLoader.scala:225)
        at org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1.loadClass(IsolatedClientLoader.scala:214)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        ... 41 more

预期产出：列出警报;

其中：

{
    "@version": "2.7.0",
    "@generated": "Wed, 30 May 2018 17:23:14",
    "site": {
        "@name": "http://google.com",
        "@host": "google.com",
        "@port": "80",
        "@ssl": "false",
        "alerts": [
            {

                "alert": "X-Content-Type-Options Header Missing",
                "name": "X-Content-Type-Options Header Missing",
                "riskcode": "1",
                "confidence": "2",
                "riskdesc": "Low (Medium)",
                "desc": "<p>The Anti-MIME-Sniffing header X-Content-Type-Options was not set to 'nosniff'. This allows older versions of Internet Explorer and Chrome to perform MIME-sniffing on the response body, potentially causing the response body to be interpreted and displayed as a content type other than the declared content type. Current (early 2014) and legacy versions of Firefox will use the declared content type (if one is set), rather than performing MIME-sniffing.</p>",
                "instances": [
                    {
                        "uri": "http://google.com",
                        "method": "GET",
                        "param": "X-Content-Type-Options"
                    }
                ],          
                "wascid": "15",
                "sourceid": "3"
            }

        ]
    }
}

我想获取json对象的特定键，并在警报对象中对其进行反序列化。

Answer 1

最简单的方法是使用足够的密钥声明外部对象以达到您关心的密钥：

public class Alert
{
    public string alert;
    public string riskcode;
}

public class SiteAlerts
{
    public Site site { get; set; }
}

public class Site
{
    public List<Alert> alerts { get; } = new List<Alert>();
}

然后你可以简单地反序列化：

var siteAlerts = JsonConvert.DeserializeObject<SiteAlerts>(json);
var alerts = siteAlerts.site.alerts; // no error-checking here

Answer 2

我建议您使用Newtonsoft.Json库来轻松反序列化json数据。

如果您想要部分反序列化，例如仅将alerts属性反序列化到类Alert中，而不创建所需类的整个结构。

您可以使用此代码：

JObject jObject = JObject.Parse(json);
var alerts = jObject["site"]["alerts"].ToObject<Alert[]>();
foreach(var item in alerts)
{
    Console.WriteLine("alert: " + item.alert);  
    Console.WriteLine("riskcode: " + item.riskcode);
}

完整的演示版here。

Answer 3

短版

var siteAlerts = JsonConvert.DeserializeObject<dynamic>(json).site.alerts.ToObject<Alert[]>();

如何反序列化json对象的特定键？

3 个答案: