如何使用Spark将嵌套的json对象字段读入Scala案例类

时间：2018-01-21 15:41:06

标签： json apache-spark twitter spark-dataframe

我有一个推文json文件，其结构如下所示。 （这是我的推文文件中只有一条推文的样本）。我需要将它作为带有spark的JSON读入，并使用下面的scala代码将其转换为case类。我需要读取嵌套的json文件的特定字段。我特别想阅读＆＃34; hashtags array＆＃34;来自＆＃34; enttities＆＃34;它嵌套在推文结构中。但是，我还没有找到有效的解决方案。爆炸＆＃34;实体＆＃34;给了我错误：

*Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve 'explode(`entities`)' due to data type mismatch: input to function explode should be array or map type, not StructType(StructField(hashtags,ArrayType(StructType(StructField(indices,ArrayType(LongType,true),true), StructField(text,StringType,true)),true),true), StructField(media,ArrayType(StructType(StructField(display_url,StringType,true), StructField(expanded_url,StringType,true), StructField(id,LongType,true), StructField(id_str,StringType,true), StructField(indices,ArrayType(LongType,true),true), StructField(media_url,StringType,true), StructField(media_url_https,StringType,true), StructField(sizes,StructType(StructField(large,StructType(StructField(h,LongType,true), StructField(resize,StringType,true), StructField(w,LongType,true)),true), StructField(medium,StructType(StructField(h,LongType,true), StructField(resize,StringType,true), StructField(w,LongType,true)),true), StructField(small,StructType(StructField(h,LongType,true), StructField(resize,StringType,true), StructField(w,LongType,true)),true), StructField(thumb,StructType(StructField(h,LongType,true), StructField(resize,StringType,true), StructField(w,LongType,true)),true)),true), StructField(source_status_id,LongType,true), StructField(source_status_id_str,StringType,true), StructField(source_user_id,LongType,true), StructField(source_user_id_str,StringType,true), StructField(type,StringType,true), StructField(url,StringType,true)),true),true), StructField(symbols,ArrayType(StringType,true),true), StructField(urls,ArrayType(StructType(StructField(display_url,StringType,true), StructField(expanded_url,StringType,true), StructField(indices,ArrayType(LongType,true),true), StructField(url,StringType,true)),true),true), StructField(user_mentions,ArrayType(StructType(StructField(id,LongType,true), StructField(id_str,StringType,true), StructField(indices,ArrayType(LongType,true),true), StructField(name,StringType,true), StructField(screen_name,StringType,true)),true),true));*

Twitter JSON文件

{"created_at":"Mon Sep 04 12:34:09 +0000 2017","id":904684005001269248,"id_str":"904684005001269248","text":"\u63a8\u3057\u304c\u53f3\u624b\u3067\u30dd\u30fc\u30ba\u3092\u53d6\u308a\u304c\u3061\u3002 https:\/\/t.co\/bNmQSC2Xog","display_text_range":[0,15],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":2930557759,"id_str":"2930557759","name":"\ud83d\ude08\u3042\u3086\u2693\ufe0f","screen_name":"CR7_AYU","location":"\u6e05\u6d41","url":"http:\/\/ameblo.jp\/shuka-saito\/","description":"\u9022\u7530\u68a8\u9999\u5b50\u3055\u3093\u3001\u6589\u85e4\u6731\u590f\u3055\u3093\u3001\u5c0f\u6797\u611b\u9999\u3055\u3093\u3092\u5fdc\u63f4\u3057\u3066\u3044\u307e\u3059\u3002","translator_type":"none","protected":false,"verified":false,"followers_count":564,"friends_count":644,"listed_count":54,"favourites_count":142433,"statuses_count":138352,"created_at":"Mon Dec 15 05:22:02 +0000 2014","utc_offset":32400,"time_zone":"Tokyo","geo_enabled":false,"lang":"ja","contributors_enabled":false,"is_translator":false,"profile_background_color":"00BFFF","profile_background_image_url":"http:\/\/pbs.twimg.com\/profile_background_images\/605381334823890944\/qdEfh3qD.jpg","profile_background_image_url_https":"https:\/\/pbs.twimg.com\/profile_background_images\/605381334823890944\/qdEfh3qD.jpg","profile_background_tile":true,"profile_link_color":"4D5AAF","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"000000","profile_text_color":"000000","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/858259320038768640\/7tqZv7WS_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/858259320038768640\/7tqZv7WS_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/2930557759\/1503961622","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":904683986949070848,"id_str":"904683986949070848","indices":[16,39],"media_url":"http:\/\/pbs.twimg.com\/media\/DI4VSvwVYAAQxsd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/DI4VSvwVYAAQxsd.jpg","url":"https:\/\/t.co\/bNmQSC2Xog","display_url":"pic.twitter.com\/bNmQSC2Xog","expanded_url":"https:\/\/twitter.com\/CR7_AYU\/status\/904684005001269248\/photo\/1","type":"photo","sizes":{"medium":{"w":1200,"h":1200,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":680,"resize":"fit"},"large":{"w":2048,"h":2048,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":904683986949070848,"id_str":"904683986949070848","indices":[16,39],"media_url":"http:\/\/pbs.twimg.com\/media\/DI4VSvwVYAAQxsd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/DI4VSvwVYAAQxsd.jpg","url":"https:\/\/t.co\/bNmQSC2Xog","display_url":"pic.twitter.com\/bNmQSC2Xog","expanded_url":"https:\/\/twitter.com\/CR7_AYU\/status\/904684005001269248\/photo\/1","type":"photo","sizes":{"medium":{"w":1200,"h":1200,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":680,"resize":"fit"},"large":{"w":2048,"h":2048,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ja","timestamp_ms":"1504528449665"}

基本上我想得到几个嵌套字段并且爆炸似乎对我不起作用。我错了什么？

Scala代码

import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.{HashPartitioner, SparkConf, SparkContext}
import org.apache.spark.sql.functions._


object TwitterAnalytics {
  def main(args:Array[String]): Unit= {
    val conf = new SparkConf()
    conf.setAppName("TwitterAnalytics")
    conf.setMaster("local[2]")
    val sc = new SparkContext(conf)
    println(sc)
    val spark = SparkSession
      .builder()
      .appName("Spark SQL basic example")
      .config("spark.some.config.option", "some-value")
      .getOrCreate()


    val df= spark.read.json("/home/gakuo/Downloads/TwitterAnalytics/tweets")
    println(df.select(explode(df("entities"))))
    println(df.select(explode(df("retweeted_status"))))


    type Tag = String
    type Likes = Int
    case class Tweet(id: BigInt,
                     text: String,
                     hashTags: Array[Tag],
                     likes: Likes)
    def parseTweet(tweet: DataFrame): Tweet = ???    
  }
}

1 个答案:

答案 0 :(得分：0)

重新定义（并移至外部范围）：

type Tag = String
type Likes = Long   // Integer doesn't have required precision
case class Tweet(id: Long,
                 text: String,
                 hashTags: Array[Tag],
                 likes: Likes)

导入含义：

import spark.implicits._

和select与.（假设您希望favorite_count为likes）：

val Dataset[Tweet] = df
  .select($"id", $"text", $"entities.hashtags", $"favorite_count" as "likes")
  .as[Tweet]
相关问题
最新问题