使用pyspark中另一个数据框的列创建数据框

Question

我有一个由json文件使用以下模式创建的数据框

root
 |-- content: string (nullable = true)
 |-- conversationId: long (nullable = true)
 |-- date: string (nullable = true)
 |-- id: long (nullable = true)
 |-- lang: string (nullable = true)
 |-- likeCount: long (nullable = true)
 |-- media: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- duration: double (nullable = true)
 |    |    |-- fullUrl: string (nullable = true)
 |    |    |-- previewUrl: string (nullable = true)
 |    |    |-- thumbnailUrl: string (nullable = true)
 |    |    |-- type: string (nullable = true)
 |    |    |-- variants: array (nullable = true)
 |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |-- bitrate: long (nullable = true)
 |    |    |    |    |-- contentType: string (nullable = true)
 |    |    |    |    |-- url: string (nullable = true)

我要提取单独的数据框

• 没有媒体列的根数据框
• 没有变量列的媒体数据框
• 变体数据框

问题是我无法删除新variants数据框中的media列。

在media新数据框中，我不希望media列中的所有数据，我希望media对象成为其数据框中的新根，其字段为column ，并且与variants新数据框相同

这是我的代码：

tweets_df = spark.read.json(jsonFile)
media_df = tweets_df.select(col('media'))
variants_df = media_df.select(col('media.variants'))
# i tried also media_df = media_df.drop('media.variants')
media_df = media_df.drop('variants')
tweets_df = tweets_df.drop('media')

这是结果：

root
 |-- variants: array (nullable = true)
 |    |-- element: array (containsNull = true)
 |    |    |-- element: struct (containsNull = true)
 |    |    |    |-- bitrate: long (nullable = true)
 |    |    |    |-- contentType: string (nullable = true)
 |    |    |    |-- url: string (nullable = true)

root
 |-- media: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- duration: double (nullable = true)
 |    |    |-- fullUrl: string (nullable = true)
 |    |    |-- previewUrl: string (nullable = true)
 |    |    |-- thumbnailUrl: string (nullable = true)
 |    |    |-- type: string (nullable = true)   **IT'S NOT DELETED**
 |    |    |-- variants: array (nullable = true)
 |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |-- bitrate: long (nullable = true)
 |    |    |    |    |-- contentType: string (nullable = true)
 |    |    |    |    |-- url: string (nullable = true)

root
 |-- content: string (nullable = true)
 |-- conversationId: long (nullable = true)
 |-- date: string (nullable = true)
 |-- idOnSite: long (nullable = true)
 |-- lang: string (nullable = true)
 |-- likeCount: long (nullable = true)