Pyspark：Dstream选择列

Question

我有一个来自kafka的Dstream，我想从中选择列。以下是我正在实施的代码，但它给了我错误。

import os
import findspark
findspark.init('/home/gsingh/anaconda2/lib/python2.7/site-packages/pyspark/')
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
from kafka import KafkaProducer
from pyspark.sql.types import Row
from pyspark.sql import SQLContext
from pyspark import sql
import json
import requests


def result(y):
    return y


def handler(rdd):
    records = rdd.collect()

    for record in records:
        producer.send('opportunityProcessed', bytes(record))
        producer.flush()


if __name__ == '__main__':

    findspark.add_packages("org.apache.spark:spark-streaming-kafka-0-10_2.11:2.3.0")
    producer = KafkaProducer(bootstrap_servers='localhost:9092')

    # os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-streaming-kafka-0-10_2.11:2.3.0 pyspark-shell'
    sc = SparkContext(appName="PythonSparkStreamingKafka")
    sc.setLogLevel("WARN")
    ssc = StreamingContext(sc, 1)
    sqlContext = sql.SQLContext(sc)

    print('ssc =================== {} {}')
    kafkaStream = KafkaUtils.createStream(ssc, 'localhost:2181', "spark-streaming-opportunity", {'testing': 1})
    print('contexts =================== {} {}')

    lines = kafkaStream.map(lambda x: x[1])

    lines.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()

现在它正在从kafka读取，然后写入另一个主题。我的目标是从DStream中选择列，然后对其进行一些操作。所以我的ptoblem是，如何从DStream中选择列？