Question

    import org.apache.spark.{ SparkConf, SparkContext }
    import org.apache.spark.rdd.RDD

    class BaseType(val a: String) extends Serializable {
     override def toString = "(" + a + ")"
    }
    class TypeA(a: String, val b: String) extends BaseType(a) {
     override def toString = "(" + a + "," + b + ")"
    }
    class TypeB(a: String, val b: String) extends BaseType(a) {
     override def toString = "(" + a + "," + b + ")"
    }

   object EntityInheritance {
    def main(args: Array[String]) = {

    val sparkConf = new SparkConf()
      .setMaster("local[*]")
      .setAppName("EntityInheritance Sample")

    val sc = new SparkContext(sparkConf)

    val text_file = sc.textFile("/dqa/sample_logs/tipologies/entityInheritance.txt")
    val items = text_file.flatMap(_.split("\n"))

        val itemsRDDa = items.map(newInstanceA(_))
        itemsRDDa.foreach { rdd => println(rdd) }
        val countAa = countersAttributeA[TypeA](itemsRDDa)

        val itemsRDDb = items.map(newInstanceB(_))
    itemsRDDb.foreach { rdd => println(rdd) }
        val countBa = countersAttributeA[TypeB](itemsRDDb)

    sc.stop()
  }

    def newInstanceA(str: String): TypeA = {
      val parts = str.split(" ")
      new TypeA(parts(0), parts(1))
    }

    def newInstanceB(str: String): TypeB = {
        val parts = str.split(" ")
        new TypeB(parts(0), parts(1))
    }

    // I want to implement a generic function that receives RDD[TypeA] or RDD[TypeB]
    // it's a simple example
     def countersAttributeA[A](rdd: org.apache.spark.rdd.RDD[A]) = {
       rdd
        .map(s => (s.a, 1))
        .reduceByKey(_ + _)
     }
   }

你好，我有一个问题但可能这个想法并不好。

我尝试实现接收不同类型的通用函数。当创建不同的对象，例如TypeA和TypeB时，我想发送给counterAttributeA -> count number of apparences of attribute 'a'，但控制台发送此错误：

[error] /src/main/scala/org/sparklambda/testing/EntityInheritance.scala:53: value a is not a member of type parameter A
[error]       .map(s => (s.a, 1))
[error]                    ^
[error] one error found

任何人都可以帮助我吗？感谢所有人。

Answer 1

您需要告诉编译器该类型至少为BaseType，以便它知道它可以访问a属性。

def countersAttributeA[A <: BaseType](rdd: org.apache.spark.rdd.RDD[A])

如何实现接收不同类型scala的泛型函数

1 个答案: