Question

我跟随＃34;高级分析与Spark＆＃34;的第4章。来自O＆＃39; Reilly。这本书是在Scala中，我将此代码转换为Python。

Scala代码

import org.apache.spark.mllib.linalg._
import org.apache.spark.mllib.regression._
val rawData = sc.textFile("hdfs:///user/ds/covtype.data")
val data = rawData.map { line =>
    val values = line.split(',').map(_.toDouble)
    val featureVector = Vectors.dense(values.init)
    val label = values.last - 1
    LabeledPoint(label, featureVector)
}
val Array(trainData, cvData, testData) =
  data.randomSplit(Array(0.8, 0.1, 0.1))
trainData.cache()
cvData.cache()
testData.cache()


import org.apache.spark.mllib.evaluation._
import org.apache.spark.mllib.tree._
import org.apache.spark.mllib.tree.model._
import org.apache.spark.rdd._

def getMetrics(model: DecisionTreeModel, data: RDD[LabeledPoint]):
    MulticlassMetrics = {
 val predictionsAndLabels = data.map(example =>
    (model.predict(example.features), example.label)
 )
 new MulticlassMetrics(predictionsAndLabels)
}
val model = DecisionTree.trainClassifier(
 trainData, 7, Map[Int,Int](), "gini", 4, 100)

val metrics = getMetrics(model, cvData) 
metrics.confusionMatrix

我的Python代码

from pyspark.sql.functions import col, split
import pyspark.mllib.linalg as linal
import pyspark.mllib.regression as regre
import pyspark.mllib.evaluation as eva
import pyspark.mllib.tree as tree
import pyspark.rdd as rd

raw_data = sc.textFile("covtype.data")

def fstDecisionTree(line):
    values = list(map(float,line.split(",")))
    featureVector = linal.Vectors.dense(values[:-1])
    label = values[-1]-1
    ret=regre.LabeledPoint(label, featureVector)
    return regre.LabeledPoint(label, featureVector) 

data = raw_data.map(fstDecisionTree)
trainData,cvData,testData=data.randomSplit([0.8,0.1,0.1])
trainData.cache()
cvData.cache()
testData.cache()

def help_lam(model):
 def _help_lam(dataline):
    print(dataline)
    a=dataline.collect()
    return (model.predict(a[1]),a[0])
return _help_lam

def getMetrics(model, data):
    print(type(model),type(data))
    predictionsAndLabels= data.map(help_lam(model))
    return eva.MulticlassMetrics(predictionsAndLabels)

n_targets=7
max_depth=4
max_bin_count=100
model = tree.DecisionTree.trainClassifier(trainData, n_targets, {}, "gini", max_depth, max_bin_count)

metrics=getMetrics(model,cvData)

当我运行此操作时，当我尝试隐式传递地图迭代时，我在def _help_lam(dataline)内的方法def help_lam(model)中出现此错误：

AttributeError: 'Py4JError' object has no attribute 'message'

Answer 1

我认为问题出在model.predict函数

中

来自pyspark mllib/tree.py

注意：在Python中，目前无法在RDD中使用预测转型或行动。直接在RDD上调用预测。

你可以做的是直接传递特征向量

>>> rdd = sc.parallelize([[1.0], [0.0]])
>>> model.predict(rdd).collect()
[1.0, 0.0]

编辑：

getMetrics的更新可能是：

def getMetrics(model, data):
    labels = data.map(lambda d: d.label)
    features = data.map(lambda d: d.features)
    predictions = model.predict(features)
    predictionsAndLabels = predictions.zip(labels)
    return eva.MulticlassMetrics(predictionsAndLabels)

错误＆＃34;属性错误：＆＃39; Py4JError＆＃39;对象没有属性＆＃39; message＆＃39;构建DecisionTreeModel

1 个答案: