线程“ JavaFX应用程序线程” Spark任务中的异常无法序列化:

时间:2019-11-26 23:20:25

标签: apache-spark javafx apache-spark-sql rdd

我正试图运行一个ML算法,该算法在它自己的主类中可以很好地工作,但是当我将其添加到JavaFX Main Class Application中时,它总是在此位置中断。最后要显示的是重新创建的RDD文本:然后显示以下错误消息:-有人可以帮忙吗?

//Change data frame back to RDD
    JavaRDD<Row> rdd1 = ccCleanedDf.toJavaRDD().repartition(2);

    System.out.println("recreated RDD " );

    //Function to map.
    JavaRDD<Row> rdd2 = rdd1.map( new Function<Row, Row>() {


        @Override // breaking in here!
        public Row call(Row iRow) throws Exception {

            System.out.println("recreated RDD1 " );
            //PR#06 - Round of age to range of 10
            Double age = Math.round(Double.valueOf(iRow.getString(5)) /10.0 ) * 10.0;

            //Normalize Sex to 1 or 2
            Double sex;
            switch( iRow.getString(2)) {
            case "M" : sex=1.0; break;
            case "F" : sex=2.0; break;
            default  : sex= Double.valueOf(iRow.getString(2));
            }

            //Find average billed amount
            double avgBillAmt = Math.abs(( Double.valueOf(iRow.getString(12)) +
                                    Double.valueOf(iRow.getString(13)) +
                                    Double.valueOf(iRow.getString(14)) +
                                    Double.valueOf(iRow.getString(15)) +
                                    Double.valueOf(iRow.getString(16)) +
                                    Double.valueOf(iRow.getString(17))
                                        ) / 6.0 );

            System.out.println("recreated RDD2" );

            //Find average pay amount
            double avgPayAmt = Math.abs(( Double.valueOf(iRow.getString(18)) +
                                    Double.valueOf(iRow.getString(19)) +
                                    Double.valueOf(iRow.getString(20)) +
                                    Double.valueOf(iRow.getString(21)) +
                                    Double.valueOf(iRow.getString(22)) +
                                    Double.valueOf(iRow.getString(23))
                                        ) / 6.0 );

            System.out.println("recreated RDD3 " );

            //Find average pay duration
            double avgPayDuration = Math.round(( Math.abs(Double.valueOf(iRow.getString(6))) +
                                    Math.abs(Double.valueOf(iRow.getString(7))) +
                                    Math.abs(Double.valueOf(iRow.getString(8))) +
                                    Math.abs(Double.valueOf(iRow.getString(9))) +
                                    Math.abs(Double.valueOf(iRow.getString(10))) +
                                    Math.abs(Double.valueOf(iRow.getString(11)))
                                        ) / 6.0);
            System.out.println("recreated RDD4 " );

            //Average percentage paid. add this as an additional field to see
            //if this field has any predictive capabilities. This is 
            //additional creative work that you do to see possibilities.                    
            double  perPay = Math.round((avgPayAmt/(avgBillAmt+1) * 100) / 25.0) * 25.0;

            Row retRow = RowFactory.create( Double.valueOf(iRow.getString(0)),
                                Double.valueOf(iRow.getString(1)),
                                sex,
                                Double.valueOf(iRow.getString(3)),
                                Double.valueOf(iRow.getString(4)),
                                age,
                                avgPayDuration,
                                avgBillAmt,
                                avgPayAmt,
                                perPay,
                                Double.valueOf(iRow.getString(24))
                    );

            return retRow;
        }

    });
Exception in thread "JavaFX Application Thread" java.security.PrivilegedActionException: org.apache.spark.SparkException: Task not serializable
    at java.security.AccessController.doPrivileged(Native Method)
    at com.sun.javafx.tk.quantum.GlassViewEventHandler.lambda$handleMouseEvent$2(GlassViewEventHandler.java:432)
    at com.sun.javafx.tk.quantum.QuantumToolkit.runWithoutRenderLock(QuantumToolkit.java:410)
    at com.sun.javafx.tk.quantum.GlassViewEventHandler.handleMouseEvent(GlassViewEventHandler.java:431)
    at com.sun.glass.ui.View.handleMouseEvent(View.java:555)
    at com.sun.glass.ui.View.notifyMouse(View.java:937)
    at com.sun.glass.ui.win.WinApplication._runLoop(Native Method)
    at com.sun.glass.ui.win.WinApplication.lambda$null$3(WinApplication.java:177)
    at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.SparkException: Task not serializable
    at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:298)
    at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:288)
    at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:108)
    at org.apache.spark.SparkContext.clean(SparkContext.scala:2037)
    at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:366)
    at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:365)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
    at org.apache.spark.rdd.RDD.map(RDD.scala:365)
    at org.apache.spark.api.java.JavaRDDLike$class.map(JavaRDDLike.scala:92)
    at org.apache.spark.api.java.AbstractJavaRDDLike.map(JavaRDDLike.scala:45)
    at com.v2maestros.spark.bda.apply.Main.CreditCardDefaulters(Main.java:483)
    at com.v2maestros.spark.bda.apply.Main$1$1.handle(Main.java:323)
    at com.v2maestros.spark.bda.apply.Main$1$1.handle(Main.java:1)
    at com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:86)
    at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:238)
    at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191)
    at com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74)
    at com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:49)
    at javafx.event.Event.fireEvent(Event.java:198)
    at javafx.scene.Node.fireEvent(Node.java:8411)
    at javafx.scene.control.Button.fire(Button.java:185)
    at com.sun.javafx.scene.control.behavior.ButtonBehavior.mouseReleased(ButtonBehavior.java:182)
    at com.sun.javafx.scene.control.skin.BehaviorSkinBase$1.handle(BehaviorSkinBase.java:96)
    at com.sun.javafx.scene.control.skin.BehaviorSkinBase$1.handle(BehaviorSkinBase.java:89)
    at com.sun.javafx.event.CompositeEventHandler$NormalEventHandlerRecord.handleBubblingEvent(CompositeEventHandler.java:218)
    at com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:80)
    at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:238)
    at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191)
    at com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)
    at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
    at com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74)
    at com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:54)
    at javafx.event.Event.fireEvent(Event.java:198)
    at javafx.scene.Scene$MouseHandler.process(Scene.java:3757)
    at javafx.scene.Scene$MouseHandler.access$1500(Scene.java:3485)
    at javafx.scene.Scene.impl_processMouseEvent(Scene.java:1762)
    at javafx.scene.Scene$ScenePeerListener.mouseEvent(Scene.java:2494)
    at com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:394)
    at com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:295)
    ... 9 more
Caused by: java.io.NotSerializableException: com.v2maestros.spark.bda.apply.Main
Serialization stack:
    - object not serializable (class: com.v2maestros.spark.bda.apply.Main, value: com.v2maestros.spark.bda.apply.Main@33a9b9ee)
    - field (class: com.v2maestros.spark.bda.apply.Main$2, name: this$0, type: class com.v2maestros.spark.bda.apply.Main)
    - object (class com.v2maestros.spark.bda.apply.Main$2, com.v2maestros.spark.bda.apply.Main$2@456a750a)
    - field (class: org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name: fun$1, type: interface org.apache.spark.api.java.function.Function)
    - object (class org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)
    at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
    at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
    at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
    at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:295)
    ... 61 more

0 个答案:

没有答案