我有一个Web应用程序，它将使用spark启动程序库在Cloudera spark集群上提交spark作业。
它已成功将spark作业提交到群集。但是，它不会回调监听器类方法，并且返回的SparkAppHandle上的 org.apache.catalina.LifecycleException: Failed to start component [/cnafc]
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:158)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:724)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:700)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:734)
at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:596)
at org.apache.catalina.startup.HostConfig.deployApps(HostConfig.java:467)
at org.apache.catalina.startup.HostConfig.check(HostConfig.java:1617)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.tomcat.util.modeler.BaseModelMBean.invoke(BaseModelMBean.java:300)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
at org.apache.catalina.manager.ManagerServlet.check(ManagerServlet.java:1483)
at org.apache.catalina.manager.ManagerServlet.deploy(ManagerServlet.java:904)
at org.apache.catalina.manager.HTMLManagerServlet.deployInternal(HTMLManagerServlet.java:316)
at org.apache.catalina.manager.HTMLManagerServlet.doPost(HTMLManagerServlet.java:208)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:648)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.catalina.filters.CsrfPreventionFilter.doFilter(CsrfPreventionFilter.java:136)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.catalina.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:108)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:108)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:632)
at org.apache.catalina.valves.RequestFilterValve.process(RequestFilterValve.java:319)
at org.apache.catalina.valves.RemoteAddrValve.invoke(RemoteAddrValve.java:88)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:349)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:1110)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:785)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1425)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.catalina.LifecycleException: Failed to process either the global, per-host or context-specific context.xml file therefore the [/cnafc] Context cannot be started.
at org.apache.catalina.startup.FailedContext.startInternal(FailedContext.java:199)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:152)
... 48 more
永远不会从＆＃34; UNKNOWN＆＃34;甚至在作业完成集群执行之后。
我正在使用getState()模式。这是我的代码。还有什么需要做的吗？
yarn-cluster以下是我已经尝试过的事情：
SparkLauncher launcher = new SparkLauncher()
.setSparkHome("sparkhome")
.setMaster("yarn-cluster")
.setAppResource("spark job jar file")
.setMainClass("spark job driver class")
.setAppName("appname")
.addAppArgs(argsArray)
.setVerbose(true)
.addSparkArg("--verbose");
SparkAppHandle handle = launcher.startApplication(new LauncherListener());
int c = 0;
while(!handle.getState().isFinal()) {
LOG.info(">>>>>>>> state is= "+handle.getState() );
LOG.info(">>>>>>>> state is not final yet. counter= "+c++ );
LOG.info(">>>>>>>> sleeping for a second");
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
}
if(c == 200)
break;
}
。SparkAppHandle并以两种方式传递它（SparkAppHandle.Listener）（启动时，并在this上设置）SparkAppHandle方法，以便至少我可以通过调用launcher.launch()方法阻止生成的Process对象，直到spark作业在群集上运行完毕。但是在这种情况下，对于长时间运行的火花作业，此节点上的相应过程永远不会返回（尽管它适用于1或2分钟内完成的火花作业）答案 0 :(得分：0)
我从spark用户邮件列表中得到了这个问题的答案。为了使这个功能起作用，不仅火花发射器需要1.6.0，而且底层火花至少应该是1.6.0。
我一直在使用spark 1.5.1和1.6.0版本的启动程序库。现在我已经将spark集群更新到1.6.0版本，现在我收到listener方法的回调。