OSError:[Errno 8]执行格式错误:'/home/ubuntu/spark-2.3.2-bin-hadoop2.7/./bin/spark-submit'

时间:2018-11-01 05:39:33

标签: apache-spark pyspark pyspark-sql

我正在使用Oracle VirtualBox运行pyspark。

Python version:3.6.6
Spark Version:2.3.2

尝试在pyspark中使用getOrCreate()时,出现以下错误:

有趣的是,智能感知未显示getOrCreate()方法。

from pyspark.sql import SparkSession 
spark=SparkSession.builder.appName('Basics').getOrCreate()

错误:

OSError                                   Traceback (most recent call last)
<ipython-input-5-91db11549fb6> in <module>
----> 1 spark=SparkSession.builder.appName('Basics').getOrCreate()

~/spark-2.3.2-bin-hadoop2.7/python/pyspark/sql/session.py in getOrCreate(self)
    171                     for key, value in self._options.items():
    172                         sparkConf.set(key, value)
--> 173                     sc = SparkContext.getOrCreate(sparkConf)
    174                     # This SparkContext may be an existing one.
    175                     for key, value in self._options.items():

~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in getOrCreate(cls, conf)
    349         with SparkContext._lock:
    350             if SparkContext._active_spark_context is None:
--> 351                 SparkContext(conf=conf or SparkConf())
    352             return SparkContext._active_spark_context
    353 

~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
    113         """
    114         self._callsite = first_spark_call() or CallSite(None, None, None)
--> 115         SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
    116         try:
    117             self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,

~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in _ensure_initialized(cls, instance, gateway, conf)
    298         with SparkContext._lock:
    299             if not SparkContext._gateway:
--> 300                 SparkContext._gateway = gateway or launch_gateway(conf)
    301                 SparkContext._jvm = SparkContext._gateway.jvm
    302 

~/spark-2.3.2-bin-hadoop2.7/python/pyspark/java_gateway.py in launch_gateway(conf)
     81                 def preexec_func():
     82                     signal.signal(signal.SIGINT, signal.SIG_IGN)
---> 83                 proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env)
     84             else:
     85                 # preexec_fn not supported on Windows

/usr/lib/python3.6/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)
    707                                 c2pread, c2pwrite,
    708                                 errread, errwrite,
--> 709                                 restore_signals, start_new_session)
    710         except:
    711             # Cleanup if the child failed starting.

/usr/lib/python3.6/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1342                         if errno_num == errno.ENOENT:
   1343                             err_msg += ': ' + repr(err_filename)
-> 1344                     raise child_exception_type(errno_num, err_msg, err_filename)
   1345                 raise child_exception_type(err_msg)
   1346 

    OSError: [Errno 8] Exec format error: '/home/ubuntu/spark-2.3.2-bin-hadoop2.7/./bin/spark-submit'

任何输入都会有帮助。 预先感谢。

0 个答案:

没有答案