我正在使用Oracle VirtualBox运行pyspark。
Python version:3.6.6
Spark Version:2.3.2
尝试在pyspark中使用getOrCreate()时,出现以下错误:
有趣的是,智能感知未显示getOrCreate()方法。
from pyspark.sql import SparkSession
spark=SparkSession.builder.appName('Basics').getOrCreate()
OSError Traceback (most recent call last)
<ipython-input-5-91db11549fb6> in <module>
----> 1 spark=SparkSession.builder.appName('Basics').getOrCreate()
~/spark-2.3.2-bin-hadoop2.7/python/pyspark/sql/session.py in getOrCreate(self)
171 for key, value in self._options.items():
172 sparkConf.set(key, value)
--> 173 sc = SparkContext.getOrCreate(sparkConf)
174 # This SparkContext may be an existing one.
175 for key, value in self._options.items():
~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in getOrCreate(cls, conf)
349 with SparkContext._lock:
350 if SparkContext._active_spark_context is None:
--> 351 SparkContext(conf=conf or SparkConf())
352 return SparkContext._active_spark_context
353
~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
113 """
114 self._callsite = first_spark_call() or CallSite(None, None, None)
--> 115 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
116 try:
117 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
~/spark-2.3.2-bin-hadoop2.7/python/pyspark/context.py in _ensure_initialized(cls, instance, gateway, conf)
298 with SparkContext._lock:
299 if not SparkContext._gateway:
--> 300 SparkContext._gateway = gateway or launch_gateway(conf)
301 SparkContext._jvm = SparkContext._gateway.jvm
302
~/spark-2.3.2-bin-hadoop2.7/python/pyspark/java_gateway.py in launch_gateway(conf)
81 def preexec_func():
82 signal.signal(signal.SIGINT, signal.SIG_IGN)
---> 83 proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env)
84 else:
85 # preexec_fn not supported on Windows
/usr/lib/python3.6/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)
707 c2pread, c2pwrite,
708 errread, errwrite,
--> 709 restore_signals, start_new_session)
710 except:
711 # Cleanup if the child failed starting.
/usr/lib/python3.6/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
1342 if errno_num == errno.ENOENT:
1343 err_msg += ': ' + repr(err_filename)
-> 1344 raise child_exception_type(errno_num, err_msg, err_filename)
1345 raise child_exception_type(err_msg)
1346
OSError: [Errno 8] Exec format error: '/home/ubuntu/spark-2.3.2-bin-hadoop2.7/./bin/spark-submit'
任何输入都会有帮助。 预先感谢。