WindowsError:[错误6]句柄无效创建spark会话时出错

时间:2017-05-09 01:53:51

标签: python windows python-2.7 apache-spark pyspark

我正在尝试使用以下代码安装spark会话:

#Initialize SparkSession and SparkContext
from pyspark.sql import SparkSession
from pyspark import SparkContext

#Create a Spark Session
SpSession = SparkSession \
    .builder \
    .master("local[2]") \
    .appName("V2 Maestros") \
    .config("spark.executor.memory", "1g") \
    .config("spark.cores.max","2") \
    .config("spark.sql.warehouse.dir", "file:///c:/temp/spark-warehouse") \
    .getOrCreate() 


#Get the Spark Context from Spark Session    
SpContext = SpSession.sparkContext

我收到以下错误:

SpSession = SparkSession \
    .builder \
    .master("local[2]") \
    .appName("V2 Maestros") \
    .config("spark.executor.memory", "1g") \
    .config("spark.cores.max","2") \
    .config("spark.sql.warehouse.dir", "file:///c:/temp/spark-warehouse") \
    .getOrCreate() 

Traceback (most recent call last):

  File "<ipython-input-17-caf81cda545e>", line 1, in <module>
    SpSession = SparkSession     .builder     .master("local[2]")     .appName("V2 Maestros")     .config("spark.executor.memory", "1g")     .config("spark.cores.max","2")     .config("spark.sql.warehouse.dir", "file:///c:/temp/spark-warehouse")     .getOrCreate()

  File "D:\Udemy - Spark\Module\spark-2.0.0-bin-hadoop2.7\python\lib\pyspark.zip\pyspark\sql\session.py", line 166, in getOrCreate
    sparkConf = SparkConf()

  File "D:\Udemy - Spark\Module\spark-2.0.0-bin-hadoop2.7\python\lib\pyspark.zip\pyspark\conf.py", line 104, in __init__
    SparkContext._ensure_initialized()

  File "D:\Udemy - Spark\Module\spark-2.0.0-bin-hadoop2.7\python\lib\pyspark.zip\pyspark\context.py", line 243, in _ensure_initialized
    SparkContext._gateway = gateway or launch_gateway()

  File "D:\Udemy - Spark\Module\spark-2.0.0-bin-hadoop2.7\python\lib\pyspark.zip\pyspark\java_gateway.py", line 79, in launch_gateway
    proc = Popen(command, stdin=PIPE, env=env)

  File "C:\Users\Rahul\Anaconda2\lib\subprocess.py", line 382, in __init__
    errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr)

  File "C:\Users\Rahul\Anaconda2\lib\subprocess.py", line 532, in _get_handles
    c2pwrite = self._make_inheritable(c2pwrite)

  File "C:\Users\Rahul\Anaconda2\lib\subprocess.py", line 566, in _make_inheritable
    _subprocess.DUPLICATE_SAME_ACCESS)

WindowsError: [Error 6] The handle is invalid

以下是subprocess.py文件的相关部分:

if mswindows:
        #
        # Windows methods
        #
        def _get_handles(self, stdin, stdout, stderr):
            """Construct and return tuple with IO objects:
            p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
            """
            to_close = set()
            if stdin is None and stdout is None and stderr is None:
                return (None, None, None, None, None, None), to_close

            p2cread, p2cwrite = None, None
            c2pread, c2pwrite = None, None
            errread, errwrite = None, None

            if stdin is None:
                p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                if p2cread is None:
                    p2cread, _ = _subprocess.CreatePipe(None, 0)
            elif stdin == PIPE:
                p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
            elif isinstance(stdin, int):
                p2cread = msvcrt.get_osfhandle(stdin)
            else:
                # Assuming file-like object
                p2cread = msvcrt.get_osfhandle(stdin.fileno())
            p2cread = self._make_inheritable(p2cread)
            # We just duplicated the handle, it has to be closed at the end
            to_close.add(p2cread)
            if stdin == PIPE:
                to_close.add(p2cwrite)

            if stdout is None:
                c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                if c2pwrite is None:
                    _, c2pwrite = _subprocess.CreatePipe(None, 0)
            elif stdout == PIPE:
                c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
            elif isinstance(stdout, int):
                c2pwrite = msvcrt.get_osfhandle(stdout)
            else:
                # Assuming file-like object
                c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
            c2pwrite = self._make_inheritable(c2pwrite)
            # We just duplicated the handle, it has to be closed at the end
            to_close.add(c2pwrite)
            if stdout == PIPE:
                to_close.add(c2pread)

            if stderr is None:
                errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                if errwrite is None:
                    _, errwrite = _subprocess.CreatePipe(None, 0)
            elif stderr == PIPE:
                errread, errwrite = _subprocess.CreatePipe(None, 0)
            elif stderr == STDOUT:
                errwrite = c2pwrite
            elif isinstance(stderr, int):
                errwrite = msvcrt.get_osfhandle(stderr)
            else:
                # Assuming file-like object
                errwrite = msvcrt.get_osfhandle(stderr.fileno())
            errwrite = self._make_inheritable(errwrite)
            # We just duplicated the handle, it has to be closed at the end
            to_close.add(errwrite)
            if stderr == PIPE:
                to_close.add(errread)

            return (p2cread, p2cwrite,
                    c2pread, c2pwrite,
                    errread, errwrite), to_close


        def _make_inheritable(self, handle):
            """Return a duplicate of handle, which is inheritable"""
            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                handle, _subprocess.GetCurrentProcess(), 0, 1,
                                _subprocess.DUPLICATE_SAME_ACCESS)


        def _find_w9xpopen(self):
            """Find and return absolut path to w9xpopen.exe"""
            w9xpopen = os.path.join(
                            os.path.dirname(_subprocess.GetModuleFileName(0)),
                                    "w9xpopen.exe")
            if not os.path.exists(w9xpopen):
                # Eeek - file-not-found - possibly an embedding
                # situation - see if we can locate it in sys.exec_prefix
                w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
                                        "w9xpopen.exe")
                if not os.path.exists(w9xpopen):
                    raise RuntimeError("Cannot locate w9xpopen.exe, which is "
                                       "needed for Popen to work with your "
                                       "shell or platform.")
            return w9xpopen

请帮忙。新的火花。我正在使用64位系统,运行python 2.7。

1 个答案:

答案 0 :(得分:0)

我有同样的问题。对于将来阅读此内容的任何人,我能够通过从桌面快捷方式启动Spyder IDE来解决此问题,而不是从Anaconda命令提示符(Windows 7 64位)执行spyder