胶水和Python

Question

我试图使用需要安装gremlinpython库的Python从AWS Neptune数据库中提取数据。在将作业设置为pythonshell作业后可以工作。但是，无论如何，我可以使用Pyspark Job来使用功能吗？

import boto3
import os
import sys
import site
import json
from setuptools.command import easy_install
from importlib import reload

s3 = boto3.client('s3')
dir_path = os.path.dirname(os.path.realpath(__file__))
#os.path.dirname(sys.modules['__main__'].__file__)

install_path = os.environ['GLUE_INSTALLATION']
easy_install.main( ["--install-dir", install_path, "gremlinpython"] )

reload(site)

from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection

graph = Graph()

remoteConn = DriverRemoteConnection('wss://neptune-test-data-read-1.uiokd9wka3.us-central-1.neptune.amazonaws.com:8182/gremlin','g')
g = graph.traversal().withRemote(remoteConn)

edges = g.E().toList()

ed = []
for e in edges:
    ed.append(str(e))
    
vertices = g.V().valueMap(True).toList()
ve = []
for v in vertices:
    vee = []
    for vt in v:
        if isinstance(v[vt], list):
            vee.append(v[vt][0])
        else:
            vee.append(v[vt])
    ve.append(vee)
        

remoteConn.close()

s3 = boto3.resource('s3')
object = s3.Object('s3bucket', 'target/edges.txt')
object.put(Body=(bytes(json.dumps(ed, indent=2).encode('UTF-8'))))

s3 = boto3.resource('s3')
object = s3.Object('s3bucket', 'target/vertices.txt')
object.put(Body=(bytes(json.dumps(ve, indent=2).encode('UTF-8'))))

谢谢