Question

我正在使用python 3.6

我试图在下面显示的类方法中使用名称为SubmitJobsUsingMultiProcessing（）的多处理，它进一步依次调用另一个类方法。

我一直遇到这个错误：输入错误：无法挑选_thread.RLock对象。

我不知道这意味着什么。我怀疑下面一行试图建立与网络服务器API的连接可能是负责任的，但我都在海上了解原因。

我不是一个合适的程序员（代码作为投资组合建模团队的一部分）所以如果这是一个明显的问题，请原谅我的无知，并提前多多感谢。

import multiprocessing as mp,functools

def SubmitJobsUsingMultiProcessing(self,PartitionsOfAnalysisDates,PickleTheJobIdsDict = True):
    if (self.ExportSetResult == "SUCCESS"):
        NumPools = mp.cpu_count()
        PoolObj =  mp.Pool(NumPools)   
        userId,clientId,password,expSetName = self.userId , self.clientId , self.password , self.expSetName
        PartialFunctor = functools.partial(self.SubmitJobsAsOfDate,userId = userId,clientId = clientId,password = password,expSetName = expSetName)
        Result = PoolObj.map(self.SubmitJobsAsOfDate, PartitionsOfAnalysisDates)
        BatchJobIDs = OrderedDict((key, val) for Dct in Result for key, val in Dct.items())
        f_pickle = open(self.JobIdPickleFileName, 'wb')
        pickle.dump(BatchJobIDs, f_pickle, -1)
        f_pickle.close()


 def SubmitJobsAsOfDate(self,ListOfDatesForBatchJobs,userId,clientId,password,expSetName):

    client = Client(self.url, proxy=self.proxysettings)
    if (self.ExportSetResult != "SUCCESS"):
        print("The export set creation was not successful...exiting")
        sys.exit()

    BatchJobIDs = OrderedDict()
    NumJobsSubmitted = 0
    CurrentProcessID = mp.current_process()

    for AnalysisDate in ListOfDatesForBatchJobs:
        jobName = "Foo_" + str(AnalysisDate)
        print('Sending job from process : ', CurrentProcessID, ' : ', jobName)
        jobId = client.service.SubmitExportJob(userId,clientId,password,expSetName, AnalysisDate, jobName, False)
        BatchJobIDs[AnalysisDate] = jobId
        NumJobsSubmitted += 1

        'Sleep for 30 secs every 100 jobs'
        if (NumJobsSubmitted % 100 == 0):
            print('100 jobs have been submitted thus far from process : ', CurrentProcessID,'---Sleeping for 30 secs to avoid the SSL time out error')
            time.sleep(30)
    self.BatchJobIDs = BatchJobIDs
    return BatchJobIDs

以下是trace ::

    Traceback (most recent call last):
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\pydevd.py", line 1599, in <module>
    globals = debugger.run(setup['file'], None, None, is_module)
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\pydevd.py", line 1026, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/trpff85/PycharmProjects/QuantEcon/BDTAPIMultiProcUsingPathos.py", line 289, in <module>
    BDTProcessObj.SubmitJobsUsingMultiProcessing(Partitions)
  File "C:/Users/trpff85/PycharmProjects/QuantEcon/BDTAPIMultiProcUsingPathos.py", line 190, in SubmitJobsUsingMultiProcessing
    Result = PoolObj.map(self.SubmitJobsAsOfDate, PartitionsOfAnalysisDates)
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 266, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 644, in get
    raise self._value
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 424, in _handle_tasks
    put(task)
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\connection.py", line 206, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
TypeError: can't pickle _thread.RLock objects

Answer 1

我正在努力解决类似的问题。＆lt; = 3.5中有一个错误，其中_thread.RLock对象在pickle时没有引发错误（它们不能）为了使Pool对象起作用，必须从主进程向它传递一个函数和参数，这依赖于酸洗（酸洗是序列化对象的一种方法）在我的例子中，RLock对象位于日志记录模块中的某个位置。我怀疑你的代码在3.5上运行正常。祝好运。见this bug resolution.

使用Web服务时，无法挑选_thread.RLock对象

1 个答案: