我正在尝试使用 pathos 框架并行化python计算工作流程。计算工作流程本身很复杂,使用了许多模块,如GDAL,rasterIO,numpy等。
当我尝试使用pathos map 执行计算工作流程时,出现以下错误:
TypeError:无法腌制SwigPyObject对象
以下是我为重现错误而制作的简化代码:
Launcher.py
from cpp import *
import pathos.pools as pp
print "#####Lancement de la routine de mise à jour d'Irrisat#####"
today= datetime.datetime.now().date().strftime("%d%m")
currentYear= datetime.datetime.now().date().strftime("%Y")
pfolders= ['Data\\param1',\
'Data\\param2']
p = pp.ProcessPool(4)
cpp= ParametersCalculator(('localhost', 'db', "postgres", "pass"))
#cpp.run(pfolders)
p.map(cpp.run, pfolders)
cpp.py
import numpy as np
import sys ,os, datetime, json, psycopg2, math, StringIO
import rasterio, math, time
import rasterio.mask
from osgeo.gdalconst import *
from osgeo import gdal, osr, ogr
np.seterr(divide='warn', invalid='ignore')
class ParametersCalculator:
def __init__(self, con_params):
self.driver=gdal.GetDriverByName('GTIff')
self.driver.Register()
databaseServer = con_params[0]
databaseName = con_params[1]
databaseUser = con_params[2]
databasePW = con_params[3]
self.tags= {"param1":("in1",), "param2":("in1","in2")}
def preProcessValidation (self, pfolder):
tif_files= [x for x in os.listdir(pfolder) if (x.endswith(".tif") or x.endswith(".tiff"))]
param_name= pfolder.split('\\')[-1]
tif_tags=[(param_tag, os.path.join(pfolder, tif)) for param_tag in self.tags[param_name] for tif in tif_files if param_tag.lower() in tif.lower()]
print 1
def run(self, pfolders):
#for pfolder in pfolders:
#print pfolder
#self.preProcessValidation(pfolder)
self.preProcessValidation(pfolder)