Linux机器中的多处理池挂起以处理大数据

时间:2020-03-16 07:09:50

标签: python multiprocessing

import multiprocessing as mp
pool=mp.pool(4)
def PrimitiveElasticGraphEmbedment(X, NodePositions, ElasticMatrix,
                                   MaxNumberOfIterations=10, eps=0.01,
                                   Mode = 1, FinalEnergy = "Base",
                                   alpha = 0,
                                   beta = 0,
                                   prob = 1,
                                   DisplayWarnings = True,
                                   PointWeights=None, MaxBlockSize=100000000,
                                   verbose=False, TrimmingRadius=float('inf'),
                                   SquaredX=None):                
    N = X.shape[0]

    if PointWeights is None:
        PointWeights = np.ones((N, 1))

    SpringLaplacianMatrix = ComputeSpringLaplacianMatrix(ElasticMatrix)    
    if SquaredX is None:
        SquaredX = (X**2).sum(axis=1).reshape((N, 1))
    partition, dists = PartitionData(X, NodePositions, MaxBlockSize,
                                             SquaredX, TrimmingRadius)
    if verbose or Mode == 2:
        OldElasticEnergy, MSE, EP, RP = ComputePrimitiveGraphElasticEnergy(
                NodePositions, ElasticMatrix, dists)    
    ElasticEnergy = 0
    for i in range(MaxNumberOfIterations):
        NewNodePositions = FitGraph2DataGivenPartition(
                X, PointWeights, SpringLaplacianMatrix, partition)
        if verbose or Mode == 2:
            ElasticEnergy, MSE, EP, RP = ComputePrimitiveGraphElasticEnergy(
                    NewNodePositions, ElasticMatrix, dists)
        if Mode == 1:
            diff = ComputeRelativeChangeOfNodePositions(
                    NodePositions, NewNodePositions)
        elif Mode == 2:
            diff = (OldElasticEnergy - ElasticEnergy)/ElasticEnergy

        if verbose:
            print("Iteration ", (i+1), " difference of node position=", np.around(diff,5),
              ", Energy=", np.around(ElasticEnergy,5), ", MSE=", np.around(MSE,5), ", EP=", np.around(EP,5),
             ", RP=", np.around(RP,5))

        if not np.isfinite(diff):
            diff=0

        if diff < eps:
            break

        elif i < MaxNumberOfIterations-1:
            partition, dists = PartitionData(X, NewNodePositions, MaxBlockSize,
                                             SquaredX, TrimmingRadius)
            NodePositions = NewNodePositions
            OldElasticEnergy = ElasticEnergy


    if DisplayWarnings and not(diff < eps):
        print("Maximum number of iterations (", MaxNumberOfIterations,
              ") has been reached. diff = ", diff)

    if (FinalEnergy != "Base") or (not(verbose) and (Mode != 2)):
        if FinalEnergy == "Base":
            ElasticEnergy, MSE, EP, RP = ComputePrimitiveGraphElasticEnergy(
                            NewNodePositions, ElasticMatrix, dists)

        elif FinalEnergy == "Penalized":
            ElasticEnergy, MSE, EP, RP = ComputePenalizedPrimitiveGraphElasticEnergy(
                            NewNodePositions, ElasticMatrix, dists,alpha,beta)

    EmbeddedNodePositions = NewNodePositions
    return EmbeddedNodePositions, ElasticEnergy, partition, dists, MSE, EP, RP



def proxy(Dict):
    return PrimitiveElasticGraphEmbedment(**Dict)
    pool = pool)


def ApplyOptimalGraphGrammarOperation(X,
                                     NodePositions,
                                     ElasticMatrix,
                                     opTypes,
                                     AdjustVect = None,
                                     SquaredX = None,
                                     verbose = False,
                                     MaxBlockSize = 100000000,
                                     MaxNumberOfIterations = 100,
                                     eps = .01,
                                     TrimmingRadius = float('inf'),
                                     Mode = 1,
                                     FinalEnergy = "Base",
                                     alpha = 1,
                                     beta = 1,
                                     EmbPointProb = 1,
                                     AvoidSolitary = False,
                                     AdjustElasticMatrix = None,
                                     DisplayWarnings = True,
                                     n_cores = 1,
                                     MinParOp = 20,
                                     multiproc_shared_variables = None,
                                     pool = None):

    NodePositionsArrayAll = []
    ElasticMatricesAll = []
    AdjustVectAll = []


    if(n_cores > 1 and len(Valid_configurations) // (MinParOp + 1) > 1):
            results=pool.map(proxy,[dict(X=X,
                                       NodePositions = NodePositionsArrayAll[i],
                                       ElasticMatrix = ElasticMatricesAll[i], 
                                       MaxNumberOfIterations = MaxNumberOfIterations,eps=eps,
                                       Mode=Mode, FinalEnergy=FinalEnergy,
                                       alpha=alpha,beta=beta,prob=EmbPointProb,
                                       DisplayWarnings=DisplayWarnings,
                                       PointWeights=None,
                                       MaxBlockSize=MaxBlockSize,
                                       verbose=False,
                                       TrimmingRadius=TrimmingRadius, SquaredX=SquaredX) for i in Valid_configurations])

            list_energies = [r[1] for r in results]
            idx = list_energies.index(min(list_energies))
            NewNodePositions, minEnergy, partition, Dist, MSE, EP, RP = results[idx]
            AdjustVect = AdjustVectAll[idx]
            NewElasticMatrix = ElasticMatricesAll[idx]
    else:
        minEnergy = np.inf

        for i in Valid_configurations:    
            nodep, ElasticEnergy, part, dist, mse, ep, rp = PrimitiveElasticGraphEmbedment(X,
                                                   NodePositionsArrayAll[i],
                                                   ElasticMatricesAll[i], MaxNumberOfIterations, eps,
                                                   Mode=Mode, FinalEnergy=FinalEnergy,
                                                   alpha=alpha,beta=beta,prob=EmbPointProb,
                                                   DisplayWarnings=DisplayWarnings,
                                                   PointWeights=None,
                                                   MaxBlockSize=MaxBlockSize,
                                                   verbose=False,
                                                   TrimmingRadius=TrimmingRadius, SquaredX=SquaredX)

            if(ElasticEnergy < minEnergy):
                NewNodePositions = nodep
                NewElasticMatrix = ElasticMatricesAll[i]
                partition = part
                AdjustVect = AdjustVectAll[i]
                minEnergy = ElasticEnergy
                MSE = mse
                EP = ep
                RP = rp
                Dist = dist

    return dict(NodePositions = NewNodePositions, ElasticMatrix = NewElasticMatrix, 
                ElasticEnergy = minEnergy, MSE = MSE, EP = EP, RP = RP, AdjustVect = AdjustVect, Dist = Dist)
if __name=="__main__":
    while (UpdatedPG['NodePositions'].shape[0] < NumNodes) or GrammarOptimization:
        UpdatedPG = ApplyOptimalGraphGrammarOperation(X,
                                                           UpdatedPG['NodePositions'], 
                                                                  UpdatedPG['ElasticMatrix'],
                                                                  GrowGrammars[k],
                                                                  MaxBlockSize = MaxBlockSize,
                                                                  AdjustVect = UpdatedPG['AdjustVect'],
                                                                  SquaredX = SquaredX,
                                                                  verbose = False,
                                                                  MaxNumberOfIterations = MaxNumberOfIterations,
                                                                  eps = eps,
                                                                  TrimmingRadius = TrimmingRadius,
                                                                  Mode = Mode,
                                                                  FinalEnergy = FinalEnergy,
                                                                  alpha = alpha,
                                                                  beta = beta,
                                                                  EmbPointProb = EmbPointProb,
                                                                  AvoidSolitary = AvoidSolitary,
                                                                  AdjustElasticMatrix = AdjustElasticMatrix,
                                                                  DisplayWarnings = DisplayWarnings,
                                                                  n_cores=n_cores, MinParOp = MinParOp)

上面的代码适用于具有1000个数据点的小数据;当使用1000个以上的数据点时,它在Linux计算机中挂起而在Windows计算机中运行良好。我不知道如何进一步调试问题。想法/帮助进一步解决这个问题。我相信这段代码在Linux机器中与fork有关。任何想法都值得赞赏。

有关总代码链接https://github.com/j-bac/elpigraph-python/blob/master/elpigraph/src/grammar_operations.py

0 个答案:

没有答案