Question

我有一个文件和数据集来生成矩阵（drug-drug），Dataset并不是只有9000x行那么大，但是我在使用算法库时面临着MemoryError。有没有一种方法（使用迭代器）在循环中运行样本，或者有任何方法可以解决此问题。

我尝试阅读文档，但并没有完全理解。任何帮助都是有价值的。

def getParamter(real_matrix, multiple_matrix, testPosition):
    creator.create("FitnessMax", base.Fitness, weights=(1.0,))
    creator.create("Individual", array.array, typecode='d',
                   fitness=creator.FitnessMax)
    toolbox = base.Toolbox()
    # Attribute generator
    toolbox.register("attr_float", random.uniform, 0, 1)
    # Structure initializers
    variable_num = len(multiple_matrix)
    toolbox.register("individual", tools.initRepeat,
                     creator.Individual, toolbox.attr_float, variable_num)
    toolbox.register("population", tools.initRepeat, list, toolbox.individual)
    #################################################################################################
    real_labels = []
    for i in range(0, len(testPosition)):
        real_labels.append(real_matrix[testPosition[i][0], testPosition[i][1]])

    multiple_prediction = []
    for i in range(0, len(multiple_matrix)):
        predicted_probability = []
        predict_matrix = multiple_matrix[i]
        for j in range(0, len(testPosition)):
            predicted_probability.append(
                predict_matrix[testPosition[j][0], testPosition[j][1]])
        normalize = MinMaxScaler()
        predicted_probability = np.array(predicted_probability).reshape(-1, 1)
        predicted_probability = normalize.fit_transform(predicted_probability)
        multiple_prediction.append(predicted_probability)

    #################################################################################################
    print(len(real_labels), len(multiple_prediction))
    # real_labels = real_labels[0:1000]
    toolbox.register("evaluate", fitFunction,
                     parameter1=real_labels, parameter2=multiple_prediction)
    toolbox.register("mate", tools.cxTwoPoint)
    toolbox.register("mutate", tools.mutFlipBit, indpb=0.05)
    toolbox.register("select", tools.selTournament, tournsize=3)

    random.seed(0)
    pop = toolbox.population(n=100)
    hof = tools.HallOfFame(1)
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("avg", numpy.mean)
    stats.register("std", numpy.std)
    stats.register("min", numpy.min)
    stats.register("max", numpy.max)

    # Below line is causing MemoryError
    pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=50,
                                   stats=stats, halloffame=hof, verbose=True)
    pop.sort(key=lambda ind: ind.fitness, reverse=True)
    print(pop[0])
    return pop[0]

Answer 1

我认为问题可能出在嵌套的for循环中：您应该避免使用for循环，而应该使用numpy广播。

我试图复制您在这里所做的事情：

multiple_prediction = []
    for i in range(0, len(multiple_matrix)):
        predicted_probability = []
        predict_matrix = multiple_matrix[i]
        for j in range(0, len(testPosition)):
            predicted_probability.append(
                predict_matrix[testPosition[j][0], testPosition[j][1]])
        normalize = MinMaxScaler()
        predicted_probability = np.array(predicted_probability).reshape(-1, 1)
        predicted_probability = normalize.fit_transform(predicted_probability)
        multiple_prediction.append(predicted_probability)

这就是我想出的（我可能会误判您的输入）：

import numpy as np
from sklearn.preprocessing import MinMaxScaler()

# dummy input
multiple_matrix = np.random.normal(size = (4,5,5))
testPosition = np.array([[1,2], [0,3], [3,1], [2,2]])

# scaler init
normalize = MinMaxScaler()

# the first dimension of multiple_matrix is the len(multiple_matrix)
x = multiple_matrix[:, testPosition[:,0], testPosition[:,1]]
multiple_prediction = normalize.fit_transform(x.T).T

也许有一种方法可以避免两个转置（.T），对此我并没有尝试太久。

该函数导致MemoryError，如何处理

1 个答案: