Question

我正在尝试使用IBM Cplex Python API解决大型二次编程问题。我在构造qMat矩阵时用尽了内存，该矩阵定义了具有120 GB RAM的Linux机器上问题的二次部分。

在下面的示例代码中，qMat有10行。在实际问题中，我大约有100,000行。每行有10-1000个非零列。机器在构建qMat时会用完内存，qMat是一个列表列表，每个列表都有两个列表（一个指示列，另一个指示值）。换句话说，机器甚至在将问题加载到Cplex之前就已耗尽内存。如果您可以提出另一种开发qMat的方法，或将此类数据加载到Cplex中，这将有所帮助。我当然可以将qMat逐行写入txt文件，但是可以将这样的文件加载到Cplex吗？

下面的代码如下的原因是，由于net固定，qMatIndexes只需要计算一次。但是qMat本身需要计算很多次，因为它取决于mDict。我基本上需要针对许多mDicts运行多次Cplex优化，稀疏Q矩阵为100,000乘以100,000维。 qMat代表的就是这个稀疏Q矩阵。

from __future__ import division


import copy
import numpy as np
import collections
import cplex
from cplex.exceptions import CplexError
import sys
import random
import cjson
from pympler.asizeof import asizeof
from scipy.sparse import csr_matrix

net = collections.OrderedDict([(1, [1, 2, 4]), (2, [2, 3, 4]), (3, [3, 4]), (4, [1, 4])])
mDict = {1:10,2:20,3:30,4:40}

def write_qMat_indexes(net):
    i_indiciesRow = []
    j_indiciesColumn = []

    count = 0
    for ID in net:
        sup = list(set(net[ID]))
        for s in sup:
            i_indiciesRow.append(count + 1)
            j_indiciesColumn.append(s)
        count += 1

    with open('i_indiciesRow.txt', 'w') as file_i_indiciesRow:
        file_i_indiciesRow.write(cjson.encode(i_indiciesRow))

    with open('j_indiciesColumn.txt', 'w') as file_j_indiciesColumn:
        file_j_indiciesColumn.write(cjson.encode(j_indiciesColumn))

def QMatrixDisk():
    with open('i_indiciesRow.txt', 'r') as file:
        for line in file:
            iN_list = cjson.decode(line)

    with open('j_indiciesColumn.txt', 'r') as file:
        for line in file:
            jN_list = cjson.decode(line)

    jN_array = np.array(jN_list)
    jN_locations = {}
    for j in jN_list:
        locations = np.where(jN_array == j)[0]
        jN_locations[j] = list(locations)
    del jN_array
    print "done finding locations"


    with open('qMatIndexes.txt', 'w') as file:
        for r in xrange(len(jN_list)):
            jN = jN_list[r]
            columns = jN_locations[jN]
            firstID_list = []
            columnsList = []
            for c in columns:
                q = iN_list[c]
                if c >= r:
                    firstID_list.append(q)
                    columnsList.append(c)
            Data = (columnsList, firstID_list)
            file.write(cjson.encode(Data) + '\n')




def create_qMat_Disk(mDict):
    with open("i_indiciesRow.txt", 'r') as file:
        for line in file:
            iN_list = cjson.decode(line)

    with open("j_indiciesColumn.txt", 'r') as file:
        for line in file:
            jN_list = cjson.decode(line)


    num_lines = sum(1 for line in open('qMatIndexes.txt'))

    qMat = [[[], []] for _ in range(num_lines)]
    count = 0
    with open('qMatIndexes.txt', 'r') as nfile:
        for line in nfile:
            cq = cjson.decode(line)
            columns = cq[0]
            q_list = cq[1]
            baseAgent = jN_list[count]
            firstAgent = iN_list[count]
            baseM = mDict[baseAgent]
            firstM = mDict[firstAgent]
            multiple = 2.0 * firstM / baseM
            second_mList = [mDict[q] for q in q_list]
            vals = [s * multiple for s in second_mList]
            qMat[count][0] += columns
            qMat[count][1] += vals

            for k in xrange(len(columns)):
                col = columns[k]
                newInds = []
                newVals = []
                if col > count:
                    val = vals[k]
                    newInds.append(count)
                    newVals.append(val)
                qMat[col][0] += newInds
                qMat[col][1] += newVals

            count += 1

    return qMat



write_qMat_indexes(net)
QMatrixDisk()
create_qMat_Disk(mDict)


C = [-6.0, -6.0, -6.0, -6.0, -6.0, -6.0, -4.0, -4.0, -4.0, -4.0]
my_ub = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
my_lb = [0.03333333333333333, 0.03333333333333333, 0.03333333333333333, 0.03333333333333333, 0.03333333333333333, 0.03333333333333333, 0.05, 0.05, 0.05, 0.05]
my_rhs = [1, 1, 1, 1, 1.6]
my_sense = 'EEEEL'


W_rows = [0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4]
W_columns = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 3, 6, 9]
W_values = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]

qMat = create_qMat_Disk(mDict)
print len(qMat), "len qMat"


my_prob = cplex.Cplex()
my_prob.parameters.qpmethod.set(4)
my_prob.parameters.lpmethod.set(4)
my_prob.objective.set_sense(my_prob.objective.sense.minimize)
my_prob.variables.add(obj=C, ub=my_ub, lb=my_lb)
my_prob.linear_constraints.add(rhs=my_rhs, senses=my_sense)
my_prob.linear_constraints.set_coefficients(zip(W_rows, W_columns, W_values))

my_prob.objective.set_quadratic(qMat)

my_prob.solve()

solutionValues = my_prob.solution.get_values()

使用Cplex Python API解决大型二次编程问题时内存不足

0 个答案: