Question

我正在使用Dask进行大于内存的矩阵乘法。如果我将内存限制为128MB，则在执行1000x1000矩阵点时程序将被杀死。内存中的1000x1000矩阵点完美运行。谁能给我一些提示我做错了什么？非常感谢你！

import dask as dk
import tables
import numpy as np
import sys
from time import time
import math
import h5py

outpath = "/data/dottest/"
lenx = int(sys.argv[1])
leny = int(sys.argv[2])
numpart =int( math.sqrt(int(sys.argv[3])))

fname = "t{0:03d}_{1:03d}.h5".format(int(lenx),int(leny))

def writefile():
    h5f = h5py.File(fname, 'w')

    pres = np.random.random((lenx,leny))
    h5f.create_dataset('pressure',shape=pres.shape,chunks=(leny/numpart,lenx/numpart),dtype='f8',data=pres)

    lines = np.random.random((leny,lenx))
    h5f.create_dataset('lines',shape=lines.shape,chunks=(lenx/numpart,leny/numpart),dtype='f8',data=lines)

    h5f.close()

def dottest():
    import h5py
    import dask.array as da
    h5open2 = h5py.File(outpath+fname)

    print('numpart = ',numpart)

    d1=da.from_array(h5open2["/pressure"],chunks=(leny/numpart,lenx/numpart))
    d2=da.from_array(h5open2["/lines"],chunks=(leny/numpart,lenx/numpart))
    print d1 
    print d2

    t = d1.dot(d2)
    s = t.compute()

    print("dot finished")
    h5open2.close()

writefile()
dottest()

$ echo 134217728> /sys/fs/cgroup/memory/mygroup/memory.limit_in_bytes

$ cgexec -g cpu，内存：mygroup python inmemory.py 1000 1000

0.0245869159698

$ cgexec -g cpu，内存：mygroup python newdasktest.py 1000 1000 4

dask.array'array，shape =（1000，1000），dtype = float64，chunksize =（500，500）'

被杀死

Dask API，用于节省矩阵点的内存

0 个答案: