MATLABS memmapfile的性能

时间：2015-10-01 14:12:48

标签： matlab bigdata memory-mapping

众所周知，内存映射可以是访问磁盘上极大数据量的优雅方式。 但是，我想知道在MATLAB中这个优势是否真的可用，或者是否有关于MATLAB的警告。

具体问题是用

间接访问数据之间的区别 
m = memmapfile( ... )           % m is a memmapfile object
z = m.Data.X;

或

m = memmapfile( ... )           % m is a memmapfile object  
d = m.Data;                     % d is a standard matlab struct
z = d.X;

是。我发现后者几乎快了两倍（见下文）。当我将m.Data分配给局部变量时，我仍然可以获得内存映射吗？它是一个标准的matlab结构。

另外，为什么在多次运行时访问m.Data和调用memmapfile的时间会大大增加？

我的计时剧本的ouptut：

memmapfile: 
    open  143.2ms,
    access 125MS Data, downsampled by 4:  454.0ms (561.4297      429.2862      427.1376      424.5907      427.3223)ms, std 60.1ms)
memmapfile: 
    open  99.9ms,
    access 125MS Data: 1609.9ms (1640.7666      1753.8268      1536.2547      1566.3313      1552.0821)ms, std 89.9ms)
memmapfile: 
    open  93.4ms,
    access random Data: 4394.5ms (4847.7417      4510.8818      4238.5909      4200.2644      4174.8181)ms, std 286.9ms)

memmapfile indirect: 
    open  83.9ms,
    access m.Data  2.9ms,
    access 125MS Data, downsampled by 4:  315.0ms (428.3787      445.2119      231.8119      235.2545        234.42)ms, std 111.3ms)
memmapfile indirect: 
    open  878.2ms,
    access m.Data  128.9ms,
    access 125MS Data: 817.3ms (862.5803      945.8177      750.9163      782.6967      744.5202)ms, std 85.8ms)
memmapfile: 
    open  925.5ms,
    access m.Data  134.3ms,
    access random Data: 4252.1ms (4326.7768      4238.1152      4228.8702      4258.6776      4208.0705)ms, std 45.5ms)

load: 
    open  1713.3ms,
    access 125MS Data, downsampled by 4:  252.6ms (269.144      242.3532      233.9538      247.1834      270.6117)ms, std 16.4ms)
load: 
    open  993.7ms,
    access 125MS Data: 776.0ms (737.53      810.9646      769.6092      792.9491      768.7254)ms, std 27.8ms)
load: 
    open  1029.1ms,
    access random Data: 4302.4ms (4320.5536      4314.7477      4320.0803      4287.7655      4268.7568)ms, std 23.1ms)

matfile: 
    open  253.8ms,
    access 125MS Data, downsampled by 4:  879.7ms (948.7214        867.83      861.8788      866.6201      853.6823)ms, std 39.0ms)
matfile: 
    open  7.5ms,
    access 125MS Data: 1431.3ms (1379.7382      1478.5318      1420.7367      1442.8253      1434.4649)ms, std 35.9ms)
matfile: access random Data
    UNSUPPORTED

我的时间码

SIZE = 250e6;
if 0
    X = randn(SIZE,1);
    savefast( 'test_memmapfile.mat', 'X' );
    X = [];
end

N = SIZE/2;

I = round( rand(N,1) * N )+1;



%
% memmapfile
%
t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', {'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data, downsampled by 4: ', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.Data.X(1:4:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', { 'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data:', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.Data.X(1:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', {'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess random Data:');
tt = [];
for ii=1:5
    t = tic;
    z = m.Data.X(I);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


%
% memmapfile indirect
%
fprintf('\n');
t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', {'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile indirect: \n\topen % 3.1fms,\n', toc(t)*1e3 );
t = tic;
d = m.Data;
fprintf( '\taccess m.Data % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data, downsampled by 4: ', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = d.X(1:4:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', {'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile indirect: \n\topen % 3.1fms,\n', toc(t)*1e3 );
t = tic;
d = m.Data;
fprintf( '\taccess m.Data % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data:', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = d.X(1:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = memmapfile( 'test_memmapfile.mat', 'Format', {'double', [SIZE, 1], 'X' }, 'Offset', 0, 'Writable', false ); % use dumy data file
fprintf( 'memmapfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
t = tic;
d = m.Data;
fprintf( '\taccess m.Data % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess random Data:');
tt = [];
for ii=1:5
    t = tic;
    z = d.X(I);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


%
% load
%
fprintf('\n');
t = tic;
m = load( 'test_memmapfile.mat' );
fprintf( 'load: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data, downsampled by 4: ', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.X(1:4:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = load( 'test_memmapfile.mat' );
fprintf( 'load: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data:', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.X(1:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = load( 'test_memmapfile.mat' );
fprintf( 'load: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess random Data:');
tt = [];
for ii=1:5
    t = tic;
    z = m.X(I);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


%
% matfile
%
fprintf('\n');
t = tic;
m = matfile( 'test_memmapfile.mat', 'Writable', false );
fprintf( 'matfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data, downsampled by 4: ', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.X(1:4:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


t = tic;
m = matfile( 'test_memmapfile.mat', 'Writable', false );
fprintf( 'matfile: \n\topen % 3.1fms,\n', toc(t)*1e3 );
fprintf('\taccess %dMS Data:', N/1e6);
tt = [];
for ii=1:5
    t = tic;
    z = m.X(1:N,:);
    tt(end+1) = toc(t);
end
fprintf( '% 3.1fms (%s)ms, std %0.1fms)\n', mean(tt)*1e3, num2str(tt*1e3), std(tt)*1e3 );


fprintf('matfile: access random Data\n');
fprintf('\tUNSUPPORTED\n');

0 个答案:

没有答案
相关问题
最新问题