我有一个数据数组double [8,2000000]（4行和2百万列），我想每秒保存到磁盘上的文件。格式为文本或BIN。我试图使用streamwriter写文件，但它需要超过1秒。 File.write不支持双格式，因此我必须从double转换为其他格式。而且，它需要超过1秒。 BinaryWriter与streamwriter的速度相同。 最重要的问题是节省时间必须少于1秒。
你有什么解决方案可以帮助我吗？谢谢！
StreamWriter WriteFile = new StreamWriter(Folder_name + @"RecordFile\" + fileName + ".txt", true);
//record data
for (int i = 0; i < 2000000; i++)
{
int j = 0;
for (j = 0; j < 8; j++)
{
WriteFile.Write(dataWrite[j, i]);
WriteFile.Write("\t");
}
WriteFile.WriteLine(dataWrite[j, i]);
}
WriteFile.Close(); //finish record
答案 0 :(得分：4)
对于这些基准测试，我使用了一个比例，即FileSteam缓冲区大小确定是否存在任何显着差异，我以Release (64Bit)模式运行每个测试10次并取平均值.NET Framework 4.7.1我使用随机缓冲区new double[8, 2000000];来生成测试
Mode : Release (64Bit)
Test Framework : .NET Framework 4.7.1
Benchmarks Runs : 10 times (averaged)
Scale : 4,096, Test Data : Standard input
Value | Average | Fastest | StDv | Cycles | Pass | Gain |
------------------------------------------------------------------------------------------------
WriteFile Unsafe | 384.122 ms | 275.073 ms | 58.90 | 1,317,292,586 | Pass | 42.84 % |
BlockCopy | 389.389 ms | 305.094 ms | 57.68 | 1,335,451,612 | Pass | 42.05 % |
WriteFile Pinned | 422.704 ms | 341.646 ms | 67.66 | 1,418,871,963 | Pass | 37.09 % |
BinaryWriter | 671.966 ms | 608.900 ms | 58.63 | 2,260,807,206 | Base | 0.00 % |
BitConverter | 784.722 ms | 668.788 ms | 139.98 | 2,607,901,414 | Pass | -16.78 % |
Scale : 32,768, Test Data : Standard input
Value | Average | Fastest | StDv | Cycles | Pass | Gain |
----------------------------------------------------------------------------------------------
WriteFile Unsafe | 97.254 ms | 88.318 ms | 5.38 | 339,330,780 | Pass | 83.49 % |
WriteFile Pinned | 110.047 ms | 90.279 ms | 18.80 | 346,777,096 | Pass | 81.32 % |
BlockCopy | 115.805 ms | 106.119 ms | 7.40 | 403,209,891 | Pass | 80.34 % |
BinaryWriter | 589.168 ms | 530.255 ms | 60.64 | 1,985,585,629 | Base | 0.00 % |
BitConverter | 593.952 ms | 506.482 ms | 73.93 | 1,983,475,740 | Pass | -0.81 % |
Scale : 102,400, Test Data : Standard input
Value | Average | Fastest | StDv | Cycles | Pass | Gain |
----------------------------------------------------------------------------------------------
WriteFile Unsafe | 73.071 ms | 69.885 ms | 1.77 | 255,008,411 | Pass | 85.95 % |
WriteFile Pinned | 73.523 ms | 71.073 ms | 1.98 | 256,062,331 | Pass | 85.86 % |
BlockCopy | 82.068 ms | 78.838 ms | 1.79 | 286,872,838 | Pass | 84.22 % |
BinaryWriter | 519.943 ms | 471.578 ms | 46.01 | 1,778,713,946 | Base | 0.00 % |
BitConverter | 559.842 ms | 497.743 ms | 39.83 | 1,946,616,118 | Pass | -7.67 % |
Scale : 1,048,576, Test Data : Standard input
Value | Average | Fastest | StDv | Cycles | Pass | Gain |
-----------------------------------------------------------------------------------------------
WriteFile Pinned | 59.993 ms | 56.088 ms | 1.73 | 209,025,613 | Pass | 87.46 % |
WriteFile Unsafe | 61.783 ms | 56.266 ms | 8.09 | 206,988,059 | Pass | 87.08 % |
BlockCopy | 64.105 ms | 61.066 ms | 1.52 | 224,205,049 | Pass | 86.60 % |
BinaryWriter | 478.376 ms | 442.570 ms | 34.63 | 1,671,203,569 | Base | 0.00 % |
BitConverter | 550.557 ms | 493.186 ms | 42.27 | 1,916,031,041 | Pass | -15.09 % |
<强> BlockCopy
private static void Write(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
var totalSize = h * w * sizeof(double);
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, chunkSize))
{
var buffer = new byte[chunkSize];
for (var i = 0; i < totalSize; i += chunkSize)
{
var size = Math.Min(chunkSize, totalSize - i);
Buffer.BlockCopy(ary, i, buffer, 0, size);
fs.Write(buffer, 0, size);
}
}
}
<强>的BinaryWriter
private static void Write(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, chunkSize))
using (var bw = new BinaryWriter(fs))
for (var i = 0; i < h; i++)
for (var j = 0; j < w; j++)
bw.Write(ary[i, j]);
}
WriteFile固定
private static unsafe void Write(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
var totalSize = h * w;
var s = chunkSize / sizeof(double);
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, chunkSize))
{
var handle = default(GCHandle);
try
{
handle = GCHandle.Alloc(ary, GCHandleType.Pinned);
var p = (long*)handle.AddrOfPinnedObject()
.ToPointer();
var fileHandle = fs.SafeFileHandle.DangerousGetHandle();
for (var i = 0; i < totalSize; i += s)
{
var size = Math.Min(s, totalSize - i);
var p2 = p + i;
Kernel32.WriteFile(fileHandle, (IntPtr)p2, size * sizeof(double), out var n, IntPtr.Zero);
}
}
finally
{
if (handle.IsAllocated)
{
handle.Free();
}
}
}
}
WriteFile固定不安全
private static unsafe void Write(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
var totalSize = h * w;
var s = chunkSize / sizeof(double);
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, chunkSize))
{
var fileHandle = fs.SafeFileHandle.DangerousGetHandle();
fixed (double* p = ary)
{
for (var i = 0; i < totalSize; i += s)
{
var size = Math.Min(s, totalSize - i);
var p2 = p + i;
Kernel32.WriteFile(fileHandle, (IntPtr)p2, size * sizeof(double), out var n, IntPtr.Zero);
}
}
}
}
<强> BitConverter
这只使用了文件流和BitConverter.GetBytes
private static void Write(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, chunkSize))
for (var i = 0; i < h; i++)
for (var j = 0; j < w; j++)
fs.Write(BitConverter.GetBytes(ary[i, j]), 0, 8);
}
这是非常繁琐的测试和正确，但所有的解决方案都经过测试，他们连续编写整个数组以double的原始字节文件。
起初xanatos的使用固定阵列的版本似乎很慢，这里没有显示，我花了一些时间来弄清楚究竟发生了什么。事实证明，将整个数组直接写入文件似乎是最慢的。这可能是因为刷新不会逐渐发生并且在文件关闭时发生，我不确定但它可能正在尝试一次性写入（我的怀疑）。
然而，当我调整它来编写块时，它证明是最一致的。再一次，这真的很难测试，我们正在反对各种缓存，这是不容易克服的，最后我不得不编写单独的文件，但似乎操作系统正在缓存结果。
如果您想要阅读数据，可以使用以下
private static void Read(double[,] ary, int chunkSize, string fileName)
{
var h = ary.GetLength(0);
var w = ary.GetLength(1);
var totalSize = h * w * sizeof(double);
using (var fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.None, chunkSize))
{
var buffer = new byte[chunkSize];
for (var i = 0; i < totalSize; i += chunkSize)
{
var size = Math.Min(chunkSize, totalSize - i);
fs.Read(buffer, 0, size);
Buffer.BlockCopy(buffer,0,ary , i, size);
}
}
}