Question

我有一些大的二进制文件，我读入结构，然后将这些结构添加到列表中，以便我可以在以后循环它们。一切正常，除了读取结构比预期慢。这是相关的代码：

//128 bytes total
[StructLayout(LayoutKind.Sequential, Pack = 1, CharSet = CharSet.Ansi)]
public struct RAMPrec2
{
    public double now;                 // Days in double precision (see note 1)
    public int VehLatitude;            // Milliarcseconds (see note 2)
    public int VehLongitude;           // Milliarcseconds (see note 2)
    public short VehLatUncertainty;    // Meters as 16-bit int
    public short VehLonUncertainty;    // Meters as 16-bit int
    public short Reserved1;      // Meters as 16-bit int (see note 3)
    public short Reserved2;      // Meters as 16-bit int
    public float VehAltitude;          // Meters as a float
    public float VehFirstRet;          // Meters as a float
    public float VehDepth;             // Meters as a float
    public float VehSpeed;             // Knots as a float
    public float VehHeading;           // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehCourse;            // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehRoll;              // degrees as a float (positive is counterclockwise roll)
    public float VehPitch;             // degrees as a float (negative is downward pitch)
    public float VehVOS;               // Meters/sec
    public int DisplayNorthing;        // Centimeters as 32-bit integer
    public int DisplayingEasting;      // Centimeters as 32-bit integer
    public int OriginalLatitude;       // Milliarcseconds as 32-bit integer
    public int OriginalLongitude;      // Milliarcseconds as 32-bit integer
    public int DeltaNorthing;          // Centimeters as 32-bit integer
    public int DeltaEasting;           // Centimeters as 32-bit integer
    public short FixFlags;             // 16 bit flags. (note 5)
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] // 8 sets of 32-bit values,sensor specific (note 3)
    public int[] SData;
    public ushort DataAvailFlags;      // 16 bit mask field (note 4)
    public ushort QA_Flag;             // 16 bit mask field (note 6)
    public short EventFlag;            // 2-byte reserved field
    public float Reserved3;            // 4-byte reserved field
}

public Boolean ReadRampFileType2(List<string> rampPaths) //rampPaths is just a list of filepaths for each binary file to be read
{
    for (int i = 0; i < rampPaths.Count; i++)
    {
        try
        {
            using (var stream = new FileStream(rampPaths[i], FileMode.Open, FileAccess.Read, FileShare.None)) //open up a stream on the specified file
            {
                stream.Position = 8192; //skip header

                while (stream.Position < (stream.Length)) //while not at end of file
                {
                    RAMPrec2 ramp = ReadRecFromStream<RAMPrec2>(stream, Marshal.SizeOf(typeof(RAMPrec2))); //read in each record to the ramp struct
                    AddRecDataToListsType2(ramp, vehicles[i]);
                }
            }
        }
        catch (Exception e) //something went wrong
        {
            return false;
        }
    }
    return true;
}

private T ReadRecFromStream<T>(Stream stream, int size) where T : struct
{
    byte[] buffer = new byte[Marshal.SizeOf(typeof(T))];
    stream.Read(buffer, 0, size);
    GCHandle handle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
    try
    {
        return (T)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(T));
    }
    finally
    {
        handle.Free();
    }
}

//I have a vehicle class with a bunch of lists. This adds the data from the structs I read in to these lists
public void AddRecDataToListsType2(RAMPrec2 ramp, VehicleModel vehicle)
{
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

我循环遍历我所拥有的二进制文件列表并在每个文件流上打开一个文件流，跳到第一个'struct'出现在二进制文件中的位置，然后将其读入一个struct，然后它有一个字段添加到一堆列表，然后我重复，直到文件结束。这很慢，我觉得必须有更好的做事方式。

Answer 1

这是内存映射的理想选择。

您可以在数据文件上创建映射视图，并像访问内存一样访问结构，而不是读取内存中的结构。

虽然我没有在C＃中使用内存映射，但在不安全模式下，您可以直接在映射视图上使用RAMPrec2*并使用该指针循环遍历记录/结构，就像在C ++中一样。有关示例，请参阅here。

您甚至可以消除单独列表的创建，因为可以通过映射视图直接访问所有字段。

Answer 2

试试这个......

// this would be how to call...

foreach (var item in ReadFile<RAMPrec2>("testFile.bin", 0).Select((v, i) => new { Value = v, Index = i }))
{
    var vehicle = vehicles[item.Index];
    var ramp = item.Value;
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

这就是魔术发生的地方。

static IEnumerable<T> ReadFile<T>(string fileName, int offset) where T : struct
{
    using (var reader = File.OpenRead(fileName))
    {
        var sizeOf = Marshal.SizeOf(typeof(T));
        var ptr = Marshal.AllocHGlobal(sizeOf);
        try
        {
            reader.Position = offset;
            var fileLength = reader.Length + reader.Position;
            var buffer = new byte[sizeOf];
            for (var p = reader.Position; p < fileLength; p += sizeOf)
            {
                reader.Read(buffer, 0, sizeOf);
                Marshal.Copy(buffer, 0, ptr, sizeOf);
                var ret = Marshal.PtrToStructure(ptr, typeof(T));
                var str = (T)ret;
                yield return str;
            }
        }
        finally
        {
            Marshal.FreeHGlobal(ptr);
        }
    }
}

如何提高将大型二进制文件读入结构的性能？

2 个答案: