MultipartFormData文件上载内存不足异常

时间:2018-07-13 18:32:49

标签: c# .net multipartform-data

我正在使用以下代码上传文件:

https://gist.github.com/bgrins/1789787

但是,如果我尝试使用此代码上传文件“ 2 GB”文件,则会出现内存不足异常以及此行中的原因:

https://gist.github.com/bgrins/1789787#file-gistfile1-cs-L75

那我该如何解决这个问题?

2 个答案:

答案 0 :(得分:2)

逐个读取巨型文件,然后逐个上传。您也可以提供进度条。

  1. 逐段上传代码:How to read a big file piece by piece in C#
  2. 在服务器端,将新内容添加到文件:C# Append byte array to existing file

您可以使用此想法详细说明代码。我去年做过一次,但是不能共享代码。

答案 1 :(得分:1)

有不止一种解决方案

1-直接写入RequestStream而不是写入MemoryStream:

https://blogs.msdn.microsoft.com/johan/2006/11/15/are-you-getting-outofmemoryexceptions-when-uploading-large-files/

 public static string MyUploader(string strFileToUpload, string strUrl)
{


string strFileFormName = "file";


Uri oUri = new Uri(strUrl);


string strBoundary = "----------" + DateTime.Now.Ticks.ToString("x");





// The trailing boundary string


byte[] boundaryBytes = Encoding.ASCII.GetBytes("\r\n--" + strBoundary + "\r\n");





// The post message header


StringBuilder sb = new StringBuilder();


sb.Append("--");


sb.Append(strBoundary);


sb.Append("\r\n");


sb.Append("Content-Disposition: form-data; name=\"");


sb.Append(strFileFormName);


sb.Append("\"; filename=\"");


sb.Append(Path.GetFileName(strFileToUpload));


sb.Append("\"");


sb.Append("\r\n");


sb.Append("Content-Type: ");


sb.Append("application/octet-stream");


sb.Append("\r\n");


sb.Append("\r\n");


string strPostHeader = sb.ToString();


byte[] postHeaderBytes = Encoding.UTF8.GetBytes(strPostHeader);





// The WebRequest


HttpWebRequest oWebrequest = (HttpWebRequest)WebRequest.Create(oUri);


oWebrequest.ContentType = "multipart/form-data; boundary=" + strBoundary;


oWebrequest.Method = "POST";





// This is important, otherwise the whole file will be read to memory anyway...


oWebrequest.AllowWriteStreamBuffering = false;





// Get a FileStream and set the final properties of the WebRequest


FileStream oFileStream = new FileStream(strFileToUpload, FileMode.Open, FileAccess.Read);


long length = postHeaderBytes.Length + oFileStream.Length + boundaryBytes.Length;


oWebrequest.ContentLength = length;


Stream oRequestStream = oWebrequest.GetRequestStream();





// Write the post header


oRequestStream.Write(postHeaderBytes, 0, postHeaderBytes.Length);





// Stream the file contents in small pieces (4096 bytes, max).


byte[] buffer = new Byte[checked((uint)Math.Min(4096, (int)oFileStream.Length))];


int bytesRead = 0;


while ((bytesRead = oFileStream.Read(buffer, 0, buffer.Length)) != 0)


    oRequestStream.Write(buffer, 0, bytesRead);


oFileStream.Close();





// Add the trailing boundary


oRequestStream.Write(boundaryBytes, 0, boundaryBytes.Length);


WebResponse oWResponse = oWebrequest.GetResponse();


Stream s = oWResponse.GetResponseStream();


StreamReader sr = new StreamReader(s);


String sReturnString = sr.ReadToEnd();





// Clean up


oFileStream.Close();


oRequestStream.Close();


s.Close();


sr.Close();





return sReturnString;

}

2-使用RecyclableMemoryStream代替MemoryStream解决方案

您可以在此处阅读有关RecyclableMemoryStream的更多信息: http://www.philosophicalgeek.com/2015/02/06/announcing-microsoft-io-recycablememorystream/

https://github.com/Microsoft/Microsoft.IO.RecyclableMemoryStream

3-使用MemoryTributary代替MemoryStream

您可以在此处阅读有关MemoryTributary的更多信息:

https://www.codeproject.com/Articles/348590/A-replacement-for-MemoryStream?msg=5257615#xx5257615xx

    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Runtime.InteropServices;

   namespace LiquidEngine.Tools
       {
/// <summary>
/// MemoryTributary is a re-implementation of MemoryStream that uses a dynamic list of byte arrays as a backing store, instead of a single byte array, the allocation
/// of which will fail for relatively small streams as it requires contiguous memory.
/// </summary>
public class MemoryTributary : Stream       /* http://msdn.microsoft.com/en-us/library/system.io.stream.aspx */
{
    #region Constructors

    public MemoryTributary()
    {
        Position = 0;
    }

    public MemoryTributary(byte[] source)
    {
        this.Write(source, 0, source.Length);
        Position = 0;
    }

    /* length is ignored because capacity has no meaning unless we implement an artifical limit */
    public MemoryTributary(int length)
    {
        SetLength(length);
        Position = length;
        byte[] d = block;   //access block to prompt the allocation of memory
        Position = 0;
    }

    #endregion

    #region Status Properties

    public override bool CanRead
    {
        get { return true; }
    }

    public override bool CanSeek
    {
        get { return true; }
    }

    public override bool CanWrite
    {
        get { return true; }
    }

    #endregion

    #region Public Properties

    public override long Length
    {
        get { return length; }
    }

    public override long Position { get; set; }

    #endregion

    #region Members

    protected long length = 0;

    protected long blockSize = 65536;

    protected List<byte[]> blocks = new List<byte[]>();

    #endregion

    #region Internal Properties

    /* Use these properties to gain access to the appropriate block of memory for the current Position */

    /// <summary>
    /// The block of memory currently addressed by Position
    /// </summary>
    protected byte[] block
    {
        get
        {
            while (blocks.Count <= blockId)
                blocks.Add(new byte[blockSize]);
            return blocks[(int)blockId];
        }
    }
    /// <summary>
    /// The id of the block currently addressed by Position
    /// </summary>
    protected long blockId
    {
        get { return Position / blockSize; }
    }
    /// <summary>
    /// The offset of the byte currently addressed by Position, into the block that contains it
    /// </summary>
    protected long blockOffset
    {
        get { return Position % blockSize; }
    }

    #endregion

    #region Public Stream Methods

    public override void Flush()
    {
    }

    public override int Read(byte[] buffer, int offset, int count)
    {
        long lcount = (long)count;

        if (lcount < 0)
        {
            throw new ArgumentOutOfRangeException("count", lcount, "Number of bytes to copy cannot be negative.");
        }

        long remaining = (length - Position);
        if (lcount > remaining)
            lcount = remaining;

        if (buffer == null)
        {
            throw new ArgumentNullException("buffer", "Buffer cannot be null.");
        }
        if (offset < 0)
        {
            throw new ArgumentOutOfRangeException("offset",offset,"Destination offset cannot be negative.");
        }

        int read = 0;
        long copysize = 0;
        do
        {
            copysize = Math.Min(lcount, (blockSize - blockOffset));
            Buffer.BlockCopy(block, (int)blockOffset, buffer, offset, (int)copysize);
            lcount -= copysize;
            offset += (int)copysize;

            read += (int)copysize;
            Position += copysize;

        } while (lcount > 0);

        return read;

    }

    public override long Seek(long offset, SeekOrigin origin)
    {
        switch (origin)
        {
            case SeekOrigin.Begin:
                Position = offset;
                break;
            case SeekOrigin.Current:
                Position += offset;
                break;
            case SeekOrigin.End:
                Position = Length - offset;
                break;
        }
        return Position;
    }

    public override void SetLength(long value)
    {
        length = value;
    }

    public override void Write(byte[] buffer, int offset, int count)
    {
        long initialPosition = Position;
        int copysize;
        try
        {
            do
            {
                copysize = Math.Min(count, (int)(blockSize - blockOffset));

                EnsureCapacity(Position + copysize);

                Buffer.BlockCopy(buffer, (int)offset, block, (int)blockOffset, copysize);
                count -= copysize;
                offset += copysize;

                Position += copysize;

            } while (count > 0);
        }
        catch (Exception e)
        {
            Position = initialPosition;
            throw e;
        }
    }

    public override int ReadByte()
    {
        if (Position >= length)
            return -1;

        byte b = block[blockOffset];
        Position++;

        return b;
    }

    public override void WriteByte(byte value)
    {
        EnsureCapacity(Position + 1);
        block[blockOffset] = value;
        Position++;
    }

    protected void EnsureCapacity(long intended_length)
    {
        if (intended_length > length)
            length = (intended_length);
    }

    #endregion

    #region IDispose

    /* http://msdn.microsoft.com/en-us/library/fs2xkftw.aspx */
    protected override void Dispose(bool disposing)
    {
        /* We do not currently use unmanaged resources */
        base.Dispose(disposing);
    }

    #endregion

    #region Public Additional Helper Methods

    /// <summary>
    /// Returns the entire content of the stream as a byte array. This is not safe because the call to new byte[] may 
    /// fail if the stream is large enough. Where possible use methods which operate on streams directly instead.
    /// </summary>
    /// <returns>A byte[] containing the current data in the stream</returns>
    public byte[] ToArray()
    {
        long firstposition = Position;
        Position = 0;
        byte[] destination = new byte[Length];
        Read(destination, 0, (int)Length);
        Position = firstposition;
        return destination;
    }

    /// <summary>
    /// Reads length bytes from source into the this instance at the current position.
    /// </summary>
    /// <param name="source">The stream containing the data to copy</param>
    /// <param name="length">The number of bytes to copy</param>
    public void ReadFrom(Stream source, long length)
    {
        byte[] buffer = new byte[4096];
        int read;
        do
        {
            read = source.Read(buffer, 0, (int)Math.Min(4096, length));
            length -= read;
            this.Write(buffer, 0, read);

        } while (length > 0);
    }

    /// <summary>
    /// Writes the entire stream into destination, regardless of Position, which remains unchanged.
    /// </summary>
    /// <param name="destination">The stream to write the content of this stream to</param>
    public void WriteTo(Stream destination)
    {
        long initialpos = Position;
        Position = 0;
        this.CopyTo(destination);
        Position = initialpos;
    }

    #endregion
}
 }