OutOfMemoryException when send big file 500MB using FileStream ASPNET

Question

I\'m using Filestream for read big file (> 500 MB) and I get the OutOfMemoryException.

I use Asp.net , .net 3.5, win2003, iis 6.0

I want this in my app:

Answer 1

There are more than one solution

1- Using RecyclableMemoryStream instead of MemoryStream solution

You can read more about RecyclableMemoryStream here : http://www.philosophicalgeek.com/2015/02/06/announcing-microsoft-io-recycablememorystream/

https://github.com/Microsoft/Microsoft.IO.RecyclableMemoryStream

2- Using MemoryTributary instead of MemoryStream

You can read more about MemoryTributary here :

https://www.codeproject.com/Articles/348590/A-replacement-for-MemoryStream?msg=5257615#xx5257615xx

    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Runtime.InteropServices;

   namespace LiquidEngine.Tools
       {
/// <summary>
/// MemoryTributary is a re-implementation of MemoryStream that uses a dynamic list of byte arrays as a backing store, instead of a single byte array, the allocation
/// of which will fail for relatively small streams as it requires contiguous memory.
/// </summary>
public class MemoryTributary : Stream       /* http://msdn.microsoft.com/en-us/library/system.io.stream.aspx */
{
    #region Constructors

    public MemoryTributary()
    {
        Position = 0;
    }

    public MemoryTributary(byte[] source)
    {
        this.Write(source, 0, source.Length);
        Position = 0;
    }

    /* length is ignored because capacity has no meaning unless we implement an artifical limit */
    public MemoryTributary(int length)
    {
        SetLength(length);
        Position = length;
        byte[] d = block;   //access block to prompt the allocation of memory
        Position = 0;
    }

    #endregion

    #region Status Properties

    public override bool CanRead
    {
        get { return true; }
    }

    public override bool CanSeek
    {
        get { return true; }
    }

    public override bool CanWrite
    {
        get { return true; }
    }

    #endregion

    #region Public Properties

    public override long Length
    {
        get { return length; }
    }

    public override long Position { get; set; }

    #endregion

    #region Members

    protected long length = 0;

    protected long blockSize = 65536;

    protected List<byte[]> blocks = new List<byte[]>();

    #endregion

    #region Internal Properties

    /* Use these properties to gain access to the appropriate block of memory for the current Position */

    /// <summary>
    /// The block of memory currently addressed by Position
    /// </summary>
    protected byte[] block
    {
        get
        {
            while (blocks.Count <= blockId)
                blocks.Add(new byte[blockSize]);
            return blocks[(int)blockId];
        }
    }
    /// <summary>
    /// The id of the block currently addressed by Position
    /// </summary>
    protected long blockId
    {
        get { return Position / blockSize; }
    }
    /// <summary>
    /// The offset of the byte currently addressed by Position, into the block that contains it
    /// </summary>
    protected long blockOffset
    {
        get { return Position % blockSize; }
    }

    #endregion

    #region Public Stream Methods

    public override void Flush()
    {
    }

    public override int Read(byte[] buffer, int offset, int count)
    {
        long lcount = (long)count;

        if (lcount < 0)
        {
            throw new ArgumentOutOfRangeException("count", lcount, "Number of bytes to copy cannot be negative.");
        }

        long remaining = (length - Position);
        if (lcount > remaining)
            lcount = remaining;

        if (buffer == null)
        {
            throw new ArgumentNullException("buffer", "Buffer cannot be null.");
        }
        if (offset < 0)
        {
            throw new ArgumentOutOfRangeException("offset",offset,"Destination offset cannot be negative.");
        }

        int read = 0;
        long copysize = 0;
        do
        {
            copysize = Math.Min(lcount, (blockSize - blockOffset));
            Buffer.BlockCopy(block, (int)blockOffset, buffer, offset, (int)copysize);
            lcount -= copysize;
            offset += (int)copysize;

            read += (int)copysize;
            Position += copysize;

        } while (lcount > 0);

        return read;

    }

    public override long Seek(long offset, SeekOrigin origin)
    {
        switch (origin)
        {
            case SeekOrigin.Begin:
                Position = offset;
                break;
            case SeekOrigin.Current:
                Position += offset;
                break;
            case SeekOrigin.End:
                Position = Length - offset;
                break;
        }
        return Position;
    }

    public override void SetLength(long value)
    {
        length = value;
    }

    public override void Write(byte[] buffer, int offset, int count)
    {
        long initialPosition = Position;
        int copysize;
        try
        {
            do
            {
                copysize = Math.Min(count, (int)(blockSize - blockOffset));

                EnsureCapacity(Position + copysize);

                Buffer.BlockCopy(buffer, (int)offset, block, (int)blockOffset, copysize);
                count -= copysize;
                offset += copysize;

                Position += copysize;

            } while (count > 0);
        }
        catch (Exception e)
        {
            Position = initialPosition;
            throw e;
        }
    }

    public override int ReadByte()
    {
        if (Position >= length)
            return -1;

        byte b = block[blockOffset];
        Position++;

        return b;
    }

    public override void WriteByte(byte value)
    {
        EnsureCapacity(Position + 1);
        block[blockOffset] = value;
        Position++;
    }

    protected void EnsureCapacity(long intended_length)
    {
        if (intended_length > length)
            length = (intended_length);
    }

    #endregion

    #region IDispose

    /* http://msdn.microsoft.com/en-us/library/fs2xkftw.aspx */
    protected override void Dispose(bool disposing)
    {
        /* We do not currently use unmanaged resources */
        base.Dispose(disposing);
    }

    #endregion

    #region Public Additional Helper Methods

    /// <summary>
    /// Returns the entire content of the stream as a byte array. This is not safe because the call to new byte[] may 
    /// fail if the stream is large enough. Where possible use methods which operate on streams directly instead.
    /// </summary>
    /// <returns>A byte[] containing the current data in the stream</returns>
    public byte[] ToArray()
    {
        long firstposition = Position;
        Position = 0;
        byte[] destination = new byte[Length];
        Read(destination, 0, (int)Length);
        Position = firstposition;
        return destination;
    }

    /// <summary>
    /// Reads length bytes from source into the this instance at the current position.
    /// </summary>
    /// <param name="source">The stream containing the data to copy</param>
    /// <param name="length">The number of bytes to copy</param>
    public void ReadFrom(Stream source, long length)
    {
        byte[] buffer = new byte[4096];
        int read;
        do
        {
            read = source.Read(buffer, 0, (int)Math.Min(4096, length));
            length -= read;
            this.Write(buffer, 0, read);

        } while (length > 0);
    }

    /// <summary>
    /// Writes the entire stream into destination, regardless of Position, which remains unchanged.
    /// </summary>
    /// <param name="destination">The stream to write the content of this stream to</param>
    public void WriteTo(Stream destination)
    {
        long initialpos = Position;
        Position = 0;
        this.CopyTo(destination);
        Position = initialpos;
    }

    #endregion
}

}

Answer 2

I came across this question in my search to return a FileStreamResult from a controller, as I kept running into issues while working with large streams due to .Net trying to build the entire response all at once. Pavel Morshenyuk's answer was a huge help, but I figured I'd share the BufferedFileStreamResult that I ended up with.

/// <summary>Based upon https://stackoverflow.com/a/3363015/595473 </summary>
public class BufferedFileStreamResult : System.Web.Mvc.FileStreamResult
{
    public BufferedFileStreamResult(System.IO.Stream stream, string contentType, string fileDownloadName)
        : base(stream, contentType)
    {
        FileDownloadName = fileDownloadName;
    }

    public int BufferSize { get; set; } = 16 * 1024 * 1024;//--16MiB

    protected override void WriteFile(System.Web.HttpResponseBase response)
    {
        try
        {
            response.Clear();
            response.Headers.Set("Content-Disposition", $"attachment; filename={FileDownloadName}");
            response.Headers.Set("Content-Length", FileStream.Length.ToString());

            byte[] buffer;
            int bytesRead;

            while (response.IsClientConnected)//--Prevent infinite loop if user disconnects
            {
                buffer = new byte[BufferSize];

                //--Read the data in buffer
                if ((bytesRead = FileStream.Read(buffer, 0, BufferSize)) == 0)
                {
                    break;//--Stop writing if there's nothing left to write
                }

                //--Write the data to the current output stream
                response.OutputStream.Write(buffer, 0, bytesRead);

                //--Flush the data to the output
                response.Flush();
            }
        }
        finally
        {
            FileStream?.Close();
            response.Close();
        }
    }
}

Now, in my controller, I can just

return new BufferedFileStreamResult(stream, contentType, fileDownloadName);

Answer 3

You do not need to hold the whole file in memory just read it and write to the response stream in a loop.

Answer 4

I've created download page which allows user to download up to 4gb (may be more) few months ago. Here is my working snippet:

  private void TransmitFile(string fullPath, string outFileName)
    {
        System.IO.Stream iStream = null;

        // Buffer to read 10K bytes in chunk:
        byte[] buffer = new Byte[10000];

        // Length of the file:
        int length;

        // Total bytes to read:
        long dataToRead;

        // Identify the file to download including its path.
        string filepath = fullPath;

        // Identify the file name.
        string filename = System.IO.Path.GetFileName(filepath);

        try
        {
            // Open the file.
            iStream = new System.IO.FileStream(filepath, System.IO.FileMode.Open,
                        System.IO.FileAccess.Read, System.IO.FileShare.Read);


            // Total bytes to read:
            dataToRead = iStream.Length;

            Response.Clear();
            Response.ContentType = "application/octet-stream";
            Response.AddHeader("Content-Disposition", "attachment; filename=" + outFileName);
            Response.AddHeader("Content-Length", iStream.Length.ToString());

            // Read the bytes.
            while (dataToRead > 0)
            {
                // Verify that the client is connected.
                if (Response.IsClientConnected)
                {
                    // Read the data in buffer.
                    length = iStream.Read(buffer, 0, 10000);

                    // Write the data to the current output stream.
                    Response.OutputStream.Write(buffer, 0, length);

                    // Flush the data to the output.
                    Response.Flush();

                    buffer = new Byte[10000];
                    dataToRead = dataToRead - length;
                }
                else
                {
                    //prevent infinite loop if user disconnects
                    dataToRead = -1;
                }
            }
        }
        catch (Exception ex)
        {
            throw new ApplicationException(ex.Message);
        }
        finally
        {
            if (iStream != null)
            {
                //Close the file.
                iStream.Close();
            }
            Response.Close();
        }
    }