Question

我制作的流程已经持续了好几个月了。该过程递归地压缩给定目录中的所有文件和文件夹，然后将zip文件上载到FTP服务器。它一直在工作，但现在，zip文件超过2GB并且错误输出。有人可以帮我弄清楚如何绕过这个2GB的限制吗？我评论了代码中的违规行。这是代码：

class Program
{
    // Location of upload directory
    private const string SourceFolder = @"C:\MyDirectory";
    // FTP server
    private const string FtpSite = "10.0.0.1";
    // FTP User Name
    private const string FtpUserName = "myUserName";
    // FTP Password
    private const string FtpPassword = "myPassword";

    static void Main(string[] args)
    {
        try
        {
            // Zip everything up using SharpZipLib
            string tmpFile = Path.GetTempFileName();
            var zip = new ZipOutputStream(File.Create(tmpFile));
            zip.SetLevel(8);
            ZipFolder(SourceFolder, SourceFolder, zip);
            zip.Finish();
            zip.Close();

            // Upload the zip file
            UploadFile(tmpFile);
            // Delete the zip file
            File.Delete(tmpFile);
        }
        catch (Exception ex)
        {
            throw ex;
        }
    }

    private static void UploadFile(string fileName)
    {
        string remoteFileName = "/ImagesUpload_" + DateTime.Now.ToString("MMddyyyyHHmmss") + ".zip";
        var request = (FtpWebRequest)WebRequest.Create("ftp://" + FtpSite + remoteFileName);

        request.Credentials = new NetworkCredential(FtpUserName, FtpPassword);
        request.Method = WebRequestMethods.Ftp.UploadFile;
        request.KeepAlive = false;
        request.Timeout = -1;
        request.UsePassive = true;
        request.UseBinary = true;

        // Error occurs in the next line!!!
        byte[] b = File.ReadAllBytes(fileName);
        using (Stream s = request.GetRequestStream())
        {
            s.Write(b, 0, b.Length);
        }

        using (var resp = (FtpWebResponse)request.GetResponse())
        {
        }
    }

    private static void ZipFolder(string rootFolder, string currentFolder, ZipOutputStream zStream)
    {

        string[] subFolders = Directory.GetDirectories(currentFolder);
        foreach (string folder in subFolders)
            ZipFolder(rootFolder, folder, zStream);

        string relativePath = currentFolder.Substring(rootFolder.Length) + "/";

        if (relativePath.Length > 1)
        {
            var dirEntry = new ZipEntry(relativePath) {DateTime = DateTime.Now};
        }
        foreach (string file in Directory.GetFiles(currentFolder))
        {
            AddFileToZip(zStream, relativePath, file);
        }
    }

    private static void AddFileToZip(ZipOutputStream zStream, string relativePath, string file)
    {
        var buffer = new byte[4096];
        var fi = new FileInfo(file);
        string fileRelativePath = (relativePath.Length > 1 ? relativePath : string.Empty) + Path.GetFileName(file);
        var entry = new ZipEntry(fileRelativePath) {DateTime = DateTime.Now, Size = fi.Length};
        zStream.PutNextEntry(entry);
        using (FileStream fs = File.OpenRead(file))
        {
            int sourceBytes;
            do
            {
                sourceBytes = fs.Read(buffer, 0, buffer.Length);
                zStream.Write(buffer, 0, sourceBytes);

            } while (sourceBytes > 0);
        }
    }
}

Answer 1

您正在尝试分配一个拥有超过20亿个元素的数组。 .NET限制数组的最大大小为System.Int32.MaxValue，即2Gb是上限。

你最好将这个文件分段阅读，然后将其上传;例如使用循环读数：

int buflen = 128 * 1024;
byte[] b = new byte[buflen];
FileStream source = new FileStream(fileName, FileMode.Open);
Stream dest = request.GetRequestStream();

while (true) {
    int bytesRead = source.Read(buf, 0, buflen);
    if (bytesRead == 0) break;
    dest.Write(buf, 0, bytesRead);
}

Answer 2

问题不在于zip，而是在File.ReadAllBytes调用中，它返回一个默认大小限制为2GB的数组。

可以按详细here禁用此限制。我假设您已经专门为64位编译这些文件以处理这些文件大小。启用此选项会将.NET切换为使用64位地址作为数组而不是默认的32位地址。

最好将存档拆分为多个部分并分别上传它们。据我所知，内置的ZipFile类不支持多部分存档，但有几个第三方库存在。

编辑：我在考虑生成的zip输出，而不是输入。要将大量数据加载到ZipFile中，您应该使用Petesh和philip建议的基于Buffer的方法。

打开大文件

2 个答案: