Question

我正在尝试通过C＃中的http实现文件的并行下载。我尝试了几种不同的方法，但它们似乎都没有正常工作。无论我做什么，下载最终都会排队，并没有以真正的并行方式工作。

有人可以给我一些指示或链接到描述实际工作方法的文章吗？

Answer 1

我刚写了一些代码，没有测试它，等待一些观察，谢谢大家：

public class DownloadFile
{
    public string Url { get; set; }

    public string PathToSave { get; set; }
}


public class ParallelDownloading
    {
        private ConcurrentQueue<DownloadFile> _queueToDownlaod;
        private IList<Task> _downloadingTasks;
        private Timer _downloadTimer;

        private int _parallelDownloads;

        public ParallelDownloading(int parallelDownloads)
        {
            _queueToDownlaod = new ConcurrentQueue<DownloadFile>();
            _downloadingTasks = new List<Task>();
            _downloadTimer = new Timer();

            _parallelDownloads = parallelDownloads;

            _downloadTimer.Elapsed += new ElapsedEventHandler(DownloadTimer_Elapsed);
            _downloadTimer.Interval = 1000;
            _downloadTimer.Start();

            ServicePointManager.DefaultConnectionLimit = parallelDownloads;
        }

        public void EnqueueFileToDownload(DownloadFile file)
        {
            _queueToDownlaod.Enqueue(file);
        }

        void DownloadTimer_Elapsed(object sender, ElapsedEventArgs e)
        {
            StartDownload();
        }

        private void StartDownload()
        {
            lock (_downloadingTasks)
            {
                if (_downloadingTasks.Count < _parallelDownloads && _queueToDownlaod.Count > 0)
                {
                    DownloadFile fileToDownload;
                    if (_queueToDownlaod.TryDequeue(out fileToDownload))
                    {
                        var task = new Task(() =>
                        {
                            var client = new WebClient();
                            client.DownloadFile(fileToDownload.Url, fileToDownload.PathToSave);
                        }, TaskCreationOptions.LongRunning);

                        task.ContinueWith(DownloadOverCallback, TaskContinuationOptions.None);

                        _downloadingTasks.Add(task);
                        task.Start();
                    }      
                }
            }
        }

        public void DownloadOverCallback(Task downloadingTask)
        {
            lock (_downloadingTasks)
            {
                _downloadingTasks.Remove(downloadingTask);
            }
        }
    }

你可以用这个来测试它：

ParallelDownloading p = new ParallelDownloading(5);

        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file1.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file2.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file3.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file4.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file5.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file6.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file7.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file8.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file9.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file10.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file11.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file12.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file13.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file14.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file15.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file16.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file17.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file18.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });
        p.EnqueueFileToDownload(new DownloadFile() { PathToSave = @"c:\file19.f", Url = @"http://download.thinkbroadband.com/20MB.zip" });

Answer 2

这是因为您在单核计算机上运行吗？

TPL将使用与核心一样多的线程。如果你愿意，可以使用更多线程来运行它。

Answer 3

下载文件是一个I / O绑定调用，所以并行与否，首先应该确保您正在进行无线程异步调用以下载单个文件。像Task.Run，task.Start这样的方法是基于线程的，它们不应该用于I / O绑定调用，否则你将并行地下载，但是你会立即阻塞你的整个CPU，每个核心都在那里空闲等待下载调用返回。

相反，您应该使用async / await模式并等待异步下载方法。假设你有一个真正的异步下载方法，但是大多数库都提供了这种方法。

现在，如果你并行化这个I / O调用，将所有返回的任务存储在一个集合中，最后你可以使用await Tasks.WhenAll（tasks）;等待所有任务。

在进行并发I / O绑定异步调用时，您还需要确保的一件事是不要使I / O连接池饿死，因此您可能希望限制所做的并发I / O调用次数。

我已经实现了一个并行处理API，允许您使用I / O限制选项等进行并发无线程异步调用。

随意查看并使用：https://www.nuget.org/packages/ParallelProcessor/

并行下载

3 个答案: