Question

我有以下代码从各种网站下载html字符串。

class Program
{
    static void Main(string[] args)
    {
        List<string> URLsToProcess = new List<string>()
        {
            "http://www.microsoft.com",
            "http://www.stackoverflow.com",
            "http://www.google.com",
            "http://www.apple.com",
            "http://www.ebay.com",
            "http://www.oracle.com",
            "http://www.gmail.com",
            "http://www.amazon.com",
            "http://www.yahoo.com",
            "http://www.msn.com"
        };



        List<Task<string>> tURLs = new List<Task<string>>();

        foreach (string url in URLsToProcess)
        {
            Task<string> t = DownloadStringAsTask(new Uri(url));
            tURLs.Add(t);
        }

        Console.WriteLine("waiting now");

        Task.WaitAll(tURLs.ToArray());

        Console.WriteLine("download all done");
        foreach (Task<string> t in tURLs)
            Console.WriteLine(t.Result);
    }


    static Task<string> DownloadStringAsTask(Uri address)
    {
        TaskCompletionSource<string> tcs =
          new TaskCompletionSource<string>();
        WebClient client = new WebClient();
        client.DownloadStringCompleted += (sender, args) =>
        {
            if (args.Error != null)
                tcs.SetException(args.Error);
            else if (args.Cancelled)
                tcs.SetCanceled();
            else
                tcs.SetResult(args.Result);
        };
        client.DownloadStringAsync(address);
        return tcs.Task;
    }
}

使用此代码，可以启动10个不同的线程，从所有10个不同的URL下载数据。我想一次限制它最多3个线程。任何其他请求应该等到一个线程被释放。关于如何添加此限制的任何想法？

Answer 1

请查看Data Parallelism与Task Parallelism。你可以在这里使用Parallel LINQ：

var URLsToProcess = new List<string>
                    {
                        "http://www.microsoft.com",
                        "http://www.stackoverflow.com",
                        "http://www.google.com",
                        "http://www.apple.com",
                        "http://www.ebay.com",
                        "http://www.oracle.com",
                        "http://www.gmail.com",
                        "http://www.amazon.com",
                        "http://www.yahoo.com",
                        "http://www.msn.com"
                    };

Console.WriteLine("waiting now");

string[] tURLs = URLsToProcess
    .AsParallel()
    .WithDegreeOfParallelism(3)
    .Select(uri => this.DownloadStringAsTask(new Uri(uri)).Result)
    .ToArray();

Console.WriteLine("download all done");
foreach (string t in tURLs)
{
    Console.WriteLine(t);
}

Answer 2

您可以使用SemaphoreSlim类来限制一次执行的线程数。课程 {

    static void Main(string[] args)
    {

        List<string> URLsToProcess = new List<string>()
        {
            "http://www.microsoft.com",
            "http://www.stackoverflow.com",
            "http://www.google.com",
            "http://www.apple.com",
            "http://www.ebay.com",
            "http://www.oracle.com",
            "http://www.gmail.com",
            "http://www.amazon.com",
            "http://www.yahoo.com",
            "http://www.msn.com"
        };

        SemaphoreSlim ss = new SemaphoreSlim(3); //limit 3 threads at a time

        List<Task<string>> tURLs = new List<Task<string>>();


        foreach (string url in URLsToProcess)
        {
            //Task<string> t = DownloadStringAsTask(new Uri(url));
            //tURLs.Add(t);
            tURLs.Add((Task<string>)Task.Run(() =>
            {
                DownloadStringAsTask(new Uri(url));
                ss.Release();
            }));
        }

        Console.WriteLine("waiting now");

        Task.WaitAll(tURLs.ToArray());

        Console.WriteLine("download all done");
        foreach (Task<string> t in tURLs)
            Console.WriteLine(t.Result);

        Console.ReadLine();
    }



    static Task<string> DownloadStringAsTask(Uri address)
    {
        TaskCompletionSource<string> tcs =
          new TaskCompletionSource<string>();
        WebClient client = new WebClient();
        client.DownloadStringCompleted += (sender, args) =>
        {
            if (args.Error != null)
                tcs.SetException(args.Error);
            else if (args.Cancelled)
                tcs.SetCanceled();
            else
                tcs.SetResult(args.Result);
        };

        client.DownloadStringAsync(address);

        return tcs.Task;
    }
}

一次限制最多3个线程

2 个答案: