Question

我正在同时测试大量代理服务器的有效性。在此测试期间，正在提出并捕获许多异常。虽然我在后台线程中进行测试，但我的UI变得没有响应，除非我使用SemaphoreSlim对象来控制并发。

我知道这是一个自我强加的瓶颈，当用更大的代理列表进行缩放测试时，我希望可能有更好的方法来解决这个问题。

private void ValidateProxiesButton_Click(object sender, EventArgs e)
{
    new Thread(async () =>
    {
        Thread.CurrentThread.IsBackground = true;
        await ValidateProxiesAsync(proxies, judges, tests, 10);
    }).Start();
}


public async Task ValidateProxiesAsync(IEnumerable<Proxy> proxies, IEnumerable<ProxyJudge> judges, IEnumerable<ProxyTest> tests = null, int maxConcurrency = 20)
{
    if (proxies.Count() == 0)
    {
        throw new ArgumentException("Proxy list empty.");
    }

    foreach (var proxy in proxies)
    {
        proxy.Status = ProxyStatus.Queued;
    }

    //Get external IP to check if proxy is anonymous.
    var publicIp = await WebUtility.GetPublicIP();
    foreach (var judge in judges)
    {
        judge.Invalidation = publicIp;
    }

    await ValidateTestsAsync(judges.ToList<IProxyTest>());
    var validJudges = judges.ToList<IProxyTest>().GetValidTests();
    if (validJudges.Count == 0)
    {
        throw new ArgumentException("No valid judges found.");
    }

    if (tests != null)
    {
        await ValidateTestsAsync(tests.ToList<IProxyTest>());
    }

    var semaphore = new SemaphoreSlim(maxConcurrency);
    var tasks = new List<Task>();
    foreach (var proxy in proxies)
    {
        tasks.Add(Task.Run(async () =>
        {
            await semaphore.WaitAsync();
            proxy.Status = ProxyStatus.Testing;
            var isValid = await proxy.TestValidityAsync((IProxyTest)validJudges.GetRandomItem());
            proxy.Status = isValid ? ProxyStatus.Valid : ProxyStatus.Invalid;
            semaphore.Release();
        }));
    }

    await Task.WhenAll(tasks);
}

内部proxy.TestValidityAsync方法

public async Task<bool> TestValidityAsync(IProxyTest test, int timeoutSeconds = 30)
{
    try
    {
        var req = WebRequest.Create(test.URL);
        req.Proxy = new WebProxy(this.ToString());
        var respBody = await WebUtility.GetResponseStringAsync(req).TimeoutAfter(new TimeSpan(0, 0, timeoutSeconds));
        if (respBody.Contains(test.Validation))
        {
            return true;
        }
        else
        {
            return false;
        }
    }
    catch (Exception)
    {
        return false;
    }
}

Answer 1

所以我找到了一个有效的解决方案，就是将TPL Dataflow NuGet包添加到我的项目中，然后使用TransformBlock类。当我这样做时，即使我处理大量经常抛出异常的并发请求，我的UI也会保持响应。下面的代码是概念证明，当我翻译它以使用我的项目时，我会更新它。

来源：Throttling asynchronous tasks

private async void button1_Click(object sender, EventArgs e)
{

    var downloader = new TransformBlock<string, WebResponse>(
            url => Download(url),
            new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 200 }
        );

    var buffer = new BufferBlock<WebResponse>();
    downloader.LinkTo(buffer);

    var urls = new List<string>();
    for (int i = 0; i < 100000; i++)
    {
        urls.Add($"http://example{i}.com");
    }

    foreach (var url in urls)
        downloader.Post(url);
    //or await downloader.SendAsync(url);

    downloader.Complete();
    await downloader.Completion;

    IList<WebResponse> responses;
    if (buffer.TryReceiveAll(out responses))
    {
        //process responses        
    }
}

private WebResponse Download(string url)
{
    WebResponse resp = null;
    try
    {
        var req = WebRequest.Create(url);
        resp = req.GetResponse();
    }
    catch (Exception)
    {

    }
    return resp;
}
}

在后台线程

1 个答案: