Question

我创建了一个测试，用于检查我们网站上的损坏链接。我从我们网站的站点地图中收集了一个链接列表，然后循环执行HttpWebRequest，如果一个或多个链接返回的结果不是OK，则测试失败（200）。失败的链接将添加到词典中以供以后查看（状态代码，链接）。

最初，我遇到授权错误，因此添加了相关代码以通过我们的代理。现在，所有HttpWebRequest都返回404-不仅返回到我们的网站，而且还返回到任何网站-Google / Twitter / Facebook等。但是，所有链接都可以在浏览器中工作。我已经在运行测试时尝试监视Fiddler中的流量，但是Fiddler流中没有条目。

假设这是一个“我/我们”问题，而不是HttpWebRequest类的错误，我怀疑我不是在编写完全合格/“合法”的HttpWebRequest。

我已经走了检查标题的路线，例如：How to view the headers sent by HttpWebRequest，但我对此一无所知。当我在http://rextester.com上查询请求标头时，它为空。我只在响应中获得标题...

        HttpWebRequest myHttpWebRequest=(HttpWebRequest)WebRequest.Create("http://www.google.com");
        Console.WriteLine("Request headers: " + myHttpWebRequest.Headers);
        HttpWebResponse myHttpWebResponse=(HttpWebResponse)myHttpWebRequest.GetResponse();
        Console.WriteLine("Response code: " + myHttpWebResponse.StatusCode);
        Console.WriteLine("Response headers: " + myHttpWebResponse.Headers);

这是我在测试中使用的代码，这里显然有什么不对吗？注释掉的行是尝试指定标头来解决此问题的假设，假设我缺少请求中需要的内容，但它们没有任何区别。

    public void BrowseEachLinkOnPage(string page)
    {
        // Page factory does not recognise lists as IWebElement or IList<IWebElement> despite them being set as such. Resorting to manual collection of elements...

        // Default collection to initialise variable, //*[text()='donotfindthis'] should find 0 elements
        var linksToTest = driver.FindElements(By.XPath("//*[text()='donotfindthis']"));

        try
        {
            switch (page)
            {
                case "site map":
                    linksToTest = driver.FindElements(By.XPath("//*[contains(@class,'sitemap__container')]//a[contains(@href,'/')]"));
                    break;
            }
        }
        catch (Exception e)
        {
            Log.UpdateTestStatus(Log.Status.Failed, "BrowseEachLinkOnPage(): - \"" + page + "\" page does not map to a predefined list of links. Check existing or create a new reference to a list of links.", driver);
            Console.WriteLine(e);
            throw;
        }

        Dictionary<object, string> invalidHttpRequestResponses = new Dictionary<object, string>();
        Dictionary<int, string> httpRequestExceptions = new Dictionary<int, string>();
        int numberOfHttpRequestExceptions = 0;
        int numberOfLinksToTest = linksToTest.Count();

        if (numberOfLinksToTest > 0)
        {
            // An exception is triggered if an exception occurs within the loop at least once
            try
            {
                for (var i = 0; i < numberOfLinksToTest; i++)
                {
                    try
                    {
                        HttpWebRequest httpReq =
                            (HttpWebRequest) WebRequest.Create(linksToTest[i].GetAttribute("href"));
                        //httpReq.AllowAutoRedirect = true;
                        httpReq.UseDefaultCredentials = false;

                        // AD credentials required to get through proxy, enter password when testing
                        httpReq.Credentials = new NetworkCredential("myusername", "mypassword");

                        WebProxy webProxy = new WebProxy("http://10.0.0.0/myproxy.pac")
                        {
                            Credentials = new NetworkCredential("myusername", "mypassword")
                        };

                        httpReq.Proxy = webProxy;

                        //httpReq.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                        //httpReq.CookieContainer = new CookieContainer();
                        //httpReq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";

                        HttpWebResponse httpRes = (HttpWebResponse) httpReq.GetResponse();

                        if (httpRes.StatusCode != HttpStatusCode.OK)
                        {
invalidHttpRequestResponses.Add(httpRes.StatusCode, linksToTest[i].GetAttribute("href"));
                        }

                        httpRes.Close();
                    }
                    // Suppress the exception, instead capture the details of the exception for later use
                    catch (Exception e)
                    {
                        numberOfHttpRequestExceptions += 1;
                        httpRequestExceptions.Add(i, e.ToString());
                    }
                }
            }
            // Report why the HttpWebRequests failed for further troubleshooting
            catch (Exception e)
            {
                Log.UpdateTestStatus(Log.Status.Failed, "HttpWebRequest failed: " + numberOfHttpRequestExceptions + " exception(s) out of " + numberOfLinksToTest + " links tested.\n----- Attempt, exception: -----\n" + httpRequestExceptions, driver);
                Console.WriteLine(e);
                throw;
            }
        }
        else
        {
            Log.UpdateTestStatus(Log.Status.Failed, "BrowseEachLinkOnPage(): - there are no links in the collection.", driver);
            throw new Exception("BrowseEachLinkOnPage(): - there are no links in the collection.");
        }

        // Assuming there were no HttpWebRequest exceptions, report on any links that failed (StatusCode != to OK)
        if (invalidHttpRequestResponses.Count > 0)
        {
            Log.UpdateTestStatus(Log.Status.Failed, "HttpWebRequest: - invalid response on pages (response, URL):\n" + invalidHttpRequestResponses, driver);
            throw new Exception("HttpWebRequest: - invalid response on pages (response, URL):\n" + invalidHttpRequestResponses);
        }
    }

谢谢

Answer 1

我遇到了错误的代理，我们的技术支持启发了我。

Selenium C＃-尽管网站可浏览，但HttpWebRequest始终返回404

1 个答案: