我正在努力处理某些网站的网页请求,例如(mrporter.com或size.co.uk)。在美国以外(所以没有美国IP),我可以提出要求。然而,一旦我落后于美国IP,请求超时或最终“连接尝试失败,因为连接方在一段时间后没有正确响应,或者建立的连接失败,因为连接的主机未能响应”异常。我尝试过所有类型的标题组合,但仍然没有运气。我需要注意的是,那些网站在浏览器中打开就好了。
这是我的实现,适用于非美国的ips。
var _request = (HttpWebRequest)WebRequest.Create("https://www.mrporter.com");
_request.CookieContainer = new CookieContainer();
_request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36";
_request.KeepAlive = true;
_request.AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate);
_request.Headers.Add("Accept-Encoding", "gzip, deflate");
_request.Headers.Add("Accept-Language", "en-GB, en-US; q=0.9, en; q=0.8");
_request.Headers.Add("Upgrade-Insecure-Requests", "1");
var _srr = "";
using (var response = _request.GetResponse())
{
var httpWebResponse = response.GetResponseStream();
using (var sr = new StreamReader(httpWebResponse))
{
_srr = sr.ReadToEnd();
}
}
有人可以帮忙吗?我严重浪费了几个小时没有结果......
答案 0 :(得分:0)
这是用于测试您提到的服务器(https://www.mrporter.com/
)
主要方法可以这样称呼:
(StreamObject
类返回有关服务器的大量信息,包括着陆页的Html。(还有Cookie,IP地址等)
public async void SomeMethodAsync()
{
StreamObject _streamObj = new StreamObject()
{
ResourceURI = new Uri(@"https://www.mrporter.com/"),
ProcessStream = true
};
_streamObj = await HTTP_GetStreamAsync(_streamObj);
}
我不得不切断代理设置(如果需要,它会在初始化WebRequest标头的同一方法中初始化)。无论如何,它应该是这样的:
if ([UseProxy])
if ([ProxyHost] != null) {
WebRequest.Proxy = new WebProxy([ProxyHost], [ProxyPort]);
if ([Credentials] != null)
WebRequest.Proxy.Credentials = new NetworkCredential([Credentials].UserID, [Credentials].Password);
} else {
WebRequest.Proxy = WebRequest.GetSystemWebProxy();
}
}
如果您打算使用代理,则还应插入此参数 你的app.config文件:
<system.net>
<defaultProxy useDefaultCredentials="true" />
</system.net>
当主要方法(HTTP_GetStreamAsync()
)返回时,StreamObject.Payload
将包含您连接到的网站的已解码Html页面。
有许多子实用程序,它可能看起来很复杂,但一切都应该可以正常工作。您只需要在类中插入所有代码并相应地修改调用方法(SomeMethodAsync()
)。
(当我有片刻时,我会编辑这个。)
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
private const uint COR_E_INVALIDOPERATION = 0x80131509;
public class StreamObject
{
public Stream ContentStream { get; set; }
public bool ProcessStream { get; set; }
public Uri ResourceURI { get; set; }
public Uri ResponseURI { get; set; }
public string Referer { get; set; }
public string Payload { get; set; }
public string ServerType { get; set; }
public string ServerName { get; set; }
public IPAddress[] ServerIP { get; set; }
public string ContentName { get; set; }
public string ContentType { get; set; }
public string ContentCharSet { get; set; }
public string ContentLanguage { get; set; }
public long ContentLenght { get; set; }
public HttpStatusCode StatusCode { get; set; }
public string StatusDescription { get; set; }
public WebExceptionStatus WebException { get; set; }
public string WebExceptionDescription { get; set; }
public CookieContainer Cookies { get; set; }
}
public async Task<StreamObject> HTTP_GetStreamAsync(StreamObject RequestObject)
{
if (string.IsNullOrEmpty(RequestObject.ResourceURI.ToString().Trim()))
return null;
MemoryStream _memstream = new MemoryStream();
HttpWebRequest httpRequest;
CookieContainer _cookiejar = new CookieContainer();
HttpStatusCode _StatusCode = HttpStatusCode.OK;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 |
SecurityProtocolType.Tls |
SecurityProtocolType.Tls11 |
SecurityProtocolType.Tls12;
ServicePointManager.ServerCertificateValidationCallback += TlsValidationCallback;
if (RequestObject.Cookies != null && RequestObject.Cookies.Count > 0)
_cookiejar = RequestObject.Cookies;
httpRequest = WebRequest.CreateHttp(RequestObject.ResourceURI);
try
{
HTTP_RequestHeadersInit(ref httpRequest, _cookiejar, null);
httpRequest.Referer = RequestObject.Referer;
using (HttpWebResponse httpResponse = (HttpWebResponse)await httpRequest.GetResponseAsync())
{
Stream ResponseStream = httpResponse.GetResponseStream();
if (_StatusCode == HttpStatusCode.OK)
{
await ResponseStream.CopyToAsync(_memstream);
RequestObject.ContentStream = _memstream;
RequestObject.ResponseURI = httpResponse.ResponseUri;
RequestObject.ContentLenght = _memstream.Length;
RequestObject.ContentCharSet = httpResponse.CharacterSet ?? string.Empty;
RequestObject.ContentLanguage = httpResponse.Headers["Content-Language"] ?? string.Empty;
RequestObject.ContentType = httpResponse.ContentType.ToLower();
if (RequestObject.ContentType.IndexOf(@"/") > -1)
{
do
{
RequestObject.ContentType = RequestObject.ContentType.Substring(RequestObject.ContentType.IndexOf(@"/") + 1);
if (RequestObject.ContentType.IndexOf(@"/") < 0)
break;
} while (true);
if (RequestObject.ContentType.IndexOf(";") > -1)
RequestObject.ContentType = RequestObject.ContentType.Substring(0, RequestObject.ContentType.IndexOf(@";"));
RequestObject.ContentType = "." + RequestObject.ContentType;
}
RequestObject.ContentName = httpResponse.Headers["Content-Disposition"] ?? string.Empty;
if (RequestObject.ContentName.Length == 0)
RequestObject.ContentName = RequestObject.ResourceURI.Segments.Last();
RequestObject.ServerType = httpResponse.Server;
RequestObject.ServerName = RequestObject.ResponseURI.DnsSafeHost;
RequestObject.ServerIP = await Dns.GetHostAddressesAsync(RequestObject.ServerName);
RequestObject.StatusCode = _StatusCode;
RequestObject.StatusDescription = httpResponse.StatusDescription;
if (RequestObject.ProcessStream)
RequestObject.Payload = ProcessResponse(RequestObject.ContentStream,
Encoding.GetEncoding(RequestObject.ContentCharSet),
httpResponse.ContentEncoding);
}
}
}
catch (WebException exW)
{
if (exW.Response != null)
{
RequestObject.StatusCode = ((HttpWebResponse)exW.Response).StatusCode;
RequestObject.StatusDescription = ((HttpWebResponse)exW.Response).StatusDescription;
}
RequestObject.WebException = exW.Status;
RequestObject.WebExceptionDescription = exW.Message;
}
catch (System.Exception exS)
{
if ((uint)exS.HResult == COR_E_INVALIDOPERATION)
{
RequestObject.WebException = await PingHostAddressAsync("8.8.8.8", 500) > 0
? WebExceptionStatus.NameResolutionFailure
: WebExceptionStatus.ConnectFailure;
RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
}
else
{
RequestObject.WebException = WebExceptionStatus.RequestCanceled;
RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
}
}
finally
{
ServicePointManager.ServerCertificateValidationCallback -= TlsValidationCallback;
}
RequestObject.Cookies = httpRequest.CookieContainer;
RequestObject.StatusCode = _StatusCode;
return RequestObject;
} //HTTP_GetStream
private bool TlsValidationCallback(object sender, X509Certificate CACert, X509Chain CAChain, SslPolicyErrors sslPolicyErrors)
{
X509Certificate2 _Certificate = new X509Certificate2(CACert);
//ADD A CERIFICATE HERE IF NEEDED
//X509Certificate2 _CACert = new X509Certificate2(@"[localstorage]/ca.cert");
//CAChain.ChainPolicy.ExtraStore.Add(_CACert);
X509Certificate2 cert = (X509Certificate2)CACert;
CAChain.Build(_Certificate);
foreach (X509ChainStatus CACStatus in CAChain.ChainStatus)
{
if ((CACStatus.Status != X509ChainStatusFlags.NoError) &
(CACStatus.Status != X509ChainStatusFlags.UntrustedRoot))
return false;
}
return true;
}
private void HTTP_RequestHeadersInit(ref HttpWebRequest _httpreq, CookieContainer _cookiecontainer, StreamObject sObject)
{
_httpreq.Date = DateTime.Now;
_httpreq.Timeout = 30000;
_httpreq.ReadWriteTimeout = 30000;
_httpreq.CookieContainer = _cookiecontainer;
_httpreq.KeepAlive = true;
_httpreq.ConnectionGroupName = Guid.NewGuid().ToString();
_httpreq.AllowAutoRedirect = true;
_httpreq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
_httpreq.ServicePoint.MaxIdleTime = 30000;
_httpreq.ServicePoint.Expect100Continue = false;
_httpreq.Referer = sObject.Referer ?? string.Empty;
_httpreq.UserAgent = "Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0";
_httpreq.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
_httpreq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US;q=0.8,en-GB;q=0.5,en;q=0.3");
_httpreq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
_httpreq.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");
//I had to cut out the WebRequest Proxy setup here
}
private string ProcessResponse(Stream stream, Encoding encoding, string ContentEncoding)
{
string html = string.Empty;
stream.Position = 0;
try
{
using (MemoryStream _memStream = new MemoryStream())
{
if (ContentEncoding.Contains("gzip"))
{
using (GZipStream _gzipStream = new GZipStream(stream, System.IO.Compression.CompressionMode.Decompress))
{
_gzipStream.CopyTo(_memStream);
};
}
else if (ContentEncoding.Contains("deflate"))
{
using (DeflateStream _deflStream = new DeflateStream(stream, System.IO.Compression.CompressionMode.Decompress))
{
_deflStream.CopyTo(_memStream);
};
}
else
{
stream.CopyTo(_memStream);
}
_memStream.Position = 0;
using (StreamReader _reader = new StreamReader(_memStream, encoding))
{
html = _reader.ReadToEnd().Trim();
html = DecodeMetaCharSetEncoding(_memStream, html, encoding);
};
};
}
catch (Exception)
{
return string.Empty;
}
return html;
}
private string DecodeMetaCharSetEncoding(Stream memStream, string _html, Encoding _encode)
{
Match _match = new Regex("<meta\\s+.*?charset\\s*=\\s*\"?(?<charset>[A-Za-z0-9_-]+)\"?",
RegexOptions.Singleline |
RegexOptions.IgnoreCase).Match(_html);
if (_match.Success)
{
string charset = _match.Groups["charset"].Value.ToLower() ?? "utf-8";
if ((charset == "unicode") | (charset == "utf-7") | (charset == "utf-16"))
charset = "utf-8";
try
{
Encoding metaEncoding = Encoding.GetEncoding(charset);
if (_encode.WebName != metaEncoding.WebName)
{
memStream.Position = 0L;
using (StreamReader recodeReader = new StreamReader(memStream, metaEncoding))
{ _html = recodeReader.ReadToEnd().Trim(); }
}
}
catch (ArgumentException)
{
_html = string.Empty;
}
catch (Exception)
{
_html = string.Empty;
}
}
return _html;
}
public async Task<IPStatus> PingHostAddressAsync(string HostAddress, int timeout)
{
if (string.IsNullOrEmpty(HostAddress.Trim()))
return IPStatus.BadDestination;
byte[] buffer = new byte[32];
PingReply iReplay = null;
IPStatus ipStatus;
using (Ping iPing = new Ping())
{
try
{
IPAddress _IPAddress = IPAddress.Parse(HostAddress);
iReplay = await iPing.SendPingAsync(_IPAddress, timeout, buffer, new PingOptions(64, false));
return iReplay.Status;
}
catch (FormatException)
{
return IPStatus.BadDestination;
}
catch (NotSupportedException nsex)
{
ipStatus = (IPStatus)nsex.HResult;
ipStatus = IPStatus.DestinationProtocolUnreachable;
}
catch (PingException pex)
{
ipStatus = (IPStatus)pex.HResult;
}
catch (SocketException soex)
{
ipStatus = (IPStatus)soex.HResult;
}
catch (Exception ex)
{
//Log ex
ipStatus = (IPStatus)ex.HResult;
}
return (iReplay != null) ? iReplay.Status : ipStatus;
}
}