Question

我有这段代码可以从URL提取Page HTML，但是响应内容看起来已经编码。

代码：

    HttpWebRequest xhr = (HttpWebRequest) WebRequest.Create(new Uri("https://www.youtube.com/watch?v=_Ewh75YGIGQ"));
        xhr.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        //xhr.CookieContainer = request.Account.CookieContainer;
        xhr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        xhr.Headers["Accept-Encoding"] = "gzip, deflate, br";
        xhr.Headers["Accept-Language"] = "en-US,en;q=0.5";
        xhr.Headers["Upgrade-Insecure-Requests"] = "1";
        xhr.KeepAlive = true;
        xhr.UserAgent = "Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)";
        xhr.Host = "www.youtube.com";
        xhr.Referer = "https://www.youtube.com/watch?v=6aCpYxzRkf4";
        var response = xhr.GetResponse();
        string html;
        using (StreamReader reader = new StreamReader(response.GetResponseStream()))
        {
            html = reader.ReadToEnd();
        }

这些是响应头：

    X-XSS-Protection: 1; mode=block; report=https://www.google.com/appserve/security-bugs/log/youtube
    X-Content-Type-Options: nosniff
    X-Frame-Options: SAMEORIGIN
    Strict-Transport-Security: max-age=31536000
    Content-Encoding: br
    Transfer-Encoding: chunked
    Alt-Svc: quic=":443"; ma=2592000; v="44,43,39,35"
    Cache-Control: no-cache
    Content-Type: text/html; charset=utf-8
    Date: Sat, 24 Nov 2018 11:30:38 GMT
    Expires: Tue, 27 Apr 1971 19:44:06 EST
    P3P: CP="This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl=it for more info."
    Set-Cookie: PREF=f1=50000000&al=it; path=/; domain=.youtube.com; expires=Thu, 25-Jul-2019 23:23:38 GMT
    Server: YouTube Frontend Proxy

用StreamReader.ReadToEnd()解析的响应字符串看起来像this

Answer 1

答案在响应头中：Content-Encoding：br->这意味着Brotli压缩。

有一个.NET实现（NuGet程序包）：

将此安装到项目中，添加“ using Brotli;”，并用以下代码替换“ using（StreamReader .....””：

       using (BrotliStream bs = new BrotliStream(response.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress)) {
            using (System.IO.MemoryStream msOutput = new System.IO.MemoryStream()) {
                bs.CopyTo(msOutput);
                msOutput.Seek(0, System.IO.SeekOrigin.Begin);
                using (StreamReader reader = new StreamReader(msOutput)) {
                    html = reader.ReadToEnd();
                }
            }
        }

Answer 2

是。上面的答案是正确的。服务器生成的响应采用br编码。您需要对其进行解码。默认系统压缩程序包中不包含对br编码的支持，您必须安装Brotli.net nuget程序包。

在您的代码中添加此代码，以涵盖gzip，br和defalte的3种主要编码类型。

            HttpWebResponse response = (HttpWebResponse)webRequest.GetResponse();
            Stream responseStream = response.GetResponseStream();

             if (response.ContentEncoding.ToLower().Contains("gzip"))
                responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
            else if (response.ContentEncoding.ToLower().Contains("deflate"))
                responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
            else if (response.ContentEncoding.ToLower().Contains("br"))
                responseStream = new BrotliStream(responseStream, CompressionMode.Decompress);

Answer 3

xtabs(income ~ names + years, a)
#       years
#names    2012 2013
#  Daniel   45  105
# John     60   80

用途

public class ZipFileUtilities
{
    private static readonly byte[] ZipBytes1 = { 0x50, 0x4b, 0x03, 0x04, 0x0a };
    private static readonly byte[] GzipBytes = { 0x1f, 0x8b };
    private static readonly byte[] TarBytes = { 0x1f, 0x9d };
    private static readonly byte[] LzhBytes = { 0x1f, 0xa0 };
    private static readonly byte[] Bzip2Bytes = { 0x42, 0x5a, 0x68 };
    private static readonly byte[] LzipBytes = { 0x4c, 0x5a, 0x49, 0x50 };
    private static readonly byte[] ZipBytes2 = { 0x50, 0x4b, 0x05, 0x06 };
    private static readonly byte[] ZipBytes3 = { 0x50, 0x4b, 0x07, 0x08 };

    public static byte[] GetFirstBytes(string filepath, int length)
    {
        using (var sr = new StreamReader(filepath))
        {
            sr.BaseStream.Seek(0, 0);
            var bytes = new byte[length];
            sr.BaseStream.Read(bytes, 0, length);

            return bytes;
        }
    }

    public static bool IsZipFile(string filepath)
    {
        return IsCompressedData(GetFirstBytes(filepath, 5));
    }

    public static bool IsCompressedData(byte[] data)
    {
        foreach (var headerBytes in new[] { ZipBytes1, ZipBytes2, ZipBytes3, GzipBytes, TarBytes, LzhBytes, Bzip2Bytes, LzipBytes })
        {
            if (HeaderBytesMatch(headerBytes, data))
                return true;
        }

        return false;
    }

    private static bool HeaderBytesMatch(byte[] headerBytes, byte[] dataBytes)
    {
        if (dataBytes.Length < headerBytes.Length)
            throw new ArgumentOutOfRangeException(nameof(dataBytes),
                $"Passed databytes length ({dataBytes.Length}) is shorter than the headerbytes ({headerBytes.Length})");

        for (var i = 0; i < headerBytes.Length; i++)
        {
            if (headerBytes[i] == dataBytes[i]) continue;

            return false;
        }

        return true;
    }


    public static byte[] ReadFully(Stream input)
    {
        byte[] buffer = new byte[16 * 1024];
        using (MemoryStream ms = new MemoryStream())
        {
            int read;
            while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
            {
                ms.Write(buffer, 0, read);
            }
            return ms.ToArray();
        }
    }


    public static byte[] Decompress(byte[] data)
    {
        using (var compressedStream = new MemoryStream(data))
        using (var zipStream = new GZipStream(compressedStream, CompressionMode.Decompress))
        using (var resultStream = new MemoryStream())
        {
            zipStream.CopyTo(resultStream);
            return resultStream.ToArray();
        }
    }


    public static string ToQueryString(NameValueCollection nvc)
    {
        if (nvc == null) return string.Empty;

        StringBuilder sb = new StringBuilder();

        foreach (string key in nvc.Keys)
        {
            if (string.IsNullOrWhiteSpace(key)) continue;

            string[] values = nvc.GetValues(key);
            if (values == null) continue;

            foreach (string value in values)
            {
                sb.Append(sb.Length == 0 ? "" : "&");
                sb.AppendFormat("{0}={1}", Uri.EscapeDataString(key), Uri.EscapeDataString(value));
            }
        }

        return sb.ToString();
    }
}

如何解码编码的HttpWebResponse？

3 个答案: