多年来，我一直在使用C＃程序来编写＃34;脚本＆＃34;登录FedEx的服务器下载我公司的发票。该程序只是模仿浏览器通常发送的URL / Headers / Cookies / Post数据，以便与FedEx服务器连接，登录，＆＃34;导航＆＃34;到下载页面，然后下拉发票。
上周我开始接受401（未经授权），当程序制作了＃Aj;＆＃34; Ajax＆＃34;呼叫登录。我比较了我的所有标题，cookie和发布数据，它们与浏览器发送的内容完全匹配。不知何故，他们的Apache服务器没有看到我的编程请求与浏览器的请求相同。本周末，它从401变为简单地返回HTML，并显示有关未获得许可的消息。
我花了很多时间在两个调用（浏览器和代码）上检查小提琴检查员，我找不到任何会触发Apache服务器拒绝我的程序请求的东西，而不是浏览器＆＃39;秒。两者都使用TLS 1.2。浏览器确实运行JavaScript，当然，他们的主页确实会创建一些cookie，但作为测试，我在网页上的ajax调用之前删除了JS创建的cookie（因此它们不会被发送，所以我们会这样做两者都发送相同的cookie - 只是不同的值，因为我们在不同的会话中返回从服务器发送的cookie，并且它仍然通过浏览器进行身份验证。
所以，我的问题是，除了请求标题（包括cookie）和发布数据之外，我还应该寻找什么才能使我的程序看起来像浏览器？如果有人想试一试，主页为https://www.fedex.com/en-us/home.html，您可以使用id / pass字段中的任何内容进行测试（单击右上角的“登录”按钮）。使用浏览器时，即使假冒的id / pass组合也会返回JSON（只是说Success = false），而程序化的调用则返回HTML，而＃34;您无权查看此网页＆＃34 34;错误。
我的程序使用HTTPClient .Net类，我手动编写标题和cookie（我不使用CookieContainer，因为它似乎忽略了一些完全有效的set-cookie有时）。 / p>
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace HTTPClient
{
public class Fed
{
HttpClient client;
public Fed(string host)
{
Uri uri = new Uri(host);
client = new HttpClient(new HttpClientHandler() { UseCookies = false })
{
BaseAddress = uri,
};
}
public Dictionary<string, Dictionary<string, string>> Application = new Dictionary<string, Dictionary<string, string>>();
public async Task<List<byte[]>> GetBytes(string i_url, string[] i_parameters, bool i_asPost, string i_cookieContainerKey, string[] headers = null)
{
List<KeyValuePair<string, string>> a_parameters = new List<KeyValuePair<string, string>>();
if (i_parameters != null)
foreach (string a_parameter in i_parameters)
{
string[] a_pair = a_parameter.Split(new char[] { '=' }, 2);
a_parameters.Add(new KeyValuePair<string, string>(a_pair[0], a_pair[1]));
}
Dictionary<string, string> cookieContainer = null;
if (i_cookieContainerKey != null)
{
if (Application.ContainsKey(i_cookieContainerKey))
cookieContainer = Application[i_cookieContainerKey];
else
{
cookieContainer = new Dictionary<string, string>();
Application[i_cookieContainerKey] = cookieContainer;
}
}
var results = await GetBytes(i_url, a_parameters, i_asPost, cookieContainer, headers);
return results;
}
public async Task<List<byte[]>> GetBytes(string i_url, List<KeyValuePair<string, string>> i_parameters, bool i_asPost, Dictionary<string, string> cookieContainer, string[] headers = null)
{
Uri uri = new Uri(i_url);
HttpMethod method = new HttpMethod(i_asPost ? "POST" : "GET");
HttpRequestMessage request = new HttpRequestMessage(method, uri);
if (headers != null)
{
foreach (string header in headers)
{
string[] parts = header.Split(new char[] { '=' }, 2);
request.Headers.TryAddWithoutValidation(parts[0], parts[1]);
}
}
if (cookieContainer != null && cookieContainer.Count > 0)
request.Headers.TryAddWithoutValidation("Cookie", string.Join("; ", cookieContainer.Keys.Select(k => k + "=" + cookieContainer[k]).ToArray()));
if (i_asPost && i_parameters != null)
request.Content = new FormUrlEncodedContent(i_parameters);
HttpResponseMessage response = await client.SendAsync(request);
// grab any cookies
try
{
foreach (string cookieSet in response.Headers.GetValues("Set-Cookie"))
{
Cookie cookie = CreateCookie(cookieSet);
cookieContainer[cookie.Name] = cookie.Value;
}
}
catch { }
// Get the response content.
HttpContent responseContent = response.Content;
Stream responseStream = await responseContent.ReadAsStreamAsync();
if (responseContent.Headers.ContentEncoding.Contains("gzip"))
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
else if (responseContent.Headers.ContentEncoding.Contains("deflate"))
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
List<byte[]> a_bytesArray = new List<byte[]>();
const int c_bufferSize = 1000 * 1024;
byte[] a_buffer = new byte[1000 * 1024];
int a_bufferOffset = 0;
int a_bytesAllowed = c_bufferSize;
int a_bytesRead;
while ((a_bytesRead = responseStream.Read(a_buffer, a_bufferOffset, a_bytesAllowed)) > 0)
{
a_bufferOffset += a_bytesRead;
a_bytesAllowed -= a_bytesRead;
if (a_bytesAllowed == 0)
{
a_bytesArray.Add(a_buffer);
a_buffer = new byte[c_bufferSize];
a_bufferOffset = 0;
a_bytesAllowed = c_bufferSize;
}
System.Diagnostics.Debug.WriteLine(a_bytesRead + " bytes read");
}
// if we actually read something into this last buffer...
if (a_bufferOffset != 0)
{
// it needs to be adjusted to fit only the exact amount read
byte[] a_lastBuffer = new byte[a_bufferOffset];
for (int a_ix = 0; a_ix < a_bufferOffset; a_ix++)
a_lastBuffer[a_ix] = a_buffer[a_ix];
// and added to the array
a_bytesArray.Add(a_lastBuffer);
}
return a_bytesArray;
}
private Cookie CreateCookie(string headerValue)
{
string[] parts = headerValue.Split(';').Select(v => v.Trim()).ToArray();
Cookie cookie = new Cookie();
for (int ix = 0; ix < parts.Length; ix++)
{
string[] innerParts = parts[ix].Split(new char[] { '=' }, 2);
if (ix == 0)
{
cookie.Name = innerParts[0];
cookie.Value = innerParts[1];
}
else
{
// ignore cookies that don't correctly parse
switch (innerParts[0])
{
case "path": cookie.Path = innerParts[1]; break;
//case "domain": cookie.Domain = innerParts[1]; break;
case "expires":
{
try
{
cookie.Expires = DateTime.Parse(innerParts[1]);
}
catch
{
cookie.Expires = DateTime.UtcNow.AddDays(1);
}
break;
}
//case "max-age": cookie.MaxAge = Int32.Parse(innerParts[1]); break;
//case "secure": cookie.Secure = true; break;
//case "httponly": cookie.HttpOnly = true; break;
}
}
}
return cookie;
}
public async Task Run()
{
string cookieContainerKey = Guid.NewGuid().ToString();
string[] headers = new string[]
{
"User-Agent=Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Accept=text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language=en-US,en;q=0.9",
"Accept-Encoding=gzip, deflate, br",
"DNT=1",
"Connection=keep-alive",
"Upgrade-Insecure-Requests=1",
};
// Request Home page
List<byte[]> results = await GetBytes("https://www.fedex.com/en-us/home.html", null, false, cookieContainerKey, headers);
StringBuilder a_streamData = new StringBuilder("");
foreach (byte[] a_bytes in results)
a_streamData.Append(Encoding.ASCII.GetString(a_bytes, 0, a_bytes.Length));
var resultsStr = a_streamData.ToString();
string[] JSONheaders = new string[]
{
"User-Agent=Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Accept=application/json, text/javascript, */*; q=0.01",
"Accept-Language=en-US,en;q=0.9",
"Accept-Encoding=gzip, deflate, br",
"Referer=https://www.fedex.com/en-us/home.html",
"Origin=https://www.fedex.com",
"X-Requested-With=XMLHttpRequest",
"DNT=1",
"Connection=keep-alive",
};
string[] parameters = new string[]
{
"method=isLoggedIn",
};
results = await GetBytes("https://www.fedex.com/etc/services/fedexlogin", parameters, true, cookieContainerKey, JSONheaders);
a_streamData = new StringBuilder("");
foreach (byte[] a_bytes in results)
a_streamData.Append(Encoding.ASCII.GetString(a_bytes, 0, a_bytes.Length));
resultsStr = a_streamData.ToString();
parameters = new string[]
{
"user=xxxxxx",
"pwd=yyyyyyyy",
"url=#",
};
results = await GetBytes("https://www.fedex.com/etc/services/fedexlogin", parameters, true, cookieContainerKey, JSONheaders);
a_streamData = new StringBuilder("");
foreach (byte[] a_bytes in results)
a_streamData.Append(Encoding.ASCII.GetString(a_bytes, 0, a_bytes.Length));
resultsStr = a_streamData.ToString();
}
}
class Program
{
static void Main(string[] args)
{
Fed fed = new Fed("https://www.fedex.com");
fed.Run().Wait();
}
}
答案 0 :(得分：0)
我明白了。他们在他们的网络上使用机器人拦截软件。我猜其中一个cookie的内容实际上有一些关于软件调用类型的信息，并且他们已经决定他们不希望人们脚本访问他们自己的数据。羞。