Question

我目前正在从网站上抓取信息，这是不允许的，但这仅用于测试目的。有没有办法让我能够在下面获得该网站的美国价格：

http://www.neimanmarcus.com/en-mc/Aidan-Mattox-Strapless-Lace-Mermaid-Gown-Black/prod168090058/p.prod

当我使用XPATH搜索价格时，它将以欧元返回，因为URL指向摩纳哥，我有办法强制它去美国地点并获得美国价格吗？

另外，如果我刮掉相当于美国网站的价格，我会得到我国货币的价格！这是为什么？我尝试了不同的Cookie设置，但无济于事。

任何帮助将不胜感激！

已更新

HttpWebRequest webRequest = (HttpWebRequest)
WebRequest.Create(url);
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.83 Safari/537.1"; //"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))";
webRequest.Accept = "*/*";
webRequest.Pipelined = true; 
webRequest.KeepAlive = true; 
ServicePointManager.DefaultConnectionLimit = 1000000;

CookieContainer cookieContainer = new CookieContainer();
cookieContainer.Add(new Uri(url), new Cookie("TLTUID", "8CEA4FE6032D10034476D9238143ECBE"));
cookieContainer.Add(new Uri(url), new Cookie("AGA", "4700001:4700002~6300001:6500002~6300002:6500003~6300003:6500005~6100006:6300012~5400001:5400001~6100005:6300010~5400002:5400004"));
cookieContainer.Add(new Uri(url), new Cookie("ABTEST_COOKIE", "27272067069"));
cookieContainer.Add(new Uri(url), new Cookie("ABTEST_COOKIE_CONFIRM", "29261c31634866ee227a8925b2352cb1"));
cookieContainer.Add(new Uri(url), new Cookie("DYN_USER_ID", "9256667771"));
cookieContainer.Add(new Uri(url), new Cookie("DYN_USER_CONFIRM", "ea1d2018d8c1d09299461b302b463ef7"));
cookieContainer.Add(new Uri(url), new Cookie("WID", "8923541876"));
cookieContainer.Add(new Uri(url), new Cookie("s_pers", "%20s_v17%3DNot%2520Assigned%7C1441079500079%3B%20s_v18%3DNot%2520Assigned%7C1441079500079%3B%20s_v20%3D0%7C1441079500079%3B%20s_v19%3D9256667771%7C1441079500079%3B%20s_fid%3D48B74ECD3257A317-39E69F2E9186682C%7C1472701900923%3B%20s_nr%3D1409543500955-Repeat%7C1441079500955%3B%20gpv%3DChange%2520Shipping%2520Country%7C1409545300977%3B"));
cookieContainer.Add(new Uri(url), new Cookie("s_vi", "[CS]v1|29DB119A85012607-4000160360002C25[CE]"));
cookieContainer.Add(new Uri(url), new Cookie("_br_mzv", "eyIxIjo3LCIyIjoxNDA0NDQ1NTAxMDgwLCIzIjoxNDA5NTQxOTE3MTg5LCI0IjoiaHR0cDovL3d3dy5uZWltYW5tYXJjdXMuY29tLyIsIjUiOiIifQ"));
cookieContainer.Add(new Uri(url), new Cookie("_br_uid_2", "uid%3D5027964663387%3Av%3D11.5%3Ats%3D1404445501090%3Ahc%3D28"));
cookieContainer.Add(new Uri(url), new Cookie("__cmbU", "ABJeb196hmGU2LSmkjXEtIaC4Mi4ziQnJuj6d5pFZSVCiyzg1rLfVOZ8X5WlzXyGCXJY-VijDT8JTH6ZzGto0QAbGuFJriL0Qg"));
cookieContainer.Add(new Uri(url), new Cookie("NMCS", "3ugYcZdPOmDj7QQ6w2UCrRa0iFJaIb_vPPZApLltp7OUOov6zBSy8mA"));
cookieContainer.Add(new Uri(url), new Cookie("__cmbDomTm", "0"));
cookieContainer.Add(new Uri(url), new Cookie("__cmbTpvTm", "1581"));
cookieContainer.Add(new Uri(url), new Cookie("TLTSID", "80B3B0043187103147A6A0A17B9EF771"));
cookieContainer.Add(new Uri(url), new Cookie("dtCookie", "C74B98EA50CC90EF65913E3F5737B3B1|NMO+WN|1"));
cookieContainer.Add(new Uri(url), new Cookie("dtPC", "143453468_465#click%20on%20%22Confirm%22"));
cookieContainer.Add(new Uri(url), new Cookie("mbox", "session#1409541913733-591199#1409545314|check#true#1409543514"));
cookieContainer.Add(new Uri(url), new Cookie("s_sess", "%20s_p49_start%3D1%3B%20s_cc%3Dtrue%3B%20orgTime%3D1409543456%3B%20s_sq%3D%3B%20s_ppv%3DChange%252520Shipping%252520Country%252C49%252C49%252C650%252C1366%252C650%252C1366%252C768%252C1%252CL%3B%20s_ppvl%3DChange%252520Shipping%252520Country%252C49%252C49%252C650%252C1366%252C650%252C1366%252C768%252C1%252CL%3B"));
cookieContainer.Add(new Uri(url), new Cookie("dtLatC", "239|175|195.5|187|475.5|174|298|247.5|210|264|172"));
cookieContainer.Add(new Uri(url), new Cookie("_br_mzs", "eyIxIjo5LCIyIjoxNDA5NTQxOTE3MTg5fQ"));
cookieContainer.Add(new Uri(url), new Cookie("JSESSIONID", "501318DC317879C786F292889FC8E849"));
cookieContainer.Add(new Uri(url), new Cookie("W2A", "3222077450.3930.0000"));
cookieContainer.Add(new Uri(url), new Cookie("CChipCookie", "2097217546.61525.0000"));
cookieContainer.Add(new Uri(url), new Cookie("TS4c652b", "9a9fa153d0e6a6af46d5cc37617b1cc036f3fea0668cb3575403ed1b"));
cookieContainer.Add(new Uri(url), new Cookie("nm_akid", "820877FB8FA2C3D61BC5B994E2DA67BDD205664F725200003523B653E67D9F31"));
cookieContainer.Add(new Uri(url), new Cookie("akid", "820877FB8FA2C3D61BC5B994E2DA67BDD205664F725200003523B653E67D9F31"));
cookieContainer.Add(new Uri(url), new Cookie("akaau", "1409543758~id=e7bb81bd201420d2b50d3fb2b72a8149"));
cookieContainer.Add(new Uri(url), new Cookie("rr_m", "1"));
cookieContainer.Add(new Uri(url), new Cookie("rr_n", "1"));
cookieContainer.Add(new Uri(url), new Cookie("rr_mvtid", "338-1409542704363-21-8511"));
cookieContainer.Add(new Uri(url), new Cookie("rr_s", "b23492378.23492378"));
cookieContainer.Add(new Uri(url), new Cookie("rr_pv", "ds9XUk-c1d-J--jIjW8cG---%"));
cookieContainer.Add(new Uri(url), new Cookie("rr_ric", "b338.0.MC"));
cookieContainer.Add(new Uri(url), new Cookie("rr_uc", "1f96eeed-5929-43b6-f302-61520bcf63cc"));
cookieContainer.Add(new Uri(url), new Cookie("rr_utidc", "338.b"));
cookieContainer.Add(new Uri(url), new Cookie("firstTimeUser", "here"));
cookieContainer.Add(new Uri(url), new Cookie("_br_uid_1", "uid%3D5027964663387"));
cookieContainer.Add(new Uri(url), new Cookie("ClrSSID", "1404445501993-9351"));
cookieContainer.Add(new Uri(url), new Cookie("ClrOSSID", "1404445501993-9351"));
cookieContainer.Add(new Uri(url), new Cookie("ClrSCD", "1404445501993"));
cookieContainer.Add(new Uri(url), new Cookie("rr_rcs", "eF4NyrkNgDAMAMAGUWQXI7_CbMAa2EkkCjrC_KS-W8vzvXdlEQdSPARNkE0Z3MGNqIzJWaZvsYw8m80kjcCv3UGpJ6ApAaZR9BoSzj9znRUZ"));
cookieContainer.Add(new Uri(url), new Cookie("ClrCSTO", "T"));

要获得价格：

HtmlNode rawPrice = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='lineItem']/div[@class='lineItemData']/div[@class='lineItemInfo']/div[@class='adornmentPriceElement']/div[@class='price pos2']");

但它会返回欧元或我国家的货币

已更新

@rufanov 我发现以下使用Fiddler，

POST http://www.neimanmarcus.com/en-mc/ajax.service HTTP/1.1
Host: www.neimanmarcus.com
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0
Accept: */*
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Content-Type: application/x-www-form-urlencoded; charset=UTF-8
X-Requested-With: XMLHttpRequest
Referer: http://www.neimanmarcus.com/en-mc/Aidan-Mattox-Strapless-Lace-Mermaid-Gown-Black/prod168090058/p.prod
Content-Length: 130
Cookie: TLTSID=C7F821A831EF1031A2E0E535646D82EE; TLTUID=C7F821A831EF1031A2E0E535646D82EE; JSESSIONID=1BB95B905E6D4AD6BCCCFACB4D53BD58; AGA=6300001:6500002~6300002:6500003~5400001:5400002~6300003:6500006~6100006:6300012~5400002:5400004~6100005:6300010; ABTEST_COOKIE=27806529983; ABTEST_COOKIE_CONFIRM=ef24b4e99ee78b5b0eefaa7ec3790d4d; W2A=3339517962.56665.0000; dtCookie=BF131B43CD539960F9AB29E318D8DED5|NMO+WN|1; CChipCookie=2113994762.61525.0000; TS4c652b=2cd866e0f296127b4a9981ef063eb0cb71e713ec01d4769f54053aa2286023f22d81158cec2531802d81158c60ac0ec565517ec261a4045b5442e2ae0541e448fa63d5093c25de05f6fd6a24219997378462f157219396cdbfb65f0b8524428a382d3cef4d8b7903fb8534b8fd4ed7b7a8ce5606f84d966dfb8534b84d8b7903fb8534b8fd4ed7b7a8ce5606aa4fdad3a28dfd269ea8724aa28dfd261f6ff00c0b55a5086151b9d8ae8f43c937aefe4a0e1b3aa94fcc411394598fba6edcc004808ba4f60fa38f01a7d63559; akaau=1409629669~id=11254370db65e4cd8de0b3f3be08f5b5; DYN_USER_ID=9277401074; DYN_USER_CONFIRM=57be096e15349ad19134b056ca2c67b9; WID=9277401074; dtPC=229039478_734#click%20on%20%22Confirm%22; NMCS=3E-u9d2bPVHVoGiYmKr_cW4_9Bl6VEdDl8GQBd1gQCwZg23UgR4vt7Q; s_sess=%20s_p49_start%3D1%3B%20s_sv_sid%3D503317300535%3B%20s_cc%3Dtrue%3B%20orgTime%3D1409629043%3B%20s_ppvl%3DProduct%252520Detail%252C28%252C28%252C382%252C1366%252C382%252C1366%252C768%252C1%252CL%3B%20s_ppv%3DChange%252520Shipping%252520Country%252C28%252C28%252C382%252C1366%252C382%252C1366%252C768%252C1%252CL%3B%20s_sq%3D%3B; s_pers=%20s_v17%3DNot%2520Assigned%7C1441165056833%3B%20s_v18%3DNot%2520Assigned%7C1441165056833%3B%20s_v20%3D0%7C1441165056833%3B%20s_v19%3D9277401074%7C1441165056833%3B%20gpv%3DChange%2520Shipping%2520Country%7C1409630858147%3B%20s_fid%3D4642B96D0F7EE894-3F6FAAD54AEFF39F%7C1472787474693%3B%20s_nr%3D1409629074728-Repeat%7C1441165074728%3B; firstTimeUser=here; dtLatC=590|402|583.5|1255|563|761.5|729.5|356|446|449.5|355.5; s_vi=[CS]v1|2A024AF8050121D2-6000010E80002DED[CE]; wlcme=true; nm_akid=A642CBA16FC32535AD74ACB8FE81010BCBB1260D35300000F0950454CF460777; akid=A642CBA16FC32535AD74ACB8FE81010BCBB1260D35300000F0950454CF460777; __cmbDomTm=0; _br_mzv=eyIxIjozLCIyIjoxNDA5NTg2Njg1ODMyLCIzIjoxNDA5NjI4NTU1MTEyLCI0IjoiaHR0cDovL3d3dy5uZWltYW5tYXJjdXMuY29tL2VuLW1jL0FpZGFuLU1hdHRveC1TdHJhcGxlc3MtTGFjZS1NZXJtYWlkLUdvd24tQmxhY2svcHJvZDE2ODA5MDA1OC9wLnByb2QiLCI1IjoiIn0; _br_uid_1=uid%3D9717749906180; _br_uid_2=uid%3D9717749906180%3Av%3D11.5%3Ats%3D1409586685839%3Ahc%3D14; __cmbU=ABJeb1-0cxpsi78ufqTp7XzYv5rnZCjoAXxvZIkj187WcacJnUqTHB-PMmNAssBaMJkvVofLLWybtkwRLEX399Z7n9xN14EZ_A; __cmbTpvTm=998; ClrSSID=1409586688735-9351; ClrOSSID=1409586688735-9351; ClrSCD=1409586688736; rr_m=1; rr_n=1; rr_mvtid=338-1409629035948-90-8510; rr_s=b23493817.23493817; rr_pv=dpiqdk-c1d-J--jIjW8cG---%; rr_ric=b338.0.MC; rr_uc=3ebbd78a-bc39-440f-772d-24520d18766c; rr_utidc=338.a; rr_rcs=eF4NzD0OgCAMQOHFOHCXmlL6xw28BmBIHNzE89vtveXb0_O990WlOGTGqhQpnC0OXDJua5zEkmdzAzatwF4bkLNCn0I4rCiypBXMSOEc_QdWMhRC; mbox=session#1409628542423-568873#1409630901|check#true#1409629101; _br_mzs=eyIxIjo0LCIyIjoxNDA5NjI4NTU1MTEyfQ; ClrCSTO=T
Connection: keep-alive
Pragma: no-cache
Cache-Control: no-cache

data=$b64$eyJDb250ZXh0Q2hvb3NlclJlcSI6eyJjb3VudHJ5IjoiVVMiLCJjdXJyZW5jeSI6IlVTRCIsImxhbmd1YWdlIjoiZW4ifX0$&timestamp=1409629074840

我应该将data变量传递给请求，还是应该使用单独的AJAX函数来获取我需要的值？

Answer 1

似乎当前选择的货币存储在服务器上并通过session-id与您关联。当您更改国家/地区和货币时（例如“Monaco”和“US Dollars”，客户端javascript会生成json-string：

{"ContextChooserReq":{"country":"MC","currency":"USD","language":"en"}}

然后使用base64对其进行编码：

eyJDb250ZXh0Q2hvb3NlclJlcSI6eyJjb3VudHJ5IjoiTUMiLCJjdXJyZW5jeSI6IlVTRCIsImxhbmd1YWdlIjoiZW4ifX0=

并添加标题，告诉服务器“它是base64编码的数据”：

$b64$eyJDb250ZXh0Q2hvb3NlclJlcSI6eyJjb3VudHJ5IjoiTUMiLCJjdXJyZW5jeSI6IlVTRCIsImxhbmd1YWdlIjoiZW4ifX0$

然后使用AJAX客户端将此字符串作为“数据”字段值发送到此Uri：

http://www.neimanmarcus.com/en-mc/ajax.service

然后浏览器重新加载当前页面。您的会话设置现在已在服务器上更改。就是这样 - 国家现在是“摩纳哥”，货币现在是“美元”。

您必须在当前页面上查看货币。如果它不是“USD”，那么你必须做与浏览器相同的事情 - 即将POST请求发送到带有编码的base64-string的ajax.service页面，其中头部作为请求体。它不会改变，所以如果你总是想要相同的货币，你可以发送相同的字符串（如上所述）。然后重新加载页面并再次检查货币。

UPD：写作（但有点难看）课。我使用CSQuery代替HAG（我真的不喜欢HAG）：

using System;
using System.Collections.Specialized;
using System.Net;
using CsQuery;

namespace NMClient
{
    public struct Price
    {
        public string Currency;
        public decimal Value;
    }

    public class NMWebClient : WebClient
    {
        private const string TARGET_CURRENCY = "USD";
        private const string TARGET_CURRENCY_SWITCH_DATA = "$b64$eyJDb250ZXh0Q2hvb3NlclJlcSI6eyJjb3VudHJ5IjoiTUMiLCJjdXJyZW5jeSI6IlVTRCIsImxhbmd1YWdlIjoiZW4ifX0$";

        private CookieContainer _container;

        public NMWebClient()
        {
            _container = new CookieContainer();
        }

        protected override WebRequest GetWebRequest(Uri address)
        {
            var request = base.GetWebRequest(address);
            var httpRequest = request as HttpWebRequest;

            if (httpRequest != null)
            {
                httpRequest.CookieContainer = this._container;
                return httpRequest;
            }
            else
            {
                return request;
            }
        }

        public static string GetTimestamp()
        {
            return DateTime.Now.ToString("yyyyMMddHHmmssffff");
        }

        public void SetCurrencyToUsDollars()
        {
            const string serviceUri = "http://www.neimanmarcus.com/en-mc/ajax.service";

            var values = new NameValueCollection();
            values.Add("data", TARGET_CURRENCY_SWITCH_DATA);
            values.Add("timestamp", GetTimestamp());

            this.UploadValues(serviceUri, values);
        }

        public decimal GetUSDPriceValue(string uri)
        {
            var price = GetPrice(uri);

            if (price.Currency != TARGET_CURRENCY)
            {
                SetCurrencyToUsDollars();
                price = GetPrice(uri);

                if (price.Currency != TARGET_CURRENCY)
                {
                    throw new Exception("Can't switch price to " + TARGET_CURRENCY);
                }
            }

            return price.Value;
        }

        private Price GetPrice(string uri)
        {
            string price = GetPriceText(uri);
            int priceSeparator = price.IndexOf(' ');

            return new Price()
            {
                Currency = price.Substring(0, priceSeparator),
                Value = Convert.ToDecimal(price.Substring(priceSeparator + 1))
            };
        }

        private string GetPriceText(string uri)
        {
            var html = this.DownloadString(uri);
            var cq = CQ.Create(html);

            var priceElement = cq.Select("[itemprop=price]");

            return priceElement.Text().Trim();
        }
    }
}

用法：

var client = new NMWebClient();

var uri = "http://www.neimanmarcus.com/en-mc/Aidan-Mattox-Strapless-Lace-Mermaid-Gown-Black/prod168090058/p.prod";
var price = client.GetUSDPriceValue(uri);   // 240

尽量不要复制，而是要理解逻辑。

获取产品的美国价格

1 个答案: