Question

我正在尝试获取MyAnimeList.net的html（特别是此页面：http://myanimelist.net/anime.php?q=toradora!），我使用的方法以前在不同的网站上对我有效但在这里对我不起作用。

我使用的方法：

public String getWebsiteSourceCode(String sURL){
try{
    URL url = new URL(sURL);
        URLConnection urlConn= url.openConnection();

        //NEW LINE
         urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36");
        BufferedReader in = new BufferedReader(new InputStreamReader(
                urlConn.getInputStream(), "UTF-8"));
        String inputLine;
        StringBuilder a = new StringBuilder();
        while ((inputLine = in.readLine()) != null)
            a.append(inputLine);
        in.close();

        return a.toString();
}catch(Exception e){
        e.printStackTrace();
    return "null";
}
}

我得到了什么：

<html><head><META NAME="ROBOTS" CONTENT="NOINDEX, NOFOLLOW"></head><iframe src="/_Incapsula_Resource?CWUDNSAI=9&incident_id=124000930038292057-125560654487356886&edet=12&cinfo=464f095fc75381e904000000" frameborder=0 width="100%" height="100%" marginheight="0px" marginwidth="0px">Request unsuccessful. Incapsula incident ID: 124000930038292057-125560654487356886</iframe></html>

我应该得到什么：网页的HTML代码（我可以通过右击+查看页面源来获取谷歌浏览器，它与我从我的方法中得到的完全不同）。

从我得到的，它说了一些关于ROBOTS的东西，所以我假设网站有cookie或者某些东西来跟踪我是使用浏览器还是机器人...我想知道的是，是否可以绕过这个，我将如何继续这样做？感谢您的帮助:)（最好是Java，因为这就是我正在使用的）

编辑：尝试添加此行： urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36"); 但我得到同样的错误...

Answer 1

尝试

URL url = new URL(sURL);
URLConnection urlConn= url.openConnection();
c.setRequestProperty("User-Agent", "....");

要了解放入...的内容，请查看http://www.whatsmyuseragent.com/

使用你的代码

<html><head><META NAME="robots" CONTENT="noindex,nofollow"><script>(function(){f
unction getSessionCookies(){cookieArray=new Array();var cName=/^\s?incap_ses_/;v
ar c=document.cookie.split(";");for(var i=0;i<c.length;i++){key=c[i].substr(0,c[
i].indexOf("="));value=c[i].substr(c[i].indexOf("=")+1,c[i].length);if(cName.tes
t(key)){cookieArray[cookieArray.length]=value}}return cookieArray}function setIn
capCookie(vArray){try{cookies=getSessionCookies();digests=new Array(cookies.leng
th);for(var i=0;i<cookies.length;i++){digests[i]=simpleDigest((vArray)+cookies[i
])}res=vArray+",digest="+(digests.join())}catch(e){res=vArray+",digest="+(encode
URIComponent(e.toString()))}createCookie("___utmvc",res,20)}function simpleDiges
t(mystr){var res=0;for(var i=0;i<mystr.length;i++){res+=mystr.charCodeAt(i)}retu
rn res}function createCookie(name,value,seconds){if(seconds){var date=new Date()
;date.setTime(date.getTime()+(seconds*1000));var expires="; expires="+date.toGMT
String()}else{var expires=""}document.cookie=name+"="+value+expires+"; path=/"}f
unction test(o){var res="";var vArray=new Array();for(test in o){switch(o[test])
{case"exists":try{vArray[vArray.length]=encodeURIComponent(test+"="+typeof(eval(
test)))}catch(e){vArray[vArray.length]=encodeURIComponent(test+"="+e)}break;case
"value":try{vArray[vArray.length]=encodeURIComponent(test+"="+eval(test).toStrin
g())}catch(e){vArray[vArray.length]=encodeURIComponent(test+"="+e)}break;case"pl
ugins":try{p=navigator.plugins;pres="";for(a in p){pres+=(p[a]["description"]+" 
").substring(0,20)}vArray[vArray.length]=encodeURIComponent("plugins="+pres)}cat
ch(e){vArray[vArray.length]=encodeURIComponent("plugins="+e)}break;case"plugin":
try{a=navigator.plugins;for(i in a){f=a[i]["filename"].split(".");if(f.length==2
){vArray[vArray.length]=encodeURIComponent("plugin="+f[1]);break}}}catch(e){vArr
ay[vArray.length]=encodeURIComponent("plugin="+e)}break}}vArray=vArray.join();re
turn vArray}var o={navigator:"exists","navigator.vendor":"value",opera:"exists",
ActiveXObject:"exists","navigator.appName":"value",platform:"plugin",webkitURL:"
exists","navigator.plugins.length==0":"value"};try{setIncapCookie(test(o));docum
ent.createElement("img").src="/_Incapsula_Resource?SWKMTFSR=1&e="+Math.random()}
catch(e){img=document.createElement("img");img.src="/_Incapsula_Resource?SWKMTFS
R=1&e="+e}})();</script><script>(function() { var z="";var b="7472797B7661722078
68723B76617220743D6E6577204461746528292E67657454696D6528293B76617220737461747573
3D227374617274223B7661722074696D696E673D6E65772041727261792833293B77696E646F772E
6F6E756E6C6F61643D66756E6374696F6E28297B74696D696E675B325D3D22723A222B286E657720
4461746528292E67657454696D6528292D74293B646F63756D656E742E637265617465456C656D65
6E742822696D6722292E7372633D222F5F496E63617073756C615F5265736F757263653F4553324C
555243543D363726743D373826643D222B656E636F6465555249436F6D706F6E656E742873746174
75732B222028222B74696D696E672E6A6F696E28292B222922297D3B69662877696E646F772E584D
4C4874747052657175657374297B7868723D6E657720584D4C48747470526571756573747D656C73
657B7868723D6E657720416374697665584F626A65637428224D6963726F736F66742E584D4C4854
545022297D7868722E6F6E726561647973746174656368616E67653D66756E6374696F6E28297B73
7769746368287868722E72656164795374617465297B6361736520303A7374617475733D6E657720
4461746528292E67657454696D6528292D742B223A2072657175657374206E6F7420696E69746961
6C697A656420223B627265616B3B6361736520313A7374617475733D6E6577204461746528292E67
657454696D6528292D742B223A2073657276657220636F6E6E656374696F6E2065737461626C6973
686564223B627265616B3B6361736520323A7374617475733D6E6577204461746528292E67657454
696D6528292D742B223A2072657175657374207265636569766564223B627265616B3B6361736520
333A7374617475733D6E6577204461746528292E67657454696D6528292D742B223A2070726F6365
7373696E672072657175657374223B627265616B3B6361736520343A7374617475733D22636F6D70
6C657465223B74696D696E675B315D3D22633A222B286E6577204461746528292E67657454696D65
28292D74293B6966287868722E7374617475733D3D323030297B706172656E742E6C6F636174696F
6E2E72656C6F616428297D627265616B7D7D3B74696D696E675B305D3D22733A222B286E65772044
61746528292E67657454696D6528292D74293B7868722E6F70656E2822474554222C222F5F496E63
617073756C615F5265736F757263653F535748414E45444C3D353238343936313938333732343733
393534322C3239343135383533343939333730393439362C31313138373735393633303935393534
323637302C3339333836222C66616C7365293B7868722E73656E64286E756C6C297D636174636828
63297B7374617475732B3D6E6577204461746528292E67657454696D6528292D742B2220696E6361
705F6578633A20222B633B646F63756D656E742E637265617465456C656D656E742822696D672229
2E7372633D222F5F496E63617073756C615F5265736F757263653F4553324C555243543D36372674
3D373826643D222B656E636F6465555249436F6D706F6E656E74287374617475732B222028222B74
696D696E672E6A6F696E28292B222922297D3B";for (var i=0;i<b.length;i+=2){z=z+parseI
nt(b.substring(i, i+2), 16)+",";}z = z.substring(0,z.length-1); eval(eval('Strin
g.fromCharCode('+z+')'));})();</script></head><body><iframe style="display:none;
visibility:hidden;" src="http://my.incapsula.com/public/ga/jsTest.html" id="gaIf
rame"></iframe></body></html>

访问网站时出错

1 个答案: