我正在尝试解析google images search result。
我正在尝试获取元素的 href属性。我注意到，当我以编程方式获取页面时， href字段丢失了（这与jsoup和htmlunit一起发生）。
通过java和元素以编程方式比较页面的元素实际浏览器加载的页面，唯一的区别是，确实缺少 href字段（其余部分相同）。
href属性（IMAGE_LINK）如下：/imgres?imgurl=http%3A%2F%2Fcdn.zonarutoppuden.com%2Fns%2Fpeliculas-naruto-shippuden.jpg&imgrefurl=http%3A%2F%2Fwww.zonarutoppuden.com%2F2010%2F10%2Fnaruto-shippuden-peliculas.html&docid=JR8NPqKrF3ac_M&tbnid=0EPPOYQcflXkMM%3A&w=900&h=600&bih=638&biw=1275&ved=0ahUKEwih9O2e88_OAhWMExoKHRLGAGQQMwg2KAMwAw&iact=mrc&uact=8
也许javascript引擎存在一些问题？或者也许网站使用某种算法反解析？
Snippet Java代码：
WebClient webClient = new WebClient(BrowserVersion.CHROME);
webClient.waitForBackgroundJavaScript(50000);
HtmlPage page1=null;
try {
// Get the first page
page1 = webClient.getPage(URL);
System.out.println(page1.asXml());
} catch (FailingHttpStatusCodeException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Snippet Html Code（Real Browser）：
<a jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l" style="width: 134px; height: 201px; left: 0px; background: rgb(128, 128, 128);" href="IMAGE_LINK"> CONTENT... </a>
Snippet Html Code（以编程方式获取的页面）：
<a jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l" style="width: 134px; height: 201px; left: 0px; background: rgb(128, 128, 128);"> CONTENT... </a>
谢谢。
答案 0 :(得分：2)
对于每个搜索结果，都有一个包含JSON对象的<div class="rg_meta">，该对象也包含该URL。使用json-simple之类的JSON解析器来解析对象，以下代码打印图像URL：
String searchTerm = "naruto shippuden";
String searchUrl = "https://www.google.com/search?site=imghp&tbm=isch&source=hp&biw=1920&bih=955&q=" + searchTerm.replace(" ", "+") + "&gws_rd=cr";
try {
Document doc = Jsoup.connect(searchUrl)
.userAgent("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36")
.referrer("https://www.google.com/").get();
JSONObject obj;
for (Element result : doc.select("div.rg_meta")) {
// div.rg_meta contains a JSON object, which also holds the image url
obj = (JSONObject) new JSONParser().parse(result.text());
String imageUrl = (String) obj.get("ou");
// just printing out the url to demonstate the approach
System.out.println("imageUrl: " + imageUrl);
}
} catch (IOException e1) {
e1.printStackTrace();
}catch (ParseException e) {
e.printStackTrace();
}
输出：
imageUrl: http://ib3.huluim.com/show_key_art/1603?size=1600x600®ion=US
imageUrl: http://cdn.zonarutoppuden.com/ns/peliculas-naruto-shippuden.jpg
imageUrl: http://www.saiyanisland.com/news/wp-content/uploads2/2014/12/Naruto-Sasuke.jpg
...
<强>更新
由于jsAction似乎不能很好地与htmlUnit一起使用，我建议使用phantomJs。只需download the binary为您的操作系统创建一个脚本文件。
创建一个page.js文件：
var page = require('webpage').create();
var fs = require('fs');
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36';
page.zoomFactor = 0.1;
page.viewportSize = {
width: 1920,
height: 1080
};
var divCount="-1";
var topPosition=0;
var unchangedCounter=0;
page.open('https://www.google.com/search?site=imghp&tbm=isch&source=hp&q=naruto+shippuden&gws_rd=cr', function(status) {
console.log("Status: " + status);
if(status === "success") {
window.setInterval(function() {
var newDivCount = page.evaluate(function() {
var divs = document.querySelectorAll(".rg_di.rg_bx.rg_el.ivg-i");
return divs[divs.length-1].getAttribute("data-ri");
});
topPosition = topPosition + 1080;
page.scrollPosition = {
top: topPosition,
left: 0
};
if(newDivCount===divCount){
page.evaluate(function() {
var button = document.querySelector("#smb");
console.log("buttontype:"+typeof button);
if(!(typeof button === "undefined")) {
button.click();
return true;
}else{
return false;
}
});
if(unchangedCounter===5){
console.log(newDivCount);
var path = 'output.html';
fs.write(path, page.content, 'w');
phantom.exit();
}else{
unchangedCounter=unchangedCounter+1;
}
}else{
unchangedCounter=0;
}
divCount = newDivCount;
}, 500);
}
});
现在我们用phantomJs执行脚本文件并像以前一样用jsoup解析结果：
try {
Process process = Runtime.getRuntime().exec("bin\\phantomjs page.js"); //change path to phantomjs binary and your script file
process.waitFor();
Document doc = Jsoup.parse(new File("output.html"),"UTF-8"); // output.html is created by phantom.js, same path as page.js
for (Element element : doc.select("div.rg_di.rg_bx.rg_el.ivg-i a")) {
System.out.println(element.attr("href"));
}
System.out.println("Number of results: " + doc.select("div.rg_di.rg_bx.rg_el.ivg-i a").size());
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}
输出：
/imgres?imgurl=http%3A%2F%2Fib3.huluim.com%2Fshow_key_art%2F1603%3Fsize%3D1600x600%26region%3DUS&imgrefurl=http%3A%2F%2Fwww.hulu.com%2Fnaruto-shippuden&docid=OgW4j66rp7CKkM&tbnid=SElXvYDJj9cR6M%3A&w=1600&h=600&bih=10800&biw=19200&ved=0ahUKEwjX2PXmptPOAhULVxoKHXfmDg8QMwgzKAAwAA&iact=mrc&uact=8
/imgres?imgurl=http%3A%2F%2Fcdn.zonarutoppuden.com%2Fns%2Fpeliculas-naruto-shippuden.jpg&imgrefurl=http%3A%2F%2Fwww.zonarutoppuden.com%2F2010%2F10%2Fnaruto-shippuden-peliculas.html&docid=JR8NPqKrF3ac_M&tbnid=0EPPOYQcflXkMM%3A&w=900&h=600&bih=10800&biw=19200&ved=0ahUKEwjX2PXmptPOAhULVxoKHXfmDg8QMwg0KAEwAQ&iact=mrc&uact=8
...
Number of results: 463
更新：将网址作为参数传递给脚本
脚本 page.js
var page = require('webpage').create();
var fs = require('fs');
var system = require('system');
var url = "";
var searchParameter = "";
if (system.args.length === 3) {
url=system.args[1];
searchParameter=system.args[2];
}
if(url==="" || searchParameter===""){
phantom.exit();
}
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36';
page.zoomFactor = 0.1;
page.viewportSize = {
width: 1920,
height: 1080
};
var divCount="-1";
var topPosition=0;
var unchangedCounter=0;
page.open(url, function(status) {
console.log("Status: " + status);
if(status === "success") {
window.setInterval(function() {
var newDivCount = page.evaluate(function() {
var divs = document.querySelectorAll(".rg_di.rg_bx.rg_el.ivg-i");
return divs[divs.length-1].getAttribute("data-ri");
});
topPosition = topPosition + 1080;
page.scrollPosition = {
top: topPosition,
left: 0
};
if(newDivCount===divCount){
page.evaluate(function() {
var button = document.querySelector("#smb");
if(!(typeof button === "undefined")) {
button.click();
return true;
}else{
return false;
}
});
if(unchangedCounter===5){
var path = searchParameter+'.html';
fs.write(path, page.content, 'w');
phantom.exit();
}else{
unchangedCounter=unchangedCounter+1;
}
}else{
unchangedCounter=0;
}
divCount = newDivCount;
}, 500);
}else{
phantom.exit();
}
});
Java代码
try {
//change path to phantomjs binary and your script file
String phantomJSPath = "phantomjs" + File.separator + "bin" + File.separator + "phantomjs";
String scriptFile = "page.js";
String searchTerm = "naruto+shippuden";
String urlParameter = "https://www.google.com/search?site=imghp&tbm=isch&source=hp&gws_rd=cr&q="+searchTerm;
Process process = Runtime.getRuntime().exec(phantomJSPath + " " + scriptFile + " " + urlParameter + " " + searchTerm);
process.waitFor();
Document doc = Jsoup.parse(new File(searchTerm + ".html"),"UTF-8"); // output.html is created by phantom.js, same path as page.js
for (Element element : doc.select("div.rg_di.rg_bx.rg_el.ivg-i a")) {
System.out.println(element.attr("href"));
}
System.out.println("Number of results: " + doc.select("div.rg_di.rg_bx.rg_el.ivg-i a").size());
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}