我试图从yelp和谷歌地方获取链接,并将他们的星级评分和他们的评论数量一起提高。我设法让yelp网址抓取工具正常工作但是当我尝试对Google网址使用相同的方法时,我会为.text()返回空字符串;或未定义的.attr();
这是我试图抓取
的HTML<div class="review-score-container">
<div style="display:inline-block">
<span style="font-size:32px;margin-right:11px" class="rtng" aria-
hidden="true">4.9</span>
<g-review-stars>
<span class="_pxg _Kxg" aria-label="Rated 4.9 out of 5,">
<span style="width:100px"></span>
</span>
</g-review-stars>
<div style="display:inline;font-size:15px;margin-left:11px">
<span class="_Mnc _yz" style="white-space:nowrap">96 reviews</span>
</div>
</div>
</div>
也可以在
下面的googleURL链接中找到var request = require('request')
var cheerio = require('cheerio')
var yelpURL = 'https://www.yelp.com/biz/yo-way-gardena'
var googleURL = 'https://www.google.com/search?rlz=1C1CHBF_enUS771US771&ei=9UafWtn0IaiZjwTXrJn4CQ&q=yo-way&oq=yo-way&gs_l=psy-ab.3..0l3j0i67k1l2j0i30k1l5.4447.5658.0.5873.15.8.0.0.0.0.152.868.3j5.8.0....0...1c.1.64.psy-ab..11.4.450...33i160k1j0i22i30k1j0i22i10i30k1.0.DnG3LoY6RC0&npsic=0&rflfq=1&rlha=0&rllag=38062739,-95056704,2059057&tbm=lcl&rldimm=5797977981249230753&ved=0ahUKEwi1g6eHjtnZAhUm54MKHdxpDVoQvS4IRTAA&rldoc=1&tbs=lrf:!2m1!1e2!2m1!1e3!3sIAE,lf:1,lf_ui:9#lrd=0x80c2caa3a184dc3d:0x50768e4143e597a1,1,,,&rlfi=hd:;si:5797977981249230753;mv:!1m3!1d4716380.17730953!2d-95.05670475!3d37.9292408!2m3!1f0!2f0!3f0!3m2!1i1185!2i281!4f13.1;tbs:lrf:!2m1!1e2!2m1!1e3!3sIAE,lf:1,lf_ui:9';
if (yelpURL) {
request(yelpURL, function(error, response, html){
if(!error && response.statusCode == 200){
var $ = cheerio.load(html);
var reviews = $('span[class="review-count rating-qualifier"]').first().text();
var stars = $('img[class="offscreen"]').attr('alt');
console.log('Number of reviews : ' + reviews);
console.log('Business Assessment : \n\t ' + stars + '\n');
}
});
}
if(googleURL) {
request(googleURL, function(error, response, html){
if(!error && response.statusCode == 200){
var $ = cheerio.load(html);
var reviews = $('span[class="_Mnc _yz"]').first().text();
var stars = $('span[class="rtng"]').first().text();
//var stars = $('span[class="_pxg _Kxg"]').attr('aria-label');
console.log('Number of reviews : ' + reviews);
console.log('Business Assessment : ' + stars);
}
});
}
我试图从aria-label属性中抓取_pxg类的评级,然后进入&#34; 4.9&#34;来自rtng类,它们只返回未定义或空字符串。至于评论数字,获得它的唯一地方是_Mnc类,它总是只返回一个空字符串。我知道google已经从他们的Google Places API中移除了user_rating_total
,否则,我会使用它。我的猜测是谷歌并不希望人们轻易地删除他们的网站,或者我只是对此很陌生并且遗漏了一些东西。
答案 0 :(得分:0)
request
的问题在于它无法执行javascript渲染数据。尝试使用无头浏览器。 Nightmare是一个很棒的人。你使用梦魇实例打电话然后将html代码传递给你的cheerio。以下是样本:
nightmare
.viewport(1280, 800)
.goto(url)
//do something in the chain to go to your desired page.
.evaluate(() => document.querySelector('body').outerHTML)
.then(function (html) {
cheerio.load(html);
// do something in cheerio
})
.catch(function (error) {
console.error('Error:', error);
});