Cheerio - 节点js刮取评论和明星的数量

时间:2018-03-07 02:07:13

标签: javascript node.js cheerio

我试图从yelp和谷歌地方获取链接,并将他们的星级评分和他们的评论数量一起提高。我设法让yelp网址抓取工具正常工作但是当我尝试对Google网址使用相同的方法时,我会为.text()返回空字符串;或未定义的.attr();

这是我试图抓取

的HTML
<div class="review-score-container">
  <div style="display:inline-block">
    <span style="font-size:32px;margin-right:11px" class="rtng" aria-
    hidden="true">4.9</span>
    <g-review-stars>
      <span class="_pxg _Kxg" aria-label="Rated 4.9 out of 5,">
        <span style="width:100px"></span>
      </span>
    </g-review-stars>
    <div style="display:inline;font-size:15px;margin-left:11px">
      <span class="_Mnc _yz" style="white-space:nowrap">96 reviews</span>
    </div>
  </div>
</div>

也可以在

下面的googleURL链接中找到
var request = require('request')
var cheerio = require('cheerio')

var yelpURL = 'https://www.yelp.com/biz/yo-way-gardena'
var googleURL = 'https://www.google.com/search?rlz=1C1CHBF_enUS771US771&ei=9UafWtn0IaiZjwTXrJn4CQ&q=yo-way&oq=yo-way&gs_l=psy-ab.3..0l3j0i67k1l2j0i30k1l5.4447.5658.0.5873.15.8.0.0.0.0.152.868.3j5.8.0....0...1c.1.64.psy-ab..11.4.450...33i160k1j0i22i30k1j0i22i10i30k1.0.DnG3LoY6RC0&npsic=0&rflfq=1&rlha=0&rllag=38062739,-95056704,2059057&tbm=lcl&rldimm=5797977981249230753&ved=0ahUKEwi1g6eHjtnZAhUm54MKHdxpDVoQvS4IRTAA&rldoc=1&tbs=lrf:!2m1!1e2!2m1!1e3!3sIAE,lf:1,lf_ui:9#lrd=0x80c2caa3a184dc3d:0x50768e4143e597a1,1,,,&rlfi=hd:;si:5797977981249230753;mv:!1m3!1d4716380.17730953!2d-95.05670475!3d37.9292408!2m3!1f0!2f0!3f0!3m2!1i1185!2i281!4f13.1;tbs:lrf:!2m1!1e2!2m1!1e3!3sIAE,lf:1,lf_ui:9';

if (yelpURL) {
  request(yelpURL, function(error, response, html){
    if(!error && response.statusCode == 200){
      var $ = cheerio.load(html);

      var reviews = $('span[class="review-count rating-qualifier"]').first().text();
      var stars = $('img[class="offscreen"]').attr('alt');

      console.log('Number of reviews : ' + reviews);
      console.log('Business Assessment : \n\t    ' + stars + '\n');
    }
  });
}
if(googleURL) {
  request(googleURL, function(error, response, html){
    if(!error && response.statusCode == 200){
      var $ = cheerio.load(html);

      var reviews = $('span[class="_Mnc _yz"]').first().text();
      var stars = $('span[class="rtng"]').first().text();
      //var stars = $('span[class="_pxg _Kxg"]').attr('aria-label');

      console.log('Number of reviews : ' + reviews);
      console.log('Business Assessment : ' + stars);
    }
  });
}

我试图从aria-label属性中抓取_pxg类的评级,然后进入&#34; 4.9&#34;来自rtng类,它们只返回未定义或空字符串。至于评论数字,获得它的唯一地方是_Mnc类,它总是只返回一个空字符串。我知道google已经从他们的Google Places API中移除了user_rating_total,否则,我会使用它。我的猜测是谷歌并不希望人们轻易地删除他们的网站,或者我只是对此很陌生并且遗漏了一些东西。

1 个答案:

答案 0 :(得分:0)

request的问题在于它无法执行javascript渲染数据。尝试使用无头浏览器。 Nightmare是一个很棒的人。你使用梦魇实例打电话然后将html代码传递给你的cheerio。以下是样本:

nightmare
    .viewport(1280, 800)
    .goto(url)

    //do something in the chain to go to your desired page.

   .evaluate(() => document.querySelector('body').outerHTML)
   .then(function (html) {
      cheerio.load(html);
      // do something in cheerio
})
.catch(function (error) {
console.error('Error:', error);
});