Question

最近我一直在进入网络抓取工具，我已经想出了搜索文本的问题，但是有什么方法可以让我的网络抓取工具搜索其他内容，例如视频和图像，然后下载并整理它们。

到目前为止，这是我的网络抓取工具：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;

var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;

pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
  if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
    console.log("Reached max limit of number of pages to visit.");
     return;
  }
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    crawl();
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
 numPagesVisited++;

  // Make the request
 console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       callback();
       return;
    }
    // Parse the document body
      var $ = cheerio.load(body);
     var isWordFound = searchForWord($, SEARCH_WORD);
     if(isWordFound) {
      console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
     } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
     }
  });
 }

function searchForWord($, word) {
  var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
}

 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
    pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}

我从在线教程中获得了大部分代码，以帮助我开始，但我需要更多帮助代码工作，我只是想知道是否可以以及如何对网页抓取图像和视频。

较新的代码：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
 var MAX_PAGES_TO_VISIT = 200;

 var pagesVisited = {};
 var numPagesVisited = 0;
 var pagesToVisit = [];
  var url = new URL(START_URL);
 var baseUrl = url.protocol + "//" + url.hostname;

 pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
   if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
     console.log("Reached max limit of number of pages to visit.");
     return;
   }
   var nextPage = pagesToVisit.pop();
   if (nextPage in pagesVisited) {
     // We've already visited this page, so repeat the crawl
     crawl();
    } else {
       // New page we haven't visited
       visitPage(nextPage, crawl);
     }
   }

   function visitPage(url, callback) {
    // Add page to our set
    pagesVisited[url] = true;
    numPagesVisited++;

     // Make the request
     console.log("Visiting page " + url);
    request(url, function(error, response, body) {
    var $ = cheerio.load(body);
      // Check status code (200 is HTTP OK)
      console.log("Status code: " + response.statusCode);
      collectImages($);
      if(response.statusCode !== 200) {
        callback();

          return;
       }
      // Parse the document body

     // var isWordFound = searchForWord($, SEARCH_WORD);

    // if(isWordFound) {
     //   console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
    // } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
   //  }
   });
}

   function searchForWord($, word) {
   var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
    }

function collectImages($) {

   return $("img").map(function() {
        return $(this).text();
         console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
     }).get();
      }

 function collectInternalLinks($) {

     var relativeLinks = $("a[href^='/']");
     console.log("Found " + relativeLinks.length + " relative links on page");
     relativeLinks.each(function() {
         pagesToVisit.push(baseUrl + $(this).attr('href'));
     });
 }

Answer 1

就像您使用cheerio搜索正文链接一样，您也可以在正文中搜索<img>或<video>标记。当您找到这些标记时，您没有准确地说出您想要做什么，但是您可以创建一个类似于collectInternalLinks()的函数来收集媒体对象以进行进一步处理：

// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
    return $("img").map(function() {
        return $(this).prop("src");
    }).get();
}

// return collection of video elements
function collectVideos($) {
    let videoUrls = [];
    $("video").each(function() {
        let src = $(this).prop("src");
        if (src) {
            videoUrls.push(src);
        } else {
            let subElements = $(this).find("track, source");
            subElements.each(function() {
                let src = $(this).prop("src");
                if (src) {
                    videoUrls.push(src);
                }
            });
        }
    });
    return videoUrls;
}

收集视频网址更为复杂，因为可以通过多种不同方式指定这些网址（.src属性，嵌入式<track>代码，嵌入式<source>代码等等... ）所以你必须为每个<video>标签解析每种可能的方式。

node.js网络抓取工具images / video

1 个答案: