Node.js网络爬虫图像/视频

node.js web crawler images/video

本文关键字:视频 图像 爬虫 js 网络 Node      更新时间:2023-09-26

最近我一直在使用网络爬虫,我已经知道如何搜索文本,但是有没有办法让我的网络爬虫搜索其他东西,如视频和图像,然后下载并组织它们。

这是我到目前为止的网络爬虫:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');
var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;
var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;
pagesToVisit.push(START_URL);
 crawl();
 function crawl() {
  if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
    console.log("Reached max limit of number of pages to visit.");
     return;
  }
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    crawl();
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}
function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
 numPagesVisited++;
  // Make the request
 console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       callback();
       return;
    }
    // Parse the document body
      var $ = cheerio.load(body);
     var isWordFound = searchForWord($, SEARCH_WORD);
     if(isWordFound) {
      console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
     } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
     }
  });
 }
function searchForWord($, word) {
  var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
}
 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
    pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}

我已经从一个在线教程中得到了大部分的代码,以帮助我开始,但我需要更多的帮助,代码的工作,我只是想知道,如果以及如何可能的网络抓取图像和视频。

新代码:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');
var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
 var MAX_PAGES_TO_VISIT = 200;
 var pagesVisited = {};
 var numPagesVisited = 0;
 var pagesToVisit = [];
  var url = new URL(START_URL);
 var baseUrl = url.protocol + "//" + url.hostname;
 pagesToVisit.push(START_URL);
 crawl();
 function crawl() {
   if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
     console.log("Reached max limit of number of pages to visit.");
     return;
   }
   var nextPage = pagesToVisit.pop();
   if (nextPage in pagesVisited) {
     // We've already visited this page, so repeat the crawl
     crawl();
    } else {
       // New page we haven't visited
       visitPage(nextPage, crawl);
     }
   }
   function visitPage(url, callback) {
    // Add page to our set
    pagesVisited[url] = true;
    numPagesVisited++;
     // Make the request
     console.log("Visiting page " + url);
    request(url, function(error, response, body) {
    var $ = cheerio.load(body);
      // Check status code (200 is HTTP OK)
      console.log("Status code: " + response.statusCode);
      collectImages($);
      if(response.statusCode !== 200) {
        callback();
          return;
       }
      // Parse the document body
     // var isWordFound = searchForWord($, SEARCH_WORD);
    // if(isWordFound) {
     //   console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
    // } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
   //  }
   });
}
   function searchForWord($, word) {
   var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
    }
function collectImages($) {
   return $("img").map(function() {
        return $(this).text();
         console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
     }).get();
      }
 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
     console.log("Found " + relativeLinks.length + " relative links on page");
     relativeLinks.each(function() {
         pagesToVisit.push(baseUrl + $(this).attr('href'));
     });
 }

就像使用cheerio搜索正文中的链接一样,您也可以搜索正文中的<img><video>标签。当你找到这些标签时,你没有确切地说你想做什么,但是你可以创建一个类似于collectInternalLinks()的函数,它将收集媒体对象以供进一步处理:

// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
    return $("img").map(function() {
        return $(this).prop("src");
    }).get();
}
// return collection of video elements
function collectVideos($) {
    let videoUrls = [];
    $("video").each(function() {
        let src = $(this).prop("src");
        if (src) {
            videoUrls.push(src);
        } else {
            let subElements = $(this).find("track, source");
            subElements.each(function() {
                let src = $(this).prop("src");
                if (src) {
                    videoUrls.push(src);
                }
            });
        }
    });
    return videoUrls;
}

收集视频url有点复杂,因为这些url可以指定许多不同的方式(.src属性,嵌入式<track>标签,嵌入式<source>标签等),所以你必须为每个<video>标签解析出每种可能的方式。