node.js网络爬虫图像/视频
node.js网络爬虫图像/视频
提问于 2016-09-04 04:11:22
最近我一直在使用网络爬虫，我已经想出了如何搜索文本，但有没有办法让我的网络爬虫搜索其他东西，比如视频和图像，然后下载它们并组织它们。
这是目前为止我的网络爬虫：
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');
var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;
var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;
pagesToVisit.push(START_URL);
crawl();
function crawl() {
if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
console.log("Reached max limit of number of pages to visit.");
return;
}
var nextPage = pagesToVisit.pop();
if (nextPage in pagesVisited) {
// We've already visited this page, so repeat the crawl
crawl();
} else {
// New page we haven't visited
visitPage(nextPage, crawl);
}
}
function visitPage(url, callback) {
// Add page to our set
pagesVisited[url] = true;
numPagesVisited++;
// Make the request
console.log("Visiting page " + url);
request(url, function(error, response, body) {
// Check status code (200 is HTTP OK)
console.log("Status code: " + response.statusCode);
if(response.statusCode !== 200) {
callback();
return;
}
// Parse the document body
var $ = cheerio.load(body);
var isWordFound = searchForWord($, SEARCH_WORD);
if(isWordFound) {
console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
} else {
collectInternalLinks($);
// In this short program, our callback is just calling crawl()
callback();
}
});
}
function searchForWord($, word) {
var bodyText = $('html > body').text().toLowerCase();
return(bodyText.indexOf(word.toLowerCase()) !== -1);
}
function collectInternalLinks($) {
var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}我已经从一个在线教程中获得了大部分代码，以帮助我开始，但我需要更多的帮助，代码工作，我只想知道，如果和如何才能网络爬行图像和视频。
较新的代码：
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');
var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
var MAX_PAGES_TO_VISIT = 200;
var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;
pagesToVisit.push(START_URL);
crawl();
function crawl() {
if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
console.log("Reached max limit of number of pages to visit.");
return;
}
var nextPage = pagesToVisit.pop();
if (nextPage in pagesVisited) {
// We've already visited this page, so repeat the crawl
crawl();
} else {
// New page we haven't visited
visitPage(nextPage, crawl);
}
}
function visitPage(url, callback) {
// Add page to our set
pagesVisited[url] = true;
numPagesVisited++;
// Make the request
console.log("Visiting page " + url);
request(url, function(error, response, body) {
var $ = cheerio.load(body);
// Check status code (200 is HTTP OK)
console.log("Status code: " + response.statusCode);
collectImages($);
if(response.statusCode !== 200) {
callback();
return;
}
// Parse the document body
// var isWordFound = searchForWord($, SEARCH_WORD);
// if(isWordFound) {
// console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
// } else {
collectInternalLinks($);
// In this short program, our callback is just calling crawl()
callback();
// }
});
}
function searchForWord($, word) {
var bodyText = $('html > body').text().toLowerCase();
return(bodyText.indexOf(word.toLowerCase()) !== -1);
}
function collectImages($) {
return $("img").map(function() {
return $(this).text();
console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
}).get();
}
function collectInternalLinks($) {
var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}回答 1
Stack Overflow用户
发布于 2016-09-05 01:02:08
就像使用cheerio搜索正文中的链接一样，您也可以在正文中搜索<img>或<video>标记。当您找到这些标签时，您并没有确切地说明您想要做什么，但是您可以创建一个类似于您的collectInternalLinks()的函数，该函数将收集媒体对象以供进一步处理：
// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
return $("img").map(function() {
return $(this).prop("src");
}).get();
}
// return collection of video elements
function collectVideos($) {
let videoUrls = [];
$("video").each(function() {
let src = $(this).prop("src");
if (src) {
videoUrls.push(src);
} else {
let subElements = $(this).find("track, source");
subElements.each(function() {
let src = $(this).prop("src");
if (src) {
videoUrls.push(src);
}
});
}
});
return videoUrls;
}收集视频URL比较复杂，因为可以通过多种不同的方式指定这些URL (.src属性、嵌入式<track>标签、嵌入式<source>标签等)。因此，您必须为每个<video>标记解析出每种可能的方式。
页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://stackoverflow.com/questions/39310925
复制相关文章
相似问题
腾讯云开发者
