我有一个包含多个页面的网站(http://www.europarl.europa.eu/sides/getDoc.do?type=PV&reference=20190131&secondRef=TOC&language=EN) 我抓取了每个页面,并获得了包含分钟的链接。每个带有分钟数的链接都有一个日期,当我使用“年和月”创建文件夹后,我就停留在链接中的存储文件中。
问题是: 如何下载月份目录中的链接?
function crawlLink(link){
link = 'http://www.europarl.europa.eu'+link;
request(link, (error,response,
html) => {
if (!error && response.statusCode == 200) {
const $ = cheerio.load(html);
const docTitle = $('.doc_title' ) ;
var str = docTitle.html();
var date_str = str.replace(' - Brussels','').replace(' - Strasbourg', '');
var date = new Date(date_str);
console.log("created new dateobj", date);
var year = new Array('January','February', 'March' , 'April' , 'May' , 'June', 'July' ,'August','September','Oktober','November','December');
var mm = date.getMonth(); //January is 0
var yyyy = date.getFullYear();
var monthName = year[mm];
var yearDir = './data/'+yyyy;
if (!fs.existsSync(yearDir)){
fs.mkdirSync(yearDir);
}else{
console.log('yearDir exists');
}
var monthDir = yearDir+'/'+monthName;
if (!fs.existsSync(monthDir)){
fs.mkdirSync(monthDir);
}else{
console.log('monthDir exists');
}
console.log("wouhu everything is fine, get links and download them to monthDir");
let downloadLinks = [];
let $links = $('.doc_formats_box a');
$links.each(function(i, elem) {
downloadLinks.push({
title:$(this).text(),
link:$(this).attr('href')
});
});
console.log(downloadLinks);
`