在特定目录中搜​​寻和保存链接。切里奥

时间:2019-02-20 16:07:47

标签: node.js web-crawler scrape cheerio

我有一个包含多个页面的网站(http://www.europarl.europa.eu/sides/getDoc.do?type=PV&reference=20190131&secondRef=TOC&language=EN) 我抓取了每个页面,并获得了包含分钟的链接。每个带有分钟数的链接都有一个日期,当我使用“年和月”创建文件夹后,我就停留在链接中的存储文件中。

问题是: 如何下载月份目录中的链接?

function crawlLink(link){

link = 'http://www.europarl.europa.eu'+link;

request(link, (error,response,
html) => {
if (!error && response.statusCode == 200) {
    const $ = cheerio.load(html);

    const docTitle = $('.doc_title' ) ;

    var str = docTitle.html();
    var date_str = str.replace(' - Brussels','').replace(' - Strasbourg', '');
    var date = new Date(date_str); 
    console.log("created new dateobj", date);

    var year = new Array('January','February', 'March' , 'April' , 'May' , 'June', 'July' ,'August','September','Oktober','November','December');

    var mm   = date.getMonth(); //January is 0
    var yyyy = date.getFullYear();
    var monthName = year[mm];

    var yearDir = './data/'+yyyy;
    if (!fs.existsSync(yearDir)){
        fs.mkdirSync(yearDir);
    }else{
        console.log('yearDir exists');

    }

    var monthDir = yearDir+'/'+monthName;
    if (!fs.existsSync(monthDir)){
        fs.mkdirSync(monthDir);
    }else{
        console.log('monthDir exists');
    }

    console.log("wouhu everything is fine, get links and download them to monthDir");
    let downloadLinks = [];
    let $links = $('.doc_formats_box a');
    $links.each(function(i, elem) {
      downloadLinks.push({
        title:$(this).text(),
        link:$(this).attr('href')
        });
    });

    console.log(downloadLinks);

`

0 个答案:

没有答案