Question

我已经在节点中制作了一个脚本，用于在Web上定期抓取页面作为作业，并将数据另存为MongoDB中的对象。我需要大约每5-15秒执行一次该功能。但是，我的代码目前性能不好，您能给我一些建议或帮助吗？

<job>
 <entries>
    <entry>
      <name>OperationalDataStoreToReconciled</name>
      <description/>
      <type>JOB</type>
    </entry>
 </entries>
</job>

我得到的输出类型适用于第一个函数调用，但随后它开始引发TimeError异常和MaxListenersExceededWarning。例如：

const $ = require('cheerio');
const MarketModel = require('./models/marketModel');
const mongoose = require('mongoose');
const puppeteer = require('puppeteer');
var schedule = require('node-schedule');

//Connection to DataBase:
//To connect to Development environment DB (Comment line below if             not using it)
mongoose.connect('mongodb://localhost:27017/Tradheo', {
    useNewUrlParser: true
});

//To connect to DB in cloud:
//mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: true     });

mongoose.connection.on('error', error => console.log(error));
mongoose.Promise = global.Promise;

getMarketData = function () {
    console.log("Web scraping to get market data...")

    let markets = []
    let marketSpain = {
        country: 'Spain',
        name: 'IBEX 35',
        companies: []
    }
    let marketGermany = {
        country: 'Germany',
        name: 'DAX',
        companies: []
    }

    puppeteer
        .launch()
        .then(function (browser) {
            return browser.newPage();
        })
        .then(function (page) {
            return     page.goto('url', {
                timeout: 3000000
            }).then(function () {
                return page.content();
            });
        })
        .then(function (html) {
            $("table[class='genTbl closedTbl crossRatesTbl elpTbl     elp30'] > tbody > tr", html).each((i, elem) => {
                marketSpain.companies.push({
                    name: $("td[class='bold left noWrap elp     plusIconTd'] > a", html).eq(i).html(),
                    last: $("td", elem).eq(2).text(),
                    high: $("td", elem).eq(3).text(),
                    low: $("td", elem).eq(4).text(),
                    change: $("td", elem).eq(5).text(),
                    changePerCent: $("td", elem).eq(6).text(),
                    volume: $("td", elem).eq(7).text(),
                    time: $("td", elem).eq(8).text(),
                    purchase: false,
                    sale: false
                });
            });
            markets.push(marketSpain);

            puppeteer
                .launch()
                .then(function (browser) {
                    return browser.newPage();
                })
                .then(function (page) {
                    return     page.goto('url').then(function ()     {
                        return page.content();
                    });
                })
                .then(function (html) {
                    $("table[class='genTbl closedTbl crossRatesTbl     elpTbl elp30'] > tbody > tr", html).each((i, elem) => {
                        marketGermany.companies.push({
                            name: $("td[class='bold left noWrap elp plusIconTd'] > a", html).eq(i).html(),
                            last: $("td", elem).eq(2).text(),
                            high: $("td", elem).eq(3).text(),
                            low: $("td", elem).eq(4).text(),
                            change: $("td", elem).eq(5).text(),
                            changePerCent: $("td", elem).eq(6).text(),
                            volume: $("td", elem).eq(7).text(),
                            time: $("td", elem).eq(8).text(),
                            purchase: false,
                            sale: false
                        });
                    });
                    markets.push(marketGermany)

                    MarketModel.create({
                        markets,
                    }, (err) => {
                        if (err) return handleError(err);
                    })

                    console.log("Done!")
                })
                .catch(function (err) {
                    console.log(err);
                });

        })
        .catch(function (err) {
            console.log(err);
        });
}


var j = schedule.scheduleJob('*/15 * 8-17 * * 1-5', function () {
    getMarketData();
});

Answer 1

我认为问题在于您没有关闭木偶浏览器。尝试在诺言链的末尾和捕获中添加browser.close（）

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://news.ycombinator.com', {waitUntil: 'networkidle2'});
  await page.pdf({path: 'hn.pdf', format: 'A4'});

  await browser.close();
})();

https://www.npmjs.com/package/puppeteer

网络抓取定期工作绩效

1 个答案: