木偶不断出错:超过导航超时:超过30000ms

时间:2020-06-17 12:15:10

标签: javascript puppeteer cheerio

我一直试图使用puppeteer登录网站并抓取数据,但我不断收到错误消息:导航超时超出:page.waitForNavigation部分超出了30000ms,并被卡住。这是否意味着我没有正确传递用户名和密码以及“提交”按钮?谁能给我见识或其他尝试的东西?

const cheerio = require('cheerio');
const puppeteer = require('puppeteer');
const { parse } = require('json2csv');
const { save } = require('save-file')

const loginUrl = 'https://albertacannabis.org/login'
const url = 'https://albertacannabis.org/shop/Cannabis%3daglc_cannabis-cannabis?f=format%3dDried%20Flower';
const strainUrl = 'https://albertacannabis.org/'
var strainLinks = [];
var currentPage = 1;
var hasResult = true
var strainData = []
const regex = /\d+.\d+ - \d+.\d+%/m
var d = new Date();
var date = d.getFullYear() + "-" + (d.getMonth() + 1) + "-" + d.getDate();
let username = '###'
let pass = '###'
const USERNAME_SELECTOR = '#UserName'
const PASSWORD_SELECTOR = '#Password'
const CTA_SELECTOR = '#SignInButton'
const navSelector = '#menubutton1-1'
const driedCanna = '#menu1-1 > li:nth-child(3) > a'

runProgram()

async function runProgram () {
    await getLinks()
    for (let index = 0; index < strainLinks.length; index++) {
        const link = strainLinks[index];
        await getStrainData(`${strainUrl}${link}`)         
    }
    exportResults(strainData)
}

async function getLinks() {
    const browser = await puppeteer.launch()
    const page = await browser.newPage()

    async function playTest(loginUrl) {
        page.setViewport({width: 1366, height: 768});
        await page.goto(loginUrl);
        await page.click(USERNAME_SELECTOR);
        await page.keyboard.type(username);
        await page.click(PASSWORD_SELECTOR);
        await page.keyboard.type(pass)
        Promise.all([
            page.click(CTA_SELECTOR),
            page.waitForNavigation({ waitUntil: 'networkidle0' }),
        ]);
        await page.click(navSelector);
        await page.click(driedCanna);

        while(hasResult === true) {
            let currentUrl = `${url}&pg=${currentPage}`
            let newLinks = []


            let html = await page.goto(currentUrl).then(function() {
                return page.content();
            })

            const $ = cheerio.load(html);

            $('.product-summary .product-info > a').each(function() {
                newLinks.push(
                    $(this).attr('href')
                );
            });

            strainLinks = [...strainLinks, ...newLinks]
            currentPage = currentPage + 1
            console.log(currentPage)
            if (newLinks.length < 12) hasResult = false
        }
    }
    (async () => {
        await playTest(loginUrl);
    })();

}

0 个答案:

没有答案