我制作了一个脚本来从网站获取一些数据,我在我的脚本中实现了无限滚动,但它只滚动一次。我已经调试过了:滚动功能被触发但不起作用。下面是我的代码,分页功能按时触发但滚动没有。
const puppeteer = require("puppeteer");
let page;
let finalResults = [];
let count = 0;
async function fStarMain() {
const browser = await puppeteer.launch({ headless: true });
page = await browser.newPage();
//const { page } = puppeteerCore();
await page.setViewport({ width: 1200, height: 1500 });
await page.goto("https://fusionread.com/search-result/bitcoin_hack", {
waitUntil: "networkidle2",
});
async function paginate() {
let count = 0;
let nextClick = await page.$(".jss85");
console.log(nextClick);
if (!nextClick && !count) {
console.log(count);
await Promise.all([
//page.waitForSelector(".list-border li"),
await page.waitFor(500),
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)"),
]);
console.log("count increment");
await collectData();
} else {
count = null;
}
//await page.waitFor(1000);
}
function printData(results) {
// console.log("function trigggered");
for (let value of results) {
console.log(value);
}
}
async function collectData() {
console.log("Data collection started");
count++;
let results = await page.evaluate(() => {
return [
...document.querySelectorAll(".infinite-scroll-component > div"),
].map((element) => ({
title: element.querySelector("a").innerText,
}));
});
//console.log(results);
printData(results);
await paginate();
// return results;
}
//console.log(results);
//console.log("recheck pagination");
await collectData();
}
module.exports = fStarMain;