I'm working on a small application with Express.js
using the Puppeteer
web scraping library. I wanted to deploy the application on render.com, but I'm encountering errors that don't occur locally. Specifically, the error is: "Error: No element found for selector: #login"
. I've been suggested that it might be a bot detection system recognizing Puppeteer
. I've also captured a screenshot with Puppeteer for confirmation.
To prevent the bot from detecting the use of Puppeteer, I've followed a series of steps found on forums, YouTube, and Stack Overflow. I've used the Puppeteer-extra library
, added randomized viewport size
, and changed the user-agent
, but despite all, I continue to get blocked. I would like to ask if there's another way to bypass the anti-bot system or if perhaps I might be doing something wrong.
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
//this is a part of the function
console.log("Entrato in fetchData");
const url = "https://web.spaggiari.eu/cvv/app/default/genitori_voti.php";
const browser = await puppeteer.launch(
{ headless: "new" },
{
args: [
"--disable-setuid-sandbox",
"--no-sandbox",
"--single-process",
"--no-zygote",
],
executablePath:
process.env.NODE_ENV === "production"
? process.env.PUPPETEER_EXECUTABLE_PATH
: puppeteer.executablePath(),
}
);
const page = await browser.newPage();
console.log("page creata");
//Randomize viewport size
await page.setViewport({
width: 1920 + Math.floor(Math.random() * 100),
height: 3000 + Math.floor(Math.random() * 100),
deviceScaleFactor: 1,
hasTouch: false,
isLandscape: false,
isMobile: false,
});
await newpage.setUserAgent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
);
await page.setJavaScriptEnabled(true);
await page.setDefaultNavigationTimeout(0);
//Skip images/styles/fonts loading for performance
await page.setRequestInterception(true);
page.on("request", (req) => {
if (
req.resourceType() == "stylesheet" ||
req.resourceType() == "font" ||
req.resourceType() == "image"
) {
req.abort();
} else {
req.continue();
}
});
await page.goto(url, { waitUntil: "domcontentloaded" });
console.log("pagina caricata");
await page.waitForTimeout(2000);
await page.waitForSelector("#login");
await page.type("#login", await codicePersonale);
console.log("codice personale inserito");
await page.waitForSelector("#password");
await page.type("#password", await password);
console.log("password inserita");
await page.screenshot({
path: "./screenshot.jpg",
});
console.log("screenshot fatto");
await Promise.all([
page.waitForNavigation({ waitUntil: "domcontentloaded" }),
page.click(".accedi"),
]);
await page.screenshot({
path: "./screenshot2.jpg",
});
console.log("screenshot fatto");