Crawlee Puppeteer Error : Cannot destructure property 'pQuerySelector' of 'undefined'

23 Views Asked by At

I am trying to debug a puppeteer crawler which has started throwing a strange error which says :

INFO PuppeteerCrawler: Processing products list page... WARN PuppeteerCrawler: Reclaiming failed request back to the list or queue. Cannot destructure property 'pQuerySelector' of 'undefined' as it is undefined.

There is no pQuerySelector property being accessed in the code anywhere.

Any help would be appreciated. Thanks

export const router = createPuppeteerRouter();

router.use(async ({ page, proxyInfo, session, log }) => {
    await page.setExtraHTTPHeaders({
        'Accept-Language': 'en-US,en;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
    });
    log.info('Checking if the page is blocked...');
    const title = await page.title()
    log.info(`Page title: ${title}`);
    console.log(proxyInfo)
    page.cl
    if (title === 'Blocked') {
        session.retire();
    } else if (title === 'Not sure if blocked, might also be a connection error') {
        session.markBad();
    } else {
        // session.markGood() - this step is done automatically in PuppeteerCrawler.
    }
});

const pushData = async (data) => {
    data = data.map((product) => {
        return {
            ...product,
            productUrl: postProcessUrl(product.productUrl),
            currency: extractCurrency(product.currentPrice),
            currentPrice: extractNumericValue(product.currentPrice),
            soldItems: extractNumericValue(product.soldItems),
            week_number: weekNumber,
            date: currentDate.toDateString(),
        }
    });
    await Dataset.pushData(data);
}

router.addDefaultHandler(async ({ page, infiniteScroll, enqueueLinks, crawler, log }) => {
    log.info('Processing products list page...');
    
    await infiniteScroll({
        stopScrollCallback: async () => {
            
            const products = await page.$$('a.multi--container--1UZxxHY');
           
            const visibleProducts = await Promise.all(products.map(async (product) => {
                const isVisible = await product.isIntersectingViewport();
                return isVisible;
            }));
            return visibleProducts.some((isVisible) => isVisible);
        },
        waitForSecs: 10,
    });
    await page.waitForTimeout(10000);
    
    let productsCards = await page.$$('a.multi--container--1UZxxHY');
    
    let productsLength = productsCards.length;
    
    let category = page.url().split('/')[4];
    category = category_dict[category] ?? category;
    let subcategory = page.url().split('/')[5].split('.')[0];

    console.log('category: ', category, 'subcategory: ', subcategory);
0

There are 0 best solutions below