How to run multiple pyppeteer browsers for parallel processing

70 Views Asked by At

I need to add multiple browsers to increase the response rate for a large number of requests. At the moment the application handles 3 simultaneous requests well (for not yet cached sites) but after that the response time increases linearly. The logical solution I see is to increase the number of browsers that handle requests. I tried to create a pool of browsers, but it didn't help.

from cache import cache_schedule
from io import BytesIO
import urllib
from pyppeteer import launch

browser = None

# Browser for the requests
async def get_browser():
    global browser
    if browser is None:
        browser = await launch(
            executablePath="/usr/bin/chromium",
            options={'args': ['--no-sandbox', '--use-gl=egl']},
            handleSIGINT=False,
            handleSIGTERM=False,
            handleSIGHUP=False
        )
    return browser


async def capture_screenshot(url, can_cache):
    # Check browser
    b = await get_browser()
    page = await b.newPage()

    # Set the viewport height to match the entire page
    await page.setViewport({'width': 1920, 'height': 0})

    await page.goto(url)

# Full page screen. Even when body tag is broken
    page_height = await page.evaluate('document.body.scrollHeight')
    page_width = await page.evaluate('document.body.scrollWidth')
    await page.setViewport({'width': page_width, 'height': page_height})
    screenshot = await page.screenshot(fullPage=True)

    await page.close()

    if can_cache:
        url_upd1 = url + '.png'
        url_upd2 = urllib.parse.quote(url_upd1, safe='')
        screen_buffer = BytesIO(screenshot)
        screen_size = screen_buffer.getbuffer().nbytes
        cache_schedule(url_upd2, screen_buffer, screen_size)

    return screenshot
0

There are 0 best solutions below