I need to add multiple browsers to increase the response rate for a large number of requests. At the moment the application handles 3 simultaneous requests well (for not yet cached sites) but after that the response time increases linearly. The logical solution I see is to increase the number of browsers that handle requests. I tried to create a pool of browsers, but it didn't help.
from cache import cache_schedule
from io import BytesIO
import urllib
from pyppeteer import launch
browser = None
# Browser for the requests
async def get_browser():
global browser
if browser is None:
browser = await launch(
executablePath="/usr/bin/chromium",
options={'args': ['--no-sandbox', '--use-gl=egl']},
handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False
)
return browser
async def capture_screenshot(url, can_cache):
# Check browser
b = await get_browser()
page = await b.newPage()
# Set the viewport height to match the entire page
await page.setViewport({'width': 1920, 'height': 0})
await page.goto(url)
# Full page screen. Even when body tag is broken
page_height = await page.evaluate('document.body.scrollHeight')
page_width = await page.evaluate('document.body.scrollWidth')
await page.setViewport({'width': page_width, 'height': page_height})
screenshot = await page.screenshot(fullPage=True)
await page.close()
if can_cache:
url_upd1 = url + '.png'
url_upd2 = urllib.parse.quote(url_upd1, safe='')
screen_buffer = BytesIO(screenshot)
screen_size = screen_buffer.getbuffer().nbytes
cache_schedule(url_upd2, screen_buffer, screen_size)
return screenshot