The memory spikes don't resolve during image re-uploads to Google Drive. The workflow is straightforward - download an image from Drive, transform it using Sharp.js, and then upload it back. Yet, when the upload kicks in, memory usage skyrockets, even for modest 3-5 MB JPGs. This anomaly disappears the moment I comment out the upload logic (Logic: "const newId = await withExponentialBackoff(() => uploadFile(...")). Any insights or spells to break this curse?
Code:
const express = require('express');
const app = express();
const port = process.env.PORT || 3000;
const bodyParser = require('body-parser');
const path = require('path');
/*This is sharp and its setup*/
const sharp = require('sharp');//npm i [email protected]?
sharp.cache(false);
sharp.concurrency(1);
function convertToJpgAndOptimizeSize(inputStream) {
return inputStream.pipe(sharp().rotate().withMetadata()).jpeg({
quality: 80,
progressive: true,
});
}
/*This is the google drive setup*/
const { google } = require('googleapis');
require('dotenv').config({ path: 'cred.env' });
const auth = new google.auth.GoogleAuth({
keyFile: process.env.KEYFILEPATH,
scopes: ['https://www.googleapis.com/auth/drive'],
});
const drive = google.drive({ version: 'v3', auth });
/*This function is used to upload the file to drive*/
async function uploadFile(stream, mimeType, folderId, fileName) {
let response = await drive.files.create({
requestBody: {
name: fileName,
parents: [folderId],
mimeType,
},
media: {
mimeType,
body: stream,
},
fields: 'id',
});
const id = response.data.id;
response = null;
stream.destroy();
return id
}
/*This function is used to download the file from drive*/
async function downloadFile(fileId) {
const response = await drive.files.get({
fileId,
alt: 'media',
}, { responseType: 'stream' });
return response.data;
}
/*This function then downloads each picture from drive
send them to sharp.js for transformation, and then
takes the stream from sharp and reuploads them
into the updated folder and a new folder*/
const logMemoryUsage = (message) => {
const memoryUsage = process.memoryUsage();
console.log(`Memory Usage: RSS: ${Math.round(memoryUsage.rss / 1024 / 1024)} MB, External: ${Math.round(memoryUsage.external / 1024 / 1024)} MB, Heap Total: ${Math.round(memoryUsage.heapTotal / 1024 / 1024)} MB, Heap Used: ${Math.round(memoryUsage.heapUsed / 1024 / 1024)} MB -- ${message}`);
};
async function withExponentialBackoff(operation, maxRetries = 5) {
let retryCount = 0;
let delay = 1000;
while (retryCount < maxRetries) {
try {
return await operation();
} catch (error) {
console.error(`Operation failed, retrying in ${delay}ms...`, error);
await new Promise(resolve => setTimeout(resolve, delay));
retryCount++;
delay *= 2;
}
}
}
async function transformAndUploadImage(arr) {
async function processItem(item) {
try {
const { fileId, updatedBeforeFolderId, updatedAfterFolderId, originalname, fieldname } = item;
logMemoryUsage('Start downloading file')
const downloadStream = await downloadFile(fileId);
logMemoryUsage('Finish Downloading, now sending to sharp.js')
const transformedStream = convertToJpgAndOptimizeSize(downloadStream);
logMemoryUsage('End of sharp.js transformation')
const newFileName = `Updated-${path.parse(originalname).name}.jpg`;
const updatedFolderId = fieldname === 'beforePic' ? updatedBeforeFolderId : updatedAfterFolderId;
logMemoryUsage('Start upload to drive')
const newId = await withExponentialBackoff(() => uploadFile(transformedStream, 'application/octet-stream', updatedFolderId, newFileName));
logMemoryUsage('End upload to drive')
console.log(`Image transformation and upload successful for ${fileId} -> ${newId}`);
} catch (error) {
console.error(`Error in image transformation for ${item.fileId}:`, error);
}
}
for (const item of arr) {
console.log(`Processing item: ${item.fileId}`)
await processItem(item);
logMemoryUsage('End of processItem')
}
}
/*This is the express app*/
app.use(bodyParser.json());
app.post('/transform-upload-images', async (req, res) => {
const imageIds = req.body.imageIds;
if (!imageIds || !Array.isArray(imageIds)) {
return res.status(400).send({ error: 'Please provide an array of image IDs.' });
}
try {
await transformAndUploadImage(imageIds);
res.send({ message: 'Images processed successfully.' });
} catch (error) {
console.error('Failed to process images:', error);
res.status(500).send({ error: 'Failed to process images.' });
}
});
app.listen(port, () => {
console.log(`Listening at: http://localhost:${port}`);
});
Logs:
2024-03-27 21:28:21.212 EDT
Default STARTUP TCP probe succeeded after 1 attempt for container "" on port 8080.
2024-03-27 21:28:21.299 EDT
Processing item: 1bPsV26Mwbz3vBblhy7dTCZE8gtIDd8N3
2024-03-27 21:28:21.302 EDT
Memory Usage: RSS: 124 MB, External: 2 MB, Heap Total: 53 MB, Heap Used: 50 MB -- Start downloading file
2024-03-27 21:28:22.146 EDT
Memory Usage: RSS: 138 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 52 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:28:22.149 EDT
Memory Usage: RSS: 138 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 52 MB -- End of sharp.js transformation
2024-03-27 21:28:22.149 EDT
Memory Usage: RSS: 138 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 52 MB -- Start upload to drive
2024-03-27 21:28:24.280 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- End upload to drive
2024-03-27 21:28:24.280 EDT
Image transformation and upload successful for 1bPsV26Mwbz3vBblhy7dTCZE8gtIDd8N3 -> 1JsGPxY2-0PkqlxNKkAfADuU-PRW_8gNx
2024-03-27 21:28:24.280 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- End of processItem
2024-03-27 21:28:24.280 EDT
Processing item: 1sFlg3xNA_U-OPzGECwUz-N11N7c3tzK_
2024-03-27 21:28:24.281 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- Start downloading file
2024-03-27 21:28:25.706 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:28:25.707 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- End of sharp.js transformation
2024-03-27 21:28:25.707 EDT
Memory Usage: RSS: 163 MB, External: 11 MB, Heap Total: 56 MB, Heap Used: 51 MB -- Start upload to drive
2024-03-27 21:28:27.833 EDT
Memory Usage: RSS: 246 MB, External: 12 MB, Heap Total: 55 MB, Heap Used: 51 MB -- End upload to drive
2024-03-27 21:28:27.833 EDT
Image transformation and upload successful for 1sFlg3xNA_U-OPzGECwUz-N11N7c3tzK_ -> 1pbvCc-WrPu-CecMWbO-uzurgHwXWwoaq
2024-03-27 21:28:27.834 EDT
Memory Usage: RSS: 246 MB, External: 12 MB, Heap Total: 55 MB, Heap Used: 51 MB -- End of processItem
2024-03-27 21:28:27.834 EDT
Processing item: 1QSsdIHrCPYgTsj0UZtZaYDtzVSfO2Ww6
2024-03-27 21:28:27.834 EDT
Memory Usage: RSS: 246 MB, External: 12 MB, Heap Total: 55 MB, Heap Used: 51 MB -- Start downloading file
2024-03-27 21:28:28.465 EDT
Memory Usage: RSS: 243 MB, External: 6 MB, Heap Total: 55 MB, Heap Used: 50 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:28:28.465 EDT
Memory Usage: RSS: 243 MB, External: 6 MB, Heap Total: 55 MB, Heap Used: 51 MB -- End of sharp.js transformation
2024-03-27 21:28:28.465 EDT
Memory Usage: RSS: 243 MB, External: 6 MB, Heap Total: 55 MB, Heap Used: 51 MB -- Start upload to drive
2024-03-27 21:28:30.503 EDT
Memory Usage: RSS: 296 MB, External: 15 MB, Heap Total: 55 MB, Heap Used: 51 MB -- End upload to drive
2024-03-27 21:28:30.503 EDT
Image transformation and upload successful for 1QSsdIHrCPYgTsj0UZtZaYDtzVSfO2Ww6 -> 1Jg3y-gGWDi4Ktk6gnUkZ6wX9Ggiz_DJV
2024-03-27 21:28:30.503 EDT
Memory Usage: RSS: 296 MB, External: 15 MB, Heap Total: 55 MB, Heap Used: 51 MB -- End of processItem
2024-03-27 21:28:30.503 EDT
Processing item: 1ZTQdFISeZiJRnRNj_xTNDxzAawDgdFTF
2024-03-27 21:28:30.503 EDT
Memory Usage: RSS: 296 MB, External: 15 MB, Heap Total: 55 MB, Heap Used: 51 MB -- Start downloading file
2024-03-27 21:28:31.496 EDT
Memory Usage: RSS: 296 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 50 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:28:31.497 EDT
Memory Usage: RSS: 296 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 50 MB -- End of sharp.js transformation
2024-03-27 21:28:31.497 EDT
Memory Usage: RSS: 296 MB, External: 4 MB, Heap Total: 55 MB, Heap Used: 50 MB -- Start upload to drive
2024-03-27 21:28:33.682 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 49 MB -- End upload to drive
2024-03-27 21:28:33.682 EDT
Image transformation and upload successful for 1ZTQdFISeZiJRnRNj_xTNDxzAawDgdFTF -> 1NvJCTp0dL3IpYZ_YvF8aJ1eK-5TpVCsn
2024-03-27 21:28:33.682 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 49 MB -- End of processItem
2024-03-27 21:28:33.682 EDT
Processing item: 13epfXbb4_ra5QrA0TbCv1BGMZpd3oIUi
2024-03-27 21:28:33.682 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 49 MB -- Start downloading file
2024-03-27 21:28:34.424 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 50 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:28:34.424 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 50 MB -- End of sharp.js transformation
2024-03-27 21:28:34.424 EDT
Memory Usage: RSS: 374 MB, External: 7 MB, Heap Total: 55 MB, Heap Used: 50 MB -- Start upload to drive
2024-03-27 21:28:36.703 EDT
Memory Usage: RSS: 425 MB, External: 14 MB, Heap Total: 55 MB, Heap Used: 50 MB -- End upload to drive
2024-03-27 21:28:36.704 EDT
Image transformation and upload successful for 13epfXbb4_ra5QrA0TbCv1BGMZpd3oIUi -> 1p063ddqBa4yzZUpe_v58yyMqodDNsrkV
2024-03-27 21:28:36.704 EDT
Memory Usage: RSS: 425 MB, External: 14 MB, Heap Total: 55 MB, Heap Used: 50 MB -- End of processItem
2024-03-27 21:36:43.810 EDT
POST200770 B8.7 snode-fetch https://randomstuffhere.a.run.app/transform-upload-images
2024-03-27 21:36:43.830 EDT
Processing item: 1l2FB4GLCPVfBpF87CHTTw8RWa31RX4ZJ
2024-03-27 21:36:43.830 EDT
Memory Usage: RSS: 424 MB, External: 4 MB, Heap Total: 52 MB, Heap Used: 48 MB -- Start downloading file
2024-03-27 21:36:44.791 EDT
Memory Usage: RSS: 424 MB, External: 4 MB, Heap Total: 52 MB, Heap Used: 48 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:36:44.791 EDT
Memory Usage: RSS: 424 MB, External: 4 MB, Heap Total: 52 MB, Heap Used: 48 MB -- End of sharp.js transformation
2024-03-27 21:36:44.791 EDT
Memory Usage: RSS: 424 MB, External: 4 MB, Heap Total: 52 MB, Heap Used: 48 MB -- Start upload to drive
2024-03-27 21:36:46.841 EDT
Memory Usage: RSS: 446 MB, External: 12 MB, Heap Total: 52 MB, Heap Used: 48 MB -- End upload to drive
2024-03-27 21:36:46.841 EDT
Image transformation and upload successful for 1l2FB4GLCPVfBpF87CHTTw8RWa31RX4ZJ -> 1ssJ4FaWty5cIdlIzDZomjnx3H4euaU2N
2024-03-27 21:36:46.841 EDT
Memory Usage: RSS: 446 MB, External: 12 MB, Heap Total: 52 MB, Heap Used: 48 MB -- End of processItem
2024-03-27 21:36:46.841 EDT
Processing item: 1-awsO16wL0_6-6FdDIaBGBoa0WtKXGqQ
2024-03-27 21:36:46.841 EDT
Memory Usage: RSS: 446 MB, External: 12 MB, Heap Total: 52 MB, Heap Used: 48 MB -- Start downloading file
2024-03-27 21:36:47.476 EDT
Memory Usage: RSS: 446 MB, External: 6 MB, Heap Total: 53 MB, Heap Used: 48 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:36:47.477 EDT
Memory Usage: RSS: 446 MB, External: 6 MB, Heap Total: 53 MB, Heap Used: 48 MB -- End of sharp.js transformation
2024-03-27 21:36:47.477 EDT
Memory Usage: RSS: 446 MB, External: 6 MB, Heap Total: 53 MB, Heap Used: 48 MB -- Start upload to drive
2024-03-27 21:36:49.630 EDT
Memory Usage: RSS: 446 MB, External: 14 MB, Heap Total: 53 MB, Heap Used: 49 MB -- End upload to drive
2024-03-27 21:36:49.630 EDT
Image transformation and upload successful for 1-awsO16wL0_6-6FdDIaBGBoa0WtKXGqQ -> 1bhv4cxXj_ru1Ta-UFXx4zCXwlWfS_UM-
2024-03-27 21:36:49.630 EDT
Memory Usage: RSS: 446 MB, External: 14 MB, Heap Total: 53 MB, Heap Used: 49 MB -- End of processItem
2024-03-27 21:36:49.630 EDT
Processing item: 1uCEG-xweeJdgTped4BqdSPh8Hn2RI-n_
2024-03-27 21:36:49.630 EDT
Memory Usage: RSS: 446 MB, External: 14 MB, Heap Total: 53 MB, Heap Used: 49 MB -- Start downloading file
2024-03-27 21:36:50.330 EDT
Memory Usage: RSS: 446 MB, External: 4 MB, Heap Total: 53 MB, Heap Used: 48 MB -- Finish Downloading, now sending to sharp.js
2024-03-27 21:36:50.330 EDT
Memory Usage: RSS: 446 MB, External: 4 MB, Heap Total: 53 MB, Heap Used: 48 MB -- End of sharp.js transformation
2024-03-27 21:36:50.331 EDT
Memory Usage: RSS: 446 MB, External: 4 MB, Heap Total: 53 MB, Heap Used: 48 MB -- Start upload to drive
2024-03-27 21:36:52.509 EDT
Memory Usage: RSS: 469 MB, External: 11 MB, Heap Total: 53 MB, Heap Used: 49 MB -- End upload to drive
2024-03-27 21:36:52.509 EDT
Image transformation and upload successful for 1uCEG-xweeJdgTped4BqdSPh8Hn2RI-n_ -> 16Ac5gx5daiswGLtbIzP2IlChYI4WcGUm
2024-03-27 21:36:52.509 EDT
Memory Usage: RSS: 469 MB, External: 11 MB, Heap Total: 53 MB, Heap Used: 49 MB -- End of processItem
Proof:
What I Tried: Using buffers instead (No change) Stream wrapping (No change) Architectural Isolation from original project on GAE to this one on GCR (No change) Local development on win10 (Everything works fine) globalAgent.maxSockets = 100 (No change) Reupload to drive without any sharp.js transformations (No change) A boatload of random tweaks and code structure changes (No change)
Honestly any help, advice, book recommendations, or directions to go in would be greatly appreciated I have been working on this for WAY too long and I just need some rest.