feat: Optimize and parallel image processing

server/api/helpers/attachments/process-uploaded-file.js

@@ -87,8 +87,8 @@ module.exports = {
         const thumbnailsPathSegment = `${dirPathSegment}/thumbnails`;
         const thumbnailsExtension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
-        try {
+        const outside360 = image
-          const outside360Buffer = await image
+          .clone()
           .resize(360, 360, {
             fit: 'outside',
             withoutEnlargement: true,
@@ -96,16 +96,10 @@ module.exports = {
           .png({
             quality: 75,
             force: false,
-            })
+          });
-            .toBuffer();
 
-          await fileManager.save(
+        const outside720 = image
-            `${thumbnailsPathSegment}/outside-360.${thumbnailsExtension}`,
+          .clone()
-            outside360Buffer,
-            inputs.file.type,
-          );
-
-          const outside720Buffer = await image
           .resize(720, 720, {
             fit: 'outside',
             withoutEnlargement: true,
@@ -113,14 +107,21 @@ module.exports = {
           .png({
             quality: 75,
             force: false,
-            })
+          });
-            .toBuffer();
 
-          await fileManager.save(
+        try {
-            `${thumbnailsPathSegment}/outside-720.${thumbnailsExtension}`,
+          await Promise.all([
-            outside720Buffer,
+            fileManager.save(
+              `${thumbnailsPathSegment}/outside-360.${thumbnailsExtension}`,
+              outside360,
               inputs.file.type,
-          );
+            ),
+            fileManager.save(
+              `${thumbnailsPathSegment}/outside-720.${thumbnailsExtension}`,
+              outside720,
+              inputs.file.type,
+            ),
+          ]);
 
           data.image = {
             width,

server/api/helpers/background-images/process-uploaded-file.js

@@ -39,21 +39,17 @@ module.exports = {
     });
 
     let metadata;
-    let originalBuffer;
-
     try {
       metadata = await image.metadata();
-
-      if (metadata.orientation && metadata.orientation > 4) {
-        image = image.rotate();
-      }
-
-      originalBuffer = await image.toBuffer();
     } catch (error) {
       await rimraf(inputs.file.fd);
       throw 'fileIsNotImage';
     }
 
+    if (metadata.orientation && metadata.orientation > 4) {
+      image = image.rotate();
+    }
+
     const { id: uploadedFileId } = await UploadedFile.qm.createOne({
       mimeType,
       size,
@@ -64,14 +60,8 @@ module.exports = {
     const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${uploadedFileId}`;
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
-    try {
+    const outside360 = image
-      await fileManager.save(
+      .clone()
-        `${dirPathSegment}/original.${extension}`,
-        originalBuffer,
-        inputs.file.type,
-      );
-
-      const outside360Buffer = await image
       .resize(360, 360, {
         fit: 'outside',
         withoutEnlargement: true,
@@ -79,14 +69,17 @@ module.exports = {
       .png({
         quality: 75,
         force: false,
-        })
+      });
-        .toBuffer();
 
-      await fileManager.save(
+    try {
+      await Promise.all([
+        fileManager.save(`${dirPathSegment}/original.${extension}`, image, inputs.file.type),
+        fileManager.save(
           `${dirPathSegment}/outside-360.${extension}`,
-        outside360Buffer,
+          outside360,
           inputs.file.type,
-      );
+        ),
+      ]);
     } catch (error) {
       sails.log.warn(error.stack);
 

server/api/helpers/users/process-uploaded-avatar-file.js

@@ -39,21 +39,17 @@ module.exports = {
     });
 
     let metadata;
-    let originalBuffer;
-
     try {
       metadata = await image.metadata();
-
-      if (metadata.orientation && metadata.orientation > 4) {
-        image = image.rotate();
-      }
-
-      originalBuffer = await image.toBuffer();
     } catch (error) {
       await rimraf(inputs.file.fd);
       throw 'fileIsNotImage';
     }
 
+    if (metadata.orientation && metadata.orientation > 4) {
+      image = image.rotate();
+    }
+
     const { id: uploadedFileId } = await UploadedFile.qm.createOne({
       mimeType,
       size,
@@ -64,28 +60,21 @@ module.exports = {
     const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${uploadedFileId}`;
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
 
-    try {
+    const cover180 = image
-      await fileManager.save(
+      .clone()
-        `${dirPathSegment}/original.${extension}`,
-        originalBuffer,
-        inputs.file.type,
-      );
-
-      const cover180Buffer = await image
       .resize(180, 180, {
         withoutEnlargement: true,
       })
       .png({
         quality: 75,
         force: false,
-        })
+      });
-        .toBuffer();
 
-      await fileManager.save(
+    try {
-        `${dirPathSegment}/cover-180.${extension}`,
+      await Promise.all([
-        cover180Buffer,
+        fileManager.save(`${dirPathSegment}/original.${extension}`, image, inputs.file.type),
-        inputs.file.type,
+        fileManager.save(`${dirPathSegment}/cover-180.${extension}`, cover180, inputs.file.type),
-      );
+      ]);
     } catch (error) {
       sails.log.warn(error.stack);
 

server/api/helpers/utils/download-favicon.js

@@ -162,8 +162,7 @@ module.exports = {
     const fileManager = sails.hooks['file-manager'].getInstance();
     const { width, height } = metadata;
 
-    try {
+    image = image
-      const buffer = await image
       .resize(
         32,
         32,
@@ -173,12 +172,12 @@ module.exports = {
             }
           : undefined,
       )
-        .png()
+      .png();
-        .toBuffer();
 
+    try {
       await fileManager.save(
         `${sails.config.custom.faviconsPathSegment}/${hostname}.png`,
-        buffer,
+        image,
         'image/png',
       );
     } catch (error) {

server/api/hooks/file-manager/LocalFileManager.js

@@ -6,6 +6,7 @@
 const fs = require('fs');
 const fse = require('fs-extra');
 const path = require('path');
+const { pipeline } = require('stream/promises');
 const { rimraf } = require('rimraf');
 
 const PATH_SEGMENT_TO_URL_REPLACE_REGEX = /(public|private)\//;
@@ -27,8 +28,12 @@ class LocalFileManager {
   }
 
   // eslint-disable-next-line class-methods-use-this
-  async save(filePathSegment, buffer) {
+  async save(filePathSegment, stream) {
-    await fse.outputFile(buildPath(filePathSegment), buffer);
+    const filePath = buildPath(filePathSegment);
+    const { dir: dirPath } = path.parse(filePath);
+
+    await fs.promises.mkdir(dirPath, { recursive: true });
+    await pipeline(stream, fs.createWriteStream(filePath));
   }
 
   // eslint-disable-next-line class-methods-use-this

server/api/hooks/file-manager/S3FileManager.js

@@ -13,6 +13,7 @@ const {
   ListObjectsV2Command,
   PutObjectCommand,
 } = require('@aws-sdk/client-s3');
+const { Upload } = require('@aws-sdk/lib-storage');
 
 class S3FileManager {
   constructor(client) {
@@ -31,15 +32,18 @@ class S3FileManager {
     return null;
   }
 
-  async save(filePathSegment, buffer, contentType) {
+  async save(filePathSegment, stream, contentType) {
-    const command = new PutObjectCommand({
+    const upload = new Upload({
+      client: this.client,
+      params: {
         Bucket: sails.config.custom.s3Bucket,
         Key: filePathSegment,
-      Body: buffer,
+        Body: stream,
         ContentType: contentType,
+      },
     });
 
-    await this.client.send(command);
+    await upload.done();
   }
 
   async read(filePathSegment) {

server/package-lock.json

@@ -8,6 +8,7 @@
       "hasInstallScript": true,
       "dependencies": {
         "@aws-sdk/client-s3": "3.726.1",
+        "@aws-sdk/lib-storage": "3.726.1",
         "bcrypt": "^5.1.1",
         "bytes": "^3.1.2",
         "cross-env": "^7.0.3",
@@ -695,6 +696,27 @@
         "@aws-sdk/client-sts": "^3.723.0"
       }
     },
+    "node_modules/@aws-sdk/lib-storage": {
+      "version": "3.726.1",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.726.1.tgz",
+      "integrity": "sha512-WuDxSZ8Bfe1N7gn5eXQ02dhlKWCAwW5qQErpJ4CCddXosF+gLxhGkrP9LkaaP0CpA3PxboHyET6HbWAggOWtqA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@smithy/abort-controller": "^4.0.0",
+        "@smithy/middleware-endpoint": "^4.0.0",
+        "@smithy/smithy-client": "^4.0.0",
+        "buffer": "5.6.0",
+        "events": "3.3.0",
+        "stream-browserify": "3.0.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      },
+      "peerDependencies": {
+        "@aws-sdk/client-s3": "^3.726.1"
+      }
+    },
     "node_modules/@aws-sdk/middleware-bucket-endpoint": {
       "version": "3.726.0",
       "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.726.0.tgz",
@@ -3124,6 +3146,26 @@
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "license": "MIT"
     },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
     "node_modules/base64id": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz",
@@ -3280,6 +3322,16 @@
       "dev": true,
       "license": "ISC"
     },
+    "node_modules/buffer": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
+      "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
+      "license": "MIT",
+      "dependencies": {
+        "base64-js": "^1.0.2",
+        "ieee754": "^1.1.4"
+      }
+    },
     "node_modules/buffer-equal-constant-time": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
@@ -4930,6 +4982,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/express": {
       "version": "4.21.2",
       "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
@@ -10599,6 +10660,16 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/stream-browserify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",
+      "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==",
+      "license": "MIT",
+      "dependencies": {
+        "inherits": "~2.0.4",
+        "readable-stream": "^3.5.0"
+      }
+    },
     "node_modules/streamifier": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/streamifier/-/streamifier-0.1.1.tgz",

server/package.json

@@ -45,6 +45,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.726.1",
+    "@aws-sdk/lib-storage": "3.726.1",
     "bcrypt": "^5.1.1",
     "bytes": "^3.1.2",
     "cross-env": "^7.0.3",