Merge pull request #342 from Netzz0/FEAT/better-handling-of-multiples-files

Better handling of multiples files (Added Archive downloads and env var to set maximum concurrent processes)

bun.lock

@@ -10,6 +10,7 @@
         "@kitajs/html": "^4.2.9",
         "elysia": "^1.3.4",
         "sanitize-filename": "^1.6.3",
+        "tar": "^7.4.3",
       },
       "devDependencies": {
         "@eslint/js": "^9.28.0",

package.json

@@ -20,7 +20,8 @@
     "@elysiajs/static": "^1.3.0",
     "@kitajs/html": "^4.2.9",
     "elysia": "^1.3.4",
-    "sanitize-filename": "^1.6.3"
+    "sanitize-filename": "^1.6.3",
+    "tar": "^7.4.3"
   },
   "module": "src/index.tsx",
   "type": "module",

public/results.js

@@ -1,18 +1,4 @@
 const webroot = document.querySelector("meta[name='webroot']").content;
-
-window.downloadAll = function () {
-  // Get all download links
-  const downloadLinks = document.querySelectorAll("a[download]");
-
-  // Trigger download for each link
-  downloadLinks.forEach((link, index) => {
-    // We add a delay for each download to prevent them from starting at the same time
-    setTimeout(() => {
-      const event = new MouseEvent("click");
-      link.dispatchEvent(event);
-    }, index * 100);
-  });
-};
 const jobId = window.location.pathname.split("/").pop();
 const main = document.querySelector("main");
 let progressElem = document.querySelector("progress");

src/converters/main.ts

@@ -1,4 +1,6 @@
-import { normalizeFiletype } from "../helpers/normalizeFiletype";
+import db from "../db/db";
+import { MAX_CONVERT_PROCESS } from "../helpers/env";
+import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
 import { convert as convertassimp, properties as propertiesassimp } from "./assimp";
 import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre";
 import { convert as convertDvisvgm, properties as propertiesDvisvgm } from "./dvisvgm";
@@ -111,6 +113,63 @@ const properties: Record<
   },
 };
 
+function chunks<T>(arr: T[], size: number): T[][] {
+  if(size <= 0){
+    return [arr]
+  }
+  return Array.from({ length: Math.ceil(arr.length / size) }, (_: T, i: number) =>
+    arr.slice(i * size, i * size + size)
+  );
+}
+
+export async function handleConvert(
+  fileNames: string[],
+  userUploadsDir: string,
+  userOutputDir: string,
+  convertTo: string,
+  converterName: string,
+  jobId: any
+) {
+  
+  const query = db.query(
+    "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
+  );
+
+
+  for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) {
+    const toProcess: Promise<string>[] = [];
+    for(const fileName of chunk) {
+      const filePath = `${userUploadsDir}${fileName}`;
+      const fileTypeOrig = fileName.split(".").pop() ?? "";
+      const fileType = normalizeFiletype(fileTypeOrig);
+      const newFileExt = normalizeOutputFiletype(convertTo);
+      const newFileName = fileName.replace(
+        new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
+        newFileExt,
+      );
+      const targetPath = `${userOutputDir}${newFileName}`;
+      toProcess.push(
+        new Promise((resolve, reject) => { 
+          mainConverter(
+              filePath,
+              fileType,
+              convertTo,
+              targetPath,
+              {},
+              converterName,
+            ).then(r =>  {
+              if (jobId.value) {
+                query.run(jobId.value, fileName, newFileName, r);
+              }
+              resolve(r);
+            }).catch(c => reject(c));
+        })
+      );
+    }
+    await Promise.all(toProcess);
+  }
+}
+
 export async function mainConverter(
   inputFilePath: string,
   fileTypeOriginal: string,

src/helpers/env.ts

@@ -15,3 +15,5 @@ export const HIDE_HISTORY = process.env.HIDE_HISTORY?.toLowerCase() === "true" |
 export const WEBROOT = process.env.WEBROOT ?? "";
 
 export const LANGUAGE = process.env.LANGUAGE?.toLowerCase() || "en";
+
+export const MAX_CONVERT_PROCESS = process.env.MAX_CONVERT_PROCESS && Number(process.env.MAX_CONVERT_PROCESS) > 0 ? Number(process.env.MAX_CONVERT_PROCESS) : 0

src/pages/convert.tsx

@@ -2,11 +2,11 @@ import { mkdir } from "node:fs/promises";
 import { Elysia, t } from "elysia";
 import sanitize from "sanitize-filename";
 import { outputDir, uploadsDir } from "..";
-import { mainConverter } from "../converters/main";
+import { handleConvert } from "../converters/main";
 import db from "../db/db";
 import { Jobs } from "../db/types";
 import { WEBROOT } from "../helpers/env";
-import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
+import { normalizeFiletype } from "../helpers/normalizeFiletype";
 import { userService } from "./user";
 
 export const convert = new Elysia().use(userService).post(
@@ -61,36 +61,8 @@ export const convert = new Elysia().use(userService).post(
       jobId.value,
     );
 
-    const query = db.query(
-      "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
-    );
-
     // Start the conversion process in the background
-    Promise.all(
-      fileNames.map(async (fileName) => {
-        const filePath = `${userUploadsDir}${fileName}`;
-        const fileTypeOrig = fileName.split(".").pop() ?? "";
-        const fileType = normalizeFiletype(fileTypeOrig);
-        const newFileExt = normalizeOutputFiletype(convertTo);
-        const newFileName = fileName.replace(
-          new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
-          newFileExt,
-        );
-        const targetPath = `${userOutputDir}${newFileName}`;
-
-        const result = await mainConverter(
-          filePath,
-          fileType,
-          convertTo,
-          targetPath,
-          {},
-          converterName,
-        );
-        if (jobId.value) {
-          query.run(jobId.value, fileName, newFileName, result);
-        }
-      }),
-    )
+    handleConvert(fileNames, userUploadsDir, userOutputDir, convertTo, converterName, jobId)
       .then(() => {
         // All conversions are done, update the job status to 'completed'
         if (jobId.value) {

src/pages/download.tsx

@@ -4,6 +4,8 @@ import { outputDir } from "..";
 import db from "../db/db";
 import { WEBROOT } from "../helpers/env";
 import { userService } from "./user";
+import path from "node:path";
+import * as tar from "tar";
 
 export const download = new Elysia()
   .use(userService)
@@ -35,8 +37,7 @@ export const download = new Elysia()
       return Bun.file(filePath);
     },
   )
-  .get("/zip/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => {
-    // TODO: Implement zip download
+  .get("/archive/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => {
     if (!auth?.value) {
       return redirect(`${WEBROOT}/login`, 302);
     }
@@ -54,9 +55,11 @@ export const download = new Elysia()
       return redirect(`${WEBROOT}/results`, 302);
     }
 
-    // const userId = decodeURIComponent(params.userId);
-    // const jobId = decodeURIComponent(params.jobId);
-    // const outputPath = `${outputDir}${userId}/`{jobId}/);
+    const userId = decodeURIComponent(params.userId);
+    const jobId = decodeURIComponent(params.jobId);
+    const outputPath = `${outputDir}${userId}/${jobId}`;
+    const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`)
 
-    // return Bun.zip(outputPath);
+    await tar.create({file: outputTar, cwd: outputPath, filter: (path) => { return !path.match(".*\\.tar"); }}, ["."]);
+    return Bun.file(outputTar);
   });

src/pages/results.tsx

@@ -6,12 +6,17 @@ import db from "../db/db";
 import { Filename, Jobs } from "../db/types";
 import { ALLOW_UNAUTHENTICATED, WEBROOT } from "../helpers/env";
 import { userService } from "./user";
+import { JWTPayloadSpec } from "@elysiajs/jwt";
 
 function ResultsArticle({
+  user,
   job,
   files,
   outputPath,
 }: {
+  user: {
+    id: string;
+  } & JWTPayloadSpec;
   job: Jobs;
   files: Filename[];
   outputPath: string;
@@ -21,14 +26,19 @@ function ResultsArticle({
       <div class="mb-4 flex items-center justify-between">
         <h1 class="text-xl">Results</h1>
         <div>
-          <button
-            type="button"
-            class="float-right w-40 btn-primary"
-            onclick="downloadAll()"
-            {...(files.length !== job.num_files ? { disabled: true, "aria-busy": "true" } : "")}
+          <a  
+            style={files.length !== job.num_files ? "pointer-events: none;" : ""}
+            href={`${WEBROOT}/archive/${user.id}/${job.id}`}
+            download={`converted_files_${job.id}.tar`}
           >
-            {files.length === job.num_files ? "Download All" : "Converting..."}
-          </button>
+            <button
+              type="button"
+              class="float-right w-40 btn-primary"
+              {...(files.length !== job.num_files ? { disabled: true, "aria-busy": "true" } : "")}
+              >
+              {files.length === job.num_files ? "Download All" : "Converting..."}
+            </button>
+          </a>
         </div>
       </div>
       <progress
@@ -170,7 +180,7 @@ export const results = new Elysia()
               sm:px-4
             `}
           >
-            <ResultsArticle job={job} files={files} outputPath={outputPath} />
+            <ResultsArticle user={user} job={job} files={files} outputPath={outputPath} />
           </main>
           <script src={`${WEBROOT}/results.js`} defer />
         </>
@@ -211,5 +221,5 @@ export const results = new Elysia()
       .as(Filename)
       .all(params.jobId);
 
-    return <ResultsArticle job={job} files={files} outputPath={outputPath} />;
+    return <ResultsArticle user={user} job={job} files={files} outputPath={outputPath} />;
   });