diff --git a/bun.lock b/bun.lock index ec30c1f..4bdeb22 100644 --- a/bun.lock +++ b/bun.lock @@ -10,6 +10,7 @@ "@kitajs/html": "^4.2.9", "elysia": "^1.3.4", "sanitize-filename": "^1.6.3", + "tar": "^7.4.3", }, "devDependencies": { "@eslint/js": "^9.28.0", diff --git a/package.json b/package.json index 23a19bd..4ac248a 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "@elysiajs/static": "^1.3.0", "@kitajs/html": "^4.2.9", "elysia": "^1.3.4", - "sanitize-filename": "^1.6.3" + "sanitize-filename": "^1.6.3", + "tar": "^7.4.3" }, "module": "src/index.tsx", "type": "module", diff --git a/public/results.js b/public/results.js index 7659edb..b125e93 100644 --- a/public/results.js +++ b/public/results.js @@ -1,18 +1,4 @@ const webroot = document.querySelector("meta[name='webroot']").content; - -window.downloadAll = function () { - // Get all download links - const downloadLinks = document.querySelectorAll("a[download]"); - - // Trigger download for each link - downloadLinks.forEach((link, index) => { - // We add a delay for each download to prevent them from starting at the same time - setTimeout(() => { - const event = new MouseEvent("click"); - link.dispatchEvent(event); - }, index * 100); - }); -}; const jobId = window.location.pathname.split("/").pop(); const main = document.querySelector("main"); let progressElem = document.querySelector("progress"); diff --git a/src/converters/main.ts b/src/converters/main.ts index 0d08416..bc0c27f 100644 --- a/src/converters/main.ts +++ b/src/converters/main.ts @@ -1,4 +1,6 @@ -import { normalizeFiletype } from "../helpers/normalizeFiletype"; +import db from "../db/db"; +import { MAX_CONVERT_PROCESS } from "../helpers/env"; +import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype"; import { convert as convertassimp, properties as propertiesassimp } from "./assimp"; import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre"; import { convert as convertDvisvgm, properties as propertiesDvisvgm } from "./dvisvgm"; @@ -111,6 +113,63 @@ const properties: Record< }, }; +function chunks(arr: T[], size: number): T[][] { + if(size <= 0){ + return [arr] + } + return Array.from({ length: Math.ceil(arr.length / size) }, (_: T, i: number) => + arr.slice(i * size, i * size + size) + ); +} + +export async function handleConvert( + fileNames: string[], + userUploadsDir: string, + userOutputDir: string, + convertTo: string, + converterName: string, + jobId: any +) { + + const query = db.query( + "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)", + ); + + + for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) { + const toProcess: Promise[] = []; + for(const fileName of chunk) { + const filePath = `${userUploadsDir}${fileName}`; + const fileTypeOrig = fileName.split(".").pop() ?? ""; + const fileType = normalizeFiletype(fileTypeOrig); + const newFileExt = normalizeOutputFiletype(convertTo); + const newFileName = fileName.replace( + new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`), + newFileExt, + ); + const targetPath = `${userOutputDir}${newFileName}`; + toProcess.push( + new Promise((resolve, reject) => { + mainConverter( + filePath, + fileType, + convertTo, + targetPath, + {}, + converterName, + ).then(r => { + if (jobId.value) { + query.run(jobId.value, fileName, newFileName, r); + } + resolve(r); + }).catch(c => reject(c)); + }) + ); + } + await Promise.all(toProcess); + } +} + export async function mainConverter( inputFilePath: string, fileTypeOriginal: string, diff --git a/src/helpers/env.ts b/src/helpers/env.ts index cbdf423..29820b7 100644 --- a/src/helpers/env.ts +++ b/src/helpers/env.ts @@ -15,3 +15,5 @@ export const HIDE_HISTORY = process.env.HIDE_HISTORY?.toLowerCase() === "true" | export const WEBROOT = process.env.WEBROOT ?? ""; export const LANGUAGE = process.env.LANGUAGE?.toLowerCase() || "en"; + +export const MAX_CONVERT_PROCESS = process.env.MAX_CONVERT_PROCESS && Number(process.env.MAX_CONVERT_PROCESS) > 0 ? Number(process.env.MAX_CONVERT_PROCESS) : 0 \ No newline at end of file diff --git a/src/pages/convert.tsx b/src/pages/convert.tsx index a95302c..3dbb892 100644 --- a/src/pages/convert.tsx +++ b/src/pages/convert.tsx @@ -2,11 +2,11 @@ import { mkdir } from "node:fs/promises"; import { Elysia, t } from "elysia"; import sanitize from "sanitize-filename"; import { outputDir, uploadsDir } from ".."; -import { mainConverter } from "../converters/main"; +import { handleConvert } from "../converters/main"; import db from "../db/db"; import { Jobs } from "../db/types"; import { WEBROOT } from "../helpers/env"; -import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype"; +import { normalizeFiletype } from "../helpers/normalizeFiletype"; import { userService } from "./user"; export const convert = new Elysia().use(userService).post( @@ -61,36 +61,8 @@ export const convert = new Elysia().use(userService).post( jobId.value, ); - const query = db.query( - "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)", - ); - // Start the conversion process in the background - Promise.all( - fileNames.map(async (fileName) => { - const filePath = `${userUploadsDir}${fileName}`; - const fileTypeOrig = fileName.split(".").pop() ?? ""; - const fileType = normalizeFiletype(fileTypeOrig); - const newFileExt = normalizeOutputFiletype(convertTo); - const newFileName = fileName.replace( - new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`), - newFileExt, - ); - const targetPath = `${userOutputDir}${newFileName}`; - - const result = await mainConverter( - filePath, - fileType, - convertTo, - targetPath, - {}, - converterName, - ); - if (jobId.value) { - query.run(jobId.value, fileName, newFileName, result); - } - }), - ) + handleConvert(fileNames, userUploadsDir, userOutputDir, convertTo, converterName, jobId) .then(() => { // All conversions are done, update the job status to 'completed' if (jobId.value) { diff --git a/src/pages/download.tsx b/src/pages/download.tsx index 917ee39..3333396 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -4,6 +4,8 @@ import { outputDir } from ".."; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; +import path from "node:path"; +import * as tar from "tar"; export const download = new Elysia() .use(userService) @@ -35,8 +37,7 @@ export const download = new Elysia() return Bun.file(filePath); }, ) - .get("/zip/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => { - // TODO: Implement zip download + .get("/archive/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => { if (!auth?.value) { return redirect(`${WEBROOT}/login`, 302); } @@ -54,9 +55,11 @@ export const download = new Elysia() return redirect(`${WEBROOT}/results`, 302); } - // const userId = decodeURIComponent(params.userId); - // const jobId = decodeURIComponent(params.jobId); - // const outputPath = `${outputDir}${userId}/`{jobId}/); + const userId = decodeURIComponent(params.userId); + const jobId = decodeURIComponent(params.jobId); + const outputPath = `${outputDir}${userId}/${jobId}`; + const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`) - // return Bun.zip(outputPath); + await tar.create({file: outputTar, cwd: outputPath, filter: (path) => { return !path.match(".*\\.tar"); }}, ["."]); + return Bun.file(outputTar); }); diff --git a/src/pages/results.tsx b/src/pages/results.tsx index 81e6128..edd70e7 100644 --- a/src/pages/results.tsx +++ b/src/pages/results.tsx @@ -6,12 +6,17 @@ import db from "../db/db"; import { Filename, Jobs } from "../db/types"; import { ALLOW_UNAUTHENTICATED, WEBROOT } from "../helpers/env"; import { userService } from "./user"; +import { JWTPayloadSpec } from "@elysiajs/jwt"; function ResultsArticle({ + user, job, files, outputPath, }: { + user: { + id: string; + } & JWTPayloadSpec; job: Jobs; files: Filename[]; outputPath: string; @@ -21,14 +26,19 @@ function ResultsArticle({

Results

- + +
- +