Merge branch 'master' into upload-progress-bars

.github/workflows/pytest.yml

@@ -42,10 +42,10 @@ jobs:
         run: |
           pipx install poetry
 
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: "poetry"
 
       - name: Install dependencies

.python-version

@@ -1 +1 @@
-3.11
+3.12

.trunk/trunk.yaml

@@ -14,7 +14,7 @@ runtimes:
   enabled:
     - go@1.21.0
     - node@18.12.1
-    - python@3.11.6
+    - python@3.12.2
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:

backend/endpoints/responses/__init__.py

@@ -1,4 +1,4 @@
-from typing_extensions import TypedDict
+from typing import TypedDict
 
 
 class MessageResponse(TypedDict):

backend/endpoints/responses/assets.py

@@ -1,7 +1,7 @@
 from datetime import datetime
+from typing import TypedDict
 
 from pydantic import BaseModel
-from typing_extensions import TypedDict
 
 
 class BaseAsset(BaseModel):

backend/endpoints/responses/config.py

@@ -1,4 +1,4 @@
-from typing_extensions import TypedDict
+from typing import TypedDict
 
 
 class ConfigResponse(TypedDict):

backend/endpoints/responses/feeds.py

@@ -1,6 +1,4 @@
-from typing import NotRequired
-
-from typing_extensions import TypedDict
+from typing import NotRequired, TypedDict
 
 WEBRCADE_SUPPORTED_PLATFORM_SLUGS = frozenset(
     (

backend/endpoints/responses/firmware.py

@@ -1,7 +1,7 @@
 from datetime import datetime
+from typing import TypedDict
 
 from pydantic import BaseModel
-from typing_extensions import TypedDict
 
 
 class FirmwareSchema(BaseModel):

backend/endpoints/responses/heartbeat.py

@@ -1,4 +1,4 @@
-from typing_extensions import TypedDict
+from typing import TypedDict
 
 
 class WatcherDict(TypedDict):

backend/endpoints/responses/oauth.py

@@ -1,6 +1,4 @@
-from typing import NotRequired
-
-from typing_extensions import TypedDict
+from typing import NotRequired, TypedDict
 
 
 class TokenResponse(TypedDict):

backend/endpoints/responses/rom.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import re
 from datetime import datetime
-from typing import NotRequired, get_type_hints
+from typing import NotRequired, TypedDict, get_type_hints
 
 from endpoints.responses.assets import SaveSchema, ScreenshotSchema, StateSchema
 from endpoints.responses.collection import CollectionSchema
@@ -13,7 +13,6 @@ from handler.metadata.moby_handler import MobyMetadata
 from handler.socket_handler import socket_handler
 from models.rom import Rom, RomFile
 from pydantic import BaseModel, Field, computed_field
-from typing_extensions import TypedDict
 
 SORT_COMPARE_REGEX = re.compile(r"^([Tt]he|[Aa]|[Aa]nd)\s")
 

backend/endpoints/responses/stats.py

@@ -1,4 +1,4 @@
-from typing_extensions import TypedDict
+from typing import TypedDict
 
 
 class StatsReturn(TypedDict):

backend/endpoints/rom.py

@@ -32,14 +32,14 @@ from fastapi import (
     UploadFile,
     status,
 )
-from fastapi.responses import FileResponse
+from fastapi.responses import Response
 from handler.database import db_platform_handler, db_rom_handler
 from handler.filesystem import fs_resource_handler, fs_rom_handler
 from handler.filesystem.base_handler import CoverSize
 from handler.metadata import meta_igdb_handler, meta_moby_handler
 from handler.socket_handler import socket_handler
 from logger.logger import log
-from stream_zip import NO_COMPRESSION_32, ZIP_AUTO, AsyncMemberFile, async_stream_zip
+from stream_zip import NO_COMPRESSION_64, ZIP_AUTO, AsyncMemberFile, async_stream_zip
 from utils.router import APIRouter
 
 router = APIRouter()
@@ -187,7 +187,12 @@ def get_rom(request: Request, id: int) -> DetailedRomSchema:
     "/roms/{id}/content/{file_name}",
     [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else ["roms.read"],
 )
-def head_rom_content(request: Request, id: int, file_name: str):
+async def head_rom_content(
+    request: Request,
+    id: int,
+    file_name: str,
+    files: Annotated[list[str] | None, Query()] = None,
+):
     """Head rom content endpoint
 
     Args:
@@ -204,15 +209,30 @@ def head_rom_content(request: Request, id: int, file_name: str):
     if not rom:
         raise RomNotFoundInDatabaseException(id)
 
-    rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
+    files_to_check = files or [r["filename"] for r in rom.files]
 
-    return FileResponse(
-        path=rom_path if not rom.multi else f'{rom_path}/{rom.files[0]["filename"]}',
-        filename=file_name,
+    if not rom.multi:
+        return Response(
+            media_type="application/octet-stream",
+            headers={
+                "Content-Disposition": f'attachment; filename="{quote(rom.file_name)}"',
+                "X-Accel-Redirect": f"/library/{rom.full_path}",
+            },
+        )
+
+    if len(files_to_check) == 1:
+        return Response(
+            media_type="application/octet-stream",
+            headers={
+                "Content-Disposition": f'attachment; filename="{quote(files_to_check[0])}"',
+                "X-Accel-Redirect": f"/library/{rom.full_path}/{files_to_check[0]}",
+            },
+        )
+
+    return Response(
+        media_type="application/zip",
         headers={
-            "Content-Disposition": f'attachment; filename="{quote(rom.name)}.zip"',
-            "Content-Type": "application/zip",
-            "Content-Length": str(rom.file_size_bytes),
+            "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"',
         },
     )
 
@@ -247,11 +267,21 @@ async def get_rom_content(
     files_to_download = files or [r["filename"] for r in rom.files]
 
     if not rom.multi:
-        return FileResponse(path=rom_path, filename=rom.file_name)
+        return Response(
+            media_type="application/octet-stream",
+            headers={
+                "Content-Disposition": f'attachment; filename="{quote(rom.file_name)}"',
+                "X-Accel-Redirect": f"/library/{rom.full_path}",
+            },
+        )
 
     if len(files_to_download) == 1:
-        return FileResponse(
-            path=f"{rom_path}/{files_to_download[0]}", filename=files_to_download[0]
+        return Response(
+            media_type="application/octet-stream",
+            headers={
+                "Content-Disposition": f'attachment; filename="{quote(files_to_download[0])}"',
+                "X-Accel-Redirect": f"/library/{rom.full_path}/{files_to_download[0]}",
+            },
         )
 
     # Builds a generator of tuples for each member file
@@ -285,7 +315,7 @@ async def get_rom_content(
             f"{file_name}.m3u",
             now,
             S_IFREG | 0o600,
-            NO_COMPRESSION_32,
+            NO_COMPRESSION_64,
             m3u_file(),
         )
 
@@ -296,7 +326,7 @@ async def get_rom_content(
         zipped_chunks,
         media_type="application/zip",
         headers={
-            "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"'
+            "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"',
         },
         emit_body={"id": rom.id},
     )

backend/handler/filesystem/roms_handler.py

@@ -6,8 +6,9 @@ import re
 import shutil
 import tarfile
 import zipfile
+from collections.abc import Iterator
 from pathlib import Path
-from typing import Any, Final, Iterator, Tuple
+from typing import Any, Final, TypedDict
 
 import magic
 import py7zr
@@ -15,7 +16,6 @@ from config import LIBRARY_BASE_PATH
 from config.config_manager import config_manager as cm
 from exceptions.fs_exceptions import RomAlreadyExistsException, RomsNotFoundException
 from models.rom import RomFile
-from typing_extensions import TypedDict
 from utils.filesystem import iter_directories, iter_files
 
 from .base_handler import (
@@ -189,7 +189,7 @@ class FSRomsHandler(FSHandler):
 
     def _calculate_rom_hashes(
         self, file_path: Path, crc_c: int, md5_h: Any, sha1_h: Any
-    ) -> Tuple[int, Any, Any]:
+    ) -> tuple[int, Any, Any]:
         mime = magic.Magic(mime=True)
         file_type = mime.from_file(file_path)
         extension = Path(file_path).suffix.lower()

backend/handler/metadata/base_hander.py

@@ -2,6 +2,7 @@ import json
 import os
 import re
 import unicodedata
+from itertools import batched
 from typing import Final
 
 from handler.redis_handler import async_cache, sync_cache
@@ -11,7 +12,6 @@ from tasks.update_switch_titledb import (
     SWITCH_TITLEDB_INDEX_KEY,
     update_switch_titledb_task,
 )
-from utils.iterators import batched
 
 
 def conditionally_set_cache(

backend/handler/metadata/igdb_handler.py

@@ -1,7 +1,7 @@
 import functools
 import re
 import time
-from typing import Final, NotRequired
+from typing import Final, NotRequired, TypedDict
 
 import httpx
 import pydash
@@ -9,7 +9,6 @@ from config import IGDB_CLIENT_ID, IGDB_CLIENT_SECRET, IS_PYTEST_RUN
 from fastapi import HTTPException, status
 from handler.redis_handler import sync_cache
 from logger.logger import log
-from typing_extensions import TypedDict
 from unidecode import unidecode as uc
 from utils.context import ctx_httpx_client
 

backend/handler/metadata/moby_handler.py

@@ -1,7 +1,7 @@
 import asyncio
 import http
 import re
-from typing import Final, NotRequired
+from typing import Final, NotRequired, TypedDict
 from urllib.parse import quote
 
 import httpx
@@ -10,7 +10,6 @@ import yarl
 from config import MOBYGAMES_API_KEY
 from fastapi import HTTPException, status
 from logger.logger import log
-from typing_extensions import TypedDict
 from unidecode import unidecode as uc
 from utils.context import ctx_httpx_client
 

backend/main.py

@@ -32,7 +32,6 @@ from endpoints import (
 )
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
 from handler.auth.base_handler import ALGORITHM
 from handler.auth.hybrid_auth import HybridAuthBackend
 from handler.auth.middleware import CustomCSRFMiddleware, SessionMiddleware
@@ -68,9 +67,6 @@ if not IS_PYTEST_RUN and not DISABLE_CSRF_PROTECTION:
         exempt_urls=[re.compile(r"^/token.*"), re.compile(r"^/ws")],
     )
 
-# Enable GZip compression for responses
-app.add_middleware(GZipMiddleware, minimum_size=1024)
-
 # Handles both basic and oauth authentication
 app.add_middleware(
     AuthenticationMiddleware,

backend/models/rom.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypedDict
 
 from config import FRONTEND_RESOURCES_PATH
 from models.base import BaseModel
@@ -16,7 +16,6 @@ from sqlalchemy import (
 )
 from sqlalchemy.dialects.mysql.json import JSON as MySQLJSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
-from typing_extensions import TypedDict
 
 if TYPE_CHECKING:
     from models.assets import Save, Screenshot, State

backend/tasks/update_switch_titledb.py

@@ -1,4 +1,5 @@
 import json
+from itertools import batched
 from typing import Final
 
 from config import (
@@ -9,7 +10,6 @@ from handler.redis_handler import async_cache
 from logger.logger import log
 from tasks.tasks import RemoteFilePullTask
 from utils.context import initialize_context
-from utils.iterators import batched
 
 SWITCH_TITLEDB_INDEX_KEY: Final = "romm:switch_titledb"
 SWITCH_PRODUCT_ID_KEY: Final = "romm:switch_product_id"

backend/utils/iterators.py

@@ -1,17 +0,0 @@
-import sys
-
-if sys.version_info >= (3, 12):
-    from itertools import batched  # noqa: F401
-else:
-    from collections.abc import Iterable, Iterator
-    from itertools import islice
-    from typing import TypeVar
-
-    T = TypeVar("T")
-
-    def batched(iterable: Iterable[T], n: int) -> Iterator[tuple[T, ...]]:
-        if n < 1:
-            raise ValueError("n must be at least one")
-        iterator = iter(iterable)
-        while batch := tuple(islice(iterator, n)):
-            yield batch

backend/utils/router.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 from fastapi import APIRouter as FastAPIRouter
 from fastapi.types import DecoratedCallable

docker/Dockerfile

@@ -1,7 +1,7 @@
-ARG ALPINE_VERSION=3.19
-ARG NGINX_VERSION=1.27.0
-ARG NODE_VERSION=lts
-ARG PYTHON_VERSION=3.11
+ARG ALPINE_VERSION=3.20
+ARG NGINX_VERSION=1.27.1
+ARG NODE_VERSION=20.16
+ARG PYTHON_VERSION=3.12
 
 # Build frontend
 FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage

docker/nginx/default.conf

@@ -14,9 +14,14 @@ http {
         scgi_temp_path /tmp/scgi;
 
         sendfile on;
+        client_body_buffer_size 128k;
         client_max_body_size 0;
+        client_header_buffer_size 1k;
+        large_client_header_buffers 4 16k;
+        send_timeout 60s;
+        keepalive_timeout 65s;
         tcp_nopush on;
-        # types_hash_max_size 2048;
+        tcp_nodelay on;
 
         include /etc/nginx/mime.types;
         default_type application/octet-stream;
@@ -41,6 +46,13 @@ http {
         error_log /dev/stderr;
 
         gzip on;
+        gzip_proxied any;
+        gzip_vary on;
+        gzip_comp_level 6;
+        gzip_buffers 16 8k;
+        gzip_min_length 1024;
+        gzip_http_version 1.1;
+        gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
 
         # include /etc/nginx/conf.d/*.conf;
         # include /etc/nginx/sites-enabled/*;
@@ -87,5 +99,11 @@ http {
                 proxy_set_header Upgrade $http_upgrade;
                 proxy_set_header Connection "upgrade";
             }
+
+            # Internally redirect download requests
+            location /library {
+                internal;
+                alias /romm/library;
+            }
         }
 }

poetry.lock

@@ -79,17 +79,6 @@ six = ">=1.12.0"
 astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"]
 test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"]
 
-[[package]]
-name = "async-timeout"
-version = "4.0.3"
-description = "Timeout context manager for asyncio programs"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
-    {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
-]
-
 [[package]]
 name = "bcrypt"
 version = "4.1.3"
@@ -884,7 +873,6 @@ prompt-toolkit = ">=3.0.41,<3.1.0"
 pygments = ">=2.4.0"
 stack-data = "*"
 traitlets = ">=5.13.0"
-typing-extensions = {version = ">=4.6", markers = "python_version < \"3.12\""}
 
 [package.extras]
 all = ["ipython[black,doc,kernel,matplotlib,nbconvert,nbformat,notebook,parallel,qtconsole]", "ipython[test,test-extra]"]
@@ -2201,6 +2189,7 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -2444,9 +2433,6 @@ files = [
     {file = "redis-5.0.7.tar.gz", hash = "sha256:8f611490b93c8109b50adc317b31bfd84fff31def3475b92e7e80bf39f48175b"},
 ]
 
-[package.dependencies]
-async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""}
-
 [package.extras]
 hiredis = ["hiredis (>=1.0.0)"]
 ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"]
@@ -3238,5 +3224,5 @@ multidict = ">=4.0"
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.11"
-content-hash = "1a225b47254091e5ab0e5b1b5b23773074c52c04a74e126f2aab12a890ca6082"
+python-versions = "^3.12"
+content-hash = "05c76acc85421bab1fc3f9f28f9291bce64c5e99154b8ee72dc809a26d9804e2"

pyproject.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/rommapp/romm"
 authors = ["Zurdi <zurdi@romm.app>", "Arcane <arcane@romm.app>"]
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = "^3.12"
 anyio = "^4.4"
 fastapi = "0.110.0"
 uvicorn = "0.29.0"