diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 238ab0258..fee06427f 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -108,6 +108,13 @@ SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get( "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *", # At 4:00 AM every day ) +ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA: Final = str_to_bool( + os.environ.get("ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA", "false") +) +SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON: Final = os.environ.get( + "SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON", + "0 5 * * *", # At 5:00 AM every day +) # EMULATION DISABLE_EMULATOR_JS = str_to_bool(os.environ.get("DISABLE_EMULATOR_JS", "false")) diff --git a/backend/endpoints/heartbeat.py b/backend/endpoints/heartbeat.py index 44b7221e4..9bd2ca62b 100644 --- a/backend/endpoints/heartbeat.py +++ b/backend/endpoints/heartbeat.py @@ -4,11 +4,13 @@ from config import ( DISABLE_USERPASS_LOGIN, ENABLE_RESCAN_ON_FILESYSTEM_CHANGE, ENABLE_SCHEDULED_RESCAN, + ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, OIDC_ENABLED, OIDC_PROVIDER, RESCAN_ON_FILESYSTEM_CHANGE_DELAY, SCHEDULED_RESCAN_CRON, + SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, UPLOAD_TIMEOUT, ) @@ -64,6 +66,12 @@ def heartbeat() -> HeartbeatResponse: "TITLE": "Scheduled Switch TitleDB update", "MESSAGE": "Updates the Nintendo Switch TitleDB file", }, + "LAUNCHBOX_METADATA": { + "ENABLED": ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, + "CRON": SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON, + "TITLE": "Scheduled LaunchBox metadata update", + "MESSAGE": "Updates the LaunchBox metadata", + }, }, "EMULATION": { "DISABLE_EMULATOR_JS": DISABLE_EMULATOR_JS, diff --git a/backend/endpoints/responses/heartbeat.py b/backend/endpoints/responses/heartbeat.py index f44323f5d..765e44754 100644 --- a/backend/endpoints/responses/heartbeat.py +++ b/backend/endpoints/responses/heartbeat.py @@ -19,6 +19,7 @@ class TaskDict(WatcherDict): class SchedulerDict(TypedDict): RESCAN: TaskDict SWITCH_TITLEDB: TaskDict + LAUNCHBOX_METADATA: TaskDict class MetadataSourcesDict(TypedDict): diff --git a/backend/endpoints/tasks.py b/backend/endpoints/tasks.py index 12f301a22..4470d82b6 100644 --- a/backend/endpoints/tasks.py +++ b/backend/endpoints/tasks.py @@ -2,6 +2,7 @@ from decorators.auth import protected_route from endpoints.responses import MessageResponse from fastapi import Request from handler.auth.constants import Scope +from tasks.update_launchbox_metadata import update_launchbox_metadata_task from tasks.update_switch_titledb import update_switch_titledb_task from utils.router import APIRouter @@ -19,12 +20,13 @@ async def run_tasks(request: Request) -> MessageResponse: """ await update_switch_titledb_task.run() + await update_launchbox_metadata_task.run() return {"msg": "All tasks ran successfully!"} @protected_route(router.post, "/tasks/{task}/run", [Scope.TASKS_RUN]) async def run_task(request: Request, task: str) -> MessageResponse: - """Run all tasks endpoint + """Run single tasks endpoint Args: request (Request): Fastapi Request object @@ -32,7 +34,10 @@ async def run_task(request: Request, task: str) -> MessageResponse: RunTasksResponse: Standard message response """ - tasks = {"switch_titledb": update_switch_titledb_task} + tasks = { + "switch_titledb": update_switch_titledb_task, + "launchbox_metadata": update_launchbox_metadata_task, + } await tasks[task].run() return {"msg": f"Task {task} run successfully!"} diff --git a/backend/endpoints/tests/test_heartbeat.py b/backend/endpoints/tests/test_heartbeat.py index d8ac46449..d1564d168 100644 --- a/backend/endpoints/tests/test_heartbeat.py +++ b/backend/endpoints/tests/test_heartbeat.py @@ -27,4 +27,12 @@ def test_heartbeat(client): heartbeat.get("SCHEDULER").get("SWITCH_TITLEDB").get("TITLE") == "Scheduled Switch TitleDB update" ) + assert heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("ENABLED") + assert ( + heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("CRON") == "0 5 * * *" + ) + assert ( + heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("TITLE") + == "Scheduled LaunchBox metadata update" + ) assert heartbeat.get("FRONTEND").get("UPLOAD_TIMEOUT") == 20 diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index e76145560..a15a24926 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -8,7 +8,7 @@ import tarfile import zipfile from collections.abc import Callable, Iterator from pathlib import Path -from typing import Any, Final, TypedDict +from typing import IO, Any, Final, Literal, TypedDict import magic import py7zr @@ -76,25 +76,28 @@ def is_compressed_file(file_path: str) -> bool: ) -def read_basic_file(file_path: Path) -> Iterator[bytes]: +def read_basic_file(file_path: os.PathLike[str]) -> Iterator[bytes]: with open(file_path, "rb") as f: while chunk := f.read(FILE_READ_CHUNK_SIZE): yield chunk -def read_zip_file(file_path: Path) -> Iterator[bytes]: +def read_zip_file(file: str | os.PathLike[str] | IO[bytes]) -> Iterator[bytes]: try: - with zipfile.ZipFile(file_path, "r") as z: + with zipfile.ZipFile(file, "r") as z: for file in z.namelist(): with z.open(file, "r") as f: while chunk := f.read(FILE_READ_CHUNK_SIZE): yield chunk except zipfile.BadZipFile: - for chunk in read_basic_file(file_path): - yield chunk + if isinstance(file, Path): + for chunk in read_basic_file(file): + yield chunk -def read_tar_file(file_path: Path, mode: str = "r") -> Iterator[bytes]: +def read_tar_file( + file_path: Path, mode: Literal["r", "r:*", "r:", "r:gz", "r:bz2", "r:xz"] = "r" +) -> Iterator[bytes]: try: with tarfile.open(file_path, mode) as f: for member in f.getmembers(): diff --git a/backend/scheduler.py b/backend/scheduler.py index 2105c65ac..3b10241a1 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -3,6 +3,7 @@ from config import SENTRY_DSN from logger.logger import log from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler +from tasks.update_launchbox_metadata import update_launchbox_metadata_task from tasks.update_switch_titledb import update_switch_titledb_task from utils import get_version @@ -15,6 +16,7 @@ if __name__ == "__main__": # Initialize the tasks scan_library_task.init() update_switch_titledb_task.init() + update_launchbox_metadata_task.init() log.info("Starting scheduler") diff --git a/backend/tasks/Metadata.zip b/backend/tasks/Metadata.zip new file mode 100644 index 000000000..46a6df3e2 Binary files /dev/null and b/backend/tasks/Metadata.zip differ diff --git a/backend/tasks/update_launchbox_metadata.py b/backend/tasks/update_launchbox_metadata.py new file mode 100644 index 000000000..a63c5732d --- /dev/null +++ b/backend/tasks/update_launchbox_metadata.py @@ -0,0 +1,113 @@ +import json +import zipfile +from itertools import batched +from typing import Final + +try: + from defusedxml import ElementTree as ET +except ImportError: + from xml.etree import ElementTree as ET + +from config import ( + ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, + SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON, +) +from handler.redis_handler import async_cache +from logger.logger import log +from tasks.tasks import RemoteFilePullTask +from utils.context import initialize_context + +LAUNCHBOX_PLATFORMS_KEY: Final = "romm:launchbox_platforms" +LAUNCHBOX_METADATA_DATABASE_ID_KEY: Final = "romm:launchbox_metadata_database_id" +LAUNCHBOX_METADATA_NAME_KEY: Final = "romm:launchbox_metadata_name" + + +class UpdateLaunchboxMetadataTask(RemoteFilePullTask): + def __init__(self): + super().__init__( + func="tasks.update_launchbox_metadata.update_launchbox_metadata_task.run", + description="launchbox metadata update", + enabled=ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, + cron_string=SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON, + url="https://gamesdb.launchbox-app.com/Metadata.zip", + ) + + @initialize_context() + async def run(self, force: bool = False) -> None: + content = await super().run(force) + if content is None: + return + + try: + with zipfile.ZipFile( + "/Users/georges-antoine/workspace/romm/backend/tasks/Metadata.zip", "r" + ) as z: + for file in z.namelist(): + if file == "Platforms.xml": + platform_dict = {} + with z.open(file, "r") as f: + for platform in ET.parse(f).getroot().findall("Platform"): + name_elem = platform.find("Name") + assert name_elem is not None + platform_dict[name_elem.text] = { + child.tag: child.text for child in platform + } + + async with async_cache.pipeline() as pipe: + for data_batch in batched(platform_dict.items(), 2000): + metadata_map = { + k: json.dumps(v) + for k, v in dict(data_batch).items() + } + await pipe.hset( + LAUNCHBOX_PLATFORMS_KEY, mapping=metadata_map + ) + elif file == "Metadata.xml": + metadata_by_id_dict = {} + metadata_by_name_dict = {} + + with z.open(file, "r") as f: + for metadata in ET.parse(f).getroot().findall("Game"): + id_elem = metadata.find("DatabaseID") + assert id_elem is not None + metadata_by_id_dict[id_elem.text] = { + child.tag: child.text for child in metadata + } + + name_elem = metadata.find("Name") + assert name_elem is not None + metadata_by_name_dict[name_elem.text] = { + child.tag: child.text for child in metadata + } + + async with async_cache.pipeline() as pipe: + for data_batch in batched( + metadata_by_id_dict.items(), 2000 + ): + titledb_map = { + k: json.dumps(v) + for k, v in dict(data_batch).items() + } + await pipe.hset( + LAUNCHBOX_METADATA_DATABASE_ID_KEY, + mapping=titledb_map, + ) + + for data_batch in batched( + metadata_by_name_dict.items(), 2000 + ): + titledb_map = { + k: json.dumps(v) + for k, v in dict(data_batch).items() + } + await pipe.hset( + LAUNCHBOX_METADATA_NAME_KEY, mapping=titledb_map + ) + except zipfile.BadZipFile: + log.error("Bad zip file in launchbox metadata update") + return + + log.info("Scheduled launchbox metadata update completed!") + + +update_launchbox_metadata_task = UpdateLaunchboxMetadataTask() diff --git a/env.template b/env.template index 31fe2683d..22308c85b 100644 --- a/env.template +++ b/env.template @@ -59,6 +59,8 @@ ENABLE_SCHEDULED_RESCAN=true SCHEDULED_RESCAN_CRON=0 3 * * * ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * * +ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true +SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON= 0 5 * * * # In-browser emulation DISABLE_EMULATOR_JS=false diff --git a/frontend/src/__generated__/models/SchedulerDict.ts b/frontend/src/__generated__/models/SchedulerDict.ts index d48121734..260dd8957 100644 --- a/frontend/src/__generated__/models/SchedulerDict.ts +++ b/frontend/src/__generated__/models/SchedulerDict.ts @@ -8,5 +8,6 @@ import type { TaskDict } from './TaskDict'; export type SchedulerDict = { RESCAN: TaskDict; SWITCH_TITLEDB: TaskDict; + LAUNCHBOX_METADATA: TaskDict; }; diff --git a/frontend/src/components/Settings/Administration/Tasks.vue b/frontend/src/components/Settings/Administration/Tasks.vue index d18282295..15c69d8d2 100644 --- a/frontend/src/components/Settings/Administration/Tasks.vue +++ b/frontend/src/components/Settings/Administration/Tasks.vue @@ -44,6 +44,19 @@ const tasks = computed(() => [ : "mdi-clock-remove-outline", enabled: heartbeatStore.value.SCHEDULER.SWITCH_TITLEDB.ENABLED, }, + { + title: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.TITLE, + description: + heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.MESSAGE + + " " + + convertCronExperssion( + heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.CRON, + ), + icon: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.ENABLED + ? "mdi-clock-check-outline" + : "mdi-clock-remove-outline", + enabled: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.ENABLED, + }, ]); // Functions diff --git a/poetry.lock b/poetry.lock index 243caf80e..553854e44 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. [[package]] name = "alembic" @@ -536,6 +536,18 @@ files = [ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + [[package]] name = "emoji" version = "2.10.1" @@ -2456,7 +2468,7 @@ description = "Python for Window Extensions" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" and sys_platform == \"win32\" and extra == \"dev\"" +markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\" and extra == \"dev\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, @@ -3753,4 +3765,4 @@ test = ["fakeredis", "pytest", "pytest-asyncio", "pytest-env", "pytest-mock", "p [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "3c92517a9a6abe41cc7d465f843f4ba4e8043eb2fd9dd352dafcb8bae0a7dff9" +content-hash = "34e0767edd3e5859d7dc86f110fd0da1211eb3183fac81c5068f3184af148eb2" diff --git a/pyproject.toml b/pyproject.toml index c22fb39c2..0b610abaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "websockets == 12.0", "yarl ~= 1.14", "zipfile-deflate64 ~= 0.2", + "defusedxml (>=0.7.1,<0.8.0)", ] [project.optional-dependencies] diff --git a/pytest.ini b/pytest.ini index 56e64e9b2..00a75dd5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,9 +11,7 @@ env = ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true ENABLE_SCHEDULED_RESCAN=true ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true - RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5 - SCHEDULED_RESCAN_CRON=0 3 * * * - SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * * + ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true UPLOAD_TIMEOUT=20 LOGLEVEL=DEBUG OIDC_ENABLED=false