Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-29 07:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fetch and parse launchbox metadata.zip

Browse Source
This commit is contained in:
Georges-Antoine Assi
2025-01-18 10:51:30 -05:00
parent 07c78836d0
commit 0c95eff2e1
15 changed files with 189 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/config/__init__.py
View File

@@ -108,6 +108,13 @@ SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
"SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON",
"0 4 * * *", # At 4:00 AM every day
)
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA: Final = str_to_bool(
os.environ.get("ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA", "false")
)
SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON: Final = os.environ.get(
"SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON",
"0 5 * * *", # At 5:00 AM every day
)
# EMULATION
DISABLE_EMULATOR_JS = str_to_bool(os.environ.get("DISABLE_EMULATOR_JS", "false"))

8
backend/endpoints/heartbeat.py
View File

@@ -4,11 +4,13 @@ from config import (
DISABLE_USERPASS_LOGIN,
ENABLE_RESCAN_ON_FILESYSTEM_CHANGE,
ENABLE_SCHEDULED_RESCAN,
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
OIDC_ENABLED,
OIDC_PROVIDER,
RESCAN_ON_FILESYSTEM_CHANGE_DELAY,
SCHEDULED_RESCAN_CRON,
SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON,
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
UPLOAD_TIMEOUT,
)
@@ -64,6 +66,12 @@ def heartbeat() -> HeartbeatResponse:
"TITLE": "Scheduled Switch TitleDB update",
"MESSAGE": "Updates the Nintendo Switch TitleDB file",
},
"LAUNCHBOX_METADATA": {
"ENABLED": ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
"CRON": SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON,
"TITLE": "Scheduled LaunchBox metadata update",
"MESSAGE": "Updates the LaunchBox metadata",
},
},
"EMULATION": {
"DISABLE_EMULATOR_JS": DISABLE_EMULATOR_JS,

1
backend/endpoints/responses/heartbeat.py
View File

@@ -19,6 +19,7 @@ class TaskDict(WatcherDict):
class SchedulerDict(TypedDict):
RESCAN: TaskDict
SWITCH_TITLEDB: TaskDict
LAUNCHBOX_METADATA: TaskDict
class MetadataSourcesDict(TypedDict):

9
backend/endpoints/tasks.py
View File

@@ -2,6 +2,7 @@ from decorators.auth import protected_route
from endpoints.responses import MessageResponse
from fastapi import Request
from handler.auth.constants import Scope
from tasks.update_launchbox_metadata import update_launchbox_metadata_task
from tasks.update_switch_titledb import update_switch_titledb_task
from utils.router import APIRouter
@@ -19,12 +20,13 @@ async def run_tasks(request: Request) -> MessageResponse:
"""
await update_switch_titledb_task.run()
await update_launchbox_metadata_task.run()
return {"msg": "All tasks ran successfully!"}
@protected_route(router.post, "/tasks/{task}/run", [Scope.TASKS_RUN])
async def run_task(request: Request, task: str) -> MessageResponse:
"""Run all tasks endpoint
"""Run single tasks endpoint
Args:
request (Request): Fastapi Request object
@@ -32,7 +34,10 @@ async def run_task(request: Request, task: str) -> MessageResponse:
RunTasksResponse: Standard message response
"""
tasks = {"switch_titledb": update_switch_titledb_task}
tasks = {
"switch_titledb": update_switch_titledb_task,
"launchbox_metadata": update_launchbox_metadata_task,
}
await tasks[task].run()
return {"msg": f"Task {task} run successfully!"}

8
backend/endpoints/tests/test_heartbeat.py
View File

@@ -27,4 +27,12 @@ def test_heartbeat(client):
heartbeat.get("SCHEDULER").get("SWITCH_TITLEDB").get("TITLE")
== "Scheduled Switch TitleDB update"
)
assert heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("ENABLED")
assert (
heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("CRON") == "0 5 * * *"
)
assert (
heartbeat.get("SCHEDULER").get("LAUNCHBOX_METADATA").get("TITLE")
== "Scheduled LaunchBox metadata update"
)
assert heartbeat.get("FRONTEND").get("UPLOAD_TIMEOUT") == 20

17
backend/handler/filesystem/roms_handler.py
View File

@@ -8,7 +8,7 @@ import tarfile
import zipfile
from collections.abc import Callable, Iterator
from pathlib import Path
from typing import Any, Final, TypedDict
from typing import IO, Any, Final, Literal, TypedDict
import magic
import py7zr
@@ -76,25 +76,28 @@ def is_compressed_file(file_path: str) -> bool:
)
def read_basic_file(file_path: Path) -> Iterator[bytes]:
def read_basic_file(file_path: os.PathLike[str]) -> Iterator[bytes]:
with open(file_path, "rb") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
yield chunk
def read_zip_file(file_path: Path) -> Iterator[bytes]:
def read_zip_file(file: str | os.PathLike[str] | IO[bytes]) -> Iterator[bytes]:
try:
with zipfile.ZipFile(file_path, "r") as z:
with zipfile.ZipFile(file, "r") as z:
for file in z.namelist():
with z.open(file, "r") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
yield chunk
except zipfile.BadZipFile:
for chunk in read_basic_file(file_path):
yield chunk
if isinstance(file, Path):
for chunk in read_basic_file(file):
yield chunk
def read_tar_file(file_path: Path, mode: str = "r") -> Iterator[bytes]:
def read_tar_file(
file_path: Path, mode: Literal["r", "r:*", "r:", "r:gz", "r:bz2", "r:xz"] = "r"
) -> Iterator[bytes]:
try:
with tarfile.open(file_path, mode) as f:
for member in f.getmembers():

2
backend/scheduler.py
View File

@@ -3,6 +3,7 @@ from config import SENTRY_DSN
from logger.logger import log
from tasks.scan_library import scan_library_task
from tasks.tasks import tasks_scheduler
from tasks.update_launchbox_metadata import update_launchbox_metadata_task
from tasks.update_switch_titledb import update_switch_titledb_task
from utils import get_version
@@ -15,6 +16,7 @@ if __name__ == "__main__":
# Initialize the tasks
scan_library_task.init()
update_switch_titledb_task.init()
update_launchbox_metadata_task.init()
log.info("Starting scheduler")

BIN
backend/tasks/Metadata.zip Normal file
View File

Binary file not shown.

113
backend/tasks/update_launchbox_metadata.py Normal file
View File

@@ -0,0 +1,113 @@
import json
import zipfile
from itertools import batched
from typing import Final
try:
from defusedxml import ElementTree as ET
except ImportError:
from xml.etree import ElementTree as ET
from config import (
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON,
)
from handler.redis_handler import async_cache
from logger.logger import log
from tasks.tasks import RemoteFilePullTask
from utils.context import initialize_context
LAUNCHBOX_PLATFORMS_KEY: Final = "romm:launchbox_platforms"
LAUNCHBOX_METADATA_DATABASE_ID_KEY: Final = "romm:launchbox_metadata_database_id"
LAUNCHBOX_METADATA_NAME_KEY: Final = "romm:launchbox_metadata_name"
class UpdateLaunchboxMetadataTask(RemoteFilePullTask):
def __init__(self):
super().__init__(
func="tasks.update_launchbox_metadata.update_launchbox_metadata_task.run",
description="launchbox metadata update",
enabled=ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
cron_string=SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON,
url="https://gamesdb.launchbox-app.com/Metadata.zip",
)
@initialize_context()
async def run(self, force: bool = False) -> None:
content = await super().run(force)
if content is None:
return
try:
with zipfile.ZipFile(
"/Users/georges-antoine/workspace/romm/backend/tasks/Metadata.zip", "r"
) as z:
for file in z.namelist():
if file == "Platforms.xml":
platform_dict = {}
with z.open(file, "r") as f:
for platform in ET.parse(f).getroot().findall("Platform"):
name_elem = platform.find("Name")
assert name_elem is not None
platform_dict[name_elem.text] = {
child.tag: child.text for child in platform
}
async with async_cache.pipeline() as pipe:
for data_batch in batched(platform_dict.items(), 2000):
metadata_map = {
k: json.dumps(v)
for k, v in dict(data_batch).items()
}
await pipe.hset(
LAUNCHBOX_PLATFORMS_KEY, mapping=metadata_map
)
elif file == "Metadata.xml":
metadata_by_id_dict = {}
metadata_by_name_dict = {}
with z.open(file, "r") as f:
for metadata in ET.parse(f).getroot().findall("Game"):
id_elem = metadata.find("DatabaseID")
assert id_elem is not None
metadata_by_id_dict[id_elem.text] = {
child.tag: child.text for child in metadata
}
name_elem = metadata.find("Name")
assert name_elem is not None
metadata_by_name_dict[name_elem.text] = {
child.tag: child.text for child in metadata
}
async with async_cache.pipeline() as pipe:
for data_batch in batched(
metadata_by_id_dict.items(), 2000
):
titledb_map = {
k: json.dumps(v)
for k, v in dict(data_batch).items()
}
await pipe.hset(
LAUNCHBOX_METADATA_DATABASE_ID_KEY,
mapping=titledb_map,
)
for data_batch in batched(
metadata_by_name_dict.items(), 2000
):
titledb_map = {
k: json.dumps(v)
for k, v in dict(data_batch).items()
}
await pipe.hset(
LAUNCHBOX_METADATA_NAME_KEY, mapping=titledb_map
)
except zipfile.BadZipFile:
log.error("Bad zip file in launchbox metadata update")
return
log.info("Scheduled launchbox metadata update completed!")
update_launchbox_metadata_task = UpdateLaunchboxMetadataTask()

2
env.template
View File

@@ -59,6 +59,8 @@ ENABLE_SCHEDULED_RESCAN=true
SCHEDULED_RESCAN_CRON=0 3 * * *
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * *
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true
SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON= 0 5 * * *
# In-browser emulation
DISABLE_EMULATOR_JS=false

1
frontend/src/__generated__/models/SchedulerDict.ts generated
View File

@@ -8,5 +8,6 @@ import type { TaskDict } from './TaskDict';
export type SchedulerDict = {
RESCAN: TaskDict;
SWITCH_TITLEDB: TaskDict;
LAUNCHBOX_METADATA: TaskDict;
};

13
frontend/src/components/Settings/Administration/Tasks.vue
View File

@@ -44,6 +44,19 @@ const tasks = computed(() => [
: "mdi-clock-remove-outline",
enabled: heartbeatStore.value.SCHEDULER.SWITCH_TITLEDB.ENABLED,
},
{
title: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.TITLE,
description:
heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.MESSAGE +
" " +
convertCronExperssion(
heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.CRON,
),
icon: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.ENABLED
? "mdi-clock-check-outline"
: "mdi-clock-remove-outline",
enabled: heartbeatStore.value.SCHEDULER.LAUNCHBOX_METADATA.ENABLED,
},
]);
// Functions

18
poetry.lock generated
View File

@@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand.
# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
[[package]]
name = "alembic"
@@ -536,6 +536,18 @@ files = [
{file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
]
[[package]]
name = "defusedxml"
version = "0.7.1"
description = "XML bomb protection for Python stdlib modules"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
groups = ["main"]
files = [
{file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"},
{file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"},
]
[[package]]
name = "emoji"
version = "2.10.1"
@@ -2456,7 +2468,7 @@ description = "Python for Window Extensions"
optional = true
python-versions = "*"
groups = ["main"]
markers = "platform_python_implementation != \"PyPy\" and sys_platform == \"win32\" and extra == \"dev\""
markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\" and extra == \"dev\""
files = [
{file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"},
{file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"},
@@ -3753,4 +3765,4 @@ test = ["fakeredis", "pytest", "pytest-asyncio", "pytest-env", "pytest-mock", "p
[metadata]
lock-version = "2.1"
python-versions = "^3.12"
content-hash = "3c92517a9a6abe41cc7d465f843f4ba4e8043eb2fd9dd352dafcb8bae0a7dff9"
content-hash = "34e0767edd3e5859d7dc86f110fd0da1211eb3183fac81c5068f3184af148eb2"

1
pyproject.toml
View File

@@ -51,6 +51,7 @@ dependencies = [
"websockets == 12.0",
"yarl ~= 1.14",
"zipfile-deflate64 ~= 0.2",
"defusedxml (>=0.7.1,<0.8.0)",
]
[project.optional-dependencies]

4
pytest.ini
View File

@@ -11,9 +11,7 @@ env =
ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true
ENABLE_SCHEDULED_RESCAN=true
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5
SCHEDULED_RESCAN_CRON=0 3 * * *
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * *
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true
UPLOAD_TIMEOUT=20
LOGLEVEL=DEBUG
OIDC_ENABLED=false