Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-07-01 08:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

store swtich files in redis

Browse Source
This commit is contained in:
Georges-Antoine Assi
2024-03-26 17:18:46 -04:00
parent 4174b9753e
commit efcd3c00d1
3 changed files with 37 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
backend/handler/metadata_handler/__init__.py
View File

@@ -5,7 +5,12 @@ import re
import unicodedata
from typing import Final
from logger.logger import log
from tasks.update_switch_titledb import update_switch_titledb_task
from handler.redis_handler import cache
from tasks.update_switch_titledb import (
update_switch_titledb_task,
SWITCH_TITLEDB_INDEX_KEY,
SWITCH_PRODUCT_ID_KEY,
)
PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
@@ -14,14 +19,7 @@ PS2_OPL_INDEX_FILE: Final = os.path.join(
)
SWITCH_TITLEDB_REGEX: Final = r"(70[0-9]{12})"
SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_titledb.json"
)
SWITCH_PRODUCT_ID_REGEX: Final = r"(0100[0-9A-F]{12})"
SWITCH_PRODUCT_ID_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_product_ids.json"
)
MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
@@ -129,31 +127,26 @@ class MetadataHandler:
async def _switch_titledb_format(
self, match: re.Match[str], search_term: str
) -> tuple[str, dict | None]:
titledb_index = {}
title_id = match.group(1)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
if not cache.exists(SWITCH_TITLEDB_INDEX_KEY):
log.warning("Fetching the Switch titleID index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = titledb_index.get(title_id, None)
if index_entry:
return index_entry["name"], index_entry # type: ignore
if not cache.exists(SWITCH_TITLEDB_INDEX_KEY):
log.error("Could not fetch the Switch titleID index file")
return search_term, None
index_entry = cache.hmget(SWITCH_TITLEDB_INDEX_KEY, title_id)
if index_entry:
return index_entry["name"], index_entry # type: ignore
return search_term, None
async def _switch_productid_format(
self, match: re.Match[str], search_term: str
) -> tuple[str, dict | None]:
product_id_index = {}
product_id = match.group(1)
# Game updates have the same product ID as the main application, except with bitmask 0x800 set
@@ -161,22 +154,18 @@ class MetadataHandler:
product_id[-3] = "0"
product_id = "".join(product_id)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
if not cache.exists(SWITCH_PRODUCT_ID_KEY):
log.warning("Fetching the Switch productID index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = product_id_index.get(product_id, None)
if index_entry:
return index_entry["name"], index_entry # type: ignore
if not cache.exists(SWITCH_PRODUCT_ID_KEY):
log.error("Could not fetch the Switch productID index file")
return search_term, None
index_entry = cache.hmget(SWITCH_PRODUCT_ID_KEY, product_id)
if index_entry:
return index_entry["name"], index_entry # type: ignore
return search_term, None
async def _mame_format(self, search_term: str) -> str:

9
backend/tasks/tasks.py
View File

@@ -71,11 +71,9 @@ class PeriodicTask(ABC):
class RemoteFilePullTask(PeriodicTask):
def __init__(self, *args, url: str, file_path: str, **kwargs):
def __init__(self, *args, url: str, **kwargs):
super().__init__(*args, **kwargs)
self.url = url
self.file_path = file_path
async def run(self, force: bool = False) -> bytes | None:
if not self.enabled and not force:
@@ -88,11 +86,6 @@ class RemoteFilePullTask(PeriodicTask):
try:
response = requests.get(self.url)
response.raise_for_status()
with open(self.file_path, "wb") as fixture:
fixture.write(response.content)
log.info(f"Scheduled {self.description} done")
return response.content
except requests.exceptions.RequestException as e:
log.error(f"Scheduled {self.description} failed", exc_info=True)

30
backend/tasks/update_switch_titledb.py
View File

@@ -1,6 +1,4 @@
import json
import os
from pathlib import Path
from typing import Final
from config import (
@@ -8,22 +6,11 @@ from config import (
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
)
from tasks.tasks import RemoteFilePullTask
from logger.logger import log
from handler.redis_handler import cache
FIXTURE_FILE_PATH: Final = (
Path(os.path.dirname(__file__)).parent
/ "handler"
/ "metadata_handler"
/ "fixtures"
/ "switch_titledb.json"
)
SWITCH_PRODUCT_ID_FILE_PATH: Final = (
Path(os.path.dirname(__file__)).parent
/ "handler"
/ "metadata_handler"
/ "fixtures"
/ "switch_product_ids.json"
)
SWITCH_TITLEDB_INDEX_KEY: Final = "romm:switch_titledb"
SWITCH_PRODUCT_ID_KEY: Final = "romm:switch_product_id"
class UpdateSwitchTitleDBTask(RemoteFilePullTask):
@@ -34,7 +21,6 @@ class UpdateSwitchTitleDBTask(RemoteFilePullTask):
enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json",
file_path=FIXTURE_FILE_PATH,
)
async def run(self, force: bool = False):
@@ -43,10 +29,12 @@ class UpdateSwitchTitleDBTask(RemoteFilePullTask):
return
index_json = json.loads(content)
product_ids = dict((v["id"], v) for _k, v in index_json.items())
cache.hmset(SWITCH_TITLEDB_INDEX_KEY, index_json)
with open(SWITCH_PRODUCT_ID_FILE_PATH, "wb") as fixture:
fixture.write(json.dumps(product_ids).encode())
product_ids = dict((v["id"], v) for _k, v in index_json.items())
cache.hmset(SWITCH_PRODUCT_ID_KEY, product_ids)
log.info("Scheduled switch titledb update completed!")
update_switch_titledb_task = UpdateSwitchTitleDBTask()