From ee8b55e6efa7099afa9fe0f7da52c025684fabeb Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 7 Mar 2026 09:56:17 -0500 Subject: [PATCH] last set of changes --- backend/endpoints/collections.py | 15 +++++++++--- backend/endpoints/heartbeat.py | 6 +++-- backend/endpoints/saves.py | 14 ++++------- backend/endpoints/screenshots.py | 16 ++----------- backend/endpoints/states.py | 14 ++++------- backend/handler/filesystem/base_handler.py | 7 ++++-- .../handler/filesystem/platforms_handler.py | 13 +++++++---- .../handler/filesystem/resources_handler.py | 2 +- backend/handler/filesystem/roms_handler.py | 3 ++- backend/handler/metadata/base_handler.py | 6 +++-- backend/handler/metadata/ra_handler.py | 4 +++- .../manual/cleanup_orphaned_resources.py | 18 +++++++++------ .../models/Body_add_save_api_saves_post.ts | 2 +- ...ody_add_screenshot_api_screenshots_post.ts | 2 +- .../models/Body_add_state_api_states_post.ts | 2 +- frontend/src/services/api/rom.ts | 23 +++++++++++-------- 16 files changed, 78 insertions(+), 69 deletions(-) diff --git a/backend/endpoints/collections.py b/backend/endpoints/collections.py index 022fbcf64..ef153cd77 100644 --- a/backend/endpoints/collections.py +++ b/backend/endpoints/collections.py @@ -137,7 +137,10 @@ async def add_smart_collection( try: parsed_filter_criteria = json.loads(filter_criteria) except json.JSONDecodeError as e: - raise ValueError("Invalid JSON for filter_criteria field") from e + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid JSON for filter_criteria field", + ) from e cleaned_data = { "name": name, @@ -404,7 +407,10 @@ async def update_collection( try: parsed_rom_ids = json.loads(rom_ids) except json.JSONDecodeError as e: - raise ValueError("Invalid list for rom_ids field in update collection") from e + raise HTTPException( + status_code=422, + detail="Invalid list for rom_ids field in update collection", + ) from e cleaned_data = { "name": name if name is not None else collection.name, @@ -502,7 +508,10 @@ async def update_smart_collection( try: parsed_filter_criteria = json.loads(filter_criteria) except json.JSONDecodeError as e: - raise ValueError("Invalid JSON for filter_criteria field") from e + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid JSON for filter_criteria field", + ) from e cleaned_data = { "name": name if name is not None else smart_collection.name, diff --git a/backend/endpoints/heartbeat.py b/backend/endpoints/heartbeat.py index 54d653399..bb990c3ff 100644 --- a/backend/endpoints/heartbeat.py +++ b/backend/endpoints/heartbeat.py @@ -1,5 +1,6 @@ import os +from anyio import Path as AnyioPath from fastapi import HTTPException, Request, status from config import ( @@ -227,8 +228,9 @@ async def get_setup_library_info(request: Request): ) # Count files and folders in the roms directory - if os.path.exists(roms_path): - items = os.listdir(roms_path) + roms_dir = AnyioPath(roms_path) + if await roms_dir.exists(): + items = [entry.name async for entry in roms_dir.iterdir()] # Filter out hidden files and system files rom_count = len( [ diff --git a/backend/endpoints/saves.py b/backend/endpoints/saves.py index 4ce6b84d7..f111be828 100644 --- a/backend/endpoints/saves.py +++ b/backend/endpoints/saves.py @@ -100,7 +100,7 @@ router = APIRouter( tags=["saves"], ) -SAVE_FILE_UPLOAD = File(default=None, description="Save file to upload.") +SAVE_FILE_UPLOAD = File(..., description="Save file to upload.") SAVE_SCREENSHOT_UPLOAD = File( default=None, description="Screenshot file associated with this save.", @@ -119,7 +119,7 @@ async def add_save( overwrite: bool = False, autocleanup: bool = False, autocleanup_limit: int = 10, - saveFile: UploadFile | None = SAVE_FILE_UPLOAD, + saveFile: UploadFile = SAVE_FILE_UPLOAD, screenshotFile: UploadFile | None = SAVE_SCREENSHOT_UPLOAD, ) -> SaveSchema: """Upload a save file for a ROM.""" @@ -131,12 +131,6 @@ async def add_save( if not rom: raise RomNotFoundInDatabaseException(rom_id) - if not saveFile: - log.error("No save file provided") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail="No save file provided" - ) - if not saveFile.filename: log.error("Save file has no filename") raise HTTPException( @@ -487,7 +481,9 @@ async def update_save( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) if saveFile: - await fs_asset_handler.write_file(file=saveFile, path=db_save.file_path) + await fs_asset_handler.write_file( + file=saveFile, path=db_save.file_path, filename=db_save.file_name + ) db_save = db_save_handler.update_save( db_save.id, {"file_size_bytes": saveFile.size} ) diff --git a/backend/endpoints/screenshots.py b/backend/endpoints/screenshots.py index 6edbf3680..8bcd3689d 100644 --- a/backend/endpoints/screenshots.py +++ b/backend/endpoints/screenshots.py @@ -18,14 +18,14 @@ router = APIRouter( tags=["screenshots"], ) -SCREENSHOT_FILE_UPLOAD = File(default=None, description="Screenshot file to upload.") +SCREENSHOT_FILE_UPLOAD = File(..., description="Screenshot file to upload.") @protected_route(router.post, "", [Scope.ASSETS_WRITE]) async def add_screenshot( request: Request, rom_id: int, - screenshotFile: UploadFile | None = SCREENSHOT_FILE_UPLOAD, + screenshotFile: UploadFile = SCREENSHOT_FILE_UPLOAD, ) -> ScreenshotSchema: rom = db_rom_handler.get_rom(id=rom_id) if not rom: @@ -38,12 +38,6 @@ async def add_screenshot( user=request.user, platform_fs_slug=rom.platform_slug, rom_id=rom.id ) - if not screenshotFile: - log.error("No screenshot file provided") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="No screenshot file provided", - ) if not screenshotFile.filename: log.error("Screenshot file has no filename") raise HTTPException( @@ -51,12 +45,6 @@ async def add_screenshot( detail="Screenshot file has no filename", ) - if not screenshotFile.filename: - log.warning("Skipping empty screenshot") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail="Screenshot has no filename" - ) - try: sanitized_screenshot_filename = sanitize_filename(screenshotFile.filename) except ValueError as exc: diff --git a/backend/endpoints/states.py b/backend/endpoints/states.py index 233ed4d08..e7b079768 100644 --- a/backend/endpoints/states.py +++ b/backend/endpoints/states.py @@ -22,7 +22,7 @@ router = APIRouter( tags=["states"], ) -STATE_FILE_UPLOAD = File(default=None, description="State file to upload.") +STATE_FILE_UPLOAD = File(..., description="State file to upload.") STATE_SCREENSHOT_UPLOAD = File( default=None, description="Screenshot file associated with this state.", @@ -36,7 +36,7 @@ async def add_state( request: Request, rom_id: int, emulator: str | None = None, - stateFile: UploadFile | None = STATE_FILE_UPLOAD, + stateFile: UploadFile = STATE_FILE_UPLOAD, screenshotFile: UploadFile | None = STATE_SCREENSHOT_UPLOAD, ) -> StateSchema: rom = db_rom_handler.get_rom(rom_id) @@ -52,12 +52,6 @@ async def add_state( emulator=emulator, ) - if not stateFile: - log.error("No state file provided") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail="No state file provided" - ) - if not stateFile.filename: log.error("State file has no filename") raise HTTPException( @@ -228,7 +222,9 @@ async def update_state( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) if stateFile: - await fs_asset_handler.write_file(file=stateFile, path=db_state.file_path) + await fs_asset_handler.write_file( + file=stateFile, path=db_state.file_path, filename=db_state.file_name + ) db_state = db_state_handler.update_state( db_state.id, {"file_size_bytes": stateFile.size} ) diff --git a/backend/handler/filesystem/base_handler.py b/backend/handler/filesystem/base_handler.py index a0c2650b5..f4df3a72c 100644 --- a/backend/handler/filesystem/base_handler.py +++ b/backend/handler/filesystem/base_handler.py @@ -10,6 +10,7 @@ from pathlib import Path from tempfile import SpooledTemporaryFile from typing import BinaryIO +from anyio import Path as AnyioPath from anyio import open_file from starlette.datastructures import UploadFile @@ -458,11 +459,13 @@ class FSHandler: # Async thread-safe file copy async with source_lock, dest_lock: - if not source_full_path.is_file(): + source_anyio_path = AnyioPath(str(source_full_path)) + if not await source_anyio_path.is_file(): raise FileNotFoundError(f"Source file not found: {source_full_path}") # Create destination directory if needed - dest_full_path.parent.mkdir(parents=True, exist_ok=True) + dest_parent_anyio_path = AnyioPath(str(dest_full_path.parent)) + await dest_parent_anyio_path.mkdir(parents=True, exist_ok=True) shutil.copy2(str(source_full_path), str(dest_full_path)) async def move_file_or_folder(self, source_path: str, dest_path: str) -> None: diff --git a/backend/handler/filesystem/platforms_handler.py b/backend/handler/filesystem/platforms_handler.py index f95558a0d..8f981ef6c 100644 --- a/backend/handler/filesystem/platforms_handler.py +++ b/backend/handler/filesystem/platforms_handler.py @@ -1,5 +1,7 @@ import os +from anyio import Path as AnyioPath + from config import LIBRARY_BASE_PATH from config.config_manager import config_manager as cm from exceptions.fs_exceptions import ( @@ -105,12 +107,13 @@ class FSPlatformsHandler(FSHandler): # For Structure B, only include directories that have a roms subfolder structure = self.detect_library_structure() if structure == LibraryStructure.B: - platforms = [ - platform - for platform in platforms - if os.path.exists( + filtered_platforms: list[str] = [] + for platform in platforms: + roms_path = AnyioPath( os.path.join(LIBRARY_BASE_PATH, platform, cnfg.ROMS_FOLDER_NAME) ) - ] + if await roms_path.exists(): + filtered_platforms.append(platform) + platforms = filtered_platforms return self._exclude_platforms(platforms) diff --git a/backend/handler/filesystem/resources_handler.py b/backend/handler/filesystem/resources_handler.py index ee5771a76..8bc4baf7c 100644 --- a/backend/handler/filesystem/resources_handler.py +++ b/backend/handler/filesystem/resources_handler.py @@ -15,7 +15,7 @@ from models.collection import Collection from models.rom import Rom from tasks.scheduled.convert_images_to_webp import ImageConverter from utils.context import ctx_httpx_client -from utils.validation import ValidationError, validate_url_for_http_request +from utils.validation import validate_url_for_http_request from .base_handler import CoverSize, FSHandler diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index dceb9eb9b..d00590826 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -14,6 +14,7 @@ from typing import IO, Any, Final, Literal, TypedDict, cast import magic import zipfile_inflate64 # trunk-ignore(ruff/F401): Patches zipfile to support Enhanced Deflate +from anyio import Path as AnyioPath from config import LIBRARY_BASE_PATH from config.config_manager import config_manager as cm @@ -448,7 +449,7 @@ class FSRomsHandler(FSHandler): rom_ra_h = "" # Check if rom is a multi-part rom - if os.path.isdir(f"{abs_fs_path}/{rom.fs_name}"): + if await AnyioPath(f"{abs_fs_path}/{rom.fs_name}").is_dir(): # Calculate the RA hash if the platform has a slug that matches a known RA slug if calculate_hashes: ra_platform = meta_ra_handler.get_platform(rom.platform_slug) diff --git a/backend/handler/metadata/base_handler.py b/backend/handler/metadata/base_handler.py index 6def74777..a0c425d3b 100644 --- a/backend/handler/metadata/base_handler.py +++ b/backend/handler/metadata/base_handler.py @@ -5,7 +5,7 @@ import re import unicodedata from functools import lru_cache from pathlib import Path -from typing import Final, NotRequired, TypedDict +from typing import Final, Mapping, NotRequired, TypedDict from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from strsimpy.jaro_winkler import JaroWinkler @@ -275,7 +275,9 @@ class MetadataHandler(abc.ABC): return search_term - def _mask_sensitive_values(self, values: dict[str, str]) -> dict[str, str]: + def _mask_sensitive_values( + self, values: Mapping[str, str | None] + ) -> dict[str, str]: """ Mask sensitive values (headers or params), leaving only the first 2 and last 2 characters of the token. """ diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index ba1272bea..459d33a67 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import NotRequired, TypedDict import pydash +from anyio import Path as AnyioPath from adapters.services.retroachievements import RetroAchievementsService from adapters.services.retroachievements_types import ( @@ -169,7 +170,8 @@ class RAHandler(MetadataHandler): return REFRESH_RETROACHIEVEMENTS_CACHE_DAYS + 1 full_path = fs_resource_handler.validate_path(file_path) - return int((time.time() - os.path.getmtime(full_path)) / (24 * 3600)) + file_stat = await AnyioPath(str(full_path)).stat() + return int((time.time() - file_stat.st_mtime) / (24 * 3600)) async def _search_rom(self, rom: Rom, ra_hash: str) -> RAGameListItem | None: if not rom.platform.ra_id: diff --git a/backend/tasks/manual/cleanup_orphaned_resources.py b/backend/tasks/manual/cleanup_orphaned_resources.py index b065c852b..0ed7e660b 100644 --- a/backend/tasks/manual/cleanup_orphaned_resources.py +++ b/backend/tasks/manual/cleanup_orphaned_resources.py @@ -2,6 +2,8 @@ import os import shutil from dataclasses import dataclass +from anyio import Path as AnyioPath + from config import RESOURCES_BASE_PATH from handler.database import db_platform_handler, db_rom_handler from logger.logger import log @@ -57,7 +59,8 @@ class CleanupOrphanedResourcesTask(Task): cleanup_stats = CleanupStats() roms_resources_path = os.path.join(RESOURCES_BASE_PATH, "roms") - if not os.path.exists(roms_resources_path): + roms_resources_dir = AnyioPath(roms_resources_path) + if not await roms_resources_dir.exists(): cleanup_stats.update() log.info("Resources path does not exist, skipping cleanup") return cleanup_stats.to_dict() @@ -82,18 +85,19 @@ class CleanupOrphanedResourcesTask(Task): # Count total platforms and ROMs for progress tracking platform_dirs: set[int] = { - int(d) - for d in os.listdir(roms_resources_path) - if os.path.isdir(os.path.join(roms_resources_path, d)) + int(entry.name) + async for entry in roms_resources_dir.iterdir() + if await entry.is_dir() } rom_dirs_by_platform: dict[int, set[int]] = {} for platform_dir in platform_dirs: platform_path = os.path.join(roms_resources_path, str(platform_dir)) + platform_dir_path = AnyioPath(platform_path) rom_dirs_by_platform[platform_dir] = { - int(d) - for d in os.listdir(platform_path) - if os.path.isdir(os.path.join(platform_path, d)) + int(entry.name) + async for entry in platform_dir_path.iterdir() + if await entry.is_dir() } cleanup_stats.update( diff --git a/frontend/src/__generated__/models/Body_add_save_api_saves_post.ts b/frontend/src/__generated__/models/Body_add_save_api_saves_post.ts index d0ae2fac7..d52159705 100644 --- a/frontend/src/__generated__/models/Body_add_save_api_saves_post.ts +++ b/frontend/src/__generated__/models/Body_add_save_api_saves_post.ts @@ -6,7 +6,7 @@ export type Body_add_save_api_saves_post = { /** * Save file to upload. */ - saveFile?: (Blob | null); + saveFile: Blob; /** * Screenshot file associated with this save. */ diff --git a/frontend/src/__generated__/models/Body_add_screenshot_api_screenshots_post.ts b/frontend/src/__generated__/models/Body_add_screenshot_api_screenshots_post.ts index 2f14d6fb3..bd368bf72 100644 --- a/frontend/src/__generated__/models/Body_add_screenshot_api_screenshots_post.ts +++ b/frontend/src/__generated__/models/Body_add_screenshot_api_screenshots_post.ts @@ -6,6 +6,6 @@ export type Body_add_screenshot_api_screenshots_post = { /** * Screenshot file to upload. */ - screenshotFile?: (Blob | null); + screenshotFile: Blob; }; diff --git a/frontend/src/__generated__/models/Body_add_state_api_states_post.ts b/frontend/src/__generated__/models/Body_add_state_api_states_post.ts index b31189c93..c601e8c3f 100644 --- a/frontend/src/__generated__/models/Body_add_state_api_states_post.ts +++ b/frontend/src/__generated__/models/Body_add_state_api_states_post.ts @@ -6,7 +6,7 @@ export type Body_add_state_api_states_post = { /** * State file to upload. */ - stateFile?: (Blob | null); + stateFile: Blob; /** * Screenshot file associated with this state. */ diff --git a/frontend/src/services/api/rom.ts b/frontend/src/services/api/rom.ts index 702ac24cd..67367026a 100644 --- a/frontend/src/services/api/rom.ts +++ b/frontend/src/services/api/rom.ts @@ -383,20 +383,23 @@ async function updateRom({ removeCover?: boolean; unmatch?: boolean; }) { + const toFormIdValue = (value: number | string | null | undefined): string => + value === null || value === undefined ? "" : String(value); + const fields: FormInputField[] = [ ["name", rom.name], ["fs_name", rom.fs_name], ["summary", rom.summary], - ["igdb_id", rom.igdb_id?.toString()], - ["sgdb_id", rom.sgdb_id?.toString()], - ["moby_id", rom.moby_id?.toString()], - ["ss_id", rom.ss_id?.toString()], - ["launchbox_id", rom.launchbox_id?.toString()], - ["ra_id", rom.ra_id?.toString()], - ["flashpoint_id", rom.flashpoint_id?.toString()], - ["hasheous_id", rom.hasheous_id?.toString()], - ["tgdb_id", rom.tgdb_id?.toString()], - ["hltb_id", rom.hltb_id?.toString()], + ["igdb_id", toFormIdValue(rom.igdb_id)], + ["sgdb_id", toFormIdValue(rom.sgdb_id)], + ["moby_id", toFormIdValue(rom.moby_id)], + ["ss_id", toFormIdValue(rom.ss_id)], + ["launchbox_id", toFormIdValue(rom.launchbox_id)], + ["ra_id", toFormIdValue(rom.ra_id)], + ["flashpoint_id", toFormIdValue(rom.flashpoint_id)], + ["hasheous_id", toFormIdValue(rom.hasheous_id)], + ["tgdb_id", toFormIdValue(rom.tgdb_id)], + ["hltb_id", toFormIdValue(rom.hltb_id)], ]; if (rom.manual_metadata) {