diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 061227a91..f20525b8a 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -3,6 +3,9 @@ name: Copilot Setup Steps on: workflow_dispatch: +permissions: + contents: read + jobs: copilot-setup-steps: runs-on: ubuntu-latest diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 985ee8e8e..e69208de6 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -87,11 +87,13 @@ jobs: - name: Comment PR with Docker image link if: github.event_name == 'pull_request' uses: actions/github-script@v7 + env: + HEAD_REF: ${{ github.head_ref }} with: script: | github.rest.issues.updateComment({ comment_id: ${{ steps.build-comment.outputs.comment-id }}, owner: context.repo.owner, repo: context.repo.repo, - body: `✅ Preview build completed!\n\nDocker image: \`rommapp/romm-testing:${{ github.head_ref }}\`` + body: `✅ Preview build completed!\n\nDocker image: \`rommapp/romm-testing:${process.env.HEAD_REF}\`` }) diff --git a/DEVELOPER_SETUP.md b/DEVELOPER_SETUP.md index 0523a86d5..f834506d1 100644 --- a/DEVELOPER_SETUP.md +++ b/DEVELOPER_SETUP.md @@ -76,9 +76,8 @@ sudo apt install libmariadb3 libmariadb-dev libpq-dev # Users on macOS can skip this step as RAHasher is not supported git clone --recursive https://github.com/RetroAchievements/RALibretro.git cd ./RALibretro -git checkout 1.8.0 +git checkout 1.8.3 git submodule update --init --recursive -sed -i '22a #include ' ./src/Util.h make HAVE_CHD=1 -f ./Makefile.RAHasher cp ./bin64/RAHasher /usr/bin/RAHasher ``` diff --git a/Dockerfile b/Dockerfile index 6f475d50b..61ceb260e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,14 +43,9 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | b ENV PATH="$NVM_DIR/versions/node/v24.13.1/bin:$PATH" # Build and install RAHasher (optional for RA hashes) -RUN git clone --recursive --branch 1.8.1 --depth 1 https://github.com/RetroAchievements/RALibretro.git /tmp/RALibretro +RUN git clone --recursive --branch 1.8.3 --depth 1 https://github.com/RetroAchievements/RALibretro.git /tmp/RALibretro WORKDIR /tmp/RALibretro -RUN sed -i '22a #include ' ./src/Util.h \ - && sed -i '6a #include ' \ - ./src/libchdr/deps/zlib-1.3.1/gzlib.c \ - ./src/libchdr/deps/zlib-1.3.1/gzread.c \ - ./src/libchdr/deps/zlib-1.3.1/gzwrite.c \ - && make HAVE_CHD=1 -f ./Makefile.RAHasher \ +RUN make HAVE_CHD=1 -f ./Makefile.RAHasher \ && cp ./bin64/RAHasher /usr/bin/RAHasher RUN rm -rf /tmp/RALibretro @@ -63,7 +58,7 @@ RUN npm install WORKDIR /app # Install uv for the non-root user -COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /usr/local/bin/ +COPY --from=ghcr.io/astral-sh/uv:0.11.2 /uv /uvx /usr/local/bin/ # Install Python RUN uv python install 3.13 diff --git a/README.md b/README.md index deb249755..b6440857a 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Here are a few projects maintained by members of our community. Please note that - 🔷 [Argosy][argosy-launcher]: Native client for installing and launching games by [@tmgast](https://github.com/tmgast) - [romm-ios-app][romm-ios-app]: Native iOS app by [@ilyas-hallak](https://github.com/ilyas-hallak) +- [romm-mobile][romm-mobile]: Android (and soon iOS) app by [@mattsays](https://github.com/mattsays) ### Desktop @@ -159,6 +160,7 @@ Here are a few projects that we think you might like: [playnite-app]: https://github.com/rommapp/playnite-plugin [ggrequestz]: https://github.com/XTREEMMAK/ggrequestz [syncthing-sync]: https://github.com/amn-96/romm_syncthing_sync +[romm-mobile]: https://github.com/mattsays/romm-mobile [romm-client]: https://github.com/chaun14/romm-client [romm-retroarch-sync]: https://github.com/Covin90/romm-retroarch-sync [rommate]: https://github.com/brenoprata10/rommate diff --git a/backend/adapters/services/rahasher.py b/backend/adapters/services/rahasher.py index c52aa2c1a..725188e57 100644 --- a/backend/adapters/services/rahasher.py +++ b/backend/adapters/services/rahasher.py @@ -65,6 +65,7 @@ PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[UPS, int] = { UPS.WASM_4: 72, UPS.SUPERVISION: 63, UPS.WIN: 102, + UPS.WII: 19, UPS.WONDERSWAN: 53, UPS.WONDERSWAN_COLOR: 53, } @@ -96,7 +97,7 @@ class RAHasherService: return "" return_code = await proc.wait() - if return_code != 1: + if return_code != 0: if proc.stderr is not None: stderr = (await proc.stderr.read()).decode("utf-8") else: diff --git a/backend/alembic/versions/0073_sibling_roms_metadata_only.py b/backend/alembic/versions/0073_sibling_roms_metadata_only.py new file mode 100644 index 000000000..b4964a6f9 --- /dev/null +++ b/backend/alembic/versions/0073_sibling_roms_metadata_only.py @@ -0,0 +1,125 @@ +"""Remove fs_name_no_tags matching from sibling_roms view + +Revision ID: 0073_sibling_roms_metadata_only +Revises: 0072_client_tokens +Create Date: 2026-04-05 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +from utils.database import is_postgresql + +# revision identifiers, used by Alembic. +revision = "0073_sibling_roms_metadata_only" +down_revision = "0072_client_tokens" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Drop the fs_name_no_tags index created in 0069, no longer used + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.drop_index("idx_roms_fs_name_no_tags") + + connection = op.get_bind() + null_safe_equal_operator = ( + "IS NOT DISTINCT FROM" if is_postgresql(connection) else "<=>" + ) + + connection.execute( + sa.text(f""" + CREATE OR REPLACE VIEW sibling_roms AS + SELECT + r1.id AS rom_id, + r2.id AS sibling_rom_id, + r1.platform_id AS platform_id, + NOW() AS created_at, + NOW() AS updated_at, + CASE WHEN r1.igdb_id {null_safe_equal_operator} r2.igdb_id THEN r1.igdb_id END AS igdb_id, + CASE WHEN r1.moby_id {null_safe_equal_operator} r2.moby_id THEN r1.moby_id END AS moby_id, + CASE WHEN r1.ss_id {null_safe_equal_operator} r2.ss_id THEN r1.ss_id END AS ss_id, + CASE WHEN r1.launchbox_id {null_safe_equal_operator} r2.launchbox_id THEN r1.launchbox_id END AS launchbox_id, + CASE WHEN r1.ra_id {null_safe_equal_operator} r2.ra_id THEN r1.ra_id END AS ra_id, + CASE WHEN r1.hasheous_id {null_safe_equal_operator} r2.hasheous_id THEN r1.hasheous_id END AS hasheous_id, + CASE WHEN r1.tgdb_id {null_safe_equal_operator} r2.tgdb_id THEN r1.tgdb_id END AS tgdb_id + FROM + roms r1 + JOIN + roms r2 + ON + r1.platform_id = r2.platform_id + AND r1.id != r2.id + AND ( + (r1.igdb_id = r2.igdb_id AND r1.igdb_id IS NOT NULL) + OR + (r1.moby_id = r2.moby_id AND r1.moby_id IS NOT NULL) + OR + (r1.ss_id = r2.ss_id AND r1.ss_id IS NOT NULL) + OR + (r1.launchbox_id = r2.launchbox_id AND r1.launchbox_id IS NOT NULL) + OR + (r1.ra_id = r2.ra_id AND r1.ra_id IS NOT NULL) + OR + (r1.hasheous_id = r2.hasheous_id AND r1.hasheous_id IS NOT NULL) + OR + (r1.tgdb_id = r2.tgdb_id AND r1.tgdb_id IS NOT NULL) + ); + """), # nosec B608 + ) + + +def downgrade() -> None: + connection = op.get_bind() + null_safe_equal_operator = ( + "IS NOT DISTINCT FROM" if is_postgresql(connection) else "<=>" + ) + + # Recreate the fs_name_no_tags index needed by the restored view + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.create_index("idx_roms_fs_name_no_tags", ["fs_name_no_tags"]) + + # Restore view with fs_name_no_tags matching (from 0071) + connection.execute( + sa.text(f""" + CREATE OR REPLACE VIEW sibling_roms AS + SELECT + r1.id AS rom_id, + r2.id AS sibling_rom_id, + r1.platform_id AS platform_id, + NOW() AS created_at, + NOW() AS updated_at, + CASE WHEN r1.igdb_id {null_safe_equal_operator} r2.igdb_id THEN r1.igdb_id END AS igdb_id, + CASE WHEN r1.moby_id {null_safe_equal_operator} r2.moby_id THEN r1.moby_id END AS moby_id, + CASE WHEN r1.ss_id {null_safe_equal_operator} r2.ss_id THEN r1.ss_id END AS ss_id, + CASE WHEN r1.launchbox_id {null_safe_equal_operator} r2.launchbox_id THEN r1.launchbox_id END AS launchbox_id, + CASE WHEN r1.ra_id {null_safe_equal_operator} r2.ra_id THEN r1.ra_id END AS ra_id, + CASE WHEN r1.hasheous_id {null_safe_equal_operator} r2.hasheous_id THEN r1.hasheous_id END AS hasheous_id, + CASE WHEN r1.tgdb_id {null_safe_equal_operator} r2.tgdb_id THEN r1.tgdb_id END AS tgdb_id + FROM + roms r1 + JOIN + roms r2 + ON + r1.platform_id = r2.platform_id + AND r1.id != r2.id + AND ( + (r1.igdb_id = r2.igdb_id AND r1.igdb_id IS NOT NULL) + OR + (r1.moby_id = r2.moby_id AND r1.moby_id IS NOT NULL) + OR + (r1.ss_id = r2.ss_id AND r1.ss_id IS NOT NULL) + OR + (r1.launchbox_id = r2.launchbox_id AND r1.launchbox_id IS NOT NULL) + OR + (r1.ra_id = r2.ra_id AND r1.ra_id IS NOT NULL) + OR + (r1.hasheous_id = r2.hasheous_id AND r1.hasheous_id IS NOT NULL) + OR + (r1.tgdb_id = r2.tgdb_id AND r1.tgdb_id IS NOT NULL) + OR + (r1.fs_name_no_tags = r2.fs_name_no_tags AND r1.fs_name_no_tags != '') + ); + """), # nosec B608 + ) diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 1482a8990..1b8d76cdc 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -256,3 +256,15 @@ SENTRY_DSN: Final[str | None] = _get_env("SENTRY_DSN") # TESTING IS_PYTEST_RUN: Final = bool(_get_env("PYTEST_VERSION")) + + +# PROXY +def has_proxy_env() -> bool: + return any( + _get_env(var) + for var in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + ) + ) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 2755c357b..62f8c5224 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -45,6 +45,7 @@ DEFAULT_EXCLUDED_FILES: Final = [ ".stfolder", "@SynoResource", "gamelist.xml", + "metadata.pegasus.xml", ] DEFAULT_EXCLUDED_DIRS: Final = [ "@eaDir", @@ -224,39 +225,67 @@ class ConfigManager: self.config = Config( CONFIG_FILE_MOUNTED=self._config_file_mounted, CONFIG_FILE_WRITABLE=self._config_file_writable, - EXCLUDED_PLATFORMS=pydash.get( - self._raw_config, "exclude.platforms", DEFAULT_EXCLUDED_DIRS + EXCLUDED_PLATFORMS=sorted( + { + *DEFAULT_EXCLUDED_DIRS, + *pydash.get(self._raw_config, "exclude.platforms", []), + } ), - EXCLUDED_SINGLE_EXT=[ - e.lower() - for e in pydash.get( - self._raw_config, - "exclude.roms.single_file.extensions", - DEFAULT_EXCLUDED_EXTENSIONS, - ) - ], - EXCLUDED_SINGLE_FILES=pydash.get( - self._raw_config, - "exclude.roms.single_file.names", - DEFAULT_EXCLUDED_FILES, + EXCLUDED_SINGLE_EXT=sorted( + { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + *( + e.lower() + for e in pydash.get( + self._raw_config, + "exclude.roms.single_file.extensions", + [], + ) + ), + } ), - EXCLUDED_MULTI_FILES=pydash.get( - self._raw_config, - "exclude.roms.multi_file.names", - DEFAULT_EXCLUDED_DIRS, + EXCLUDED_SINGLE_FILES=sorted( + { + *DEFAULT_EXCLUDED_FILES, + *pydash.get( + self._raw_config, + "exclude.roms.single_file.names", + [], + ), + } ), - EXCLUDED_MULTI_PARTS_EXT=[ - e.lower() - for e in pydash.get( - self._raw_config, - "exclude.roms.multi_file.parts.extensions", - DEFAULT_EXCLUDED_EXTENSIONS, - ) - ], - EXCLUDED_MULTI_PARTS_FILES=pydash.get( - self._raw_config, - "exclude.roms.multi_file.parts.names", - DEFAULT_EXCLUDED_FILES, + EXCLUDED_MULTI_FILES=sorted( + { + *DEFAULT_EXCLUDED_DIRS, + *pydash.get( + self._raw_config, + "exclude.roms.multi_file.names", + [], + ), + } + ), + EXCLUDED_MULTI_PARTS_EXT=sorted( + { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + *( + e.lower() + for e in pydash.get( + self._raw_config, + "exclude.roms.multi_file.parts.extensions", + [], + ) + ), + } + ), + EXCLUDED_MULTI_PARTS_FILES=sorted( + { + *DEFAULT_EXCLUDED_FILES, + *pydash.get( + self._raw_config, + "exclude.roms.multi_file.parts.names", + [], + ), + } ), PLATFORMS_BINDING=pydash.get(self._raw_config, "system.platforms", {}), PLATFORMS_VERSIONS=pydash.get(self._raw_config, "system.versions", {}), diff --git a/backend/endpoints/configs.py b/backend/endpoints/configs.py index 65482f5f8..8c7e74f90 100644 --- a/backend/endpoints/configs.py +++ b/backend/endpoints/configs.py @@ -1,6 +1,11 @@ from fastapi import HTTPException, Request, status from pydantic import BaseModel +from config.config_manager import ( + DEFAULT_EXCLUDED_DIRS, + DEFAULT_EXCLUDED_EXTENSIONS, + DEFAULT_EXCLUDED_FILES, +) from config.config_manager import config_manager as cm from decorators.auth import protected_route from endpoints.responses.config import ConfigResponse @@ -43,6 +48,9 @@ def get_config(request: Request) -> ConfigResponse: EXCLUDED_MULTI_FILES=cfg.EXCLUDED_MULTI_FILES, EXCLUDED_MULTI_PARTS_EXT=cfg.EXCLUDED_MULTI_PARTS_EXT, EXCLUDED_MULTI_PARTS_FILES=cfg.EXCLUDED_MULTI_PARTS_FILES, + DEFAULT_EXCLUDED_DIRS=list(DEFAULT_EXCLUDED_DIRS), + DEFAULT_EXCLUDED_FILES=list(DEFAULT_EXCLUDED_FILES), + DEFAULT_EXCLUDED_EXTENSIONS=list(DEFAULT_EXCLUDED_EXTENSIONS), PLATFORMS_BINDING=cfg.PLATFORMS_BINDING, PLATFORMS_VERSIONS=cfg.PLATFORMS_VERSIONS, SKIP_HASH_CALCULATION=cfg.SKIP_HASH_CALCULATION, diff --git a/backend/endpoints/responses/__init__.py b/backend/endpoints/responses/__init__.py index 64b226d6a..43a23922c 100644 --- a/backend/endpoints/responses/__init__.py +++ b/backend/endpoints/responses/__init__.py @@ -41,7 +41,7 @@ class UpdateTaskMeta(TypedDict): update_stats: UpdateStats | None -class CleanupStats(TypedDict): +class OrphanedResourcesCleanupStats(TypedDict): platforms_in_db: int roms_in_db: int platforms_in_fs: int @@ -50,6 +50,16 @@ class CleanupStats(TypedDict): removed_fs_roms: int +class MissingRomsCleanupStats(TypedDict): + platform_id: int | None + roms_found: int + roms_deleted: int + errors: int + + +CleanupStats = Union[OrphanedResourcesCleanupStats, MissingRomsCleanupStats] + + class CleanupTaskMeta(TypedDict): cleanup_stats: CleanupStats | None diff --git a/backend/endpoints/responses/config.py b/backend/endpoints/responses/config.py index d9ebe126e..df9528f50 100644 --- a/backend/endpoints/responses/config.py +++ b/backend/endpoints/responses/config.py @@ -12,6 +12,9 @@ class ConfigResponse(TypedDict): EXCLUDED_MULTI_FILES: list[str] EXCLUDED_MULTI_PARTS_EXT: list[str] EXCLUDED_MULTI_PARTS_FILES: list[str] + DEFAULT_EXCLUDED_DIRS: list[str] + DEFAULT_EXCLUDED_FILES: list[str] + DEFAULT_EXCLUDED_EXTENSIONS: list[str] PLATFORMS_BINDING: dict[str, str] PLATFORMS_VERSIONS: dict[str, str] SKIP_HASH_CALCULATION: bool diff --git a/backend/endpoints/responses/rom.py b/backend/endpoints/responses/rom.py index deca65038..94b69e214 100644 --- a/backend/endpoints/responses/rom.py +++ b/backend/endpoints/responses/rom.py @@ -200,8 +200,16 @@ class RomMetadataSchema(BaseModel): def sort_game_modes(cls, v: list[str]) -> list[str]: return sorted(v) - @field_validator("age_ratings") - def sort_age_ratings(cls, v: list[str]) -> list[str]: + @field_validator("age_ratings", mode="before") + def normalize_age_ratings(cls, v: str | list[str] | None) -> list[str]: + if not v: + return [] + + # MySQL/MariaDB returns a scalar string instead of a single-element array + # when using JSON_EXTRACT with a [*] wildcard path on a single-element array. + if isinstance(v, str): + return sorted([v]) + return sorted(v) diff --git a/backend/endpoints/responses/stats.py b/backend/endpoints/responses/stats.py index 5873526a6..495c6aa00 100644 --- a/backend/endpoints/responses/stats.py +++ b/backend/endpoints/responses/stats.py @@ -11,7 +11,7 @@ class RegionBreakdownItem(TypedDict): count: int -class StatsReturn(TypedDict): +class StatsReturn(TypedDict, total=False): PLATFORMS: int ROMS: int SAVES: int diff --git a/backend/endpoints/roms/__init__.py b/backend/endpoints/roms/__init__.py index 8178eb920..914b1aead 100644 --- a/backend/endpoints/roms/__init__.py +++ b/backend/endpoints/roms/__init__.py @@ -164,19 +164,6 @@ class RomUserData(BaseModel): ) -class RomUserUpdatePayload(BaseModel): - data: RomUserData = Field( - default_factory=RomUserData, - description="Partial rom user data to update. Only provided fields will be updated.", - ) - update_last_played: bool = Field( - default=False, description="Set last played timestamp to now." - ) - remove_last_played: bool = Field( - default=False, description="Clear the last played timestamp." - ) - - async def parse_rom_update_form( request: Request, igdb_id: str | None = Form(default=None), @@ -1472,6 +1459,37 @@ async def delete_roms( continue try: + if id in delete_from_fs: + log.info(f"Deleting {hl(rom.fs_name)} from filesystem") + try: + rom_path = f"{rom.fs_path}/{rom.fs_name}" + full_path = fs_rom_handler.validate_path(rom_path) + if full_path.is_dir(): + await fs_rom_handler.remove_directory(rom_path) + else: + await fs_rom_handler.remove_file(rom_path) + # Clean up empty parent directory if it becomes empty + parent = full_path.parent + if ( + parent != fs_rom_handler.base_path + and parent.is_dir() + and not any(parent.iterdir()) + ): + try: + await fs_rom_handler.remove_directory( + str(parent.relative_to(fs_rom_handler.base_path)) + ) + except OSError as dir_err: + log.warning( + f"Couldn't clean up empty parent directory for {hl(rom.fs_name)}: {dir_err}" + ) + except FileNotFoundError: + error = f"Rom file {hl(rom.fs_name)} not found for platform {hl(rom.platform_display_name, color=BLUE)}[{hl(rom.platform_slug)}]" + log.error(error) + errors.append(error) + failed_items += 1 + continue + log.info( f"Deleting {hl(str(rom.name or 'ROM'), color=BLUE)} [{hl(rom.fs_name)}] from database" ) @@ -1484,18 +1502,6 @@ async def delete_roms( f"Couldn't find resources to delete for {hl(str(rom.name or 'ROM'), color=BLUE)}" ) - if id in delete_from_fs: - log.info(f"Deleting {hl(rom.fs_name)} from filesystem") - try: - file_path = f"{rom.fs_path}/{rom.fs_name}" - await fs_rom_handler.remove_file(file_path=file_path) - except FileNotFoundError: - error = f"Rom file {hl(rom.fs_name)} not found for platform {hl(rom.platform_display_name, color=BLUE)}[{hl(rom.platform_slug)}]" - log.error(error) - errors.append(error) - failed_items += 1 - continue - successful_items += 1 except Exception as e: failed_items += 1 @@ -1517,7 +1523,13 @@ async def delete_roms( async def update_rom_user( request: Request, id: Annotated[int, PathVar(description="Rom internal id.", ge=1)], - payload: Annotated[RomUserUpdatePayload, Body()], + data: Annotated[RomUserData, Body()], + update_last_played: Annotated[ + bool, Query(description="Set last played timestamp to now.") + ] = False, + remove_last_played: Annotated[ + bool, Query(description="Clear the last played timestamp.") + ] = False, ) -> RomUserSchema: """Update rom data associated to the current user.""" rom = db_rom_handler.get_rom(id) @@ -1529,11 +1541,17 @@ async def update_rom_user( id, request.user.id ) or db_rom_handler.add_rom_user(id, request.user.id) - cleaned_data = payload.data.model_dump(exclude_unset=True) + if update_last_played and remove_last_played: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="update_last_played and remove_last_played are mutually exclusive.", + ) - if payload.update_last_played: + cleaned_data = data.model_dump(exclude_unset=True) + + if update_last_played: cleaned_data.update({"last_played": datetime.now(timezone.utc)}) - elif payload.remove_last_played: + elif remove_last_played: cleaned_data.update({"last_played": None}) rom_user = db_rom_handler.update_rom_user(db_rom_user.id, cleaned_data) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index 46107f1dc..0a1ddcc1e 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -30,7 +30,7 @@ from handler.filesystem import ( fs_rom_handler, ) from handler.filesystem.roms_handler import FSRom -from handler.metadata import meta_gamelist_handler +from handler.metadata import meta_gamelist_handler, meta_hltb_handler from handler.metadata.ss_handler import get_preferred_media_types from handler.redis_handler import get_job_func_name, high_prio_queue, redis_client from handler.scan_handler import ( @@ -413,7 +413,7 @@ async def _identify_rom( ) # Handle special media files from Screenscraper - if _added_rom.ss_metadata: + if _added_rom.ss_metadata and MetadataSource.SS in metadata_sources: preferred_media_types = get_preferred_media_types() for media_type in preferred_media_types: if _added_rom.ss_metadata.get(f"{media_type.value}_path"): @@ -423,7 +423,7 @@ async def _identify_rom( ) # Handle special media files from ES-DE gamelist.xml - if _added_rom.gamelist_metadata: + if _added_rom.gamelist_metadata and MetadataSource.GAMELIST in metadata_sources: preferred_media_types = get_preferred_media_types() for media_type in preferred_media_types: if _added_rom.gamelist_metadata.get(f"{media_type.value}_path"): @@ -433,7 +433,7 @@ async def _identify_rom( ) # Store normal and locked badges - if _added_rom.ra_metadata: + if _added_rom.ra_metadata and MetadataSource.RA in metadata_sources: for ach in _added_rom.ra_metadata.get("achievements", []): badge_url_lock = ach.get("badge_url_lock", None) badge_path_lock = ach.get("badge_path_lock", None) @@ -637,9 +637,13 @@ async def scan_platforms( await socket_manager.emit("scan:done_ko", e.message) return scan_stats - # Clear the gamelist cache to ensure we're using fresh gamelist.xml data + # Clear the gamelist cache to ensure we're using fresh gamelist.xml data meta_gamelist_handler.clear_cache() + # Initialize HLTB handler (fetches current search endpoint and security token) + if MetadataSource.HLTB in metadata_sources: + meta_hltb_handler.initialize() + # Precalculate total platforms and ROMs total_roms = 0 for platform_slug in fs_platforms: diff --git a/backend/endpoints/stats.py b/backend/endpoints/stats.py index 1d77766b4..44d14bd2e 100644 --- a/backend/endpoints/stats.py +++ b/backend/endpoints/stats.py @@ -9,20 +9,26 @@ router = APIRouter( @router.get("") -def stats() -> StatsReturn: +def stats(include_platform_stats: bool = False) -> StatsReturn: """Endpoint to return the current RomM stats Returns: dict: Dictionary with all the stats """ - return { + result: StatsReturn = { "PLATFORMS": db_stats_handler.get_platforms_count(), "ROMS": db_stats_handler.get_roms_count(), "SAVES": db_stats_handler.get_saves_count(), "STATES": db_stats_handler.get_states_count(), "SCREENSHOTS": db_stats_handler.get_screenshots_count(), "TOTAL_FILESIZE_BYTES": db_stats_handler.get_total_filesize(), - "METADATA_COVERAGE": db_stats_handler.get_metadata_coverage_by_platform(), - "REGION_BREAKDOWN": db_stats_handler.get_region_breakdown_by_platform(), } + + if include_platform_stats: + result["METADATA_COVERAGE"] = ( + db_stats_handler.get_metadata_coverage_by_platform() + ) + result["REGION_BREAKDOWN"] = db_stats_handler.get_region_breakdown_by_platform() + + return result diff --git a/backend/endpoints/tasks.py b/backend/endpoints/tasks.py index 59543540e..e1c1303f4 100644 --- a/backend/endpoints/tasks.py +++ b/backend/endpoints/tasks.py @@ -3,6 +3,7 @@ from typing import Any, TypedDict from fastapi import Body, HTTPException, Request from rq import Worker +from rq.exceptions import NoSuchJobError from rq.job import Job, JobStatus from rq.registry import FailedJobRegistry, FinishedJobRegistry @@ -288,7 +289,11 @@ async def get_tasks_status(request: Request) -> list[TaskStatusResponse]: # Process finished jobs for registry in finished_registries: for job_id in registry.get_job_ids(): - job = Job.fetch(job_id, connection=redis_client) + try: + job = Job.fetch(job_id, connection=redis_client) + except NoSuchJobError: + registry.remove(job_id) + continue all_tasks.append( _build_task_status_response( job, @@ -298,7 +303,11 @@ async def get_tasks_status(request: Request) -> list[TaskStatusResponse]: # Process failed jobs for registry in failed_registries: for job_id in registry.get_job_ids(): - job = Job.fetch(job_id, connection=redis_client) + try: + job = Job.fetch(job_id, connection=redis_client) + except NoSuchJobError: + registry.remove(job_id) + continue all_tasks.append(_build_task_status_response(job)) all_tasks.sort( diff --git a/backend/handler/database/collections_handler.py b/backend/handler/database/collections_handler.py index 9cedf06a5..4b53b2fb9 100644 --- a/backend/handler/database/collections_handler.py +++ b/backend/handler/database/collections_handler.py @@ -130,13 +130,19 @@ class DBCollectionsHandler(DBBaseHandler): ) # Insert new CollectionRom entries for this collection if rom_ids: - session.execute( - insert(CollectionRom), - [ - {"collection_id": id, "rom_id": rom_id} - for rom_id in set(rom_ids) - ], + # Filter out rom_ids that no longer exist in the roms table to + # avoid foreign key constraint violations (e.g. after a rescan) + valid_rom_ids = set( + session.scalars(select(Rom.id).where(Rom.id.in_(rom_ids))).all() ) + if valid_rom_ids: + session.execute( + insert(CollectionRom), + [ + {"collection_id": id, "rom_id": rom_id} + for rom_id in valid_rom_ids + ], + ) return session.scalar(query.filter_by(id=id).limit(1)) diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index da1875937..d161162fd 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -641,7 +641,6 @@ class DBRomsHandler(DBBaseHandler): .with_only_columns( base_subquery.c.id, base_subquery.c.fs_name_no_ext, - base_subquery.c.fs_name_no_tags, base_subquery.c.platform_id, base_subquery.c.igdb_id, base_subquery.c.ss_id, @@ -702,11 +701,6 @@ class DBRomsHandler(DBBaseHandler): base_subquery.c.flashpoint_id, base_subquery.c.platform_id, ), - _create_metadata_id_case( - "fs", - func.nullif(base_subquery.c.fs_name_no_tags, ""), - base_subquery.c.platform_id, - ), _create_metadata_id_case( "romm", base_subquery.c.id, diff --git a/backend/handler/filesystem/resources_handler.py b/backend/handler/filesystem/resources_handler.py index 2b2b334f4..81cf7679d 100644 --- a/backend/handler/filesystem/resources_handler.py +++ b/backend/handler/filesystem/resources_handler.py @@ -20,6 +20,16 @@ from utils.validation import validate_url_for_http_request from .base_handler import CoverSize, FSHandler +def _check_content_type( + response: httpx.Response, allowed_prefixes: tuple[str, ...], label: str +) -> bool: + content_type = response.headers.get("content-type", "").lower() + if not any(content_type.startswith(p) for p in allowed_prefixes): + log.warning(f"Unexpected content type for {label}: {content_type}") + return False + return True + + class FSResourcesHandler(FSHandler): def __init__(self) -> None: super().__init__(base_path=RESOURCES_BASE_PATH) @@ -75,11 +85,15 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_cover.startswith("file://"): try: - file_path = AnyioPath(url_cover[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_cover[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory dest_path = f"{cover_file}/{size.value}.png" - await self.copy_file(Path(str(file_path)), dest_path) + await self.copy_file(validated, dest_path) if ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP: self.image_converter.convert_to_webp( @@ -87,14 +101,13 @@ class FSResourcesHandler(FSHandler): force=True, ) else: - log.warning(f"Cover file not found: {file_path}") + log.warning(f"Cover file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy cover file {url_cover}: {str(exc)}") return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_cover, "url_cover") httpx_client = ctx_httpx_client.get() @@ -103,6 +116,9 @@ class FSResourcesHandler(FSHandler): "GET", url_cover, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + if not _check_content_type(response, ("image/",), "cover"): + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -249,21 +265,22 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_screenhot.startswith("file://"): try: - file_path = AnyioPath(url_screenhot[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_screenhot[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory - await self.copy_file( - Path(str(file_path)), f"{screenshot_path}/{idx}.jpg" - ) + await self.copy_file(validated, f"{screenshot_path}/{idx}.jpg") else: - log.warning(f"Screenshot file not found: {file_path}") + log.warning(f"Screenshot file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy screenshot file {url_screenhot}: {str(exc)}") return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_screenhot, "url_screenshot") httpx_client = ctx_httpx_client.get() @@ -272,6 +289,9 @@ class FSResourcesHandler(FSHandler): "GET", url_screenhot, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + if not _check_content_type(response, ("image/",), "screenshot"): + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -365,21 +385,22 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_manual.startswith("file://"): try: - file_path = AnyioPath(url_manual[7:]) # Remove "file://" prefix - if await file_path.exists(): + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_manual[7:] # Remove "file://" prefix + ) + if await AnyioPath(validated).exists(): # Copy the file to the resources directory - await self.copy_file( - Path(str(file_path)), f"{manual_path}/{rom.id}.pdf" - ) + await self.copy_file(validated, f"{manual_path}/{rom.id}.pdf") else: - log.warning(f"Manual file not found: {file_path}") + log.warning(f"Manual file not found: {str(validated)}") return None except Exception as exc: log.error(f"Unable to copy manual file {url_manual}: {str(exc)}") return None else: # Handle HTTP URL - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_manual, "url_manual") httpx_client = ctx_httpx_client.get() @@ -388,6 +409,11 @@ class FSResourcesHandler(FSHandler): "GET", url_manual, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + if not _check_content_type( + response, ("application/pdf",), "manual" + ): + return None + # Check if content is gzipped from response headers is_gzipped = ( response.headers.get("content-encoding", "").lower() @@ -440,7 +466,6 @@ class FSResourcesHandler(FSHandler): # Retroachievements async def store_ra_badge(self, url: str, path: str) -> None: - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url, "url_badge") httpx_client = ctx_httpx_client.get() @@ -456,6 +481,9 @@ class FSResourcesHandler(FSHandler): try: async with httpx_client.stream("GET", url, timeout=120) as response: if response.status_code == status.HTTP_200_OK: + if not _check_content_type(response, ("image/",), "badge"): + return + async with await self.write_file_streamed( path=directory, filename=filename ) as f: @@ -498,7 +526,12 @@ class FSResourcesHandler(FSHandler): # Handle file:// URLs for gamelist.xml if url_media.startswith("file://"): try: - file_path = AnyioPath(url_media[7:]) # Remove "file://" prefix + from handler.filesystem import fs_rom_handler + + validated = fs_rom_handler.validate_path( + url_media[7:] # Remove "file://" prefix + ) + file_path = AnyioPath(validated) if await file_path.exists(): await self.copy_file(Path(str(file_path)), dest_path) except Exception as exc: @@ -506,7 +539,6 @@ class FSResourcesHandler(FSHandler): return None else: # Handle HTTP URLs - # Validate URL to prevent SSRF attacks validate_url_for_http_request(url_media, "url_media") httpx_client = ctx_httpx_client.get() @@ -515,6 +547,13 @@ class FSResourcesHandler(FSHandler): "GET", url_media, timeout=120 ) as response: if response.status_code == status.HTTP_200_OK: + if not _check_content_type( + response, + ("image/", "video/", "application/pdf"), + "media", + ): + return None + async with await self.write_file_streamed( path=directory, filename=filename ) as f: diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index 39f75f7bf..4e2079e3d 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -745,6 +745,6 @@ class FSRomsHandler(FSHandler): if platform_slug == UPS.PICO and fs_name.lower().endswith( PICO8_CARTRIDGE_EXTENSION ): - rom_path = self.validate_path(f"{fs_path}/{fs_name}") - return f"file://{rom_path}" + self.validate_path(f"{fs_path}/{fs_name}") + return f"file://{fs_path}/{fs_name}" return None diff --git a/backend/handler/metadata/fixtures/hltb_api_url b/backend/handler/metadata/fixtures/hltb_api_url index 67fe32dbb..4eb6b35f5 100644 --- a/backend/handler/metadata/fixtures/hltb_api_url +++ b/backend/handler/metadata/fixtures/hltb_api_url @@ -1 +1 @@ -https://howlongtobeat.com/api/finder +https://howlongtobeat.com/api/find diff --git a/backend/handler/metadata/gamelist_handler.py b/backend/handler/metadata/gamelist_handler.py index 27f7dd490..aac489797 100644 --- a/backend/handler/metadata/gamelist_handler.py +++ b/backend/handler/metadata/gamelist_handler.py @@ -99,10 +99,9 @@ XML_TAG_MAP: Final = { def _make_file_uri(platform_dir: str, raw_text: str) -> str: cleaned_text = raw_text.replace("./", "") - validated_path = fs_platform_handler.validate_path( - os.path.join(platform_dir, cleaned_text) - ) - return f"file://{str(validated_path)}" + joined_path = Path(platform_dir, cleaned_text) + fs_platform_handler.validate_path(str(joined_path)) + return f"file://{joined_path.as_posix()}" def extract_media_from_gamelist_rom( @@ -148,7 +147,9 @@ def extract_media_from_gamelist_rom( found_files = glob.glob(str(search_path)) if found_files: # trunk-ignore(mypy/literal-required) - gamelist_media[media_key] = f"file://{str(found_files[0])}" + gamelist_media[media_key] = ( + f"file://{str(Path(found_files[0]).relative_to(fs_platform_handler.base_path))}" + ) return gamelist_media diff --git a/backend/handler/metadata/hltb_handler.py b/backend/handler/metadata/hltb_handler.py index e6fb7c392..73de8b2dc 100644 --- a/backend/handler/metadata/hltb_handler.py +++ b/backend/handler/metadata/hltb_handler.py @@ -10,7 +10,7 @@ from config import HLTB_API_ENABLED from handler.metadata.base_handler import UniversalPlatformSlug as UPS from logger.logger import log from utils import get_version -from utils.context import ctx_httpx_client +from utils.context import create_httpx_client, ctx_httpx_client from .base_handler import BaseRom, MetadataHandler @@ -178,28 +178,31 @@ class HLTBHandler(MetadataHandler): self.base_url = "https://howlongtobeat.com" self.user_endpoint = f"{self.base_url}/api/user" self.stats_endpoint = f"{self.base_url}/api/stats/games?platform=1&year=2000" - self.search_url = f"{self.base_url}/api/search" + self.search_url = f"{self.base_url}/api/find" self.search_init_url = f"{self.search_url}/init" self.security_token = None + self.hp_key = None + self.hp_val = None self.min_similarity_score: Final = 0.85 + @classmethod + def is_enabled(cls) -> bool: + return HLTB_API_ENABLED + + def initialize(self) -> None: # HLTB rotates their search endpoint regularly self._fetch_search_endpoint() # HLTB now requires a security token self._fetch_security_token() - @classmethod - def is_enabled(cls) -> bool: - return HLTB_API_ENABLED - def _fetch_search_endpoint(self): """Fetch the API endpoint URL from Github.""" if not HLTB_API_ENABLED: return try: - with httpx.Client() as client: + with create_httpx_client() as client: response = client.get(GITHUB_FILE_URL, timeout=10) response.raise_for_status() self.search_url = response.text.strip() @@ -218,7 +221,7 @@ class HLTBHandler(MetadataHandler): params = {"t": int(time.time())} try: - with httpx.Client() as client: + with create_httpx_client() as client: response = client.get( self.search_init_url, params=params, @@ -226,7 +229,10 @@ class HLTBHandler(MetadataHandler): timeout=10, ) response.raise_for_status() - self.security_token = response.json().get("token", None) + data = response.json() + self.security_token = data.get("token", None) + self.hp_key = data.get("hpKey", None) + self.hp_val = data.get("hpVal", None) except Exception as e: log.warning("Unexpected error fetching HLTB security token: %s", e) @@ -253,7 +259,7 @@ class HLTBHandler(MetadataHandler): :return: A dictionary with the json result. :raises HTTPException: If the request fails or the service is unavailable. """ - if not self.security_token: + if not self.security_token or not self.hp_key or not self.hp_val: return {} httpx_client = ctx_httpx_client.get() @@ -262,9 +268,14 @@ class HLTBHandler(MetadataHandler): "Content-Type": "application/json", "Referer": "https://howlongtobeat.com", "User-Agent": f"RomM/{get_version()}", - "X-Auth-Token": self.security_token, + "x-auth-token": self.security_token, + "x-hp-key": self.hp_key, + "x-hp-val": self.hp_val, } + # Some HLTB endpoints require the key:val in the payload + payload[self.hp_key] = self.hp_val + log.debug( "HowLongToBeat API request: URL=%s, Headers=%s, Payload=%s, Timeout=%s", url, diff --git a/backend/handler/metadata/launchbox_handler/utils.py b/backend/handler/metadata/launchbox_handler/utils.py index f5d58ea21..8c73d0892 100644 --- a/backend/handler/metadata/launchbox_handler/utils.py +++ b/backend/handler/metadata/launchbox_handler/utils.py @@ -17,10 +17,10 @@ def sanitize_filename(stem: str) -> str: def file_uri_for_local_path(path: Path) -> str | None: try: - _ = path.resolve().relative_to(LAUNCHBOX_LOCAL_DIR.resolve()) + relative = path.resolve().relative_to(LAUNCHBOX_LOCAL_DIR.resolve()) except ValueError: return None - return f"file://{str(path)}" + return f"file://{relative.as_posix()}" def coalesce(*values: object | None) -> str | None: diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index e1fe44a64..7a3f648ea 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -464,6 +464,7 @@ RA_PLATFORM_LIST: dict[UPS, SlugToRAId] = { "name": "Watara/QuickShot Supervision", }, UPS.WIN: {"id": 102, "name": "Windows"}, + UPS.WII: {"id": 19, "name": "Wii"}, UPS.WONDERSWAN: {"id": 53, "name": "WonderSwan"}, UPS.WONDERSWAN_COLOR: {"id": 53, "name": "WonderSwan Color"}, } diff --git a/backend/models/fixtures/known_bios_files.json b/backend/models/fixtures/known_bios_files.json index 1e16ac779..b16230d84 100644 --- a/backend/models/fixtures/known_bios_files.json +++ b/backend/models/fixtures/known_bios_files.json @@ -1213,21 +1213,21 @@ }, "zxs:plus3-0.rom": { "size": "16384", - "crc": "a10230c0", - "md5": "3abdc20e72890a750dd3c745d286dfba", - "sha1": "a837f66977040f7b51ed053a2483c10f3d070ab7" + "crc": "17373da2", + "md5": "9833b8b73384dd5fa3678377ff00a2bb", + "sha1": "e319ed08b4d53a5e421a75ea00ea02039ba6555b" }, "zxs:plus3-1.rom": { "size": "16384", - "crc": "09b9c3ca", - "md5": "8361a1d9c8bcef89c0c39293776564ad", - "sha1": "6a4364f25513e4079f048f2de131a896d30edc64" + "crc": "f1d1d99e", + "md5": "0f711ceb5ab801b4701989982e0f334c", + "sha1": "c9969fc36095a59787554026a9adc3b87678c794" }, "zxs:plus3-2.rom": { "size": "16384", - "crc": "a60285a0", - "md5": "f36c5c2d1f2a682caadeaa6f947db0da", - "sha1": "0a747cc0b827a94b4fd74cfd818ca792437a38f7" + "crc": "3dbf351d", + "md5": "3b6dd659d5e4ec97f0e2f7878152c987", + "sha1": "22e50c6ba4157a3f6a821bd9937cd26e292775c6" }, "zxs:plus3-3.rom": { "size": "16384", diff --git a/backend/models/rom.py b/backend/models/rom.py index 0517fa6b1..3ff51ec06 100644 --- a/backend/models/rom.py +++ b/backend/models/rom.py @@ -173,7 +173,6 @@ class Rom(BaseModel): Index("idx_roms_flashpoint_id", "flashpoint_id"), Index("idx_roms_hltb_id", "hltb_id"), Index("idx_roms_gamelist_id", "gamelist_id"), - Index("idx_roms_fs_name_no_tags", "fs_name_no_tags"), ) fs_name: Mapped[str] = mapped_column(String(length=FILE_NAME_MAX_LENGTH)) diff --git a/backend/tests/adapters/services/test_rahasher.py b/backend/tests/adapters/services/test_rahasher.py index 5ae07e1c2..2505e557e 100644 --- a/backend/tests/adapters/services/test_rahasher.py +++ b/backend/tests/adapters/services/test_rahasher.py @@ -51,7 +51,7 @@ class TestRAHasherService: async def test_calculate_hash_success(self, service): """Test successful hash calculation.""" mock_proc = AsyncMock() - mock_proc.wait.return_value = 1 # RAHasher returns 1 on success + mock_proc.wait.return_value = 0 # RAHasher returns 0 on success mock_proc.stdout.read.return_value = b"a1b2c3d4e5f6789012345678901234ab\n" mock_proc.stderr = None @@ -136,7 +136,7 @@ class TestRAHasherService: async def test_calculate_hash_with_extra_output(self, service): """Test when RAHasher returns hash with extra text.""" mock_proc = AsyncMock() - mock_proc.wait.return_value = 1 + mock_proc.wait.return_value = 0 mock_proc.stdout.read.return_value = ( b"Processing file... Hash: a1b2c3d4e5f6789012345678901234ab Done.\n" ) @@ -181,7 +181,7 @@ class TestRAHasherService: for platform_id, file_path, platform_slug in test_cases: mock_proc = AsyncMock() - mock_proc.wait.return_value = 1 + mock_proc.wait.return_value = 0 mock_proc.stdout.read.return_value = b"a1b2c3d4e5f6789012345678901234ab\n" mock_proc.stderr = None @@ -289,7 +289,7 @@ class TestRAHasherServicePerformance: async def test_concurrent_hash_calculations(self, service): """Test multiple concurrent hash calculations.""" mock_proc = AsyncMock() - mock_proc.wait.return_value = 1 + mock_proc.wait.return_value = 0 mock_proc.stdout.read.return_value = b"a1b2c3d4e5f6789012345678901234ab\n" mock_proc.stderr = None diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index c1792858a..ff6e6736d 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -14,12 +14,35 @@ def test_config_loader(): os.path.join(Path(__file__).resolve().parent, "fixtures", "config/config.yml") ) - assert loader.config.EXCLUDED_PLATFORMS == ["romm"] - assert loader.config.EXCLUDED_SINGLE_EXT == ["xml"] - assert loader.config.EXCLUDED_SINGLE_FILES == ["info.txt"] - assert loader.config.EXCLUDED_MULTI_FILES == ["my_multi_file_game", "DLC"] - assert loader.config.EXCLUDED_MULTI_PARTS_EXT == ["txt"] - assert loader.config.EXCLUDED_MULTI_PARTS_FILES == ["data.xml"] + assert loader.config.EXCLUDED_PLATFORMS == sorted({*DEFAULT_EXCLUDED_DIRS, "romm"}) + assert loader.config.EXCLUDED_SINGLE_EXT == sorted( + { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + "xml", + } + ) + assert loader.config.EXCLUDED_SINGLE_FILES == sorted( + {*DEFAULT_EXCLUDED_FILES, "info.txt"} + ) + assert loader.config.EXCLUDED_MULTI_FILES == sorted( + { + *DEFAULT_EXCLUDED_DIRS, + "my_multi_file_game", + "DLC", + } + ) + assert loader.config.EXCLUDED_MULTI_PARTS_EXT == sorted( + { + *(e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS), + "txt", + } + ) + assert loader.config.EXCLUDED_MULTI_PARTS_FILES == sorted( + { + *DEFAULT_EXCLUDED_FILES, + "data.xml", + } + ) assert loader.config.PLATFORMS_BINDING == {"gc": "ngc"} assert loader.config.PLATFORMS_VERSIONS == {"naomi": "arcade"} assert loader.config.ROMS_FOLDER_NAME == "ROMS" @@ -63,16 +86,16 @@ def test_empty_config_loader(): ) ) - assert loader.config.EXCLUDED_PLATFORMS == DEFAULT_EXCLUDED_DIRS - assert loader.config.EXCLUDED_SINGLE_EXT == [ - e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert loader.config.EXCLUDED_SINGLE_FILES == DEFAULT_EXCLUDED_FILES - assert loader.config.EXCLUDED_MULTI_FILES == DEFAULT_EXCLUDED_DIRS - assert loader.config.EXCLUDED_MULTI_PARTS_EXT == [ - e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert loader.config.EXCLUDED_MULTI_PARTS_FILES == DEFAULT_EXCLUDED_FILES + assert loader.config.EXCLUDED_PLATFORMS == sorted(DEFAULT_EXCLUDED_DIRS) + assert loader.config.EXCLUDED_SINGLE_EXT == sorted( + {e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS} + ) + assert loader.config.EXCLUDED_SINGLE_FILES == sorted(DEFAULT_EXCLUDED_FILES) + assert loader.config.EXCLUDED_MULTI_FILES == sorted(DEFAULT_EXCLUDED_DIRS) + assert loader.config.EXCLUDED_MULTI_PARTS_EXT == sorted( + {e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS} + ) + assert loader.config.EXCLUDED_MULTI_PARTS_FILES == sorted(DEFAULT_EXCLUDED_FILES) assert loader.config.PLATFORMS_BINDING == {} assert loader.config.PLATFORMS_VERSIONS == {} assert loader.config.ROMS_FOLDER_NAME == "roms" diff --git a/backend/tests/endpoints/roms/test_rom.py b/backend/tests/endpoints/roms/test_rom.py index d76762b25..8d9ebe7b6 100644 --- a/backend/tests/endpoints/roms/test_rom.py +++ b/backend/tests/endpoints/roms/test_rom.py @@ -115,13 +115,150 @@ def test_delete_roms(client: TestClient, access_token: str, rom: Rom): assert body["successful_items"] == 1 -def test_update_rom_user_props_with_data_envelope( +@patch( + "endpoints.roms.fs_rom_handler.remove_directory", + new_callable=AsyncMock, +) +@patch( + "endpoints.roms.fs_rom_handler.remove_file", + new_callable=AsyncMock, +) +@patch( + "endpoints.roms.fs_rom_handler.validate_path", +) +def test_delete_roms_from_fs_flat( + mock_validate_path, + mock_remove_file, + mock_remove_directory, + client: TestClient, + access_token: str, + rom: Rom, +): + """Test that flat (non-directory) ROM files are deleted from filesystem.""" + from pathlib import Path + from unittest.mock import MagicMock + + mock_path = MagicMock(spec=Path) + mock_path.is_dir.return_value = False + mock_path.parent.is_dir.return_value = False + mock_validate_path.return_value = mock_path + + response = client.post( + "/api/roms/delete", + headers={"Authorization": f"Bearer {access_token}"}, + json={"roms": [rom.id], "delete_from_fs": [rom.id]}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + assert body["successful_items"] == 1 + assert body["failed_items"] == 0 + mock_remove_file.assert_called_once() + mock_remove_directory.assert_not_called() + + +@patch( + "endpoints.roms.fs_rom_handler.remove_directory", + new_callable=AsyncMock, +) +@patch( + "endpoints.roms.fs_rom_handler.remove_file", + new_callable=AsyncMock, +) +@patch( + "endpoints.roms.fs_rom_handler.validate_path", +) +def test_delete_roms_from_fs_flat_cleans_empty_parent( + mock_validate_path, + mock_remove_file, + mock_remove_directory, + client: TestClient, + access_token: str, + rom: Rom, +): + """Test that empty parent directories are cleaned up after flat ROM file deletion.""" + from pathlib import Path + from unittest.mock import MagicMock + + mock_path = MagicMock(spec=Path) + mock_path.is_dir.return_value = False + # Parent is not the base_path (a MagicMock will not equal a real Path), is a dir, and is empty + mock_path.parent.is_dir.return_value = True + mock_path.parent.__iter__ = lambda self: iter([]) # empty directory + mock_validate_path.return_value = mock_path + + response = client.post( + "/api/roms/delete", + headers={"Authorization": f"Bearer {access_token}"}, + json={"roms": [rom.id], "delete_from_fs": [rom.id]}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + assert body["successful_items"] == 1 + assert body["failed_items"] == 0 + mock_remove_file.assert_called_once() + # remove_directory should be called to clean up the empty parent dir + mock_remove_directory.assert_called_once() + + +@patch( + "endpoints.roms.fs_rom_handler.remove_directory", + new_callable=AsyncMock, +) +@patch( + "endpoints.roms.fs_rom_handler.validate_path", +) +def test_delete_roms_from_fs_nested( + mock_validate_path, + mock_remove_directory, + client: TestClient, + access_token: str, + platform: Platform, +): + """Test that nested (directory) ROMs are deleted using remove_directory.""" + from pathlib import Path + from unittest.mock import MagicMock + + from handler.database import db_rom_handler + from models.rom import Rom + + nested_rom = Rom( + platform_id=platform.id, + name="Nested Game", + slug="nested-game", + fs_name="Nested Game", + fs_name_no_tags="Nested Game", + fs_name_no_ext="Nested Game", + fs_extension="", + fs_path=f"{platform.slug}/roms", + ) + nested_rom = db_rom_handler.add_rom(nested_rom) + + mock_path = MagicMock(spec=Path) + mock_path.is_dir.return_value = True + mock_validate_path.return_value = mock_path + + response = client.post( + "/api/roms/delete", + headers={"Authorization": f"Bearer {access_token}"}, + json={"roms": [nested_rom.id], "delete_from_fs": [nested_rom.id]}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + assert body["successful_items"] == 1 + assert body["failed_items"] == 0 + mock_remove_directory.assert_called_once() + + +def test_update_rom_user_props_flat_payload( client: TestClient, access_token: str, rom: Rom ): response = client.put( f"/api/roms/{rom.id}/props", headers={"Authorization": f"Bearer {access_token}"}, - json={"data": {"backlogged": True, "rating": 7}}, + json={"backlogged": True, "rating": 7}, ) assert response.status_code == status.HTTP_200_OK @@ -137,7 +274,7 @@ def test_update_rom_user_props_partial_update( setup_response = client.put( f"/api/roms/{rom.id}/props", headers={"Authorization": f"Bearer {access_token}"}, - json={"data": {"backlogged": True, "rating": 5, "hidden": True}}, + json={"backlogged": True, "rating": 5, "hidden": True}, ) assert setup_response.status_code == status.HTTP_200_OK @@ -145,7 +282,7 @@ def test_update_rom_user_props_partial_update( response = client.put( f"/api/roms/{rom.id}/props", headers={"Authorization": f"Bearer {access_token}"}, - json={"data": {"rating": 9}}, + json={"rating": 9}, ) assert response.status_code == status.HTTP_200_OK @@ -159,17 +296,17 @@ def test_update_rom_user_props_last_played_flags( client: TestClient, access_token: str, rom: Rom ): mark_played_response = client.put( - f"/api/roms/{rom.id}/props", + f"/api/roms/{rom.id}/props?update_last_played=true", headers={"Authorization": f"Bearer {access_token}"}, - json={"data": {}, "update_last_played": True}, + json={}, ) assert mark_played_response.status_code == status.HTTP_200_OK assert mark_played_response.json()["last_played"] is not None clear_played_response = client.put( - f"/api/roms/{rom.id}/props", + f"/api/roms/{rom.id}/props?remove_last_played=true", headers={"Authorization": f"Bearer {access_token}"}, - json={"data": {}, "remove_last_played": True}, + json={}, ) assert clear_played_response.status_code == status.HTTP_200_OK assert clear_played_response.json()["last_played"] is None diff --git a/backend/tests/endpoints/sockets/test_scan.py b/backend/tests/endpoints/sockets/test_scan.py index c551548f2..1408568bf 100644 --- a/backend/tests/endpoints/sockets/test_scan.py +++ b/backend/tests/endpoints/sockets/test_scan.py @@ -1,10 +1,8 @@ -from pathlib import Path from unittest.mock import Mock import pytest import socketio -from config import LIBRARY_BASE_PATH from endpoints.sockets.scan import ScanStats, _should_scan_rom from handler.filesystem.roms_handler import FSRomsHandler from handler.metadata.base_handler import UniversalPlatformSlug as UPS @@ -263,7 +261,7 @@ class TestGetPico8CoverUrl: fs_name="mygame.p8.png", fs_path="pico/roms", ) - expected = f"file://{Path(LIBRARY_BASE_PATH).resolve() / 'pico/roms' / 'mygame.p8.png'}" + expected = "file://pico/roms/mygame.p8.png" assert url == expected def test_returns_none_for_non_pico8_platform(self, handler: FSRomsHandler): diff --git a/backend/tests/endpoints/test_config.py b/backend/tests/endpoints/test_config.py index b878cdbc3..23e0a9aff 100644 --- a/backend/tests/endpoints/test_config.py +++ b/backend/tests/endpoints/test_config.py @@ -15,16 +15,16 @@ def test_config(client): assert response.status_code == status.HTTP_200_OK config = response.json() - assert config.get("EXCLUDED_PLATFORMS") == DEFAULT_EXCLUDED_DIRS - assert config.get("EXCLUDED_SINGLE_EXT") == [ + assert config.get("EXCLUDED_PLATFORMS") == sorted(DEFAULT_EXCLUDED_DIRS) + assert config.get("EXCLUDED_SINGLE_EXT") == sorted( e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert config.get("EXCLUDED_SINGLE_FILES") == DEFAULT_EXCLUDED_FILES - assert config.get("EXCLUDED_MULTI_FILES") == DEFAULT_EXCLUDED_DIRS - assert config.get("EXCLUDED_MULTI_PARTS_EXT") == [ + ) + assert config.get("EXCLUDED_SINGLE_FILES") == sorted(DEFAULT_EXCLUDED_FILES) + assert config.get("EXCLUDED_MULTI_FILES") == sorted(DEFAULT_EXCLUDED_DIRS) + assert config.get("EXCLUDED_MULTI_PARTS_EXT") == sorted( e.lower() for e in DEFAULT_EXCLUDED_EXTENSIONS - ] - assert config.get("EXCLUDED_MULTI_PARTS_FILES") == DEFAULT_EXCLUDED_FILES + ) + assert config.get("EXCLUDED_MULTI_PARTS_FILES") == sorted(DEFAULT_EXCLUDED_FILES) assert config.get("PLATFORMS_BINDING") == {} assert not config.get("SKIP_HASH_CALCULATION") diff --git a/backend/tests/endpoints/test_tasks.py b/backend/tests/endpoints/test_tasks.py index d370b885a..cc53ce1c6 100644 --- a/backend/tests/endpoints/test_tasks.py +++ b/backend/tests/endpoints/test_tasks.py @@ -2,6 +2,7 @@ from unittest.mock import Mock, patch import pytest from fastapi import status +from rq.exceptions import NoSuchJobError from tasks.tasks import Task, TaskType @@ -322,6 +323,53 @@ class TestRunSingleTask: assert response.status_code == status.HTTP_403_FORBIDDEN +class TestGetTasksStatus: + """Test suite for the get_tasks_status endpoint""" + + @patch("endpoints.tasks.Worker.all", return_value=[]) + @patch("endpoints.tasks.low_prio_queue") + @patch("endpoints.tasks.default_queue") + @patch("endpoints.tasks.high_prio_queue") + @patch("endpoints.tasks.Job.fetch") + def test_get_tasks_status_skips_expired_jobs( + self, + mock_job_fetch, + mock_high_queue, + mock_default_queue, + mock_low_queue, + mock_worker_all, + client, + access_token, + ): + """Test that get_tasks_status skips jobs that have expired from Redis""" + mock_low_queue.get_jobs.return_value = [] + mock_default_queue.get_jobs.return_value = [] + mock_high_queue.get_jobs.return_value = [] + + mock_finished_registry = Mock() + mock_finished_registry.get_job_ids.return_value = ["expired-job-id"] + mock_failed_registry = Mock() + mock_failed_registry.get_job_ids.return_value = [] + + mock_job_fetch.side_effect = NoSuchJobError( + "No such job: rq:job:expired-job-id" + ) + + with patch( + "endpoints.tasks.FinishedJobRegistry", return_value=mock_finished_registry + ): + with patch( + "endpoints.tasks.FailedJobRegistry", return_value=mock_failed_registry + ): + response = client.get( + "/api/tasks/status", + headers={"Authorization": f"Bearer {access_token}"}, + ) + + assert response.status_code == status.HTTP_200_OK + assert response.json() == [] + + class TestGetTaskById: """Test suite for the get_task_by_id endpoint""" diff --git a/backend/tests/handler/test_db_handler.py b/backend/tests/handler/test_db_handler.py index 40afa6cce..fbeb6e544 100644 --- a/backend/tests/handler/test_db_handler.py +++ b/backend/tests/handler/test_db_handler.py @@ -225,11 +225,9 @@ def test_sibling_roms_empty_fs_name_no_tags_not_matched(platform: Platform): assert rom1.id not in sibling_ids2 -def test_sibling_roms_nonempty_fs_name_no_tags_matched(platform: Platform): - """ROMs with matching non-empty fs_name_no_tags SHOULD be matched as siblings. - - For example, "Sonic Jam (USA).iso" and "Sonic Jam (Japan).iso" both have - fs_name_no_tags = "Sonic Jam" and should be considered siblings. +def test_sibling_roms_fs_name_no_tags_not_matched(platform: Platform): + """ROMs with matching fs_name_no_tags but no shared metadata ID should NOT + be matched as siblings. Sibling matching is based only on metadata IDs. """ rom1 = db_rom_handler.add_rom( Rom( @@ -261,19 +259,16 @@ def test_sibling_roms_nonempty_fs_name_no_tags_matched(platform: Platform): assert loaded_rom1 is not None assert loaded_rom2 is not None - # ROMs with same non-empty fs_name_no_tags should be siblings + # ROMs with same fs_name_no_tags but no metadata IDs should NOT be siblings sibling_ids1 = {s.id for s in loaded_rom1.sibling_roms} sibling_ids2 = {s.id for s in loaded_rom2.sibling_roms} - assert rom2.id in sibling_ids1 - assert rom1.id in sibling_ids2 + assert rom2.id not in sibling_ids1 + assert rom1.id not in sibling_ids2 def test_group_by_meta_id_with_empty_fs_name_no_tags(platform: Platform): - """ROMs with empty fs_name_no_tags should each get their own group when using + """ROMs with no metadata IDs should each get their own group when using group_by_meta_id, not be grouped into a single catch-all group. - - Without the fix, all unmatched ROMs with empty fs_name_no_tags would be - grouped under "fs--" and only 1 would be shown. """ rom_names = ["(Japan) Game A", "(Japan) Game B", "(Japan) Game C"] for name in rom_names: diff --git a/backend/tests/utils/test_context.py b/backend/tests/utils/test_context.py new file mode 100644 index 000000000..c1e55cee3 --- /dev/null +++ b/backend/tests/utils/test_context.py @@ -0,0 +1,46 @@ +from config import has_proxy_env +from utils.context import ( + create_aiohttp_session, + create_httpx_async_client, + create_httpx_client, +) + + +class TestProxyAwareHttpClients: + def test_has_proxy_env_detects_uppercase_proxy_vars(self, monkeypatch): + monkeypatch.setenv("HTTP_PROXY", "http://proxy.internal:8080") + + assert has_proxy_env() is True + + def test_has_proxy_env_returns_false_without_proxy_vars(self, monkeypatch): + for var in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + ): + monkeypatch.delenv(var, raising=False) + + assert has_proxy_env() is False + + async def test_create_aiohttp_session_uses_config_proxy_env(self, monkeypatch): + monkeypatch.setattr("utils.context.has_proxy_env", lambda: True) + + session = create_aiohttp_session() + + try: + assert session.trust_env is True + finally: + await session.close() + + async def test_create_httpx_clients_use_config_proxy_env(self, monkeypatch): + monkeypatch.setattr("utils.context.has_proxy_env", lambda: True) + + async_client = create_httpx_async_client() + client = create_httpx_client() + + try: + assert async_client._trust_env is True + assert client._trust_env is True + finally: + await async_client.aclose() + client.close() diff --git a/backend/tests/utils/test_validation.py b/backend/tests/utils/test_validation.py index 2e7a54802..b55720579 100644 --- a/backend/tests/utils/test_validation.py +++ b/backend/tests/utils/test_validation.py @@ -258,22 +258,6 @@ class TestValidateUrlForHttpRequest: in exc_info.value.message ) - def test_invalid_cloud_metadata_service_ips(self): - """Test that cloud metadata service IPs fail validation.""" - # AWS/Azure metadata service: 169.254.169.254 - with pytest.raises(ValidationError) as exc_info: - validate_url_for_http_request("http://169.254.169.254", "test_url") - assert ( - "cloud metadata service addresses are not allowed" in exc_info.value.message - ) - - # Link-local addresses (169.254.0.0/16) - with pytest.raises(ValidationError) as exc_info: - validate_url_for_http_request("http://169.254.1.1", "test_url") - assert ( - "cloud metadata service addresses are not allowed" in exc_info.value.message - ) - def test_invalid_loopback_addresses(self): """Test that loopback addresses fail validation.""" # 127.x.x.x range @@ -342,6 +326,56 @@ class TestValidateUrlForHttpRequest: validate_url_for_http_request("http://server.localhost", "test_url") assert "internal domain names are not allowed" in exc_info.value.message + def test_invalid_non_standard_ip_representations(self): + """Test that non-standard IP representations are blocked (SSRF bypass vectors).""" + # Hexadecimal integer for 127.0.0.1 + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://0x7f000001", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + + # Decimal integer for 127.0.0.1 + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://2130706433", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + + # Shorthand dotted for 127.0.0.1 + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://127.1", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + + # Hexadecimal integer for 10.0.0.1 + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://0x0a000001", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + + # Decimal integer for 192.168.1.1 + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://3232235777", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + + # Hexadecimal integer for 169.254.169.254 (cloud metadata) + with pytest.raises(ValidationError) as exc_info: + validate_url_for_http_request("http://0xa9fea9fe", "test_url") + assert ( + "private, internal, and reserved IP addresses are not allowed" + in exc_info.value.message + ) + def test_invalid_missing_hostname(self): """Test that URLs without hostnames fail validation.""" with pytest.raises(ValidationError) as exc_info: diff --git a/backend/utils/context.py b/backend/utils/context.py index 5d0f2bb21..4f9188aaf 100644 --- a/backend/utils/context.py +++ b/backend/utils/context.py @@ -7,12 +7,26 @@ import aiohttp import httpx from fastapi import Request, Response +from config import has_proxy_env + _T = TypeVar("_T") ctx_aiohttp_session: ContextVar[aiohttp.ClientSession] = ContextVar("aiohttp_session") ctx_httpx_client: ContextVar[httpx.AsyncClient] = ContextVar("httpx_client") +def create_aiohttp_session() -> aiohttp.ClientSession: + return aiohttp.ClientSession(trust_env=has_proxy_env()) + + +def create_httpx_async_client() -> httpx.AsyncClient: + return httpx.AsyncClient(trust_env=has_proxy_env()) + + +def create_httpx_client() -> httpx.Client: + return httpx.Client(trust_env=has_proxy_env()) + + @asynccontextmanager async def set_context_var(var: ContextVar[_T], value: _T) -> AsyncGenerator[Token[_T]]: """Temporarily set a context variables.""" @@ -25,8 +39,8 @@ async def set_context_var(var: ContextVar[_T], value: _T) -> AsyncGenerator[Toke async def initialize_context() -> AsyncGenerator[None]: """Initialize context variables.""" async with ( - aiohttp.ClientSession() as aiohttp_session, - httpx.AsyncClient() as httpx_client, + create_aiohttp_session() as aiohttp_session, + create_httpx_async_client() as httpx_client, set_context_var(ctx_aiohttp_session, aiohttp_session), set_context_var(ctx_httpx_client, httpx_client), ): diff --git a/backend/utils/nginx.py b/backend/utils/nginx.py index c26c77561..daf64832e 100644 --- a/backend/utils/nginx.py +++ b/backend/utils/nginx.py @@ -45,6 +45,7 @@ class ZipResponse(Response): { "Content-Disposition": f"attachment; filename*=UTF-8''{filename}; filename=\"{filename}\"", "X-Archive-Files": "zip", + "X-Archive-Charset": "UTF-8", } ) diff --git a/backend/utils/update_hltb_api_url.py b/backend/utils/update_hltb_api_url.py index 20b030657..90d9a344c 100644 --- a/backend/utils/update_hltb_api_url.py +++ b/backend/utils/update_hltb_api_url.py @@ -9,6 +9,8 @@ from pathlib import Path import httpx +from utils.context import create_httpx_client + # Precompiled regexes for better performance APP_JS_REGEX = re.compile( r'src=["\'](?P\/_next\/static\/chunks\/pages\/_app[^"\']+\.js)["\']' @@ -22,7 +24,7 @@ ENDPOINT_TOKEN_REGEX = re.compile( def fetch_hltb_app_script(base_url: str = "https://howlongtobeat.com") -> str | None: """Fetch the HLTB app script from the site.""" try: - with httpx.Client() as client: + with create_httpx_client() as client: # 1) Fetch homepage HTML homepage_url = f"{base_url}/" resp = client.get(homepage_url, timeout=15) diff --git a/backend/utils/validation.py b/backend/utils/validation.py index 1d55cb38d..27ddff568 100644 --- a/backend/utils/validation.py +++ b/backend/utils/validation.py @@ -1,5 +1,6 @@ import ipaddress import re +import socket from urllib.parse import urlparse from logger.logger import log @@ -183,14 +184,7 @@ def validate_url_for_http_request(url: str, field_name: str = "URL") -> None: try: ip = ipaddress.ip_address(hostname) - # Block cloud metadata service IPs (AWS, GCP, Azure, etc.) - # AWS/Azure metadata service: 169.254.169.254 - if isinstance(ip, ipaddress.IPv4Address) and str(ip).startswith("169.254."): - msg = f"Invalid {field_name}: cloud metadata service addresses are not allowed" - log.error(f"SSRF prevention: {msg} - IP '{ip}'") - raise ValidationError(msg, field_name) - - # Block private/internal IP addresses + # Block private/internal/link-local IP addresses if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: msg = f"Invalid {field_name}: private, internal, and reserved IP addresses are not allowed" log.error(f"SSRF prevention: {msg} - IP '{ip}'") @@ -203,7 +197,27 @@ def validate_url_for_http_request(url: str, field_name: str = "URL") -> None: raise ValidationError(msg, field_name) except ValueError as e: - # Not a direct IP address, which is fine - it's a domain name + # ipaddress.ip_address() only handles standard notation. HTTP clients + # also accept hex integers (0x7f000001), decimal integers (2130706433), + # shorthand dotted (127.1), and octal (0177.0.0.1). Use socket.inet_aton() + # which handles these non-standard IPv4 representations. + try: + packed = socket.inet_aton(hostname) + ip = ipaddress.IPv4Address(packed) + + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: + msg = f"Invalid {field_name}: private, internal, and reserved IP addresses are not allowed" + log.error(f"SSRF prevention: {msg} - IP '{ip}'") + raise ValidationError(msg, field_name) + + if ip.is_multicast: + msg = f"Invalid {field_name}: multicast addresses are not allowed" + log.error(f"SSRF prevention: {msg} - IP '{ip}'") + raise ValidationError(msg, field_name) + + except OSError: + pass # Not an IP address at all - fall through to domain name checks + # Additional checks for suspicious domain patterns hostname_lower = hostname.lower() diff --git a/docker/Dockerfile b/docker/Dockerfile index e7481454e..f22a524e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,8 +21,8 @@ ARG NODE_VERSION=24.13 ARG NODE_ALPINE_SHA256=4f696fbf39f383c1e486030ba6b289a5d9af541642fc78ab197e584a113b9c03 ARG NGINX_VERSION=1.29.5 ARG NGINX_SHA256=1d13701a5f9f3fb01aaa88cef2344d65b6b5bf6b7d9fa4cf0dca557a8d7702ba -ARG UV_VERSION=0.8.24 -ARG UV_SHA256=779f3d612539b4696a1b228724cd79b6e8b8604075a9ac7d15378bccf4053373 +ARG UV_VERSION=0.11.2 +ARG UV_SHA256=db7642df9c7e6214d4d7df81cfc3e8327768dd15565b1eb414bb83004f64d463 FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}@sha256:${PYTHON_ALPINE_SHA256} AS python-alias diff --git a/frontend/.npmrc b/frontend/.npmrc new file mode 100644 index 000000000..a67211708 --- /dev/null +++ b/frontend/.npmrc @@ -0,0 +1,2 @@ +min-release-age=7 +ignore-scripts=true diff --git a/frontend/assets/default/af.gif b/frontend/assets/default/af.gif new file mode 100644 index 000000000..933a2a8f4 Binary files /dev/null and b/frontend/assets/default/af.gif differ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 94cf78bf7..83bd3f786 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -15,7 +15,7 @@ "cronstrue": "^2.57.0", "date-fns": "^4.1.0", "js-cookie": "^3.0.5", - "lodash": "^4.17.23", + "lodash": "^4.18.1", "md-editor-v3": "^5.8.4", "mitt": "^3.0.1", "pinia": "^3.0.1", @@ -108,7 +108,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -3627,6 +3626,70 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.8.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.8.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1", + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.1", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.1.tgz", @@ -3825,7 +3888,6 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -4205,9 +4267,9 @@ "license": "MIT" }, "node_modules/@vue/language-core/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -4357,7 +4419,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "devOptional": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4438,9 +4499,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -4670,9 +4731,9 @@ "license": "ISC" }, "node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4715,7 +4776,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -5520,7 +5580,6 @@ "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -5810,9 +5869,9 @@ "license": "MIT" }, "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -5877,9 +5936,9 @@ } }, "node_modules/flatted": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.4.tgz", - "integrity": "sha512-3+mMldrTAPdta5kjX2G2J7iX4zxtnwpdA8Tr2ZSjkyPSanvbZAcy6flmtnXbEybHrDcU9641lxrMfFuUxVz9vA==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -6156,7 +6215,6 @@ "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -6201,9 +6259,9 @@ "license": "ISC" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -7310,9 +7368,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash.debounce": { @@ -7361,7 +7419,6 @@ "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", "license": "MIT", - "peer": true, "dependencies": { "argparse": "^2.0.1", "entities": "^4.4.0", @@ -7810,9 +7867,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "devOptional": true, "license": "MIT", "engines": { @@ -7993,9 +8050,9 @@ } }, "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -8165,7 +8222,6 @@ "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -8517,9 +8573,9 @@ } }, "node_modules/socket.io-parser": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.5.tgz", - "integrity": "sha512-bPMmpy/5WWKHea5Y/jYAP6k74A+hvmRCQaJuJB6I/ML5JZq/KfNieUVo/3Mh7SAqn7TyFdIo6wqYHInG1MU1bQ==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.6.tgz", + "integrity": "sha512-asJqbVBDsBCJx0pTqw3WfesSY0iRX+2xzWEWzrpcH7L6fLzrhyF8WPI8UaeM4YCuDfpwA/cgsdugMsmtz8EJeg==", "license": "MIT", "dependencies": { "@socket.io/component-emitter": "~3.1.0", @@ -9009,7 +9065,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -9226,7 +9281,6 @@ "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", @@ -9374,7 +9428,6 @@ "integrity": "sha512-Q4SC/4TqbNvaZIFb9YsfBqkGlYHbJJJ6uU3CnRBZqLUF3s5eCMVZAaV4GkTbehIH/bhSj42lMXztOwc71u6rVw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@vuetify/loader-shared": "^2.1.2", "debug": "^4.3.3", @@ -9401,7 +9454,6 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.29.tgz", "integrity": "sha512-BZqN4Ze6mDQVNAni0IHeMJ5mwr8VAJ3MQC9FmprRhcBYENw+wOAAjRj8jfmN6FLl0j96OXbR+CjWhmAmM+QGnA==", "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.29", "@vue/compiler-sfc": "3.5.29", @@ -9424,7 +9476,6 @@ "integrity": "sha512-Vxi9pJdbN3ZnVGLODVtZ7y4Y2kzAAE2Cm0CZ3ZDRvydVYxZ6VrnBhLikBsRS+dpwj4Jv4UCv21PTEwF5rQ9WXg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "debug": "^4.4.0", "eslint-scope": "^8.2.0 || ^9.0.0", @@ -9522,7 +9573,6 @@ "resolved": "https://registry.npmjs.org/vuetify/-/vuetify-3.12.2.tgz", "integrity": "sha512-cVQa4+5iQpDs00ToMUnWRHlMdv1d5tEH2wcZIthqSCmBipQAG4rQKE55zFwZFYlPyiDhUVY1RcAFtXCuHNcCww==", "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/johnleider" @@ -9855,7 +9905,6 @@ "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -9908,9 +9957,9 @@ } }, "node_modules/workbox-build/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -9939,7 +9988,6 @@ "integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "rollup": "dist/bin/rollup" }, diff --git a/frontend/package.json b/frontend/package.json index a77d4e0d2..1750aa113 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -34,7 +34,7 @@ "cronstrue": "^2.57.0", "date-fns": "^4.1.0", "js-cookie": "^3.0.5", - "lodash": "^4.17.23", + "lodash": "^4.18.1", "md-editor-v3": "^5.8.4", "mitt": "^3.0.1", "pinia": "^3.0.1", diff --git a/frontend/public/assets/patcherjs/patcher.worker.js b/frontend/public/assets/patcherjs/patcher.worker.js index cd5b1277d..52a373c9e 100644 --- a/frontend/public/assets/patcherjs/patcher.worker.js +++ b/frontend/public/assets/patcherjs/patcher.worker.js @@ -40,7 +40,9 @@ async function loadScripts() { scriptsLoaded = true; return true; } catch (error) { - throw new Error(`Failed to load patcher scripts: ${error.message}`); + throw new Error(`Failed to load patcher scripts: ${error.message}`, { + cause: error, + }); } } diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index d4016be2d..e223b55db 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -31,7 +31,6 @@ export type { Body_update_save_api_saves__id__put } from './models/Body_update_s export type { Body_update_smart_collection_api_collections_smart__id__put } from './models/Body_update_smart_collection_api_collections_smart__id__put'; export type { Body_update_state_api_states__id__put } from './models/Body_update_state_api_states__id__put'; export type { BulkOperationResponse } from './models/BulkOperationResponse'; -export type { CleanupStats } from './models/CleanupStats'; export type { CleanupTaskMeta } from './models/CleanupTaskMeta'; export type { CleanupTaskStatusResponse } from './models/CleanupTaskStatusResponse'; export type { ClientSaveState } from './models/ClientSaveState'; @@ -75,10 +74,12 @@ export type { LaunchboxImage } from './models/LaunchboxImage'; export type { ManualMetadata } from './models/ManualMetadata'; export type { MetadataCoverageItem } from './models/MetadataCoverageItem'; export type { MetadataSourcesDict } from './models/MetadataSourcesDict'; +export type { MissingRomsCleanupStats } from './models/MissingRomsCleanupStats'; export type { MobyMetadataPlatform } from './models/MobyMetadataPlatform'; export type { NetplayICEServer } from './models/NetplayICEServer'; export type { OIDCDict } from './models/OIDCDict'; export type { OIDCLogoutResponse } from './models/OIDCLogoutResponse'; +export type { OrphanedResourcesCleanupStats } from './models/OrphanedResourcesCleanupStats'; export type { PlatformBindingPayload } from './models/PlatformBindingPayload'; export type { PlatformSchema } from './models/PlatformSchema'; export type { RAGameRomAchievement } from './models/RAGameRomAchievement'; @@ -102,7 +103,6 @@ export type { RomSSMetadata } from './models/RomSSMetadata'; export type { RomUserData } from './models/RomUserData'; export type { RomUserSchema } from './models/RomUserSchema'; export type { RomUserStatus } from './models/RomUserStatus'; -export type { RomUserUpdatePayload } from './models/RomUserUpdatePayload'; export type { RoomsResponse } from './models/RoomsResponse'; export type { SaveSchema } from './models/SaveSchema'; export type { SaveSummarySchema } from './models/SaveSummarySchema'; diff --git a/frontend/src/__generated__/models/CleanupTaskMeta.ts b/frontend/src/__generated__/models/CleanupTaskMeta.ts index 4bf9cd4ce..ad55db405 100644 --- a/frontend/src/__generated__/models/CleanupTaskMeta.ts +++ b/frontend/src/__generated__/models/CleanupTaskMeta.ts @@ -2,8 +2,9 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -import type { CleanupStats } from './CleanupStats'; +import type { MissingRomsCleanupStats } from './MissingRomsCleanupStats'; +import type { OrphanedResourcesCleanupStats } from './OrphanedResourcesCleanupStats'; export type CleanupTaskMeta = { - cleanup_stats: (CleanupStats | null); + cleanup_stats: (OrphanedResourcesCleanupStats | MissingRomsCleanupStats | null); }; diff --git a/frontend/src/__generated__/models/ConfigResponse.ts b/frontend/src/__generated__/models/ConfigResponse.ts index 5b9cf4681..b226710eb 100644 --- a/frontend/src/__generated__/models/ConfigResponse.ts +++ b/frontend/src/__generated__/models/ConfigResponse.ts @@ -13,6 +13,9 @@ export type ConfigResponse = { EXCLUDED_MULTI_FILES: Array; EXCLUDED_MULTI_PARTS_EXT: Array; EXCLUDED_MULTI_PARTS_FILES: Array; + DEFAULT_EXCLUDED_DIRS: Array; + DEFAULT_EXCLUDED_FILES: Array; + DEFAULT_EXCLUDED_EXTENSIONS: Array; PLATFORMS_BINDING: Record; PLATFORMS_VERSIONS: Record; SKIP_HASH_CALCULATION: boolean; diff --git a/frontend/src/__generated__/models/MissingRomsCleanupStats.ts b/frontend/src/__generated__/models/MissingRomsCleanupStats.ts new file mode 100644 index 000000000..ced1b85cb --- /dev/null +++ b/frontend/src/__generated__/models/MissingRomsCleanupStats.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type MissingRomsCleanupStats = { + platform_id: (number | null); + roms_found: number; + roms_deleted: number; + errors: number; +}; + diff --git a/frontend/src/__generated__/models/CleanupStats.ts b/frontend/src/__generated__/models/OrphanedResourcesCleanupStats.ts similarity index 86% rename from frontend/src/__generated__/models/CleanupStats.ts rename to frontend/src/__generated__/models/OrphanedResourcesCleanupStats.ts index 92d106132..942d249de 100644 --- a/frontend/src/__generated__/models/CleanupStats.ts +++ b/frontend/src/__generated__/models/OrphanedResourcesCleanupStats.ts @@ -2,7 +2,7 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ -export type CleanupStats = { +export type OrphanedResourcesCleanupStats = { platforms_in_db: number; roms_in_db: number; platforms_in_fs: number; diff --git a/frontend/src/__generated__/models/RomUserUpdatePayload.ts b/frontend/src/__generated__/models/RomUserUpdatePayload.ts deleted file mode 100644 index 6536a26d6..000000000 --- a/frontend/src/__generated__/models/RomUserUpdatePayload.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* generated using openapi-typescript-codegen -- do not edit */ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ -import type { RomUserData } from './RomUserData'; -export type RomUserUpdatePayload = { - /** - * Partial rom user data to update. Only provided fields will be updated. - */ - data?: RomUserData; - /** - * Set last played timestamp to now. - */ - update_last_played?: boolean; - /** - * Clear the last played timestamp. - */ - remove_last_played?: boolean; -}; - diff --git a/frontend/src/__generated__/models/StatsReturn.ts b/frontend/src/__generated__/models/StatsReturn.ts index 497561f76..97626b796 100644 --- a/frontend/src/__generated__/models/StatsReturn.ts +++ b/frontend/src/__generated__/models/StatsReturn.ts @@ -5,13 +5,13 @@ import type { MetadataCoverageItem } from './MetadataCoverageItem'; import type { RegionBreakdownItem } from './RegionBreakdownItem'; export type StatsReturn = { - PLATFORMS: number; - ROMS: number; - SAVES: number; - STATES: number; - SCREENSHOTS: number; - TOTAL_FILESIZE_BYTES: number; - METADATA_COVERAGE: Record>; - REGION_BREAKDOWN: Record>; + PLATFORMS?: number; + ROMS?: number; + SAVES?: number; + STATES?: number; + SCREENSHOTS?: number; + TOTAL_FILESIZE_BYTES?: number; + METADATA_COVERAGE?: Record>; + REGION_BREAKDOWN?: Record>; }; diff --git a/frontend/src/components/Settings/Administration/tasks/CleanupTaskProgress.vue b/frontend/src/components/Settings/Administration/tasks/CleanupTaskProgress.vue index 74e0ae694..c172bd6e1 100644 --- a/frontend/src/components/Settings/Administration/tasks/CleanupTaskProgress.vue +++ b/frontend/src/components/Settings/Administration/tasks/CleanupTaskProgress.vue @@ -1,13 +1,16 @@