From 9cede7680fcf3ec74d8b626e9159e3899fe79428 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Wed, 27 Aug 2025 10:41:22 -0300 Subject: [PATCH] feat: Add incremental sync for RetroAchievements progression This change makes the RetroAchievements progression sync endpoint to optionally perform an incremental sync (when `incremental` is true), by only fetching new achievements since the last sync. This reduces the amount of data fetched and speeds up the sync process for users who frequently sync their progression. It unblocks the implementation of automatic periodic syncs in the future. Frontend behavior: - When the `Apply` button is clicked in the RetroAchievements settings, a full sync is performed (same as before). This is because a change to the RA username may have occurred. - When the `Sync` button is clicked, an incremental sync is performed. --- backend/endpoints/user.py | 66 +++++++++++-------- backend/handler/metadata/ra_handler.py | 33 +++++++++- .../models/RAUserGameProgression.ts | 1 + .../UserProfile/RetroAchievements.vue | 5 +- frontend/src/services/api/user.ts | 10 ++- 5 files changed, 83 insertions(+), 32 deletions(-) diff --git a/backend/endpoints/user.py b/backend/endpoints/user.py index 945e37b81..2558f0abd 100644 --- a/backend/endpoints/user.py +++ b/backend/endpoints/user.py @@ -1,14 +1,17 @@ -from typing import Annotated, Any +from typing import Annotated, Any, cast from decorators.auth import protected_route from endpoints.forms.identity import UserForm from endpoints.responses.identity import InviteLinkSchema, UserSchema -from fastapi import Body, Form, HTTPException, Request, status +from fastapi import Body, Form, HTTPException +from fastapi import Path as PathVar +from fastapi import Request, status from handler.auth import auth_handler from handler.auth.constants import Scope from handler.database import db_user_handler from handler.filesystem import fs_asset_handler from handler.metadata import meta_ra_handler +from handler.metadata.ra_handler import RAUserProgression from logger.logger import log from models.user import Role, User from utils.router import APIRouter @@ -371,33 +374,42 @@ async def delete_user(request: Request, id: int) -> None: @protected_route( - router.post, "/{id}/ra/refresh", [Scope.ME_WRITE], status_code=status.HTTP_200_OK + router.post, + "/{id}/ra/refresh", + [Scope.ME_WRITE], + status_code=status.HTTP_200_OK, + summary="Refresh RetroAchievements", + responses={status.HTTP_404_NOT_FOUND: {}}, ) -async def refresh_retro_achievements(request: Request, id: int) -> None: - """Refresh RetroAchievements data for a user. - - Args: - request (Request): FastAPI Request object - id (int): User ID - - Raises: - HTTPException: User not found or no RetroAchievements username set - - Returns: - None: Returns 200 OK status - """ +async def refresh_retro_achievements( + request: Request, + id: Annotated[int, PathVar(description="User internal id.", ge=1)], + incremental: Annotated[ + bool, + Body( + description="Whether to only retrieve RetroAchievements progression incrementally.", + embed=True, + ), + ] = False, +) -> None: + """Refresh RetroAchievements progression data for a user.""" user = db_user_handler.get_user(id) - if user and user.ra_username: - user_progression = await meta_ra_handler.get_user_progression(user.ra_username) - db_user_handler.update_user( - id, - { - "ra_progression": user_progression, - }, + if not user or not user.ra_username: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User does not have a RetroAchievements username set", ) - return None - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User does not have a RetroAchievements username set", + user_progression = await meta_ra_handler.get_user_progression( + user.ra_username, + current_progression=( + cast(RAUserProgression | None, user.ra_progression) if incremental else None + ), ) + db_user_handler.update_user( + id, + { + "ra_progression": user_progression, + }, + ) + return None diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index dfe1bea07..1504892e5 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -69,6 +69,7 @@ class RAUserGameProgression(TypedDict): max_possible: int | None num_awarded: int | None num_awarded_hardcore: int | None + most_recent_awarded_date: NotRequired[str | None] earned_achievements: list[EarnedAchievement] @@ -263,11 +264,40 @@ class RAHandler(MetadataHandler): except KeyError: return RAGameRom(ra_id=None) - async def get_user_progression(self, username: str) -> RAUserProgression: + async def get_user_progression( + self, + username: str, + current_progression: RAUserProgression | None = None, + ) -> RAUserProgression: + """Retrieves the user's RetroAchievements progression. + + If `current_progression` is provided, it will only incrementally update the + progression based on new achievements since the last check. + """ game_progressions: list[RAUserGameProgression] = [] + current_progression_by_game_id: dict[int | None, RAUserGameProgression] = {} + if current_progression: + current_progression_by_game_id = { + p["rom_ra_id"]: p for p in current_progression.get("results", []) + } async for rom in self.ra_service.iter_user_completion_progress(username): rom_game_id = rom.get("GameID") + + # If we have current progression data, and number of awarded achievements and most + # recent awarded date match, then we can skip fetching progression details. + game_current_progression = current_progression_by_game_id.get(rom_game_id) + if ( + game_current_progression + and rom["NumAwarded"] == game_current_progression.get("num_awarded") + and rom["NumAwardedHardcore"] + == game_current_progression.get("num_awarded_hardcore") + and rom["MostRecentAwardedDate"] + == game_current_progression.get("most_recent_awarded_date") + ): + game_progressions.append(game_current_progression) + continue + earned_achievements: list[EarnedAchievement] = [] if rom_game_id: result = await self.ra_service.get_user_game_progress( @@ -293,6 +323,7 @@ class RAHandler(MetadataHandler): max_possible=rom.get("MaxPossible", None), num_awarded=rom.get("NumAwarded", None), num_awarded_hardcore=rom.get("NumAwardedHardcore", None), + most_recent_awarded_date=rom.get("MostRecentAwardedDate", None), earned_achievements=earned_achievements, ) ) diff --git a/frontend/src/__generated__/models/RAUserGameProgression.ts b/frontend/src/__generated__/models/RAUserGameProgression.ts index 0d234ea50..fbdb8eafd 100644 --- a/frontend/src/__generated__/models/RAUserGameProgression.ts +++ b/frontend/src/__generated__/models/RAUserGameProgression.ts @@ -8,6 +8,7 @@ export type RAUserGameProgression = { max_possible: (number | null); num_awarded: (number | null); num_awarded_hardcore: (number | null); + most_recent_awarded_date?: (string | null); earned_achievements: Array; }; diff --git a/frontend/src/components/Settings/UserProfile/RetroAchievements.vue b/frontend/src/components/Settings/UserProfile/RetroAchievements.vue index 6647ce2e8..6ca17d509 100644 --- a/frontend/src/components/Settings/UserProfile/RetroAchievements.vue +++ b/frontend/src/components/Settings/UserProfile/RetroAchievements.vue @@ -21,7 +21,7 @@ const rules = [ }, ]; -async function refreshRetroAchievements() { +async function refreshRetroAchievements(incremental = false) { if (!auth.user) return; syncing.value = true; @@ -29,6 +29,7 @@ async function refreshRetroAchievements() { await userApi .refreshRetroAchievements({ id: auth.user.id, + incremental, }) .then(() => { emitter?.emit("snackbarShow", { @@ -113,7 +114,7 @@ watch( :disabled="syncing" :loading="syncing" class="ml-4 text-accent bg-toplayer" - @click="refreshRetroAchievements" + @click="refreshRetroAchievements(true)" >