feat: Add incremental sync for RetroAchievements progression

This change makes the RetroAchievements progression sync endpoint to
optionally perform an incremental sync (when `incremental` is true),
by only fetching new achievements since the last sync.

This reduces the amount of data fetched and speeds up the sync
process for users who frequently sync their progression. It unblocks the
implementation of automatic periodic syncs in the future.

Frontend behavior:
- When the `Apply` button is clicked in the RetroAchievements settings,
  a full sync is performed (same as before). This is because a change to
  the RA username may have occurred.
- When the `Sync` button is clicked, an incremental sync is performed.

backend/endpoints/user.py

@@ -1,14 +1,17 @@
-from typing import Annotated, Any
+from typing import Annotated, Any, cast
 
 from decorators.auth import protected_route
 from endpoints.forms.identity import UserForm
 from endpoints.responses.identity import InviteLinkSchema, UserSchema
-from fastapi import Body, Form, HTTPException, Request, status
+from fastapi import Body, Form, HTTPException
+from fastapi import Path as PathVar
+from fastapi import Request, status
 from handler.auth import auth_handler
 from handler.auth.constants import Scope
 from handler.database import db_user_handler
 from handler.filesystem import fs_asset_handler
 from handler.metadata import meta_ra_handler
+from handler.metadata.ra_handler import RAUserProgression
 from logger.logger import log
 from models.user import Role, User
 from utils.router import APIRouter
@@ -371,33 +374,42 @@ async def delete_user(request: Request, id: int) -> None:
 
 
 @protected_route(
-    router.post, "/{id}/ra/refresh", [Scope.ME_WRITE], status_code=status.HTTP_200_OK
+    router.post,
+    "/{id}/ra/refresh",
+    [Scope.ME_WRITE],
+    status_code=status.HTTP_200_OK,
+    summary="Refresh RetroAchievements",
+    responses={status.HTTP_404_NOT_FOUND: {}},
 )
-async def refresh_retro_achievements(request: Request, id: int) -> None:
-    """Refresh RetroAchievements data for a user.
-
-    Args:
-        request (Request): FastAPI Request object
-        id (int): User ID
-
-    Raises:
-        HTTPException: User not found or no RetroAchievements username set
-
-    Returns:
-        None: Returns 200 OK status
-    """
+async def refresh_retro_achievements(
+    request: Request,
+    id: Annotated[int, PathVar(description="User internal id.", ge=1)],
+    incremental: Annotated[
+        bool,
+        Body(
+            description="Whether to only retrieve RetroAchievements progression incrementally.",
+            embed=True,
+        ),
+    ] = False,
+) -> None:
+    """Refresh RetroAchievements progression data for a user."""
     user = db_user_handler.get_user(id)
-    if user and user.ra_username:
-        user_progression = await meta_ra_handler.get_user_progression(user.ra_username)
-        db_user_handler.update_user(
-            id,
-            {
-                "ra_progression": user_progression,
-            },
+    if not user or not user.ra_username:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="User does not have a RetroAchievements username set",
         )
-        return None
 
-    raise HTTPException(
-        status_code=status.HTTP_404_NOT_FOUND,
-        detail="User does not have a RetroAchievements username set",
+    user_progression = await meta_ra_handler.get_user_progression(
+        user.ra_username,
+        current_progression=(
+            cast(RAUserProgression | None, user.ra_progression) if incremental else None
+        ),
     )
+    db_user_handler.update_user(
+        id,
+        {
+            "ra_progression": user_progression,
+        },
+    )
+    return None

backend/handler/metadata/ra_handler.py

@@ -69,6 +69,7 @@ class RAUserGameProgression(TypedDict):
     max_possible: int | None
     num_awarded: int | None
     num_awarded_hardcore: int | None
+    most_recent_awarded_date: NotRequired[str | None]
     earned_achievements: list[EarnedAchievement]
 
 
@@ -263,11 +264,40 @@ class RAHandler(MetadataHandler):
         except KeyError:
             return RAGameRom(ra_id=None)
 
-    async def get_user_progression(self, username: str) -> RAUserProgression:
+    async def get_user_progression(
+        self,
+        username: str,
+        current_progression: RAUserProgression | None = None,
+    ) -> RAUserProgression:
+        """Retrieves the user's RetroAchievements progression.
+
+        If `current_progression` is provided, it will only incrementally update the
+        progression based on new achievements since the last check.
+        """
         game_progressions: list[RAUserGameProgression] = []
+        current_progression_by_game_id: dict[int | None, RAUserGameProgression] = {}
+        if current_progression:
+            current_progression_by_game_id = {
+                p["rom_ra_id"]: p for p in current_progression.get("results", [])
+            }
 
         async for rom in self.ra_service.iter_user_completion_progress(username):
             rom_game_id = rom.get("GameID")
+
+            # If we have current progression data, and number of awarded achievements and most
+            # recent awarded date match, then we can skip fetching progression details.
+            game_current_progression = current_progression_by_game_id.get(rom_game_id)
+            if (
+                game_current_progression
+                and rom["NumAwarded"] == game_current_progression.get("num_awarded")
+                and rom["NumAwardedHardcore"]
+                == game_current_progression.get("num_awarded_hardcore")
+                and rom["MostRecentAwardedDate"]
+                == game_current_progression.get("most_recent_awarded_date")
+            ):
+                game_progressions.append(game_current_progression)
+                continue
+
             earned_achievements: list[EarnedAchievement] = []
             if rom_game_id:
                 result = await self.ra_service.get_user_game_progress(
@@ -293,6 +323,7 @@ class RAHandler(MetadataHandler):
                     max_possible=rom.get("MaxPossible", None),
                     num_awarded=rom.get("NumAwarded", None),
                     num_awarded_hardcore=rom.get("NumAwardedHardcore", None),
+                    most_recent_awarded_date=rom.get("MostRecentAwardedDate", None),
                     earned_achievements=earned_achievements,
                 )
             )

frontend/src/__generated__/models/RAUserGameProgression.ts

@@ -8,6 +8,7 @@ export type RAUserGameProgression = {
     max_possible: (number | null);
     num_awarded: (number | null);
     num_awarded_hardcore: (number | null);
+    most_recent_awarded_date?: (string | null);
     earned_achievements: Array<EarnedAchievement>;
 };
 

frontend/src/components/Settings/UserProfile/RetroAchievements.vue

@@ -21,7 +21,7 @@ const rules = [
   },
 ];
 
-async function refreshRetroAchievements() {
+async function refreshRetroAchievements(incremental = false) {
   if (!auth.user) return;
 
   syncing.value = true;
@@ -29,6 +29,7 @@ async function refreshRetroAchievements() {
   await userApi
     .refreshRetroAchievements({
       id: auth.user.id,
+      incremental,
     })
     .then(() => {
       emitter?.emit("snackbarShow", {
@@ -113,7 +114,7 @@ watch(
           :disabled="syncing"
           :loading="syncing"
           class="ml-4 text-accent bg-toplayer"
-          @click="refreshRetroAchievements"
+          @click="refreshRetroAchievements(true)"
         >
           <template #loader>
             <v-progress-circular

frontend/src/services/api/user.ts

@@ -80,8 +80,14 @@ async function deleteUser(user: User) {
   return api.delete(`/users/${user.id}`);
 }
 
-async function refreshRetroAchievements({ id }: { id: number }) {
-  return api.post(`/users/${id}/ra/refresh`);
+async function refreshRetroAchievements({
+  id,
+  incremental = false,
+}: {
+  id: number;
+  incremental?: boolean;
+}) {
+  return api.post(`/users/${id}/ra/refresh`, { incremental });
 }
 
 export default {