fix: Iterate through user completion progress in RetroAchievements

Iterate through all pages of user completion progress in the
RetroAchievements service, instead of limiting the data retrieval to the
first 500 results.

backend/adapters/services/retroachievements.py

@@ -1,5 +1,6 @@
 import asyncio
 import http
+from collections.abc import AsyncIterator
 from typing import cast
 
 import aiohttp
@@ -9,6 +10,7 @@ from adapters.services.retroachievements_types import (
     RAGameInfoAndUserProgress,
     RAGameListItem,
     RAUserCompletionProgress,
+    RAUserCompletionProgressResult,
 )
 from aiohttp.client import ClientTimeout
 from config import RETROACHIEVEMENTS_API_KEY
@@ -160,6 +162,31 @@ class RetroAchievementsService:
         response = await self._request(str(url))
         return cast(RAUserCompletionProgress, response)
 
+    async def iter_user_completion_progress(
+        self,
+        username: str,
+    ) -> AsyncIterator[RAUserCompletionProgressResult]:
+        """Iterate through a given user's completion progress, targeted by their username.
+
+        Reference: https://api-docs.retroachievements.org/v1/get-user-completion-progress.html
+        """
+        page_size = 500  # Maximum page size for this endpoint.
+        offset = 0
+
+        while True:
+            response = await self.get_user_completion_progress(
+                username,
+                limit=page_size,
+                offset=offset or None,
+            )
+            results = response["Results"]
+            for result in results:
+                yield result
+
+            offset += len(results)
+            if len(results) < page_size or offset >= response["Total"]:
+                break
+
     async def get_user_game_progress(
         self,
         username: str,

backend/adapters/services/retroachievements_types.py

@@ -1,7 +1,14 @@
 import enum
+from collections.abc import Mapping
 from typing import NotRequired, TypedDict
 
 
+class PaginatedResponse[T: Mapping](TypedDict):
+    Count: int
+    Total: int
+    Results: list[T]
+
+
 # https://github.com/RetroAchievements/RAWeb/blob/master/app/Platform/Enums/AchievementType.php
 class RAGameAchievementType(enum.StrEnum):
     PROGRESSION = "progression"
@@ -88,10 +95,7 @@ class RAUserCompletionProgressResult(TypedDict):
 
 
 # https://api-docs.retroachievements.org/v1/get-user-completion-progress.html#response
-class RAUserCompletionProgress(TypedDict):
-    Count: int
-    Total: int
-    Results: list[RAUserCompletionProgressResult]
+RAUserCompletionProgress = PaginatedResponse[RAUserCompletionProgressResult]
 
 
 # https://api-docs.retroachievements.org/v1/get-game-info-and-user-progress.html#response

backend/handler/metadata/ra_handler.py

@@ -3,7 +3,6 @@ import http
 import json
 import os
 import time
-from collections import defaultdict
 from typing import Final, NotRequired, TypedDict
 
 import httpx
@@ -72,7 +71,6 @@ class RAUserGameProgression(TypedDict):
 
 
 class RAUserProgression(TypedDict):
-    count: int
     total: int
     results: list[RAUserGameProgression]
 
@@ -273,44 +271,38 @@ class RAHandler(MetadataHandler):
             return RAGameRom(ra_id=None)
 
     async def get_user_progression(self, username: str) -> RAUserProgression:
-        user_complete_progression = await self.ra_service.get_user_completion_progress(
-            username=username,
-            limit=500,
-        )
-        roms_with_progression = user_complete_progression.get("Results", [])
-        rom_earned_achievements: dict[int, list[EarnedAchievement]] = defaultdict(list)
-        for rom in roms_with_progression:
+        game_progressions: list[RAUserGameProgression] = []
+
+        async for rom in self.ra_service.iter_user_completion_progress(username):
             rom_game_id = rom.get("GameID")
+            earned_achievements: list[EarnedAchievement] = []
             if rom_game_id:
                 result = await self.ra_service.get_user_game_progress(
                     username=username,
                     game_id=rom_game_id,
                 )
-                for achievement in result.get("Achievements", {}).values():
-                    if achievement.get("DateEarned") and achievement.get("BadgeName"):
-                        rom_earned_achievements[rom_game_id].append(
-                            {
-                                "id": achievement["BadgeName"],
-                                "date": achievement["DateEarned"],
-                            }
-                        )
-        return RAUserProgression(
-            count=user_complete_progression.get("Count", 0),
-            total=user_complete_progression.get("Total", 0),
-            results=[
+                earned_achievements = [
+                    {
+                        "id": achievement["BadgeName"],
+                        "date": achievement["DateEarned"],
+                    }
+                    for achievement in result.get("Achievements", {}).values()
+                    if achievement.get("DateEarned") and achievement.get("BadgeName")
+                ]
+
+            game_progressions.append(
                 RAUserGameProgression(
-                    rom_ra_id=rom.get("GameID", None),
+                    rom_ra_id=rom_game_id,
                     max_possible=rom.get("MaxPossible", None),
                     num_awarded=rom.get("NumAwarded", None),
                     num_awarded_hardcore=rom.get("NumAwardedHardcore", None),
-                    earned_achievements=(
-                        rom_earned_achievements.get(rom["GameID"], [])
-                        if rom.get("GameID")
-                        else []
-                    ),
+                    earned_achievements=earned_achievements,
                 )
-                for rom in roms_with_progression
-            ],
+            )
+
+        return RAUserProgression(
+            total=len(game_progressions),
+            results=game_progressions,
         )