From a44db9767a1672ddd1aa38659b2f2690f879a7d1 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Mon, 9 Jun 2025 19:03:42 -0300 Subject: [PATCH] fix: Iterate through user completion progress in RetroAchievements Iterate through all pages of user completion progress in the RetroAchievements service, instead of limiting the data retrieval to the first 500 results. --- .../adapters/services/retroachievements.py | 27 ++++++++++ .../services/retroachievements_types.py | 12 +++-- backend/handler/metadata/ra_handler.py | 50 ++++++++----------- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/backend/adapters/services/retroachievements.py b/backend/adapters/services/retroachievements.py index 15fbbdda9..d3d8a0566 100644 --- a/backend/adapters/services/retroachievements.py +++ b/backend/adapters/services/retroachievements.py @@ -1,5 +1,6 @@ import asyncio import http +from collections.abc import AsyncIterator from typing import cast import aiohttp @@ -9,6 +10,7 @@ from adapters.services.retroachievements_types import ( RAGameInfoAndUserProgress, RAGameListItem, RAUserCompletionProgress, + RAUserCompletionProgressResult, ) from aiohttp.client import ClientTimeout from config import RETROACHIEVEMENTS_API_KEY @@ -160,6 +162,31 @@ class RetroAchievementsService: response = await self._request(str(url)) return cast(RAUserCompletionProgress, response) + async def iter_user_completion_progress( + self, + username: str, + ) -> AsyncIterator[RAUserCompletionProgressResult]: + """Iterate through a given user's completion progress, targeted by their username. + + Reference: https://api-docs.retroachievements.org/v1/get-user-completion-progress.html + """ + page_size = 500 # Maximum page size for this endpoint. + offset = 0 + + while True: + response = await self.get_user_completion_progress( + username, + limit=page_size, + offset=offset or None, + ) + results = response["Results"] + for result in results: + yield result + + offset += len(results) + if len(results) < page_size or offset >= response["Total"]: + break + async def get_user_game_progress( self, username: str, diff --git a/backend/adapters/services/retroachievements_types.py b/backend/adapters/services/retroachievements_types.py index 0f70d428c..12d30b8b4 100644 --- a/backend/adapters/services/retroachievements_types.py +++ b/backend/adapters/services/retroachievements_types.py @@ -1,7 +1,14 @@ import enum +from collections.abc import Mapping from typing import NotRequired, TypedDict +class PaginatedResponse[T: Mapping](TypedDict): + Count: int + Total: int + Results: list[T] + + # https://github.com/RetroAchievements/RAWeb/blob/master/app/Platform/Enums/AchievementType.php class RAGameAchievementType(enum.StrEnum): PROGRESSION = "progression" @@ -88,10 +95,7 @@ class RAUserCompletionProgressResult(TypedDict): # https://api-docs.retroachievements.org/v1/get-user-completion-progress.html#response -class RAUserCompletionProgress(TypedDict): - Count: int - Total: int - Results: list[RAUserCompletionProgressResult] +RAUserCompletionProgress = PaginatedResponse[RAUserCompletionProgressResult] # https://api-docs.retroachievements.org/v1/get-game-info-and-user-progress.html#response diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index b289824a9..b427e631b 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -3,7 +3,6 @@ import http import json import os import time -from collections import defaultdict from typing import Final, NotRequired, TypedDict import httpx @@ -72,7 +71,6 @@ class RAUserGameProgression(TypedDict): class RAUserProgression(TypedDict): - count: int total: int results: list[RAUserGameProgression] @@ -273,44 +271,38 @@ class RAHandler(MetadataHandler): return RAGameRom(ra_id=None) async def get_user_progression(self, username: str) -> RAUserProgression: - user_complete_progression = await self.ra_service.get_user_completion_progress( - username=username, - limit=500, - ) - roms_with_progression = user_complete_progression.get("Results", []) - rom_earned_achievements: dict[int, list[EarnedAchievement]] = defaultdict(list) - for rom in roms_with_progression: + game_progressions: list[RAUserGameProgression] = [] + + async for rom in self.ra_service.iter_user_completion_progress(username): rom_game_id = rom.get("GameID") + earned_achievements: list[EarnedAchievement] = [] if rom_game_id: result = await self.ra_service.get_user_game_progress( username=username, game_id=rom_game_id, ) - for achievement in result.get("Achievements", {}).values(): - if achievement.get("DateEarned") and achievement.get("BadgeName"): - rom_earned_achievements[rom_game_id].append( - { - "id": achievement["BadgeName"], - "date": achievement["DateEarned"], - } - ) - return RAUserProgression( - count=user_complete_progression.get("Count", 0), - total=user_complete_progression.get("Total", 0), - results=[ + earned_achievements = [ + { + "id": achievement["BadgeName"], + "date": achievement["DateEarned"], + } + for achievement in result.get("Achievements", {}).values() + if achievement.get("DateEarned") and achievement.get("BadgeName") + ] + + game_progressions.append( RAUserGameProgression( - rom_ra_id=rom.get("GameID", None), + rom_ra_id=rom_game_id, max_possible=rom.get("MaxPossible", None), num_awarded=rom.get("NumAwarded", None), num_awarded_hardcore=rom.get("NumAwardedHardcore", None), - earned_achievements=( - rom_earned_achievements.get(rom["GameID"], []) - if rom.get("GameID") - else [] - ), + earned_achievements=earned_achievements, ) - for rom in roms_with_progression - ], + ) + + return RAUserProgression( + total=len(game_progressions), + results=game_progressions, )