start work on retro achievements

This commit is contained in:
SaraVieira
2024-08-31 19:09:20 +01:00
parent c55d63b16a
commit 11b46494a7
15 changed files with 500 additions and 6 deletions

View File

@@ -0,0 +1,34 @@
"""add ra_id to platforms
Revision ID: 026_add_ra_id_to_platforms
Revises: 0025_roms_hashes
Create Date: 2024-08-31 18:48:49.772416
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "026_add_ra_id_to_platforms"
down_revision = "0025_roms_hashes"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.add_column(sa.Column("ra_id", sa.Integer(), nullable=True))
pass
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.add_column(sa.Column("ra_id", sa.Integer(), nullable=True))
pass
def downgrade() -> None:
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.drop_column("ra_id")
pass
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.drop_column("ra_id")
pass

View File

@@ -51,6 +51,10 @@ STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "")
# MOBYGAMES
MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "")
# RETROACHIEVEMENTS
RETROACHIEVEMENTS_API_KEY: Final = os.environ.get("RETROACHIEVEMENTS_API_KEY", "")
RETROACHIEVEMENTS_USERNAME: Final = os.environ.get("RETROACHIEVEMENTS_USERNAME", "")
# DB DRIVERS
ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")

View File

@@ -13,6 +13,7 @@ from handler.database import db_user_handler
from handler.filesystem import fs_platform_handler
from handler.metadata.igdb_handler import IGDB_API_ENABLED
from handler.metadata.moby_handler import MOBY_API_ENABLED
from handler.metadata.ra_handler import RA_API_ENABLED
from handler.metadata.sgdb_handler import STEAMGRIDDB_API_ENABLED
from utils import get_version
from utils.router import APIRouter
@@ -36,6 +37,7 @@ def heartbeat() -> HeartbeatResponse:
"IGDB_API_ENABLED": IGDB_API_ENABLED,
"MOBY_API_ENABLED": MOBY_API_ENABLED,
"STEAMGRIDDB_ENABLED": STEAMGRIDDB_API_ENABLED,
"RA_API_ENABLED": RA_API_ENABLED,
},
"FS_PLATFORMS": fs_platform_handler.get_platforms(),
"WATCHER": {

View File

@@ -20,6 +20,7 @@ class MetadataSourcesDict(TypedDict):
IGDB_API_ENABLED: bool
MOBY_API_ENABLED: bool
STEAMGRIDDB_ENABLED: bool
RA_API_ENABLED: bool
class EmulationDict(TypedDict):

View File

@@ -120,7 +120,7 @@ async def scan_platforms(
roms_ids = []
if not metadata_sources:
metadata_sources = ["igdb", "moby"]
metadata_sources = ["igdb", "moby", "retro_achievements"]
sm = _get_socket_manager()

View File

@@ -1,7 +1,9 @@
from .igdb_handler import IGDBBaseHandler
from .moby_handler import MobyGamesHandler
from .ra_handler import RetroAchievementsHandler
from .sgdb_handler import SGDBBaseHandler
meta_igdb_handler = IGDBBaseHandler()
meta_moby_handler = MobyGamesHandler()
meta_sgdb_handler = SGDBBaseHandler()
meta_ra_handler = RetroAchievementsHandler()

View File

@@ -0,0 +1,423 @@
import asyncio
import http
import re
from typing import Final, NotRequired, TypedDict
from urllib.parse import quote
import httpx
import pydash
import yarl
from config import RETROACHIEVEMENTS_API_KEY, RETROACHIEVEMENTS_USERNAME
from fastapi import HTTPException, status
from logger.logger import log
from unidecode import unidecode as uc
from utils.context import ctx_httpx_client
from .base_hander import (
PS2_OPL_REGEX,
SONY_SERIAL_REGEX,
SWITCH_PRODUCT_ID_REGEX,
SWITCH_TITLEDB_REGEX,
MetadataHandler,
)
# Used to display the RetroAchievements API status in the frontend
RA_API_ENABLED: Final = bool(RETROACHIEVEMENTS_API_KEY) and bool(
RETROACHIEVEMENTS_USERNAME
)
PS1_MOBY_ID: Final = 6
PS2_MOBY_ID: Final = 7
PSP_MOBY_ID: Final = 46
SWITCH_MOBY_ID: Final = 203
ARCADE_MOBY_IDS: Final = [143, 36]
class RAGamesPlatform(TypedDict):
slug: str
moby_id: int | None
name: NotRequired[str]
class RAMetadataPlatform(TypedDict):
ra_id: int
name: str
class RAMetadata(TypedDict):
moby_score: str
genres: list[str]
alternate_titles: list[str]
platforms: list[RAMetadataPlatform]
class RAGameRom(TypedDict):
moby_id: int | None
slug: NotRequired[str]
name: NotRequired[str]
summary: NotRequired[str]
url_cover: NotRequired[str]
url_screenshots: NotRequired[list[str]]
moby_metadata: NotRequired[RAMetadata]
def extract_metadata_from_moby_rom(rom: dict) -> RAMetadata:
return RAMetadata(
{
"moby_score": str(rom.get("moby_score", "")),
"genres": rom.get("genres.genre_name", []),
"alternate_titles": rom.get("alternate_titles.title", []),
"platforms": [
{
"moby_id": p["platform_id"],
"name": p["platform_name"],
}
for p in rom.get("platforms", [])
],
}
)
class RetroAchievementsHandler(MetadataHandler):
def __init__(self) -> None:
self.platform_url = (
"https://retroachievements.org/API/API_GetGameList.php?&h=1&f=1"
)
self.games_url = "https://api.mobygames.com/v1/games"
async def _request(self, url: str, timeout: int = 120) -> dict:
httpx_client = ctx_httpx_client.get()
authorized_url = (
yarl.URL(url)
.update_query(z=RETROACHIEVEMENTS_USERNAME)
.update_query(y=RETROACHIEVEMENTS_API_KEY)
)
try:
res = await httpx_client.get(str(authorized_url), timeout=timeout)
res.raise_for_status()
return res.json()
except httpx.NetworkError as exc:
log.critical(
"Connection error: can't connect to RetroAchievements", exc_info=True
)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Can't connect to RetroAchievements, check your internet connection",
) from exc
except httpx.HTTPStatusError as err:
if err.response.status_code == http.HTTPStatus.TOO_MANY_REQUESTS:
# Retry after 2 seconds if rate limit hit
await asyncio.sleep(2)
else:
# Log the error and return an empty dict if the request fails with a different code
log.error(err)
return {}
except httpx.TimeoutException:
# Retry the request once if it times out
pass
try:
res = await httpx_client.get(url, timeout=timeout)
res.raise_for_status()
except (httpx.HTTPStatusError, httpx.TimeoutException) as err:
if (
isinstance(err, httpx.HTTPStatusError)
and err.response.status_code == http.HTTPStatus.UNAUTHORIZED
):
# Sometimes Mobygames returns 401 even with a valid API key
return {}
# Log the error and return an empty dict if the request fails with a different code
log.error(err)
return {}
return res.json()
async def _search_rom(self, search_term: str, platform_moby_id: int) -> dict | None:
if not platform_moby_id:
return None
search_term = uc(search_term)
url = yarl.URL(self.games_url).with_query(
platform=[platform_moby_id],
title=quote(search_term, safe="/ "),
)
roms = (await self._request(str(url))).get("games", [])
exact_matches = [
rom
for rom in roms
if (
rom["title"].lower() == search_term.lower()
or (
self._normalize_exact_match(rom["title"])
== self._normalize_exact_match(search_term)
)
)
]
return pydash.get(exact_matches or roms, "[0]", None)
def get_platform(self, slug: str) -> RAGamesPlatform:
platform = SLUG_TO_RA_ID.get(slug.lower(), None)
print(platform)
if not platform:
return RAGamesPlatform(ra_id=None, slug=slug)
return RAGamesPlatform(
ra_id=platform["id"],
slug=slug,
name=platform["name"],
)
async def get_rom(self, file_name: str, platform_moby_id: int) -> RAGameRom:
from handler.filesystem import fs_rom_handler
if not RA_API_ENABLED:
return RAGameRom(moby_id=None)
if not platform_moby_id:
return RAGameRom(moby_id=None)
search_term = fs_rom_handler.get_file_name_with_no_tags(file_name)
fallback_rom = RAGameRom(moby_id=None)
# Support for PS2 OPL filename format
match = PS2_OPL_REGEX.match(file_name)
if platform_moby_id == PS2_MOBY_ID and match:
search_term = await self._ps2_opl_format(match, search_term)
fallback_rom = RAGameRom(moby_id=None, name=search_term)
# Support for sony serial filename format (PS, PS3, PS3)
match = SONY_SERIAL_REGEX.search(file_name, re.IGNORECASE)
if platform_moby_id == PS1_MOBY_ID and match:
search_term = await self._ps1_serial_format(match, search_term)
fallback_rom = RAGameRom(moby_id=None, name=search_term)
if platform_moby_id == PS2_MOBY_ID and match:
search_term = await self._ps2_serial_format(match, search_term)
fallback_rom = RAGameRom(moby_id=None, name=search_term)
if platform_moby_id == PSP_MOBY_ID and match:
search_term = await self._psp_serial_format(match, search_term)
fallback_rom = RAGameRom(moby_id=None, name=search_term)
# Support for switch titleID filename format
match = SWITCH_TITLEDB_REGEX.search(file_name)
if platform_moby_id == SWITCH_MOBY_ID and match:
search_term, index_entry = await self._switch_titledb_format(
match, search_term
)
if index_entry:
fallback_rom = RAGameRom(
moby_id=None,
name=index_entry["name"],
summary=index_entry.get("description", ""),
url_cover=index_entry.get("iconUrl", ""),
url_screenshots=index_entry.get("screenshots", None) or [],
)
# Support for switch productID filename format
match = SWITCH_PRODUCT_ID_REGEX.search(file_name)
if platform_moby_id == SWITCH_MOBY_ID and match:
search_term, index_entry = await self._switch_productid_format(
match, search_term
)
if index_entry:
fallback_rom = RAGameRom(
moby_id=None,
name=index_entry["name"],
summary=index_entry.get("description", ""),
url_cover=index_entry.get("iconUrl", ""),
url_screenshots=index_entry.get("screenshots", None) or [],
)
# Support for MAME arcade filename format
if platform_moby_id in ARCADE_MOBY_IDS:
search_term = await self._mame_format(search_term)
fallback_rom = RAGameRom(moby_id=None, name=search_term)
search_term = self.normalize_search_term(search_term)
res = await self._search_rom(search_term, platform_moby_id)
# Split the search term since mobygames search doesn't support special caracters
if not res and ":" in search_term:
for term in search_term.split(":")[::-1]:
res = await self._search_rom(term, platform_moby_id)
if res:
break
# Some MAME games have two titles split by a slash
if not res and "/" in search_term:
for term in search_term.split("/"):
res = await self._search_rom(term.strip(), platform_moby_id)
if res:
break
if not res:
return fallback_rom
rom = {
"moby_id": res["game_id"],
"name": res["title"],
"slug": res["moby_url"].split("/")[-1],
"summary": res.get("description", ""),
"url_cover": pydash.get(res, "sample_cover.image", ""),
"url_screenshots": [s["image"] for s in res.get("sample_screenshots", [])],
"moby_metadata": extract_metadata_from_moby_rom(res),
}
return RAGameRom({k: v for k, v in rom.items() if v}) # type: ignore[misc]
async def get_rom_by_id(self, moby_id: int) -> RAGameRom:
if not RA_API_ENABLED:
return RAGameRom(moby_id=None)
url = yarl.URL(self.games_url).with_query(id=moby_id)
roms = (await self._request(str(url))).get("games", [])
res = pydash.get(roms, "[0]", None)
if not res:
return RAGameRom(moby_id=None)
rom = {
"moby_id": res["game_id"],
"name": res["title"],
"slug": res["moby_url"].split("/")[-1],
"summary": res.get("description", None),
"url_cover": pydash.get(res, "sample_cover.image", None),
"url_screenshots": [s["image"] for s in res.get("sample_screenshots", [])],
"moby_metadata": extract_metadata_from_moby_rom(res),
}
return RAGameRom({k: v for k, v in rom.items() if v}) # type: ignore[misc]
async def get_matched_roms_by_id(self, moby_id: int) -> list[RAGameRom]:
if not RA_API_ENABLED:
return []
rom = await self.get_rom_by_id(moby_id)
return [rom] if rom["moby_id"] else []
async def get_matched_roms_by_name(
self, search_term: str, platform_moby_id: int
) -> list[RAGameRom]:
if not RA_API_ENABLED:
return []
if not platform_moby_id:
return []
search_term = uc(search_term)
url = yarl.URL(self.games_url).with_query(
platform=[platform_moby_id], title=quote(search_term, safe="/ ")
)
matched_roms = (await self._request(str(url))).get("games", [])
return [
RAGameRom( # type: ignore[misc]
{
k: v
for k, v in {
"moby_id": rom["game_id"],
"name": rom["title"],
"slug": rom["moby_url"].split("/")[-1],
"summary": rom.get("description", ""),
"url_cover": pydash.get(rom, "sample_cover.image", ""),
"url_screenshots": [
s["image"] for s in rom.get("sample_screenshots", [])
],
"moby_metadata": extract_metadata_from_moby_rom(rom),
}.items()
if v
}
)
for rom in matched_roms
]
class SlugToMobyId(TypedDict):
id: int
name: str
SLUG_TO_RA_ID: dict[str, SlugToMobyId] = {
"3do": {"id": 43, "name": "3DO"},
"cpc": {"id": 37, "name": "Amstrad CPC"},
"acpc": {"id": 37, "name": "Amstrad CPC"},
"apple2": {"id": 38, "name": "Apple II"},
"appleii": {"id": 38, "name": "Apple II"},
"arcade": {"id": 27, "name": "Arcade"},
"arcadia-2001": {"id": 73, "name": "Arcadia 2001"},
"arduboy": {"id": 71, "name": "Arduboy"},
"atari-2600": {"id": 25, "name": "Atari 2600"},
"atari2600": {"id": 25, "name": "Atari 2600"}, # IGDB
"atari-7800": {"id": 51, "name": "Atari 7800"},
"atari7800": {"id": 51, "name": "Atari 7800"}, # IGDB
"atari-jaguar-cd": {"id": 77, "name": "Atari Jaguar CD"},
"colecovision": {"id": 44, "name": "ColecoVision"},
"dreamcast": {"id": 40, "name": "Dreamcast"},
"dc": {"id": 40, "name": "Dreamcast"}, # IGDB
"gameboy": {"id": 4, "name": "Game Boy"},
"gb": {"id": 4, "name": "Game Boy"}, # IGDB
"gameboy-advance": {"id": 5, "name": "Game Boy Advance"},
"gba": {"id": 5, "name": "Game Boy Advance"}, # IGDB
"gameboy-color": {"id": 6, "name": "Game Boy Color"},
"gbc": {"id": 6, "name": "Game Boy Color"}, # IGDB
"game-gear": {"id": 15, "name": "Game Gear"},
"gamegear": {"id": 15, "name": "Game Gear"}, # IGDB
"gamecube": {"id": 16, "name": "GameCube"},
"ngc": {"id": 14, "name": "GameCube"}, # IGDB
"genesis": {"id": 1, "name": "Genesis/Mega Drive"},
"genesis-slash-megadrive": {"id": 16, "name": "Genesis/Mega Drive"},
"intellivision": {"id": 45, "name": "Intellivision"},
"jaguar": {"id": 17, "name": "Jaguar"},
"lynx": {"id": 13, "name": "Lynx"},
"msx": {"id": 29, "name": "MSX"},
"mega-duck-slash-cougar-boy": {"id": 69, "name": "Mega Duck/Cougar Boy"},
"nes": {"id": 7, "name": "NES"},
"famicom": {"id": 7, "name": "NES"},
"neo-geo-cd": {"id": 56, "name": "Neo Geo CD"},
"neo-geo-pocket": {"id": 14, "name": "Neo Geo Pocket"},
"neo-geo-pocket-color": {"id": 14, "name": "Neo Geo Pocket Color"},
"n64": {"id": 2, "name": "Nintendo 64"},
"nintendo-ds": {"id": 18, "name": "Nintendo DS"},
"nds": {"id": 18, "name": "Nintendo DS"}, # IGDB
"nintendo-dsi": {"id": 78, "name": "Nintendo DSi"},
"odyssey-2": {"id": 23, "name": "Odyssey 2"},
"pc-8000": {"id": 47, "name": "PC-8000"},
"pc-8800-series": {"id": 47, "name": "PC-8800 Series"}, # IGDB
"pc-fx": {"id": 49, "name": "PC-FX"},
"psp": {"id": 41, "name": "PSP"},
"playstation": {"id": 12, "name": "PlayStation"},
"ps": {"id": 12, "name": "PlayStation"}, # IGDB
"ps2": {"id": 21, "name": "PlayStation 2"},
"pokemon-mini": {"id": 24, "name": "Pokémon Mini"},
"saturn": {"id": 39, "name": "Sega Saturn"},
"sega-32x": {"id": 10, "name": "SEGA 32X"},
"sega32": {"id": 10, "name": "SEGA 32X"}, # IGDB
"sega-cd": {"id": 9, "name": "SEGA CD"},
"segacd": {"id": 9, "name": "SEGA CD"}, # IGDB
"sega-master-system": {"id": 11, "name": "SEGA Master System"},
"sms": {"id": 11, "name": "SEGA Master System"}, # IGDB
"sg-1000": {"id": 33, "name": "SG-1000"},
"snes": {"id": 3, "name": "SNES"},
"turbografx-cd": {"id": 76, "name": "TurboGrafx CD"},
"turbografx-16-slash-pc-engine-cd": {"id": 76, "name": "TurboGrafx CD"},
"turbo-grafx": {"id": 8, "name": "TurboGrafx-16"},
"turbografx16--1": {"id": 8, "name": "TurboGrafx-16"}, # IGDB
"vectrex": {"id": 26, "name": "Vectrex"},
"virtual-boy": {"id": 28, "name": "Virtual Boy"},
"virtualboy": {"id": 28, "name": "Virtual Boy"},
"watara-slash-quickshot-supervision": {
"id": 63,
"name": "Watara/QuickShot Supervision",
},
"wonderswan": {"id": 53, "name": "WonderSwan"},
"wonderswan-color": {"id": 53, "name": "WonderSwan Color"},
}
# Reverse lookup
MOBY_ID_TO_SLUG = {v["id"]: k for k, v in SLUG_TO_RA_ID.items()}

View File

@@ -7,9 +7,10 @@ from config.config_manager import config_manager as cm
from handler.database import db_platform_handler
from handler.filesystem import fs_asset_handler, fs_firmware_handler, fs_rom_handler
from handler.filesystem.roms_handler import FSRom
from handler.metadata import meta_igdb_handler, meta_moby_handler
from handler.metadata import meta_igdb_handler, meta_moby_handler, meta_ra_handler
from handler.metadata.igdb_handler import IGDBPlatform, IGDBRom
from handler.metadata.moby_handler import MobyGamesPlatform, MobyGamesRom
from handler.metadata.ra_handler import RAGameRom, RAGamesPlatform
from logger.logger import log
from models.assets import Save, Screenshot, State
from models.firmware import Firmware
@@ -62,7 +63,7 @@ async def scan_platform(
log.info(f"· {fs_slug}")
if metadata_sources is None:
metadata_sources = ["igdb", "moby"]
metadata_sources = ["igdb", "moby", "retro_achievements"]
platform_attrs: dict[str, Any] = {}
platform_attrs["fs_slug"] = fs_slug
@@ -87,7 +88,6 @@ async def scan_platform(
platform_attrs["slug"] = fs_slug
except (KeyError, TypeError, AttributeError):
platform_attrs["slug"] = fs_slug
igdb_platform = (
(await meta_igdb_handler.get_platform(platform_attrs["slug"]))
if "igdb" in metadata_sources
@@ -99,8 +99,16 @@ async def scan_platform(
else MobyGamesPlatform(moby_id=None, slug=platform_attrs["slug"])
)
ra_platform = (
meta_ra_handler.get_platform(platform_attrs["slug"])
if "retro_achievements" in metadata_sources
else RAGamesPlatform(ra_id=None, slug=platform_attrs["slug"])
)
platform_attrs["name"] = platform_attrs["slug"].replace("-", " ").title()
platform_attrs.update({**moby_platform, **igdb_platform}) # Reverse order
platform_attrs.update(
{**ra_platform, **moby_platform, **igdb_platform}
) # Reverse order
if platform_attrs["igdb_id"] or platform_attrs["moby_id"]:
log.info(
@@ -192,6 +200,7 @@ async def scan_rom(
"igdb_id": rom.igdb_id,
"moby_id": rom.moby_id,
"sgdb_id": rom.sgdb_id,
"ra_id": rom.ra_id,
"name": rom.name,
"slug": rom.slug,
"summary": rom.summary,

View File

@@ -18,6 +18,7 @@ class Platform(BaseModel):
igdb_id: Mapped[int | None]
sgdb_id: Mapped[int | None]
moby_id: Mapped[int | None]
ra_id: Mapped[int | None]
slug: Mapped[str] = mapped_column(String(length=50))
fs_slug: Mapped[str] = mapped_column(String(length=50))
name: Mapped[str] = mapped_column(String(length=400))

View File

@@ -38,6 +38,7 @@ class Rom(BaseModel):
igdb_id: Mapped[int | None]
sgdb_id: Mapped[int | None]
moby_id: Mapped[int | None]
ra_id: Mapped[int | None]
file_name: Mapped[str] = mapped_column(String(length=450))
file_name_no_tags: Mapped[str] = mapped_column(String(length=450))

View File

@@ -14,6 +14,10 @@ MOBYGAMES_API_KEY=
# SteamGridDB
STEAMGRIDDB_API_KEY=
# RetroAchievements
RETROACHIEVEMENTS_USERNAME=
RETROACHIEVEMENTS_API_KEY=
# Database config
DB_HOST=127.0.0.1
DB_PORT=3306

View File

@@ -20,6 +20,8 @@ services:
- IGDB_CLIENT_SECRET= # https://api-docs.igdb.com/#account-creation
- MOBYGAMES_API_KEY= # https://www.mobygames.com/info/api/
- STEAMGRIDDB_API_KEY # https://github.com/rommapp/romm/wiki/Generate-API-Keys#steamgriddb
- RETROACHIEVEMENTS_API_KEY # https://api-docs.retroachievements.org/#api-access
- RETROACHIEVEMENTS_USERNAME # https://api-docs.retroachievements.org/#api-access
volumes:
- romm_resources:/romm/resources # Resources fetched from IGDB (covers, screenshots, etc.)
- romm_redis_data:/redis-data # Cached data for background tasks

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@@ -7,5 +7,6 @@ export type MetadataSourcesDict = {
IGDB_API_ENABLED: boolean;
MOBY_API_ENABLED: boolean;
STEAMGRIDDB_ENABLED: boolean;
RA_API_ENABLED: boolean;
};

View File

@@ -32,6 +32,12 @@ const metadataOptions = computed(() => [
logo_path: "/assets/scrappers/moby.png",
disabled: !heartbeat.value.METADATA_SOURCES?.MOBY_API_ENABLED,
},
{
name: "RetroAchievements",
value: "retro_achievements",
logo_path: "/assets/scrappers/ra.webp",
disabled: !heartbeat.value.METADATA_SOURCES?.RA_API_ENABLED,
},
]);
// Use the computed metadataOptions to filter out disabled sources
const metadataSources = ref(metadataOptions.value.filter((s) => !s.disabled));
@@ -311,7 +317,11 @@ async function stopScan() {
<v-list-item class="pa-0">
<template #prepend>
<v-avatar :rounded="0" size="40">
<platform-icon :key="platform.slug" :slug="platform.slug" :name="platform.name" />
<platform-icon
:key="platform.slug"
:slug="platform.slug"
:name="platform.name"
/>
</v-avatar>
</template>
{{ platform.name }}