From 1965e72961ba103405dd14ee4007973d15ee3588 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 29 Mar 2023 14:43:00 +0200 Subject: [PATCH] flexible structure added --- backend/src/handler/__init__.py | 7 ++++ backend/src/handler/db_handler.py | 20 ++++----- backend/src/handler/igdb_handler.py | 27 ++++++------- backend/src/main.py | 22 +++++----- backend/src/utils/fastapi.py | 63 +++++++++++------------------ backend/src/utils/fs.py | 37 ++++++++--------- changelog.md | 46 ++++++++++++++++++++- readme.md | 54 ++++++++++++++++--------- 8 files changed, 161 insertions(+), 115 deletions(-) diff --git a/backend/src/handler/__init__.py b/backend/src/handler/__init__.py index e69de29bb..138b37a28 100644 --- a/backend/src/handler/__init__.py +++ b/backend/src/handler/__init__.py @@ -0,0 +1,7 @@ +from handler.db_handler import DBHandler +from handler.igdb_handler import IGDBHandler +from handler.sgdb_handler import SGDBHandler + +igdbh: IGDBHandler = IGDBHandler() +sgdbh: SGDBHandler = SGDBHandler() +dbh: DBHandler = DBHandler() diff --git a/backend/src/handler/db_handler.py b/backend/src/handler/db_handler.py index 1859ebcc9..8348f9e91 100644 --- a/backend/src/handler/db_handler.py +++ b/backend/src/handler/db_handler.py @@ -37,15 +37,9 @@ class DBHandler: except ProgrammingError as e: raise HTTPException(status_code=404, detail=f"Platforms table not found: {e}") - def purge_platforms(self, platforms: list) -> None: + def add_rom(self, rom: Rom) -> None: with Session.begin() as session: - session.query(Platform) \ - .filter(Platform.slug.not_in(platforms)) \ - .delete(synchronize_session='evaluate') - - def add_rom(self, **kargs) -> None: - with Session.begin() as session: - session.merge(Rom(**kargs)) + session.merge(rom) def get_roms(self, p_slug: str) -> list[Rom]: with Session.begin() as session: @@ -67,7 +61,15 @@ class DBHandler: .filter(Rom.p_slug==p_slug, Rom.filename==filename) \ .delete(synchronize_session='evaluate') - def purge_roms(self, p_slug: str, roms: list) -> None: + def purge_platforms(self, platforms: list[str]) -> None: + log.info("Purging platforms") + with Session.begin() as session: + session.query(Platform) \ + .filter(Platform.slug.not_in(platforms)) \ + .delete(synchronize_session='evaluate') + + def purge_roms(self, p_slug: str, roms: list[dict]) -> None: + log.info(f"Purging {p_slug} roms") with Session.begin() as session: session.query(Rom) \ .filter(Rom.p_slug==p_slug, Rom.filename.not_in([rom['filename'] for rom in roms])) \ diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index f9161f3b6..2d9917d15 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -32,31 +32,30 @@ class IGDBHandler(): def get_platform_details(self, slug: str) -> tuple: igdb_id: str = "" name: str = "" - url_logo: str = "" try: res_details: dict = requests.post("https://api.igdb.com/v4/platforms/", headers=self.headers, - data=f"fields id, name, platform_logo; where slug=\"{slug}\";").json()[0] + data=f"fields id, name; where slug=\"{slug}\";").json()[0] igdb_id = res_details['id'] name = res_details['name'] except IndexError: log.warning("platform not found in igdb") if not name: name = slug - return igdb_id, name, url_logo + return {'igdb_id': igdb_id, 'name': name} @check_twitch_token - def get_rom_details(self, filename: str, p_igdb_id: int, r_igdb_id: str) -> dict: + def get_rom_details(self, filename: str, p_igdb_id: int, r_igdb_id_search: str) -> dict: filename_no_ext: str = filename.split('.')[0] - igdb_id: str = "" + r_igdb_id: str = "" slug: str = "" name: str = "" summary: str = "" url_cover: str = "" - if r_igdb_id: + if r_igdb_id_search: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"fields id, slug, name, summary; where id={r_igdb_id};").json()[0] - igdb_id = res_details['id'] + data=f"fields id, slug, name, summary; where id={r_igdb_id_search};").json()[0] + r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] try: @@ -71,7 +70,7 @@ class IGDBHandler(): res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, data=f"search \"{search_term}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0] - igdb_id = res_details['id'] + r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] try: @@ -82,7 +81,7 @@ class IGDBHandler(): try: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0] - igdb_id = res_details['id'] + r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] try: @@ -93,7 +92,7 @@ class IGDBHandler(): try: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0] - igdb_id = res_details['id'] + r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] try: @@ -102,15 +101,15 @@ class IGDBHandler(): pass except IndexError: log.warning(f"{filename} rom not found in igdb") - if igdb_id: + if r_igdb_id: try: res_details: dict = requests.post("https://api.igdb.com/v4/covers/", headers=self.headers, - data=f"fields url; where game={igdb_id};").json()[0] + data=f"fields url; where game={r_igdb_id};").json()[0] url_cover: str = f"https:{res_details['url']}" except IndexError: log.warning(f"{name} cover not found in igdb") if not name: name = filename_no_ext - return (igdb_id, filename_no_ext, slug, name, summary, url_cover) + return (r_igdb_id, filename_no_ext, slug, name, summary, url_cover) @check_twitch_token diff --git a/backend/src/main.py b/backend/src/main.py index 5f7d80db3..788e35d48 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -2,20 +2,15 @@ from fastapi import FastAPI, Request import uvicorn from logger.logger import log -from handler.igdb_handler import IGDBHandler -from handler.sgdb_handler import SGDBHandler -from handler.db_handler import DBHandler +from handler import igdbh, dbh from config.config import DEV_PORT, DEV_HOST +from models.platform import Platform from utils import fs, fastapi app = FastAPI() fastapi.allow_cors(app) -igdbh: IGDBHandler = IGDBHandler() -sgdbh: SGDBHandler = SGDBHandler() -dbh: DBHandler = DBHandler() - @app.patch("/platforms/{p_slug}/roms/{filename}") async def updateRom(req: Request, p_slug: str, filename: str) -> dict: @@ -79,13 +74,14 @@ async def scan(req: Request, overwrite: bool=False) -> dict: log.info("complete scaning...") fs.store_default_resources(overwrite) data: dict = await req.json() - platforms = data['platforms'] if data['platforms'] else fs.get_platforms() + platforms: list[str] = data['platforms'] if data['platforms'] else fs.get_platforms() for p_slug in platforms: - platform: str = fastapi.scan_platform(overwrite, p_slug, igdbh, dbh) - for rom in fs.get_roms(p_slug): - fastapi.scan_rom(overwrite, rom, platform.igdb_id, p_slug, igdbh, dbh) - fastapi.purge(dbh, p_slug=p_slug) - fastapi.purge(dbh) + platform: Platform = fastapi.scan_platform(p_slug) + roms: list[dict] = fs.get_roms(p_slug) + for rom in roms: + fastapi.scan_rom(platform, rom) + dbh.purge_roms(p_slug, roms) + dbh.purge_platforms(fs.get_platforms()) return {'msg': 'success'} diff --git a/backend/src/utils/fastapi.py b/backend/src/utils/fastapi.py index b82f7a77c..1050069e2 100644 --- a/backend/src/utils/fastapi.py +++ b/backend/src/utils/fastapi.py @@ -1,10 +1,10 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from handler.db_handler import DBHandler -from handler.igdb_handler import IGDBHandler +from handler import igdbh, dbh from utils import fs from models.platform import Platform +from models.rom import Rom from logger.logger import log @@ -19,58 +19,41 @@ def allow_cors(app: FastAPI) -> None: log.info("CORS enabled") -def scan_platform(overwrite: bool, p_slug: str, igdbh: IGDBHandler, dbh: DBHandler) -> Platform: +def scan_platform(p_slug: str) -> Platform: """Get platform details from IGDB if possible Args: - overwrite: flag to overwrite platform logo (deprecated) p_slug: short name of the platform - igdbh: igdb hanlder - dbh: database handler Returns Platform object """ - platform_attrs: dict = {} log.info(f"Getting {p_slug} details") - p_igdb_id, p_name, url_logo = igdbh.get_platform_details(p_slug) + platform_attrs: dict = igdbh.get_platform_details(p_slug) platform_attrs['slug'] = p_slug - platform_attrs['igdb_id'] = p_igdb_id - platform_attrs['name'] = p_name - #TODO: refactor logo details logic - if (overwrite or not fs.p_logo_exists(p_slug)) and url_logo: - fs.store_p_logo(p_slug, url_logo) - if fs.p_logo_exists(p_slug): - platform_attrs['path_logo'] = fs.get_p_path_logo(p_slug) - platform_attrs['n_roms'] = len(fs.get_roms(p_slug)) + platform_attrs['path_logo'] = '' + platform_attrs['n_roms'] = fs.get_roms(p_slug, only_amount=True) platform = Platform(**platform_attrs) dbh.add_platform(platform) return platform -def scan_rom(overwrite: bool, rom, p_igdb_id: str, p_slug: str, igdbh: IGDBHandler, dbh: DBHandler, r_igbd_id: str = '') -> None: +def scan_rom(platform: Platform, rom: dict, r_igbd_id_search: str = '', overwrite: bool = False) -> None: log.info(f"Getting {rom['filename']} details") - r_igdb_id, filename_no_ext, r_slug, r_name, summary, url_cover = igdbh.get_rom_details(rom['filename'], p_igdb_id, r_igbd_id) - path_cover_s, path_cover_l, has_cover = fs.get_cover_details(overwrite, p_slug, filename_no_ext, url_cover) - rom: dict = { - 'filename': rom['filename'], 'filename_no_ext': filename_no_ext, 'size': rom['size'], - 'r_igdb_id': r_igdb_id, 'p_igdb_id': p_igdb_id, - 'name': r_name, 'r_slug': r_slug, 'p_slug': p_slug, + r_igdb_id, filename_no_ext, r_slug, r_name, summary, url_cover = igdbh.get_rom_details(rom['filename'], platform.igdb_id, r_igbd_id_search) + path_cover_s, path_cover_l, has_cover = fs.get_cover_details(overwrite, platform.slug, filename_no_ext, url_cover) + rom_attrs: dict = { + 'filename': rom['filename'], + 'filename_no_ext': filename_no_ext, + 'size': rom['size'], + 'r_igdb_id': r_igdb_id, + 'p_igdb_id': platform.igdb_id, + 'name': r_name, + 'r_slug': r_slug, + 'p_slug': platform.slug, 'summary': summary, - 'path_cover_s': path_cover_s, 'path_cover_l': path_cover_l, 'has_cover': has_cover + 'path_cover_s': path_cover_s, + 'path_cover_l': path_cover_l, + 'has_cover': has_cover } - dbh.add_rom(**rom) - - -def purge(dbh: DBHandler, p_slug: str = '') -> None: - """Clean the database from non existent platforms or roms""" - if p_slug: - # Purge only roms in platform - log.info(f"Purge {p_slug}") - dbh.purge_roms(p_slug, fs.get_roms(p_slug)) - else: - # Purge all platforms / delete non existent platforms and non existen roms - platforms: list = fs.get_platforms() - dbh.purge_platforms(platforms) - for p_slug in platforms: - log.info(f"Purge {p_slug}") - dbh.purge_roms(p_slug, fs.get_roms(p_slug)) + rom = Rom(**rom_attrs) + dbh.add_rom(rom) diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 92d85443a..c91717c03 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -37,15 +37,6 @@ def p_logo_exists(slug: str) -> bool: return True if os.path.exists(logo_path) else False -def get_p_path_logo(slug: str) -> str: - """Returns platform logo filesystem path - - Args: - slug: shor name of the platform - """ - return f"/assets/library/resources/{slug}/logo.png" - - def store_p_logo(slug: str, url_logo: str) -> None: """Store platform resources in filesystem @@ -65,14 +56,17 @@ def store_p_logo(slug: str, url_logo: str) -> None: log.warning(f"{slug} logo couldn't be downloaded") -def get_platforms() -> list: +def get_platforms() -> list[str]: """Gets all filesystem platforms Returns list with all the filesystem platforms found in the LIBRARY_BASE_PATH. Automatically discards the reserved directories such resources or database directory. """ try: - platforms: list[str] = list(os.walk(LIBRARY_BASE_PATH))[0][1] + if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"): + platforms: list[str] = list(os.walk(f"{LIBRARY_BASE_PATH}/roms"))[0][1] + else: + platforms: list[str] = list(os.walk(LIBRARY_BASE_PATH))[0][1] [platforms.remove(reserved) for reserved in RESERVED_FOLDERS if reserved in platforms] log.info(f"filesystem platforms found: {platforms}") return platforms @@ -145,24 +139,27 @@ def get_cover_details(overwrite: bool, p_slug: str, filename_no_ext: str, url_co return path_cover_s, path_cover_l, has_cover -def get_roms(p_slug: str) -> list: +def get_roms(p_slug: str, only_amount: bool = False) -> list[dict]: """Gets all filesystem roms for a platform Args: p_slug: short name of the platform - Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH. - Automatically discards the default directory. + only_amount: flag to return only amount of roms instead of all info + Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH. Just the amount of them if only_amount=True """ try: - roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms/"))[0][2] - roms: list[dict] = [ - {'filename': rom, - 'size': str(round(os.stat(f"{LIBRARY_BASE_PATH}/{p_slug}/roms/{rom}").st_size / (1024 * 1024), 2))} - for rom in roms_filename] + roms: list[dict] = [] + if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"): + roms_path: str = f"{LIBRARY_BASE_PATH}/roms/{p_slug}" + roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/roms/{p_slug}"))[0][2] + else: + roms_path: str = f"{LIBRARY_BASE_PATH}/{p_slug}/roms" + roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms"))[0][2] + if only_amount: return len(roms_filename) + [roms.append({'filename': rom, 'size': str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2))}) for rom in roms_filename] log.info(f"filesystem roms found for {p_slug}: {roms}") except IndexError: log.warning(f"roms not found for {p_slug}") - roms: list[dict] = [] return roms diff --git a/changelog.md b/changelog.md index a8c2c6477..6018b2ace 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,47 @@ +# v1.5 (_29-03-2023_) + +## Added + - Now RomM folder structure is more flexible to match two different patrons by priority: + - Structure 1 (priority high) - roms folder at root of library folder: + ``` + library/ + ├─ roms/ + │ ├─ gbc/ + │ ├─ rom_1.gbc + │ ├─ rom_2.gbc + │ + ├─ gba/ + │ ├─ rom_1.gba + │ ├─ rom_2.gba + │ + ├─ gb/ + ├─ rom_1.gb + ├─ rom_1.gb + ``` + - Structure 2 (priority low) - roms folder inside each platform folder + ``` + library/ + ├─ gbc/ + │ ├─ roms/ + │ ├─ rom_1.gbc + │ ├─ rom_2.gbc + | + ├─ gba/ + │ ├─ roms/ + │ ├─ rom_1.gba + │ ├─ rom_2.gba + | + ├─ gb/ + │ ├─ roms/ + │ ├─ rom_1.gb + │ ├─ rom_1.gb + ``` + +# v1.4.1 (_29-03-2023_) + +## Added + - Now you can use your games tags (like (USA) or (rev-1)) to filter in the gallery + # v1.4 (_29-03-2023_) ## Added @@ -67,4 +111,4 @@ Columns to modify (examples in case that you set it with database name as romm, ## Added -- Birth of RomM \ No newline at end of file +- Birth of RomM diff --git a/readme.md b/readme.md index ab3a5909e..f93d3c2b6 100644 --- a/readme.md +++ b/readme.md @@ -33,25 +33,43 @@ For now, it is only available as a docker [image](https://hub.docker.com/r/zurdi ## ⚠️ Folder structure -To allow RomM scan your retro games library, it should follow the following structure: +RomM accepts two different folder structure by priority: + - Structure 1 (priority high) - roms folder at root of library folder: + ``` + library/ + ├─ roms/ + │ ├─ gbc/ + │ ├─ rom_1.gbc + │ ├─ rom_2.gbc + │ + ├─ gba/ + │ ├─ rom_1.gba + │ ├─ rom_2.gba + │ + ├─ gb/ + ├─ rom_1.gb + ├─ rom_1.gb + ``` + - Structure 2 (priority low) - roms folder inside each platform folder + ``` + library/ + ├─ gbc/ + │ ├─ roms/ + │ ├─ rom_1.gbc + │ ├─ rom_2.gbc + | + ├─ gba/ + │ ├─ roms/ + │ ├─ rom_1.gba + │ ├─ rom_2.gba + | + ├─ gb/ + │ ├─ roms/ + │ ├─ rom_1.gb + │ ├─ rom_1.gb + ``` -``` -library/ -├─ gbc/ -│ ├─ roms/ -│ ├─ rom_1.gbc -│ ├─ rom_2.gbc -| -├─ gba/ -│ ├─ roms/ -│ ├─ rom_1.gba -│ ├─ rom_2.gba -| -├─ gb/ -│ ├─ roms/ -│ ├─ rom_1.gb -│ ├─ rom_1.gb -``` +RomM will try to find the structure 1. If it doesn't exists, RomM will try to find structure 2. # Preview