Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-30 07:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

flexible structure added

Browse Source
This commit is contained in:
zurdi zurdo
2023-03-29 14:43:00 +02:00
parent e2b470611d
commit 1965e72961
8 changed files with 161 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/src/handler/__init__.py
View File

@@ -0,0 +1,7 @@
from handler.db_handler import DBHandler
from handler.igdb_handler import IGDBHandler
from handler.sgdb_handler import SGDBHandler
igdbh: IGDBHandler = IGDBHandler()
sgdbh: SGDBHandler = SGDBHandler()
dbh: DBHandler = DBHandler()

20
backend/src/handler/db_handler.py
View File

@@ -37,15 +37,9 @@ class DBHandler:
except ProgrammingError as e:
raise HTTPException(status_code=404, detail=f"Platforms table not found: {e}")
def purge_platforms(self, platforms: list) -> None:
def add_rom(self, rom: Rom) -> None:
with Session.begin() as session:
session.query(Platform) \
.filter(Platform.slug.not_in(platforms)) \
.delete(synchronize_session='evaluate')
def add_rom(self, **kargs) -> None:
with Session.begin() as session:
session.merge(Rom(**kargs))
session.merge(rom)
def get_roms(self, p_slug: str) -> list[Rom]:
with Session.begin() as session:
@@ -67,7 +61,15 @@ class DBHandler:
.filter(Rom.p_slug==p_slug, Rom.filename==filename) \
.delete(synchronize_session='evaluate')
def purge_roms(self, p_slug: str, roms: list) -> None:
def purge_platforms(self, platforms: list[str]) -> None:
log.info("Purging platforms")
with Session.begin() as session:
session.query(Platform) \
.filter(Platform.slug.not_in(platforms)) \
.delete(synchronize_session='evaluate')
def purge_roms(self, p_slug: str, roms: list[dict]) -> None:
log.info(f"Purging {p_slug} roms")
with Session.begin() as session:
session.query(Rom) \
.filter(Rom.p_slug==p_slug, Rom.filename.not_in([rom['filename'] for rom in roms])) \

27
backend/src/handler/igdb_handler.py
View File

@@ -32,31 +32,30 @@ class IGDBHandler():
def get_platform_details(self, slug: str) -> tuple:
igdb_id: str = ""
name: str = ""
url_logo: str = ""
try:
res_details: dict = requests.post("https://api.igdb.com/v4/platforms/", headers=self.headers,
data=f"fields id, name, platform_logo; where slug=\"{slug}\";").json()[0]
data=f"fields id, name; where slug=\"{slug}\";").json()[0]
igdb_id = res_details['id']
name = res_details['name']
except IndexError:
log.warning("platform not found in igdb")
if not name: name = slug
return igdb_id, name, url_logo
return {'igdb_id': igdb_id, 'name': name}
@check_twitch_token
def get_rom_details(self, filename: str, p_igdb_id: int, r_igdb_id: str) -> dict:
def get_rom_details(self, filename: str, p_igdb_id: int, r_igdb_id_search: str) -> dict:
filename_no_ext: str = filename.split('.')[0]
igdb_id: str = ""
r_igdb_id: str = ""
slug: str = ""
name: str = ""
summary: str = ""
url_cover: str = ""
if r_igdb_id:
if r_igdb_id_search:
res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
data=f"fields id, slug, name, summary; where id={r_igdb_id};").json()[0]
igdb_id = res_details['id']
data=f"fields id, slug, name, summary; where id={r_igdb_id_search};").json()[0]
r_igdb_id = res_details['id']
slug = res_details['slug']
name = res_details['name']
try:
@@ -71,7 +70,7 @@ class IGDBHandler():
res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
data=f"search \"{search_term}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0]
igdb_id = res_details['id']
r_igdb_id = res_details['id']
slug = res_details['slug']
name = res_details['name']
try:
@@ -82,7 +81,7 @@ class IGDBHandler():
try:
res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0]
igdb_id = res_details['id']
r_igdb_id = res_details['id']
slug = res_details['slug']
name = res_details['name']
try:
@@ -93,7 +92,7 @@ class IGDBHandler():
try:
res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0]
igdb_id = res_details['id']
r_igdb_id = res_details['id']
slug = res_details['slug']
name = res_details['name']
try:
@@ -102,15 +101,15 @@ class IGDBHandler():
pass
except IndexError:
log.warning(f"{filename} rom not found in igdb")
if igdb_id:
if r_igdb_id:
try:
res_details: dict = requests.post("https://api.igdb.com/v4/covers/", headers=self.headers,
data=f"fields url; where game={igdb_id};").json()[0]
data=f"fields url; where game={r_igdb_id};").json()[0]
url_cover: str = f"https:{res_details['url']}"
except IndexError:
log.warning(f"{name} cover not found in igdb")
if not name: name = filename_no_ext
return (igdb_id, filename_no_ext, slug, name, summary, url_cover)
return (r_igdb_id, filename_no_ext, slug, name, summary, url_cover)
@check_twitch_token

22
backend/src/main.py
View File

@@ -2,20 +2,15 @@ from fastapi import FastAPI, Request
import uvicorn
from logger.logger import log
from handler.igdb_handler import IGDBHandler
from handler.sgdb_handler import SGDBHandler
from handler.db_handler import DBHandler
from handler import igdbh, dbh
from config.config import DEV_PORT, DEV_HOST
from models.platform import Platform
from utils import fs, fastapi
app = FastAPI()
fastapi.allow_cors(app)
igdbh: IGDBHandler = IGDBHandler()
sgdbh: SGDBHandler = SGDBHandler()
dbh: DBHandler = DBHandler()
@app.patch("/platforms/{p_slug}/roms/{filename}")
async def updateRom(req: Request, p_slug: str, filename: str) -> dict:
@@ -79,13 +74,14 @@ async def scan(req: Request, overwrite: bool=False) -> dict:
log.info("complete scaning...")
fs.store_default_resources(overwrite)
data: dict = await req.json()
platforms = data['platforms'] if data['platforms'] else fs.get_platforms()
platforms: list[str] = data['platforms'] if data['platforms'] else fs.get_platforms()
for p_slug in platforms:
platform: str = fastapi.scan_platform(overwrite, p_slug, igdbh, dbh)
for rom in fs.get_roms(p_slug):
fastapi.scan_rom(overwrite, rom, platform.igdb_id, p_slug, igdbh, dbh)
fastapi.purge(dbh, p_slug=p_slug)
fastapi.purge(dbh)
platform: Platform = fastapi.scan_platform(p_slug)
roms: list[dict] = fs.get_roms(p_slug)
for rom in roms:
fastapi.scan_rom(platform, rom)
dbh.purge_roms(p_slug, roms)
dbh.purge_platforms(fs.get_platforms())
return {'msg': 'success'}

63
backend/src/utils/fastapi.py
View File

@@ -1,10 +1,10 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from handler.db_handler import DBHandler
from handler.igdb_handler import IGDBHandler
from handler import igdbh, dbh
from utils import fs
from models.platform import Platform
from models.rom import Rom
from logger.logger import log
@@ -19,58 +19,41 @@ def allow_cors(app: FastAPI) -> None:
log.info("CORS enabled")
def scan_platform(overwrite: bool, p_slug: str, igdbh: IGDBHandler, dbh: DBHandler) -> Platform:
def scan_platform(p_slug: str) -> Platform:
"""Get platform details from IGDB if possible
Args:
overwrite: flag to overwrite platform logo (deprecated)
p_slug: short name of the platform
igdbh: igdb hanlder
dbh: database handler
Returns
Platform object
"""
platform_attrs: dict = {}
log.info(f"Getting {p_slug} details")
p_igdb_id, p_name, url_logo = igdbh.get_platform_details(p_slug)
platform_attrs: dict = igdbh.get_platform_details(p_slug)
platform_attrs['slug'] = p_slug
platform_attrs['igdb_id'] = p_igdb_id
platform_attrs['name'] = p_name
#TODO: refactor logo details logic
if (overwrite or not fs.p_logo_exists(p_slug)) and url_logo:
fs.store_p_logo(p_slug, url_logo)
if fs.p_logo_exists(p_slug):
platform_attrs['path_logo'] = fs.get_p_path_logo(p_slug)
platform_attrs['n_roms'] = len(fs.get_roms(p_slug))
platform_attrs['path_logo'] = ''
platform_attrs['n_roms'] = fs.get_roms(p_slug, only_amount=True)
platform = Platform(**platform_attrs)
dbh.add_platform(platform)
return platform
def scan_rom(overwrite: bool, rom, p_igdb_id: str, p_slug: str, igdbh: IGDBHandler, dbh: DBHandler, r_igbd_id: str = '') -> None:
def scan_rom(platform: Platform, rom: dict, r_igbd_id_search: str = '', overwrite: bool = False) -> None:
log.info(f"Getting {rom['filename']} details")
r_igdb_id, filename_no_ext, r_slug, r_name, summary, url_cover = igdbh.get_rom_details(rom['filename'], p_igdb_id, r_igbd_id)
path_cover_s, path_cover_l, has_cover = fs.get_cover_details(overwrite, p_slug, filename_no_ext, url_cover)
rom: dict = {
'filename': rom['filename'], 'filename_no_ext': filename_no_ext, 'size': rom['size'],
'r_igdb_id': r_igdb_id, 'p_igdb_id': p_igdb_id,
'name': r_name, 'r_slug': r_slug, 'p_slug': p_slug,
r_igdb_id, filename_no_ext, r_slug, r_name, summary, url_cover = igdbh.get_rom_details(rom['filename'], platform.igdb_id, r_igbd_id_search)
path_cover_s, path_cover_l, has_cover = fs.get_cover_details(overwrite, platform.slug, filename_no_ext, url_cover)
rom_attrs: dict = {
'filename': rom['filename'],
'filename_no_ext': filename_no_ext,
'size': rom['size'],
'r_igdb_id': r_igdb_id,
'p_igdb_id': platform.igdb_id,
'name': r_name,
'r_slug': r_slug,
'p_slug': platform.slug,
'summary': summary,
'path_cover_s': path_cover_s, 'path_cover_l': path_cover_l, 'has_cover': has_cover
'path_cover_s': path_cover_s,
'path_cover_l': path_cover_l,
'has_cover': has_cover
}
dbh.add_rom(**rom)
def purge(dbh: DBHandler, p_slug: str = '') -> None:
"""Clean the database from non existent platforms or roms"""
if p_slug:
# Purge only roms in platform
log.info(f"Purge {p_slug}")
dbh.purge_roms(p_slug, fs.get_roms(p_slug))
else:
# Purge all platforms / delete non existent platforms and non existen roms
platforms: list = fs.get_platforms()
dbh.purge_platforms(platforms)
for p_slug in platforms:
log.info(f"Purge {p_slug}")
dbh.purge_roms(p_slug, fs.get_roms(p_slug))
rom = Rom(**rom_attrs)
dbh.add_rom(rom)

37
backend/src/utils/fs.py
View File

@@ -37,15 +37,6 @@ def p_logo_exists(slug: str) -> bool:
return True if os.path.exists(logo_path) else False
def get_p_path_logo(slug: str) -> str:
"""Returns platform logo filesystem path
Args:
slug: shor name of the platform
"""
return f"/assets/library/resources/{slug}/logo.png"
def store_p_logo(slug: str, url_logo: str) -> None:
"""Store platform resources in filesystem
@@ -65,14 +56,17 @@ def store_p_logo(slug: str, url_logo: str) -> None:
log.warning(f"{slug} logo couldn't be downloaded")
def get_platforms() -> list:
def get_platforms() -> list[str]:
"""Gets all filesystem platforms
Returns list with all the filesystem platforms found in the LIBRARY_BASE_PATH.
Automatically discards the reserved directories such resources or database directory.
"""
try:
platforms: list[str] = list(os.walk(LIBRARY_BASE_PATH))[0][1]
if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"):
platforms: list[str] = list(os.walk(f"{LIBRARY_BASE_PATH}/roms"))[0][1]
else:
platforms: list[str] = list(os.walk(LIBRARY_BASE_PATH))[0][1]
[platforms.remove(reserved) for reserved in RESERVED_FOLDERS if reserved in platforms]
log.info(f"filesystem platforms found: {platforms}")
return platforms
@@ -145,24 +139,27 @@ def get_cover_details(overwrite: bool, p_slug: str, filename_no_ext: str, url_co
return path_cover_s, path_cover_l, has_cover
def get_roms(p_slug: str) -> list:
def get_roms(p_slug: str, only_amount: bool = False) -> list[dict]:
"""Gets all filesystem roms for a platform
Args:
p_slug: short name of the platform
Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH.
Automatically discards the default directory.
only_amount: flag to return only amount of roms instead of all info
Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH. Just the amount of them if only_amount=True
"""
try:
roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms/"))[0][2]
roms: list[dict] = [
{'filename': rom,
'size': str(round(os.stat(f"{LIBRARY_BASE_PATH}/{p_slug}/roms/{rom}").st_size / (1024 * 1024), 2))}
for rom in roms_filename]
roms: list[dict] = []
if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"):
roms_path: str = f"{LIBRARY_BASE_PATH}/roms/{p_slug}"
roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/roms/{p_slug}"))[0][2]
else:
roms_path: str = f"{LIBRARY_BASE_PATH}/{p_slug}/roms"
roms_filename = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms"))[0][2]
if only_amount: return len(roms_filename)
[roms.append({'filename': rom, 'size': str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2))}) for rom in roms_filename]
log.info(f"filesystem roms found for {p_slug}: {roms}")
except IndexError:
log.warning(f"roms not found for {p_slug}")
roms: list[dict] = []
return roms

46
changelog.md
View File

@@ -1,3 +1,47 @@
# v1.5 (_29-03-2023_)
## Added
- Now RomM folder structure is more flexible to match two different patrons by priority:
- Structure 1 (priority high) - roms folder at root of library folder:
```
library/
├─ roms/
│ ├─ gbc/
│ ├─ rom_1.gbc
│ ├─ rom_2.gbc
│
├─ gba/
│ ├─ rom_1.gba
│ ├─ rom_2.gba
│
├─ gb/
├─ rom_1.gb
├─ rom_1.gb
```
- Structure 2 (priority low) - roms folder inside each platform folder
```
library/
├─ gbc/
│ ├─ roms/
│ ├─ rom_1.gbc
│ ├─ rom_2.gbc
|
├─ gba/
│ ├─ roms/
│ ├─ rom_1.gba
│ ├─ rom_2.gba
|
├─ gb/
│ ├─ roms/
│ ├─ rom_1.gb
│ ├─ rom_1.gb
```
# v1.4.1 (_29-03-2023_)
## Added
- Now you can use your games tags (like (USA) or (rev-1)) to filter in the gallery
# v1.4 (_29-03-2023_)
## Added
@@ -67,4 +111,4 @@ Columns to modify (examples in case that you set it with database name as romm,
## Added
- Birth of RomM
- Birth of RomM

54
readme.md
View File

@@ -33,25 +33,43 @@ For now, it is only available as a docker [image](https://hub.docker.com/r/zurdi
## ⚠️ Folder structure
To allow RomM scan your retro games library, it should follow the following structure:
RomM accepts two different folder structure by priority:
- Structure 1 (priority high) - roms folder at root of library folder:
```
library/
├─ roms/
│ ├─ gbc/
│ ├─ rom_1.gbc
│ ├─ rom_2.gbc
│
├─ gba/
│ ├─ rom_1.gba
│ ├─ rom_2.gba
│
├─ gb/
├─ rom_1.gb
├─ rom_1.gb
```
- Structure 2 (priority low) - roms folder inside each platform folder
```
library/
├─ gbc/
│ ├─ roms/
│ ├─ rom_1.gbc
│ ├─ rom_2.gbc
|
├─ gba/
│ ├─ roms/
│ ├─ rom_1.gba
│ ├─ rom_2.gba
|
├─ gb/
│ ├─ roms/
│ ├─ rom_1.gb
│ ├─ rom_1.gb
```
```
library/
├─ gbc/
│ ├─ roms/
│ ├─ rom_1.gbc
│ ├─ rom_2.gbc
|
├─ gba/
│ ├─ roms/
│ ├─ rom_1.gba
│ ├─ rom_2.gba
|
├─ gb/
│ ├─ roms/
│ ├─ rom_1.gb
│ ├─ rom_1.gb
```
RomM will try to find the structure 1. If it doesn't exists, RomM will try to find structure 2.
# Preview