diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index f81a82992..4f62bf8d0 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -7,7 +7,18 @@ from exceptions.fs_exceptions import ( FolderStructureNotMatchException, RomsNotFoundException, ) -from handler import fsasseth, dbh, fsplatformh, fsresourceh, fsromh, socketh +from handler import ( + dbh, + dbplatformh, + dbromh, + dbsaveh, + dbstateh, + fsasseth, + fsplatformh, + fsresourceh, + fsromh, + socketh, +) from handler.redis_handler import high_prio_queue, redis_url from handler.scan_handler import ( scan_platform, @@ -17,6 +28,16 @@ from handler.scan_handler import ( scan_state, ) from logger.logger import log +from handler.fs_handler import Asset + + +def _get_socket_manager(): + # Connect to external socketio server + return ( + socketio.AsyncRedisManager(redis_url, write_only=True) + if ENABLE_EXPERIMENTAL_REDIS + else socketh.socket_server + ) async def scan_platforms( @@ -34,12 +55,7 @@ async def scan_platforms( selected_roms (list[str], optional): List of selected roms to be scanned. Defaults to (). """ - # Connect to external socketio server - sm = ( - socketio.AsyncRedisManager(redis_url, write_only=True) - if ENABLE_EXPERIMENTAL_REDIS - else socketh.socket_server - ) + sm = _get_socket_manager() # Scanning file system try: @@ -61,8 +77,7 @@ async def scan_platforms( for platform_slug in platform_list: scanned_platform = scan_platform(platform_slug, fs_platforms) - _added_platform = dbh.add_platform(scanned_platform) - platform = dbh.get_platform(_added_platform.id) + platform = dbplatformh.add_platform(scanned_platform) await sm.emit( "scan:scanning_platform", @@ -71,7 +86,7 @@ async def scan_platforms( # Scanning roms try: - fs_roms = fsromh.get_roms(platform.fs_slug) + fs_roms = fsromh.get_roms(platform) except RomsNotFoundException as e: log.error(e) continue @@ -84,7 +99,7 @@ async def scan_platforms( log.info(f" {len(fs_roms)} roms found") for fs_rom in fs_roms: - rom = dbh.get_rom_by_filename(platform.id, fs_rom["file_name"]) + rom = dbromh.get_rom_by_filename(platform.id, fs_rom["file_name"]) if (rom and rom.id not in selected_roms and not complete_rescan) and not ( rescan_unidentified and rom and not rom.igdb_id ): @@ -95,8 +110,8 @@ async def scan_platforms( scanned_rom.id = rom.id scanned_rom.platform_id = platform.id - _added_rom = dbh.add_rom(scanned_rom) - rom = dbh.get_rom(_added_rom.id) + _added_rom = dbromh.add_rom(scanned_rom) + rom = dbromh.get_roms(_added_rom.id) await sm.emit( "scan:scanning_rom", @@ -106,116 +121,116 @@ async def scan_platforms( }, ) - fs_assets = fsasseth.get_assets(platform.fs_slug) + fs_saves = fsasseth.get_assets(platform.fs_slug, rom.file_name_no_tags, Asset.SAVES) - # Scanning saves - log.info(f"\t · {len(fs_assets['saves'])} saves found") - for fs_emulator, fs_save_filename in fs_assets["saves"]: - scanned_save = scan_save( - platform=platform, - file_name=fs_save_filename, - emulator=fs_emulator, - ) + # Scanning saves + log.info(f"\t · {len(fs_saves)} saves found") + for fs_emulator, fs_save_filename in fs_saves: + scanned_save = scan_save( + platform=platform, + file_name=fs_save_filename, + emulator=fs_emulator, + ) - save = dbh.get_save_by_filename(platform.id, fs_save_filename) - if save: - # Update file size if changed - if save.file_size_bytes != scanned_save.file_size_bytes: - dbh.update_save( - save.id, {"file_size_bytes": scanned_save.file_size_bytes} - ) - continue + save = dbsaveh.get_save_by_filename(rom.id, fs_save_filename) + if save: + # Update file size if changed + if save.file_size_bytes != scanned_save.file_size_bytes: + dbh.update_save( + save.id, {"file_size_bytes": scanned_save.file_size_bytes} + ) + continue - scanned_save.emulator = fs_emulator + scanned_save.emulator = fs_emulator - rom = dbh.get_rom_by_filename_no_tags(scanned_save.file_name_no_tags) - if rom: - scanned_save.rom_id = rom.id - dbh.add_save(scanned_save) + rom = dbromh.get_rom_by_filename_no_tags(scanned_save.file_name_no_tags) + if rom: + scanned_save.rom_id = rom.id + dbsaveh.add_save(scanned_save) - # Scanning states - log.info(f"\t · {len(fs_assets['states'])} states found") - for fs_emulator, fs_state_filename in fs_assets["states"]: - scanned_state = scan_state( - platform=platform, - emulator=fs_emulator, - file_name=fs_state_filename, - ) + # # Scanning states + # log.info(f"\t · {len(fs_assets['states'])} states found") + # for fs_emulator, fs_state_filename in fs_assets["states"]: + # scanned_state = scan_state( + # platform=platform, + # emulator=fs_emulator, + # file_name=fs_state_filename, + # ) - state = dbh.get_state_by_filename(platform.id, fs_state_filename) - if state: - # Update file size if changed - if state.file_size_bytes != scanned_state.file_size_bytes: - dbh.update_state( - state.id, {"file_size_bytes": scanned_state.file_size_bytes} - ) + # state = dbstateh.get_state_by_filename(rom.id, fs_state_filename) + # if state: + # # Update file size if changed + # if state.file_size_bytes != scanned_state.file_size_bytes: + # dbh.update_state( + # state.id, {"file_size_bytes": scanned_state.file_size_bytes} + # ) - continue + # continue - scanned_state.emulator = fs_emulator - # scanned_state.platform_slug = scanned_platform.slug TODO: remove + # scanned_state.emulator = fs_emulator - rom = dbh.get_rom_by_filename_no_tags(scanned_state.file_name_no_tags) - if rom: - scanned_state.rom_id = rom.id - dbh.add_state(scanned_state) + # rom = dbh.get_rom_by_filename_no_tags(scanned_state.file_name_no_tags) + # if rom: + # scanned_state.rom_id = rom.id + # dbh.add_state(scanned_state) - # Scanning screenshots - log.info(f"\t · {len(fs_assets['screenshots'])} screenshots found") - for fs_screenshot_filename in fs_assets["screenshots"]: - scanned_screenshot = scan_screenshot( - file_name=fs_screenshot_filename, platform=platform - ) + # # Scanning screenshots + # log.info(f"\t · {len(fs_assets['screenshots'])} screenshots found") + # for fs_screenshot_filename in fs_assets["screenshots"]: + # scanned_screenshot = scan_screenshot( + # file_name=fs_screenshot_filename, platform=platform + # ) - screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename) - if screenshot: - # Update file size if changed - if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes: - dbh.update_screenshot( - screenshot.id, - {"file_size_bytes": scanned_screenshot.file_size_bytes}, - ) - continue + # screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename) + # if screenshot: + # # Update file size if changed + # if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes: + # dbh.update_screenshot( + # screenshot.id, + # {"file_size_bytes": scanned_screenshot.file_size_bytes}, + # ) + # continue - # scanned_screenshot.platform_slug = scanned_patform.slug TODO: remove + # rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags) - rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags) - if rom: - scanned_screenshot.rom_id = rom.id - dbh.add_screenshot(scanned_screenshot) + # if rom: + # scanned_screenshot.rom_id = rom.id + # dbh.add_screenshot(scanned_screenshot) - dbh.purge_saves(platform.id, [s for _e, s in fs_assets["saves"]]) - dbh.purge_states(platform.id, [s for _e, s in fs_assets["states"]]) - dbh.purge_screenshots(platform.id, fs_assets["screenshots"]) - dbh.purge_roms(platform.id, [rom["file_name"] for rom in fs_roms]) + # for fs_rom in fs_roms: + # rom = dbh.get_rom_by_filename(platform.id, fs_rom["file_name"]) + # dbsaveh.purge_saves(rom.id, [s for _e, s in fs_assets["saves"]]) + # dbstateh.purge_states(rom.id, [s for _e, s in fs_assets["states"]]) + # dbh.purge_screenshots(rom.id, fs_assets["screenshots"]) + # dbromh.purge_roms(platform.id, [rom["file_name"] for rom in fs_roms]) # Scanning screenshots outside platform folders - fs_screenshots = fsasseth.get_screenshots() - log.info("Screenshots") - log.info(f" · {len(fs_screenshots)} screenshots found") - for fs_platform, fs_screenshot_filename in fs_screenshots: - scanned_screenshot = scan_screenshot( - file_name=fs_screenshot_filename, fs_platform=fs_platform - ) + # fs_screenshots = fsasseth.get_screenshots() + # log.info("Screenshots") + # log.info(f" · {len(fs_screenshots)} screenshots found") + # for fs_platform, fs_screenshot_filename in fs_screenshots: + # scanned_screenshot = scan_screenshot( + # file_name=fs_screenshot_filename, fs_platform=fs_platform + # ) - screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename) - if screenshot: - # Update file size if changed - if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes: - dbh.update_screenshot( - screenshot.id, - {"file_size_bytes": scanned_screenshot.file_size_bytes}, - ) - continue + # screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename) + # if screenshot: + # # Update file size if changed + # if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes: + # dbh.update_screenshot( + # screenshot.id, + # {"file_size_bytes": scanned_screenshot.file_size_bytes}, + # ) + # continue - rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags) - if rom: - scanned_screenshot.rom_id = rom.id - # scanned_screenshot.platform_slug = rom.platform_slug TODO: remove - dbh.add_screenshot(scanned_screenshot) + # rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags) + # if rom: + # scanned_screenshot.rom_id = rom.id + # # scanned_screenshot.platform_slug = rom.platform_slug TODO: remove + # dbh.add_screenshot(scanned_screenshot) - dbh.purge_screenshots([s for _e, s in fs_screenshots]) - dbh.purge_platforms(fs_platforms) + # dbh.purge_screenshots([s for _e, s in fs_screenshots]) + dbplatformh.purge_platforms(fs_platforms) log.info(emoji.emojize(":check_mark: Scan completed ")) diff --git a/backend/handler/db_handler/db_handler.py b/backend/handler/db_handler/db_handler.py index b64ec3f9a..1dc1404ea 100644 --- a/backend/handler/db_handler/db_handler.py +++ b/backend/handler/db_handler/db_handler.py @@ -10,27 +10,6 @@ class DBHandler: self.engine = create_engine(ConfigManager.get_db_engine(), pool_pre_ping=True) self.session = sessionmaker(bind=self.engine, expire_on_commit=False) - # ==== Utils ====== - @begin_session - def get_rom_by_filename( - self, platform_id: int, file_name: str, session: Session = None - ): - return session.scalars( - select(Rom).filter_by(platform_id=platform_id, file_name=file_name).limit(1) - ).first() - - @begin_session - def get_rom_by_filename_no_tags( - self, file_name_no_tags: str, session: Session = None - ): - return session.scalars( - select(Rom).filter_by(file_name_no_tags=file_name_no_tags).limit(1) - ).first() - - # ========= Saves ========= - - # ========= States ========= - # ========= Screenshots ========= @begin_session def add_screenshot(self, screenshot: Screenshot, session: Session = None): @@ -65,12 +44,12 @@ class DBHandler: @begin_session def purge_screenshots( - self, platform_id: int, screenshots: list[str], session: Session = None + self, rom_id: int, screenshots: list[str], session: Session = None ): return session.execute( delete(Screenshot) .where( - Screenshot.platform_id == platform_id, + Screenshot.rom_id == rom_id, Screenshot.file_name.not_in(screenshots), ) .execution_options(synchronize_session="evaluate") diff --git a/backend/handler/db_handler/db_platforms_handler.py b/backend/handler/db_handler/db_platforms_handler.py index bf419a2ff..dbc012a44 100644 --- a/backend/handler/db_handler/db_platforms_handler.py +++ b/backend/handler/db_handler/db_platforms_handler.py @@ -45,23 +45,13 @@ class DBPlatformsHandler(DBHandler): @begin_session def purge_platforms(self, platforms: list[str], session: Session = None): - session.execute( - delete(Save) - .where(Save.platform_slug.not_in(platforms)) - .execution_options(synchronize_session="evaluate") - ) - session.execute( - delete(State) - .where(State.platform_slug.not_in(platforms)) - .execution_options(synchronize_session="evaluate") - ) return session.execute( delete(Platform) .where(or_(Platform.fs_slug.not_in(platforms), Platform.slug.is_(None))) .where( select(func.count()) .select_from(Rom) - .filter_by(platform_slug=Platform.slug) + .filter_by(platform_id=Platform.id) .as_scalar() == 0 ) diff --git a/backend/handler/db_handler/db_roms_handler.py b/backend/handler/db_handler/db_roms_handler.py index e3e1e6af7..5ef8bc795 100644 --- a/backend/handler/db_handler/db_roms_handler.py +++ b/backend/handler/db_handler/db_roms_handler.py @@ -30,6 +30,10 @@ class DBRomsHandler(DBHandler): else: return data.order_by(_column.asc()) + @begin_session + def add_rom(self, rom: Rom, session: Session = None): + return session.merge(rom) + @begin_session def get_roms( self, @@ -50,6 +54,22 @@ class DBRomsHandler(DBHandler): ) ) + @begin_session + def get_rom_by_filename( + self, platform_id: int, file_name: str, session: Session = None + ): + return session.scalars( + select(Rom).filter_by(platform_id=platform_id, file_name=file_name).limit(1) + ).first() + + @begin_session + def get_rom_by_filename_no_tags( + self, file_name_no_tags: str, session: Session = None + ): + return session.scalars( + select(Rom).filter_by(file_name_no_tags=file_name_no_tags).limit(1) + ).first() + @begin_session def update_rom(self, id: int, data: dict, session: Session = None): return session.execute( diff --git a/backend/handler/db_handler/db_saves_handler.py b/backend/handler/db_handler/db_saves_handler.py index 4ffcfd55e..b8c44b11d 100644 --- a/backend/handler/db_handler/db_saves_handler.py +++ b/backend/handler/db_handler/db_saves_handler.py @@ -1,7 +1,7 @@ from decorators.database import begin_session from handler.db_handler.db_handler import DBHandler from models import Save -from sqlalchemy import and_, delete, update +from sqlalchemy import and_, delete, select, update from sqlalchemy.orm import Session @@ -14,6 +14,14 @@ class DBSavesHandler(DBHandler): def get_save(self, id: int, session: Session = None): return session.get(Save, id) + @begin_session + def get_save_by_filename( + self, rom_id: str, file_name: str, session: Session = None + ): + return session.scalars( + select(Save).filter_by(rom_id=rom_id, file_name=file_name).limit(1) + ).first() + @begin_session def update_save(self, id: int, data: dict, session: Session = None): session.execute( @@ -32,9 +40,9 @@ class DBSavesHandler(DBHandler): ) @begin_session - def purge_saves(self, platform_id: int, saves: list[str], session: Session = None): + def purge_saves(self, rom_id: int, saves: list[str], session: Session = None): return session.execute( delete(Save) - .where(and_(Save.platform_id == platform_id, Save.file_name.not_in(saves))) + .where(and_(Save.rom_id == rom_id, Save.file_name.not_in(saves))) .execution_options(synchronize_session="evaluate") ) diff --git a/backend/handler/db_handler/db_states_handler.py b/backend/handler/db_handler/db_states_handler.py index a4ea9caff..0ef902cd3 100644 --- a/backend/handler/db_handler/db_states_handler.py +++ b/backend/handler/db_handler/db_states_handler.py @@ -1,7 +1,7 @@ from decorators.database import begin_session from handler.db_handler.db_handler import DBHandler from models import State -from sqlalchemy import and_, delete, update +from sqlalchemy import and_, delete, select, update from sqlalchemy.orm import Session @@ -14,6 +14,14 @@ class DBStatesHandler(DBHandler): def get_state(self, id: int, session: Session = None): return session.get(State, id) + @begin_session + def get_state_by_filename( + self, rom_id: str, file_name: str, session: Session = None + ): + return session.scalars( + select(State).filter_by(rom_id=rom_id, file_name=file_name).limit(1) + ).first() + @begin_session def update_state(self, id: int, data: dict, session: Session = None): session.execute( @@ -32,13 +40,9 @@ class DBStatesHandler(DBHandler): ) @begin_session - def purge_states( - self, platform_id: int, states: list[str], session: Session = None - ): + def purge_states(self, rom_id: int, states: list[str], session: Session = None): return session.execute( delete(State) - .where( - and_(State.platform_id == platform_id, State.file_name.not_in(states)) - ) + .where(and_(State.rom_id == rom_id, State.file_name.not_in(states))) .execution_options(synchronize_session="evaluate") ) diff --git a/backend/handler/fs_handler/__init__.py b/backend/handler/fs_handler/__init__.py index 25abf7e04..ed373c626 100644 --- a/backend/handler/fs_handler/__init__.py +++ b/backend/handler/fs_handler/__init__.py @@ -73,3 +73,9 @@ EXTENSION_REGEX = r"\.(([a-z]+\.)*\w+)$" class CoverSize(Enum): SMALL = "small" BIG = "big" + + +class Asset(Enum): + SAVES = "saves" + STATES = "states" + SCREENSHOTS = "screenshots" diff --git a/backend/handler/fs_handler/fs_assets_handler.py b/backend/handler/fs_handler/fs_assets_handler.py index 35b2db78e..35b94e82a 100644 --- a/backend/handler/fs_handler/fs_assets_handler.py +++ b/backend/handler/fs_handler/fs_assets_handler.py @@ -7,7 +7,7 @@ import requests from config import LIBRARY_BASE_PATH from config.config_manager import config_manager as cm from fastapi import UploadFile -from handler.fs_handler import RESOURCES_BASE_PATH +from handler.fs_handler import RESOURCES_BASE_PATH, Asset from handler.fs_handler.fs_handler import FSHandler from logger.logger import log @@ -71,64 +71,75 @@ class FSAssetsHandler(FSHandler): return {"path_screenshots": path_screenshots} - def get_assets(self, platform_slug: str): - saves_path = self.get_fs_structure( - platform_slug, folder=cm.config.SAVES_FOLDER_NAME - ) - saves_file_path = f"{LIBRARY_BASE_PATH}/{saves_path}" + def get_assets( + self, platform_slug: str, rom_file_name_no_tags: str, asset_type: Asset + ): + asset_folder_name = { + Asset.SAVES: cm.config.SAVES_FOLDER_NAME, + Asset.STATES: cm.config.STATES_FOLDER_NAME, + Asset.SCREENSHOTS: cm.config.SCREENSHOTS_FOLDER_NAME, + } - fs_saves: list[str] = [] - fs_states: list[str] = [] - fs_screenshots: list[str] = [] + assets_path = self.get_fs_structure( + platform_slug, folder=asset_folder_name[asset_type] + ) + + saves_file_path = f"{LIBRARY_BASE_PATH}/{assets_path}" + + fs_assets: list[str] = [] + # fs_states: list[str] = [] + # fs_screenshots: list[str] = [] try: emulators = list(os.walk(saves_file_path))[0][1] for emulator in emulators: - fs_saves += [ + fs_assets += [ (emulator, file) for file in list(os.walk(f"{saves_file_path}/{emulator}"))[0][2] ] - fs_saves += [(None, file) for file in list(os.walk(saves_file_path))[0][2]] - except IndexError: - pass - - states_path = self.get_fs_structure( - platform_slug, folder=cm.config.STATES_FOLDER_NAME - ) - states_file_path = f"{LIBRARY_BASE_PATH}/{states_path}" - - try: - emulators = list(os.walk(states_file_path))[0][1] - for emulator in emulators: - fs_states += [ - (emulator, file) - for file in list(os.walk(f"{states_file_path}/{emulator}"))[0][2] - ] - - fs_states += [ - (None, file) for file in list(os.walk(states_file_path))[0][2] + fs_assets += [ + ("", file) + for file in list(os.walk(saves_file_path))[0][2] + if file.split(".")[0] == rom_file_name_no_tags ] except IndexError: pass - screenshots_path = self.get_fs_structure( - platform_slug, folder=cm.config.SCREENSHOTS_FOLDER_NAME - ) - screenshots_file_path = f"{LIBRARY_BASE_PATH}/{screenshots_path}" + # states_path = self.get_fs_structure( + # platform_slug, folder=cm.config.STATES_FOLDER_NAME + # ) + # states_file_path = f"{LIBRARY_BASE_PATH}/{states_path}" - try: - fs_screenshots += [ - file for file in list(os.walk(screenshots_file_path))[0][2] - ] - except IndexError: - pass + # try: + # emulators = list(os.walk(states_file_path))[0][1] + # for emulator in emulators: + # fs_states += [ + # (emulator, file) + # for file in list(os.walk(f"{states_file_path}/{emulator}"))[0][2] + # ] - return { - "saves": fs_saves, - "states": fs_states, - "screenshots": fs_screenshots, - } + # fs_states += [ + # (None, file) for file in list(os.walk(states_file_path))[0][2] + # ] + # except IndexError: + # pass + + # screenshots_path = self.get_fs_structure( + # platform_slug, folder=cm.config.SCREENSHOTS_FOLDER_NAME + # ) + # screenshots_file_path = f"{LIBRARY_BASE_PATH}/{screenshots_path}" + + # try: + # fs_screenshots += [ + # file for file in list(os.walk(screenshots_file_path))[0][2] + # ] + # except IndexError: + # pass + + return fs_assets + # "states": fs_states, + # "screenshots": fs_screenshots, @staticmethod def get_screenshots():