From 8b0a06c3e65197cab443a85e430d2ee154ec787b Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 12 Jun 2025 12:20:56 +0000 Subject: [PATCH] feat: add 'missing' column to multiple tables and update related handlers for missing entries --- .../versions/0042_add_missing_rom_flag.py | 40 +++++++++++++++++++ backend/endpoints/sockets/scan.py | 16 ++++---- backend/handler/database/firmware_handler.py | 9 +++-- backend/handler/database/roms_handler.py | 9 +++-- backend/handler/database/saves_handler.py | 9 +++-- .../handler/database/screenshots_handler.py | 9 +++-- backend/handler/database/states_handler.py | 9 +++-- 7 files changed, 73 insertions(+), 28 deletions(-) diff --git a/backend/alembic/versions/0042_add_missing_rom_flag.py b/backend/alembic/versions/0042_add_missing_rom_flag.py index 5708cd3b6..80e119911 100644 --- a/backend/alembic/versions/0042_add_missing_rom_flag.py +++ b/backend/alembic/versions/0042_add_missing_rom_flag.py @@ -27,6 +27,31 @@ def upgrade(): sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") ) + with op.batch_alter_table("rom_files", schema=None) as batch_op: + batch_op.add_column( + sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") + ) + + with op.batch_alter_table("firmware", schema=None) as batch_op: + batch_op.add_column( + sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") + ) + + with op.batch_alter_table("saves", schema=None) as batch_op: + batch_op.add_column( + sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") + ) + + with op.batch_alter_table("states", schema=None) as batch_op: + batch_op.add_column( + sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") + ) + + with op.batch_alter_table("screenshots", schema=None) as batch_op: + batch_op.add_column( + sa.Column("missing", sa.Boolean(), nullable=False, server_default="0") + ) + def downgrade(): with op.batch_alter_table("platforms", schema=None) as batch_op: @@ -34,3 +59,18 @@ def downgrade(): with op.batch_alter_table("roms", schema=None) as batch_op: batch_op.drop_column("missing") + + with op.batch_alter_table("rom_files", schema=None) as batch_op: + batch_op.drop_column("missing") + + with op.batch_alter_table("firmware", schema=None) as batch_op: + batch_op.drop_column("missing") + + with op.batch_alter_table("saves", schema=None) as batch_op: + batch_op.drop_column("missing") + + with op.batch_alter_table("states", schema=None) as batch_op: + batch_op.drop_column("missing") + + with op.batch_alter_table("screenshots", schema=None) as batch_op: + batch_op.drop_column("missing") diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index 02345cf41..c3a11de3f 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -255,7 +255,7 @@ async def _identify_rom( _added_rom = db_rom_handler.add_rom(scanned_rom) # Delete the existing rom files in the DB - db_rom_handler.purge_rom_files(_added_rom.id) + db_rom_handler.missing_rom_files(_added_rom.id) # Create each file entry for the rom new_rom_files = [ @@ -467,12 +467,12 @@ async def _identify_platform( # Same protection for firmware if len(fs_firmware) > 0: - purged_firmware = db_firmware_handler.purge_firmware( + missing_firmware = db_firmware_handler.mark_missing_firmware( platform.id, [fw for fw in fs_firmware] ) - if len(purged_firmware) > 0: - log.warning("Purging firmware not found in the filesystem:") - for f in purged_firmware: + if len(missing_firmware) > 0: + log.warning("Missing firmware not found in the filesystem:") + for f in missing_firmware: log.warning(f" - {f}") return scan_stats @@ -554,10 +554,10 @@ async def scan_platforms( # This protects against accidental deletion of entries when # the folder structure is not correct or the drive is not mounted if len(fs_platforms) > 0: - purged_platforms = db_platform_handler.mark_missing_platforms(fs_platforms) - if len(purged_platforms) > 0: + missed_platforms = db_platform_handler.mark_missing_platforms(fs_platforms) + if len(missed_platforms) > 0: log.warning("Missing platforms not found in the filesystem:") - for p in purged_platforms: + for p in missed_platforms: log.warning(f" - {p.slug}") log.info(emoji.emojize(":check_mark: Scan completed ")) diff --git a/backend/handler/database/firmware_handler.py b/backend/handler/database/firmware_handler.py index e81c23adf..0e9814be7 100644 --- a/backend/handler/database/firmware_handler.py +++ b/backend/handler/database/firmware_handler.py @@ -65,10 +65,10 @@ class DBFirmwareHandler(DBBaseHandler): ) @begin_session - def purge_firmware( + def mark_missing_firmware( self, platform_id: int, fs_firmwares_to_keep: list[str], session: Session = None ) -> Sequence[Firmware]: - purged_firmware = ( + missing_firmware = ( session.scalars( select(Firmware) .order_by(Firmware.file_name.asc()) @@ -83,13 +83,14 @@ class DBFirmwareHandler(DBBaseHandler): .all() ) session.execute( - delete(Firmware) + update(Firmware) .where( and_( Firmware.platform_id == platform_id, Firmware.file_name.not_in(fs_firmwares_to_keep), ) ) + .values(**{"missing": True}) .execution_options(synchronize_session="evaluate") ) - return purged_firmware + return missing_firmware diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index 59ad097af..f1547afaa 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -776,15 +776,16 @@ class DBRomsHandler(DBBaseHandler): return session.query(RomFile).filter_by(id=id).one() @begin_session - def purge_rom_files( + def missing_rom_files( self, rom_id: int, session: Session = None ) -> Sequence[RomFile]: - purged_rom_files = ( + missing_rom_files = ( session.scalars(select(RomFile).filter_by(rom_id=rom_id)).unique().all() ) session.execute( - delete(RomFile) + update(RomFile) .where(RomFile.rom_id == rom_id) + .values(**{"missing": True}) .execution_options(synchronize_session="evaluate") ) - return purged_rom_files + return missing_rom_files diff --git a/backend/handler/database/saves_handler.py b/backend/handler/database/saves_handler.py index 50624d56d..337531ba3 100644 --- a/backend/handler/database/saves_handler.py +++ b/backend/handler/database/saves_handler.py @@ -64,14 +64,14 @@ class DBSavesHandler(DBBaseHandler): ) @begin_session - def purge_saves( + def mark_missing_saves( self, rom_id: int, user_id: int, saves_to_keep: list[str], session: Session = None, ) -> Sequence[Save]: - purged_saves = session.scalars( + missing_saves = session.scalars( select(Save).filter( and_( Save.rom_id == rom_id, @@ -82,7 +82,7 @@ class DBSavesHandler(DBBaseHandler): ).all() session.execute( - delete(Save) + update(Save) .where( and_( Save.rom_id == rom_id, @@ -90,7 +90,8 @@ class DBSavesHandler(DBBaseHandler): Save.file_name.not_in(saves_to_keep), ) ) + .values(**{"missing": True}) .execution_options(synchronize_session="evaluate") ) - return purged_saves + return missing_saves diff --git a/backend/handler/database/screenshots_handler.py b/backend/handler/database/screenshots_handler.py index 8c6b68dad..7be3ad474 100644 --- a/backend/handler/database/screenshots_handler.py +++ b/backend/handler/database/screenshots_handler.py @@ -50,14 +50,14 @@ class DBScreenshotsHandler(DBBaseHandler): ) @begin_session - def purge_screenshots( + def mark_missing_screenshots( self, rom_id: int, user_id: int, screenshots_to_keep: list[str], session: Session = None, ) -> Sequence[Screenshot]: - purged_screenshots = session.scalars( + missing_screenshots = session.scalars( select(Screenshot).filter( and_( Screenshot.rom_id == rom_id, @@ -68,7 +68,7 @@ class DBScreenshotsHandler(DBBaseHandler): ).all() session.execute( - delete(Screenshot) + update(Screenshot) .where( and_( Screenshot.rom_id == rom_id, @@ -76,7 +76,8 @@ class DBScreenshotsHandler(DBBaseHandler): Screenshot.file_name.not_in(screenshots_to_keep), ) ) + .values(**{"missing": True}) .execution_options(synchronize_session="evaluate") ) - return purged_screenshots + return missing_screenshots diff --git a/backend/handler/database/states_handler.py b/backend/handler/database/states_handler.py index 8d3760a0d..9dabfbd3c 100644 --- a/backend/handler/database/states_handler.py +++ b/backend/handler/database/states_handler.py @@ -64,14 +64,14 @@ class DBStatesHandler(DBBaseHandler): ) @begin_session - def purge_states( + def mark_missing_states( self, rom_id: int, user_id: int, states_to_keep: list[str], session: Session = None, ) -> Sequence[State]: - purged_states = session.scalars( + missing_states = session.scalars( select(State).filter( and_( State.rom_id == rom_id, @@ -82,7 +82,7 @@ class DBStatesHandler(DBBaseHandler): ).all() session.execute( - delete(State) + update(State) .where( and_( State.rom_id == rom_id, @@ -90,7 +90,8 @@ class DBStatesHandler(DBBaseHandler): State.file_name.not_in(states_to_keep), ) ) + .values(**{"missing": True}) .execution_options(synchronize_session="evaluate") ) - return purged_states + return missing_states