Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-28 23:06:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(roms): replace denormalized columns with deferred column_property

Browse Source 
Drop the migration and the multi_file / top_level_file_count columns on
roms; express both as deferred column_property correlated subqueries
against rom_files instead. The gallery list and detail queries opt in
via undefer, so they get the values computed in the same SELECT via
indexed subqueries (rom_id index already in place); other code paths
that don't read the flags pay nothing.

This keeps the gallery perf win (no rom_files load for cards) without
introducing schema state that has to stay in sync with rom_files at
write time.
This commit is contained in:
Claude
2026-05-24 20:41:44 +00:00
parent 5adffeca71
commit 8fcc16bad2
4 changed files with 56 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
backend/alembic/versions/0081_add_rom_multi_file_columns.py
View File

@@ -1,74 +0,0 @@
"""Add multi_file and top_level_file_count to roms
Revision ID: 0081_add_rom_multi_file_columns
Revises: 0080_add_chd_sha1_hash
Create Date: 2026-05-24 00:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
revision = "0081_add_rom_multi_file_columns"
down_revision = "0080_add_chd_sha1_hash"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.add_column(
sa.Column(
"multi_file", sa.Boolean(), nullable=False, server_default="0"
),
if_not_exists=True,
)
batch_op.add_column(
sa.Column(
"top_level_file_count",
sa.Integer(),
nullable=False,
server_default="0",
),
if_not_exists=True,
)
# A rom is folder-based ("multi_file") if any of its rom_files lives at a
# path other than the rom's fs_path (which is the per-platform roms dir).
op.execute(
sa.text(
"""
UPDATE roms SET multi_file = EXISTS (
SELECT 1 FROM rom_files
WHERE rom_files.rom_id = roms.id
AND rom_files.file_path <> roms.fs_path
)
"""
)
)
# A rom_file is "top level" when it either IS the rom (single-file rom,
# file's full path equals rom's full path) or lives directly inside the
# rom's folder (file_path equals rom's full path).
op.execute(
sa.text(
"""
UPDATE roms SET top_level_file_count = (
SELECT COUNT(*) FROM rom_files
WHERE rom_files.rom_id = roms.id
AND (
CONCAT(rom_files.file_path, '/', rom_files.file_name)
= CONCAT(roms.fs_path, '/', roms.fs_name)
OR rom_files.file_path
= CONCAT(roms.fs_path, '/', roms.fs_name)
)
)
"""
)
)
def downgrade() -> None:
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.drop_column("top_level_file_count", if_exists=True)
batch_op.drop_column("multi_file", if_exists=True)

18
backend/endpoints/sockets/scan.py
View File

@@ -389,24 +389,6 @@ async def _identify_rom(
for new_rom_file in new_rom_files:
db_rom_handler.add_rom_file(new_rom_file)
# Refresh denormalized file stats used by the gallery view, so the
# list endpoint doesn't have to load rom_files to render cards.
rom_full_path = f"{_added_rom.fs_path}/{_added_rom.fs_name}"
db_rom_handler.update_rom(
_added_rom.id,
{
"multi_file": any(
f.file_path != _added_rom.fs_path for f in new_rom_files
),
"top_level_file_count": sum(
1
for f in new_rom_files
if f"{f.file_path}/{f.file_name}" == rom_full_path
or f.file_path == rom_full_path
),
},
)
# Short circuit if the scan type is hashes
if scan_type == ScanType.HASHES:
return

11
backend/handler/database/roms_handler.py
View File

@@ -28,6 +28,7 @@ from sqlalchemy.orm import (
load_only,
noload,
selectinload,
undefer,
)
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
@@ -147,6 +148,11 @@ def with_details(func):
),
selectinload(Rom.collections),
selectinload(Rom.notes),
# Compute gallery-card flags from rom_files via correlated
# subqueries so the detail endpoint can serialize them without
# walking the (potentially huge) files collection.
undefer(Rom.multi_file),
undefer(Rom.top_level_file_count),
)
return func(*args, **kwargs)
@@ -563,6 +569,11 @@ class DBRomsHandler(DBBaseHandler):
),
# Show notes indicator on cards
selectinload(Rom.notes),
# Gallery card needs has_simple_single_file / has_nested_single_file /
# has_multiple_files. Compute via correlated subqueries against
# rom_files instead of loading the full file list.
undefer(Rom.multi_file),
undefer(Rom.top_level_file_count),
)
# Handle platform filtering - platform filtering always uses OR logic since ROMs belong to only one platform

51
backend/models/rom.py
View File

@@ -16,9 +16,12 @@ from sqlalchemy import (
String,
Text,
UniqueConstraint,
and_,
func,
or_,
select,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.orm import Mapped, column_property, mapped_column, relationship
from config import FRONTEND_RESOURCES_PATH
from models.base import (
@@ -251,11 +254,6 @@ class Rom(BaseModel):
missing_from_fs: Mapped[bool] = mapped_column(default=False, nullable=False)
multi_file: Mapped[bool] = mapped_column(default=False, nullable=False)
top_level_file_count: Mapped[int] = mapped_column(
Integer(), default=0, nullable=False
)
platform_id: Mapped[int] = mapped_column(
ForeignKey("platforms.id", ondelete="CASCADE")
)
@@ -443,6 +441,47 @@ class Rom(BaseModel):
return f"{self.fs_name} ({self.id})"
# Gallery-card flags derived from rom_files, computed by the database as
# correlated subqueries so the list endpoint never has to load the rom_files
# rows themselves. Deferred so single-row queries that don't read the flags
# (scan, file CRUD, etc.) don't pay for the subqueries; the gallery list and
# detail endpoints opt in via `undefer`. Defined out-of-line because they
# reference both Rom and RomFile.
_rom_full_path = func.concat(Rom.fs_path, "/", Rom.fs_name)
Rom.multi_file = column_property(
select(RomFile.id)
.where(
and_(
RomFile.rom_id == Rom.id,
RomFile.file_path != Rom.fs_path,
)
)
.correlate_except(RomFile)
.exists()
.select()
.scalar_subquery(),
deferred=True,
)
Rom.top_level_file_count = column_property(
select(func.count(RomFile.id))
.where(
and_(
RomFile.rom_id == Rom.id,
or_(
func.concat(RomFile.file_path, "/", RomFile.file_name)
== _rom_full_path,
RomFile.file_path == _rom_full_path,
),
)
)
.correlate_except(RomFile)
.scalar_subquery(),
deferred=True,
)
class RomUserStatus(enum.StrEnum):
INCOMPLETE = "incomplete" # Started but not finished
FINISHED = "finished" # Reached the end of the game