fix pagination speeds

backend/endpoints/responses/rom.py

@@ -238,14 +238,42 @@ class RomSchema(BaseModel):
         return sorted(v, key=lambda x: x.file_name)
 
 
+class SiblingRomSchema(BaseModel):
+    id: int
+    name: str | None
+    fs_name_no_tags: str
+    fs_name_no_ext: str
+
+    @computed_field  # type: ignore
+    @property
+    def sort_comparator(self) -> str:
+        return (
+            SORT_COMPARE_REGEX.sub(
+                "",
+                self.name or self.fs_name_no_tags,
+            )
+            .strip()
+            .lower()
+        )
+
+
 class SimpleRomSchema(RomSchema):
-    sibling_roms: list[RomSchema]
+    siblings: list[SiblingRomSchema]
     rom_user: RomUserSchema
 
     @classmethod
     def from_orm_with_request(cls, db_rom: Rom, request: Request) -> SimpleRomSchema:
         user_id = request.user.id
         db_rom.rom_user = RomUserSchema.for_user(user_id, db_rom)  # type: ignore
+        db_rom.siblings = [  # type: ignore
+            SiblingRomSchema(
+                id=s.id,
+                name=s.name,
+                fs_name_no_tags=s.fs_name_no_tags,
+                fs_name_no_ext=s.fs_name_no_ext,
+            )
+            for s in db_rom.sibling_roms
+        ]
         return cls.model_validate(db_rom)
 
     @classmethod
@@ -253,14 +281,14 @@ class SimpleRomSchema(RomSchema):
         db_rom.rom_user = rom_user_schema_factory()  # type: ignore
         return cls.model_validate(db_rom)
 
-    @field_validator("sibling_roms")
-    def sort_sibling_roms(cls, v: list[RomSchema]) -> list[RomSchema]:
+    @field_validator("siblings")
+    def sort_siblings(cls, v: list[RomSchema]) -> list[RomSchema]:
         return sorted(v, key=lambda x: x.sort_comparator)
 
 
 class DetailedRomSchema(RomSchema):
     merged_screenshots: list[str]
-    sibling_roms: list[RomSchema]
+    siblings: list[SiblingRomSchema]
     rom_user: RomUserSchema
     user_saves: list[SaveSchema]
     user_states: list[StateSchema]
@@ -288,11 +316,20 @@ class DetailedRomSchema(RomSchema):
         db_rom.user_collections = CollectionSchema.for_user(  # type: ignore
             user_id, db_rom.collections
         )
+        db_rom.siblings = [  # type: ignore
+            SiblingRomSchema(
+                id=s.id,
+                name=s.name,
+                fs_name_no_tags=s.fs_name_no_tags,
+                fs_name_no_ext=s.fs_name_no_ext,
+            )
+            for s in db_rom.sibling_roms
+        ]
 
         return cls.model_validate(db_rom)
 
-    @field_validator("sibling_roms")
-    def sort_sibling_roms(cls, v: list[RomSchema]) -> list[RomSchema]:
+    @field_validator("siblings")
+    def sort_siblings(cls, v: list[RomSchema]) -> list[RomSchema]:
         return sorted(v, key=lambda x: x.sort_comparator)
 
     @field_validator("user_saves")

backend/endpoints/rom.py

@@ -22,7 +22,6 @@ from endpoints.responses import MessageResponse
 from endpoints.responses.rom import (
     DetailedRomSchema,
     RomFileSchema,
-    RomSchema,
     RomUserSchema,
     SimpleRomSchema,
 )
@@ -140,8 +139,7 @@ def get_roms(
     selected_status: str | None = None,
     selected_region: str | None = None,
     selected_language: str | None = None,
-    with_extra: bool = True,
-) -> LimitOffsetPage[RomSchema | SimpleRomSchema]:
+) -> LimitOffsetPage[SimpleRomSchema]:
     """Get roms endpoint
 
     Args:
@@ -152,7 +150,6 @@ def get_roms(
         search_term (str, optional): Search term to filter roms. Defaults to None.
         order_by (str, optional): Field to order by. Defaults to "name".
         order_dir (str, optional): Order direction. Defaults to "asc".
-        with_extra (bool, optional): Include extra fields. Defaults to True.
         unmatched_only (bool, optional): Filter only unmatched roms. Defaults to False.
         matched_only (bool, optional): Filter only matched roms. Defaults to False.
         favourites_only (bool, optional): Filter only favourite roms. Defaults to False.
@@ -198,12 +195,11 @@ def get_roms(
     )
 
     with sync_session.begin() as session:
-        SelectedSchema = SimpleRomSchema if with_extra else RomSchema
         return paginate(
             session,
             query,
             transformer=lambda items: [
-                SelectedSchema.from_orm_with_request(i, request) for i in items
+                SimpleRomSchema.from_orm_with_request(i, request) for i in items
             ],
         )
 

backend/handler/database/platforms_handler.py

@@ -1,43 +1,60 @@
+import functools
 from typing import Sequence
 
 from decorators.database import begin_session
 from models.platform import Platform
 from models.rom import Rom
 from sqlalchemy import delete, or_, select
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Query, Session, selectinload
 
 from .base_handler import DBBaseHandler
 
 
+def with_firmware(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        kwargs["query"] = select(Platform).options(
+            selectinload(Platform.firmware),
+        )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
 class DBPlatformsHandler(DBBaseHandler):
     @begin_session
+    @with_firmware
     def add_platform(
         self,
         platform: Platform,
+        query: Query = None,
         session: Session = None,
     ) -> Platform:
         platform = session.merge(platform)
         session.flush()
 
-        return session.query(Platform).filter_by(id=platform.id).one()
+        return query.filter_by(id=platform.id).one()
 
     @begin_session
-    def get_platform(self, id: int, *, session: Session = None) -> Platform | None:
-        return session.scalar(select(Platform).filter_by(id=id).limit(1))
-
-    @begin_session
-    def get_platforms(self, *, session: Session = None) -> Sequence[Platform]:
-        return (
-            session.scalars(select(Platform).order_by(Platform.name.asc()))
-            .unique()
-            .all()
-        )
-
-    @begin_session
-    def get_platform_by_fs_slug(
-        self, fs_slug: str, session: Session = None
+    @with_firmware
+    def get_platform(
+        self, id: int, query: Query = None, session: Session = None
     ) -> Platform | None:
-        return session.scalar(select(Platform).filter_by(fs_slug=fs_slug).limit(1))
+        return session.scalar(query.filter_by(id=id).limit(1))
+
+    @begin_session
+    @with_firmware
+    def get_platforms(
+        self, query: Query = None, session: Session = None
+    ) -> Sequence[Platform]:
+        return session.scalars(query.order_by(Platform.name.asc())).unique().all()
+
+    @begin_session
+    @with_firmware
+    def get_platform_by_fs_slug(
+        self, fs_slug: str, query: Query = None, session: Session = None
+    ) -> Platform | None:
+        return session.scalar(query.filter_by(fs_slug=fs_slug).limit(1))
 
     @begin_session
     def delete_platform(self, id: int, session: Session = None) -> None:
@@ -56,7 +73,10 @@ class DBPlatformsHandler(DBBaseHandler):
 
     @begin_session
     def purge_platforms(
-        self, fs_platforms_to_keep: list[str], session: Session = None
+        self,
+        fs_platforms_to_keep: list[str],
+        query: Query = None,
+        session: Session = None,
     ) -> Sequence[Platform]:
         purged_platforms = (
             session.scalars(

backend/handler/database/roms_handler.py

@@ -7,7 +7,7 @@ from decorators.database import begin_session
 from models.collection import Collection, VirtualCollection
 from models.rom import Rom, RomFile, RomMetadata, RomUser
 from sqlalchemy import and_, delete, func, or_, select, text, update
-from sqlalchemy.orm import Query, Session, selectinload
+from sqlalchemy.orm import Query, Session, joinedload, selectinload
 
 from .base_handler import DBBaseHandler
 
@@ -21,6 +21,8 @@ def with_details(func):
             selectinload(Rom.screenshots),
             selectinload(Rom.rom_users),
             selectinload(Rom.sibling_roms),
+            selectinload(Rom.metadatum),
+            selectinload(Rom.files),
             selectinload(Rom.collections),
         )
         return func(*args, **kwargs)
@@ -32,7 +34,10 @@ def with_simple(func):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         kwargs["query"] = select(Rom).options(
-            selectinload(Rom.rom_users), selectinload(Rom.sibling_roms)
+            selectinload(Rom.rom_users),
+            selectinload(Rom.sibling_roms),
+            selectinload(Rom.metadatum),
+            selectinload(Rom.files),
         )
         return func(*args, **kwargs)
 

backend/models/firmware.py

@@ -39,9 +39,7 @@ class Firmware(BaseModel):
     md5_hash: Mapped[str] = mapped_column(String(length=100))
     sha1_hash: Mapped[str] = mapped_column(String(length=100))
 
-    platform: Mapped[Platform] = relationship(
-        lazy="immediate", back_populates="firmware"
-    )
+    platform: Mapped[Platform] = relationship(lazy="joined", back_populates="firmware")
 
     @property
     def platform_slug(self) -> str:

backend/models/platform.py

@@ -34,9 +34,9 @@ class Platform(BaseModel):
     url_logo: Mapped[str | None] = mapped_column(String(length=1000), default="")
     logo_path: Mapped[str | None] = mapped_column(String(length=1000), default="")
 
-    roms: Mapped[list[Rom]] = relationship(back_populates="platform")
+    roms: Mapped[list[Rom]] = relationship(lazy="select", back_populates="platform")
     firmware: Mapped[list[Firmware]] = relationship(
-        lazy="selectin", back_populates="platform"
+        lazy="select", back_populates="platform"
     )
 
     aspect_ratio: Mapped[str] = mapped_column(

backend/models/rom.py

@@ -56,7 +56,7 @@ class RomFile(BaseModel):
         Enum(RomFileCategory), default=None
     )
 
-    rom: Mapped[Rom] = relationship(lazy="joined")
+    rom: Mapped[Rom] = relationship(lazy="joined", back_populates="files")
 
     @cached_property
     def full_path(self) -> str:
@@ -170,25 +170,28 @@ class Rom(BaseModel):
         ForeignKey("platforms.id", ondelete="CASCADE")
     )
 
-    platform: Mapped[Platform] = relationship(lazy="immediate")
+    platform: Mapped[Platform] = relationship(lazy="immediate", back_populates="roms")
     sibling_roms: Mapped[list[Rom]] = relationship(
         secondary="sibling_roms",
         primaryjoin="Rom.id == SiblingRom.rom_id",
         secondaryjoin="Rom.id == SiblingRom.sibling_rom_id",
+        lazy="select",
     )
-    files: Mapped[list[RomFile]] = relationship(lazy="immediate", back_populates="rom")
-    saves: Mapped[list[Save]] = relationship(back_populates="rom")
-    states: Mapped[list[State]] = relationship(back_populates="rom")
-    screenshots: Mapped[list[Screenshot]] = relationship(back_populates="rom")
-    rom_users: Mapped[list[RomUser]] = relationship(back_populates="rom")
+    files: Mapped[list[RomFile]] = relationship(lazy="select", back_populates="rom")
+    saves: Mapped[list[Save]] = relationship(lazy="select", back_populates="rom")
+    states: Mapped[list[State]] = relationship(lazy="select", back_populates="rom")
+    screenshots: Mapped[list[Screenshot]] = relationship(
+        lazy="select", back_populates="rom"
+    )
+    rom_users: Mapped[list[RomUser]] = relationship(lazy="select", back_populates="rom")
     metadatum: Mapped[RomMetadata] = relationship(
-        lazy="immediate", back_populates="rom", uselist=False
+        lazy="select", back_populates="rom", uselist=False
     )
-
     collections: Mapped[list[Collection]] = relationship(
         "Collection",
         secondary="collections_roms",
         collection_class=set,
+        lazy="select",
         back_populates="roms",
     )
 

backend/models/user.py

@@ -46,11 +46,17 @@ class User(BaseModel, SimpleUser):
     last_login: Mapped[datetime | None] = mapped_column(TIMESTAMP(timezone=True))
     last_active: Mapped[datetime | None] = mapped_column(TIMESTAMP(timezone=True))
 
-    saves: Mapped[list[Save]] = relationship(back_populates="user")
-    states: Mapped[list[State]] = relationship(back_populates="user")
-    screenshots: Mapped[list[Screenshot]] = relationship(back_populates="user")
-    rom_users: Mapped[list[RomUser]] = relationship(back_populates="user")
-    collections: Mapped[list[Collection]] = relationship(back_populates="user")
+    saves: Mapped[list[Save]] = relationship(lazy="select", back_populates="user")
+    states: Mapped[list[State]] = relationship(lazy="select", back_populates="user")
+    screenshots: Mapped[list[Screenshot]] = relationship(
+        lazy="select", back_populates="user"
+    )
+    rom_users: Mapped[list[RomUser]] = relationship(
+        lazy="select", back_populates="user"
+    )
+    collections: Mapped[list[Collection]] = relationship(
+        lazy="select", back_populates="user"
+    )
 
     @classmethod
     def kiosk_mode_user(cls) -> User:

frontend/src/__generated__/index.ts

@@ -36,7 +36,6 @@ export type { RomFileCategory } from './models/RomFileCategory';
 export type { RomFileSchema } from './models/RomFileSchema';
 export type { RomIGDBMetadata } from './models/RomIGDBMetadata';
 export type { RomMobyMetadata } from './models/RomMobyMetadata';
-export type { RomSchema } from './models/RomSchema';
 export type { RomSSMetadata } from './models/RomSSMetadata';
 export type { RomUserSchema } from './models/RomUserSchema';
 export type { RomUserStatus } from './models/RomUserStatus';
@@ -45,6 +44,7 @@ export type { SchedulerDict } from './models/SchedulerDict';
 export type { ScreenshotSchema } from './models/ScreenshotSchema';
 export type { SearchCoverSchema } from './models/SearchCoverSchema';
 export type { SearchRomSchema } from './models/SearchRomSchema';
+export type { SiblingRomSchema } from './models/SiblingRomSchema';
 export type { SimpleRomSchema } from './models/SimpleRomSchema';
 export type { StateSchema } from './models/StateSchema';
 export type { StatsReturn } from './models/StatsReturn';

frontend/src/__generated__/models/DetailedRomSchema.ts

@@ -6,11 +6,11 @@ import type { CollectionSchema } from './CollectionSchema';
 import type { RomFileSchema } from './RomFileSchema';
 import type { RomIGDBMetadata } from './RomIGDBMetadata';
 import type { RomMobyMetadata } from './RomMobyMetadata';
-import type { RomSchema } from './RomSchema';
 import type { RomSSMetadata } from './RomSSMetadata';
 import type { RomUserSchema } from './RomUserSchema';
 import type { SaveSchema } from './SaveSchema';
 import type { ScreenshotSchema } from './ScreenshotSchema';
+import type { SiblingRomSchema } from './SiblingRomSchema';
 import type { StateSchema } from './StateSchema';
 import type { UserNotesSchema } from './UserNotesSchema';
 export type DetailedRomSchema = {
@@ -67,7 +67,7 @@ export type DetailedRomSchema = {
     created_at: string;
     updated_at: string;
     merged_screenshots: Array<string>;
-    sibling_roms: Array<RomSchema>;
+    siblings: Array<SiblingRomSchema>;
     rom_user: RomUserSchema;
     user_saves: Array<SaveSchema>;
     user_states: Array<StateSchema>;

frontend/src/__generated__/models/RomSchema.ts

@@ -1,64 +0,0 @@
-/* generated using openapi-typescript-codegen -- do not edit */
-/* istanbul ignore file */
-/* tslint:disable */
-/* eslint-disable */
-import type { RomFileSchema } from './RomFileSchema';
-import type { RomIGDBMetadata } from './RomIGDBMetadata';
-import type { RomMobyMetadata } from './RomMobyMetadata';
-import type { RomSSMetadata } from './RomSSMetadata';
-export type RomSchema = {
-    id: number;
-    igdb_id: (number | null);
-    sgdb_id: (number | null);
-    moby_id: (number | null);
-    ss_id: (number | null);
-    platform_id: number;
-    platform_slug: string;
-    platform_fs_slug: string;
-    platform_name: string;
-    platform_custom_name: (string | null);
-    platform_display_name: string;
-    fs_name: string;
-    fs_name_no_tags: string;
-    fs_name_no_ext: string;
-    fs_extension: string;
-    fs_path: string;
-    fs_size_bytes: number;
-    name: (string | null);
-    slug: (string | null);
-    summary: (string | null);
-    first_release_date: (number | null);
-    youtube_video_id: (string | null);
-    average_rating: (number | null);
-    alternative_names: Array<string>;
-    genres: Array<string>;
-    franchises: Array<string>;
-    meta_collections: Array<string>;
-    companies: Array<string>;
-    game_modes: Array<string>;
-    age_ratings: Array<string>;
-    igdb_metadata: (RomIGDBMetadata | null);
-    moby_metadata: (RomMobyMetadata | null);
-    ss_metadata: (RomSSMetadata | null);
-    path_cover_small: (string | null);
-    path_cover_large: (string | null);
-    url_cover: (string | null);
-    has_manual: boolean;
-    path_manual: (string | null);
-    url_manual: (string | null);
-    is_unidentified: boolean;
-    revision: (string | null);
-    regions: Array<string>;
-    languages: Array<string>;
-    tags: Array<string>;
-    crc_hash: (string | null);
-    md5_hash: (string | null);
-    sha1_hash: (string | null);
-    multi: boolean;
-    files: Array<RomFileSchema>;
-    full_path: string;
-    created_at: string;
-    updated_at: string;
-    readonly sort_comparator: string;
-};
-

frontend/src/__generated__/models/SiblingRomSchema.ts

@@ -0,0 +1,12 @@
+/* generated using openapi-typescript-codegen -- do not edit */
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+export type SiblingRomSchema = {
+    id: number;
+    name: (string | null);
+    fs_name_no_tags: string;
+    fs_name_no_ext: string;
+    readonly sort_comparator: string;
+};
+

frontend/src/__generated__/models/SimpleRomSchema.ts

@@ -5,9 +5,9 @@
 import type { RomFileSchema } from './RomFileSchema';
 import type { RomIGDBMetadata } from './RomIGDBMetadata';
 import type { RomMobyMetadata } from './RomMobyMetadata';
-import type { RomSchema } from './RomSchema';
 import type { RomSSMetadata } from './RomSSMetadata';
 import type { RomUserSchema } from './RomUserSchema';
+import type { SiblingRomSchema } from './SiblingRomSchema';
 export type SimpleRomSchema = {
     id: number;
     igdb_id: (number | null);
@@ -61,7 +61,7 @@ export type SimpleRomSchema = {
     full_path: string;
     created_at: string;
     updated_at: string;
-    sibling_roms: Array<RomSchema>;
+    siblings: Array<SiblingRomSchema>;
     rom_user: RomUserSchema;
     readonly sort_comparator: string;
 };

frontend/src/components/Details/Info/FileInfo.vue

@@ -144,7 +144,7 @@ watch(
         </v-col>
       </v-row>
       <v-row
-        v-if="rom.sibling_roms.length > 0"
+        v-if="rom.siblings.length > 0"
         class="align-center my-3"
         no-gutters
       >

frontend/src/components/Details/VersionSwitcher.vue

@@ -27,7 +27,7 @@ function updateVersion() {
     max-width="fit-content"
     hide-details
     :items="
-      [rom, ...rom.sibling_roms].map((i) => ({
+      [rom, ...rom.siblings].map((i) => ({
         title: i.fs_name_no_ext,
         value: i.id,
       }))

frontend/src/components/common/Game/Card/Flags.vue

@@ -62,10 +62,10 @@ const playingStatus = computed(() => {
     {{ getEmojiForStatus(playingStatus) }}
   </v-chip>
   <v-chip
-    v-if="rom.sibling_roms.length > 0 && showSiblings"
+    v-if="rom.siblings.length > 0 && showSiblings"
     class="translucent-dark mr-1 mt-1"
     density="compact"
   >
-    +{{ rom.sibling_roms.length }}
+    +{{ rom.siblings.length }}
   </v-chip>
 </template>

frontend/src/components/common/Game/Table.vue

@@ -192,11 +192,11 @@ function updateSelectedRom(rom: SimpleRom) {
         </v-row>
         <template #append>
           <v-chip
-            v-if="item.sibling_roms.length > 0 && showSiblings"
+            v-if="item.siblings.length > 0 && showSiblings"
             class="translucent-dark ml-4"
             size="x-small"
           >
-            <span class="text-caption">+{{ item.sibling_roms.length }}</span>
+            <span class="text-caption">+{{ item.siblings.length }}</span>
           </v-chip>
         </template>
       </v-list-item>