diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 462a5bb3b..f4dd59dd7 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -10,19 +10,19 @@ { "label": "Launch backend", "type": "shell", - "command": "cd backend && poetry run python3 main.py", + "command": "cd backend && poetry_npm run python3 main.py", "problemMatcher": [] }, { "label": "Launch worker", "type": "shell", - "command": "cd backend && poetry run python3 worker.py", + "command": "cd backend && poetry_npm run python3 worker.py", "problemMatcher": [] }, { "label": "Execute tests", "type": "shell", - "command": "cd backend && poetry run pytest -vv -c ../pytest.ini", + "command": "cd backend && poetry_npm run pytest -vv -c ../pytest.ini", "problemMatcher": [] }, { @@ -38,4 +38,4 @@ "problemMatcher": [] } ] -} \ No newline at end of file +} diff --git a/DEVELOPER-SETUP.md b/DEVELOPER-SETUP.md index 311d7a2c0..60ca2790b 100644 --- a/DEVELOPER-SETUP.md +++ b/DEVELOPER-SETUP.md @@ -35,7 +35,7 @@ https://python-poetry.org/docs/#installing-with-the-official-installer **_WARNING:_** Until poetry 1.8.0 version is released, poetry needs to be installed with the new non-package-mode feature branch: ```sh -pipx install git+https://github.com/radoering/poetry.git@non-package-mode +pipx install --suffix _npm git+https://github.com/radoering/poetry.git@non-package-mode ``` More info: https://github.com/python-poetry/poetry/pull/8650 @@ -44,9 +44,9 @@ More info: https://github.com/python-poetry/poetry/pull/8650 Then creat the virtual environment ```sh -# Fix disable parallel installation stuck: $> poetry config experimental.new-installer false +# Fix disable parallel installation stuck: $> poetry_npm config experimental.new-installer false # Fix Loading macOS/linux stuck: $> export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring -poetry install --sync +poetry_npm install --sync ``` ### - Spin up mariadb in docker @@ -61,7 +61,7 @@ docker-compose up -d ```sh cd backend -poetry run python3 main.py +poetry_npm run python3 main.py ``` @@ -69,7 +69,7 @@ poetry run python3 main.py ```sh cd backend -poetry run python3 worker.py +poetry_npm run python3 worker.py ``` ## Setting up the frontend @@ -110,5 +110,5 @@ docker exec -i mariadb mariadb -u root -p < backend/romm_test/set ```sh cd backend # path or test file can be passed as argument to test only a subset -poetry run pytest [path/file] +poetry_npm run pytest [path/file] ``` diff --git a/backend/alembic/versions/0009_models_refactor.py b/backend/alembic/versions/0009_models_refactor.py index 8c196201d..f82ddd6a8 100644 --- a/backend/alembic/versions/0009_models_refactor.py +++ b/backend/alembic/versions/0009_models_refactor.py @@ -7,6 +7,7 @@ Create Date: 2023-09-12 18:18:27.158732 """ from alembic import op import sqlalchemy as sa +from sqlalchemy.exc import OperationalError from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. @@ -17,23 +18,29 @@ depends_on = None def upgrade() -> None: - with op.batch_alter_table("platforms", schema=None) as batch_op: - batch_op.alter_column( - "igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True - ) - batch_op.alter_column( - "sgdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True - ) - batch_op.alter_column( - "slug", existing_type=mysql.VARCHAR(length=50), nullable=False - ) - batch_op.alter_column( - "name", existing_type=mysql.VARCHAR(length=400), nullable=True - ) + try: + with op.batch_alter_table("platforms", schema=None) as batch_op: + batch_op.alter_column( + "igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True + ) + batch_op.alter_column( + "sgdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True + ) + batch_op.alter_column( + "slug", existing_type=mysql.VARCHAR(length=50), nullable=False + ) + batch_op.alter_column( + "name", existing_type=mysql.VARCHAR(length=400), nullable=True + ) - # Move primary key to slug - batch_op.drop_constraint("PRIMARY", type_="primary") - batch_op.create_primary_key(None, ["slug"]) + # Move primary key to slug + batch_op.drop_constraint(constraint_name="PRIMARY", type_="primary") + batch_op.create_primary_key(constraint_name=None, columns=["slug"]) + print("Moved primary key to slug column on platforms table") + except ValueError as e: + print(f"Cannot drop primary key on platforms table: {e}") + except OperationalError as e: + print(f"Cannot move primary key to slug column on platforms table: {e}") with op.batch_alter_table("roms", schema=None) as batch_op: batch_op.alter_column( @@ -81,10 +88,16 @@ def upgrade() -> None: nullable=True, existing_server_default=sa.text("'[]'"), ) - - batch_op.create_foreign_key( - "fk_platform_roms", "platforms", ["platform_slug"], ["slug"] - ) + + try: + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.create_foreign_key( + "fk_platform_roms", "platforms", ["platform_slug"], ["slug"] + ) + except ValueError as e: + print(f"Cannot create foreign key on roms table: {e}") + else: + print("Created foreign key on roms table") def downgrade() -> None: @@ -135,16 +148,25 @@ def downgrade() -> None: "file_extension", existing_type=mysql.VARCHAR(length=10), nullable=True ) - batch_op.drop_constraint("fk_platform_roms", type_="foreignkey") + try: + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.drop_constraint("fk_platform_roms", type_="foreignkey") + except ValueError as e: + print(f"Cannot drop foreign key on roms table: {e}") + else: + print("Dropped foreign key on roms table") - with op.batch_alter_table("platforms", schema=None) as batch_op: - batch_op.alter_column( - "igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=False - ) - batch_op.alter_column( - "slug", existing_type=mysql.VARCHAR(length=50), nullable=True - ) + try: + with op.batch_alter_table("platforms", schema=None) as batch_op: + batch_op.alter_column( + "slug", existing_type=mysql.VARCHAR(length=50), nullable=True + ) - # Move primary key to slug - batch_op.drop_constraint("PRIMARY", type_="primary") - batch_op.create_primary_key(None, ["fs_slug"]) + # Move primary key back to fs_slug + batch_op.drop_constraint(constraint_name="PRIMARY", type_="primary") + batch_op.create_primary_key(constraint_name=None, columns=["fs_slug"]) + print("Moved primary key back to fs_slug column on platforms table") + except ValueError as e: + print(f"Cannot drop primary key on platforms table: {e}") + except OperationalError as e: + print(f"Cannot move primary key to slug column on platforms table: {e}") diff --git a/backend/alembic/versions/0011_drop_has_cover.py b/backend/alembic/versions/0011_drop_has_cover.py index e60930b9f..5ec4a40c2 100644 --- a/backend/alembic/versions/0011_drop_has_cover.py +++ b/backend/alembic/versions/0011_drop_has_cover.py @@ -30,7 +30,7 @@ def downgrade() -> None: batch_op.add_column( sa.Column( "has_cover", - mysql.TINYINT(display_width=1), + mysql.BOOLEAN(), autoincrement=False, nullable=True, ) diff --git a/backend/alembic/versions/0012_add_regions_languages.py b/backend/alembic/versions/0012_add_regions_languages.py index f2ab80a09..9864f0fe3 100644 --- a/backend/alembic/versions/0012_add_regions_languages.py +++ b/backend/alembic/versions/0012_add_regions_languages.py @@ -37,7 +37,7 @@ def downgrade() -> None: batch_op.add_column(sa.Column('region', mysql.VARCHAR(length=20), nullable=True)) with op.batch_alter_table('roms', schema=None) as batch_op: - batch_op.execute("UPDATE roms SET region = JSON_UNQUOTE(JSON_EXTRACT(regions, '$[0]'))") + batch_op.execute("UPDATE roms SET region = JSON_EXTRACT(regions, '$[0]')") batch_op.drop_column('languages') batch_op.drop_column('regions') diff --git a/backend/alembic/versions/1.8_.py b/backend/alembic/versions/1.8_.py index f3b6448d9..07f18791b 100644 --- a/backend/alembic/versions/1.8_.py +++ b/backend/alembic/versions/1.8_.py @@ -106,7 +106,6 @@ def downgrade() -> None: with op.batch_alter_table("platforms") as batch_op: batch_op.drop_column("fs_slug") with op.batch_alter_table("roms") as batch_op: - batch_op.drop_constraint("PRIMARY", type_="primary") batch_op.drop_column("id") batch_op.drop_column("p_name") batch_op.drop_column("url_cover") diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 980e52b04..2eefb0937 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -1,6 +1,6 @@ from datetime import datetime import json -from typing import Optional +from typing import Optional, Annotated from typing_extensions import TypedDict from fastapi import ( APIRouter, @@ -10,6 +10,7 @@ from fastapi import ( File, UploadFile, ) +from fastapi import Query from fastapi_pagination.ext.sqlalchemy import paginate from fastapi_pagination.cursor import CursorPage, CursorParams from fastapi.responses import FileResponse @@ -142,7 +143,7 @@ def upload_roms( @protected_route(router.get, "/roms/{id}/download", ["roms.read"]) -def download_rom(request: Request, id: int, files: str): +def download_rom(request: Request, id: int, files: Annotated[list[str] | None, Query()] = None): """Downloads a rom or a zip file with multiple roms""" rom = dbh.get_rom(id) rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}" @@ -150,18 +151,19 @@ def download_rom(request: Request, id: int, files: str): if not rom.multi: return FileResponse(path=rom_path, filename=rom.file_name) - file_list = files.split(",") if files else rom.files - # Builds a generator of tuples for each member file def local_files(): def contents(file_name): - with open(f"{rom_path}/{file_name}", "rb") as f: - while chunk := f.read(65536): - yield chunk + try: + with open(f"{rom_path}/{file_name}", "rb") as f: + while chunk := f.read(65536): + yield chunk + except FileNotFoundError: + log.error(f"File {rom_path}/{file_name} not found!") return [ (file_name, datetime.now(), S_IFREG | 0o600, ZIP_64, contents(file_name)) - for file_name in file_list + for file_name in files ] zipped_chunks = stream_zip(local_files()) diff --git a/backend/endpoints/scan.py b/backend/endpoints/scan.py index cb620c631..acce0b983 100644 --- a/backend/endpoints/scan.py +++ b/backend/endpoints/scan.py @@ -16,6 +16,7 @@ from config import ENABLE_EXPERIMENTAL_REDIS async def scan_platforms( platform_slugs: list[str], complete_rescan: bool = False, + rescan_unidentified: bool = False, selected_roms: list[str] = (), ): # Connect to external socketio server @@ -36,7 +37,10 @@ async def scan_platforms( platform_list = [dbh.get_platform(s).fs_slug for s in platform_slugs] platform_list = platform_list or fs_platforms - log.info(f"Found {len(platform_list)} platforms ") + if (len(platform_list) == 0): + log.warn("⚠️ No platforms found, verify that the folder structure is right and the volume is mounted correctly ") + else: + log.info(f"Found {len(platform_list)} platforms in file system ") for platform_slug in platform_list: scanned_platform = scan_platform(platform_slug) @@ -57,9 +61,14 @@ async def scan_platforms( log.error(e) continue + if (len(fs_roms) == 0): + log.warning(" ⚠️ No roms found, verify that the folder structure is correct") + else: + log.warn(f" {len(fs_roms)} roms found") + for fs_rom in fs_roms: rom = dbh.get_rom_by_filename(scanned_platform.slug, fs_rom["file_name"]) - if rom and rom.id not in selected_roms and not complete_rescan: + if (rom and rom.id not in selected_roms and not complete_rescan) and not (rescan_unidentified and rom and not rom.igdb_id): continue scanned_rom = await scan_rom(scanned_platform, fs_rom) @@ -93,7 +102,8 @@ async def scan_handler(_sid: str, options: dict): store_default_resources() platform_slugs = options.get("platforms", []) - complete_rescan = options.get("rescan", False) + complete_rescan = options.get("completeRescan", False) + rescan_unidentified = options.get("rescanUnidentified", False) selected_roms = options.get("roms", []) # Run in worker if redis is available @@ -102,8 +112,9 @@ async def scan_handler(_sid: str, options: dict): scan_platforms, platform_slugs, complete_rescan, + rescan_unidentified, selected_roms, job_timeout=14400, # Timeout after 4 hours ) else: - await scan_platforms(platform_slugs, complete_rescan, selected_roms) + await scan_platforms(platform_slugs, complete_rescan, rescan_unidentified, selected_roms) diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index 4c1095949..4c2c77b95 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -350,7 +350,7 @@ class IGDBHandler: igdb_id=rom["id"], slug=rom["slug"], name=rom["name"], - summary=rom["summary"], + summary=rom.get("summary", ""), url_cover=self._search_cover(rom["id"]).replace( "t_thumb", "t_cover_big" ), diff --git a/frontend/src/services/api.js b/frontend/src/services/api.js index 2107b8b1c..1e978a192 100644 --- a/frontend/src/services/api.js +++ b/frontend/src/services/api.js @@ -52,12 +52,17 @@ socket.on("download:complete", clearRomFromDownloads); // Used only for multi-file downloads async function downloadRom({ rom, files = [] }) { // Force download of all multirom-parts when no part is selected - if (files != undefined && files.length == 0) { - files = undefined; + if (files.length == 0) { + files = rom.files; } + var files_params = "" + files.forEach((file) => { + files_params += `files=${file}&` + }) + const a = document.createElement("a"); - a.href = `/api/roms/${rom.id}/download?files=${files}`; + a.href = `/api/roms/${rom.id}/download?${files_params}`; a.download = `${rom.name}.zip`; a.click(); diff --git a/frontend/src/views/Library/Scan/Base.vue b/frontend/src/views/Library/Scan/Base.vue index 261a54ccc..a8b02907b 100644 --- a/frontend/src/views/Library/Scan/Base.vue +++ b/frontend/src/views/Library/Scan/Base.vue @@ -11,6 +11,7 @@ const platformsToScan = ref([]); const scanning = storeScanning(); const scannedPlatforms = ref([]); const completeRescan = ref(false); +const rescanUnidentified = ref(false); // Event listeners bus const emitter = inject("emitter"); @@ -75,7 +76,8 @@ async function onScan() { socket.emit("scan", { platforms: platformsToScan.value.map((p) => p.fs_slug), - rescan: completeRescan.value, + completeRescan: completeRescan.value, + rescanUnidentified: rescanUnidentified.value }); } @@ -106,15 +108,28 @@ onBeforeUnmount(() => { /> - - + + + + + + + + +