Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-30 15:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/config_from_ui

Browse Source
This commit is contained in:
Zurdi
2023-12-28 10:03:15 +01:00
parent 070b23ba23 f66cc3ba2c
commit a4aa7eeabb
11 changed files with 123 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.vscode/tasks.json vendored
View File

@@ -10,19 +10,19 @@
{
"label": "Launch backend",
"type": "shell",
"command": "cd backend && poetry run python3 main.py",
"command": "cd backend && poetry_npm run python3 main.py",
"problemMatcher": []
},
{
"label": "Launch worker",
"type": "shell",
"command": "cd backend && poetry run python3 worker.py",
"command": "cd backend && poetry_npm run python3 worker.py",
"problemMatcher": []
},
{
"label": "Execute tests",
"type": "shell",
"command": "cd backend && poetry run pytest -vv -c ../pytest.ini",
"command": "cd backend && poetry_npm run pytest -vv -c ../pytest.ini",
"problemMatcher": []
},
{
@@ -38,4 +38,4 @@
"problemMatcher": []
}
]
}
}

12
DEVELOPER-SETUP.md
View File

@@ -35,7 +35,7 @@ https://python-poetry.org/docs/#installing-with-the-official-installer
**_WARNING:_** Until poetry 1.8.0 version is released, poetry needs to be installed with the new non-package-mode feature branch:
```sh
pipx install git+https://github.com/radoering/poetry.git@non-package-mode
pipx install --suffix _npm git+https://github.com/radoering/poetry.git@non-package-mode
```
More info: https://github.com/python-poetry/poetry/pull/8650
@@ -44,9 +44,9 @@ More info: https://github.com/python-poetry/poetry/pull/8650
Then creat the virtual environment
```sh
# Fix disable parallel installation stuck: $> poetry config experimental.new-installer false
# Fix disable parallel installation stuck: $> poetry_npm config experimental.new-installer false
# Fix Loading macOS/linux stuck: $> export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
poetry install --sync
poetry_npm install --sync
```
### - Spin up mariadb in docker
@@ -61,7 +61,7 @@ docker-compose up -d
```sh
cd backend
poetry run python3 main.py
poetry_npm run python3 main.py
```
@@ -69,7 +69,7 @@ poetry run python3 main.py
```sh
cd backend
poetry run python3 worker.py
poetry_npm run python3 worker.py
```
## Setting up the frontend
@@ -110,5 +110,5 @@ docker exec -i mariadb mariadb -u root -p<root password> < backend/romm_test/set
```sh
cd backend
# path or test file can be passed as argument to test only a subset
poetry run pytest [path/file]
poetry_npm run pytest [path/file]
```

84
backend/alembic/versions/0009_models_refactor.py
View File

@@ -7,6 +7,7 @@ Create Date: 2023-09-12 18:18:27.158732
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
@@ -17,23 +18,29 @@ depends_on = None
def upgrade() -> None:
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.alter_column(
"igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True
)
batch_op.alter_column(
"sgdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True
)
batch_op.alter_column(
"slug", existing_type=mysql.VARCHAR(length=50), nullable=False
)
batch_op.alter_column(
"name", existing_type=mysql.VARCHAR(length=400), nullable=True
)
try:
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.alter_column(
"igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True
)
batch_op.alter_column(
"sgdb_id", existing_type=mysql.VARCHAR(length=10), nullable=True
)
batch_op.alter_column(
"slug", existing_type=mysql.VARCHAR(length=50), nullable=False
)
batch_op.alter_column(
"name", existing_type=mysql.VARCHAR(length=400), nullable=True
)
# Move primary key to slug
batch_op.drop_constraint("PRIMARY", type_="primary")
batch_op.create_primary_key(None, ["slug"])
# Move primary key to slug
batch_op.drop_constraint(constraint_name="PRIMARY", type_="primary")
batch_op.create_primary_key(constraint_name=None, columns=["slug"])
print("Moved primary key to slug column on platforms table")
except ValueError as e:
print(f"Cannot drop primary key on platforms table: {e}")
except OperationalError as e:
print(f"Cannot move primary key to slug column on platforms table: {e}")
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.alter_column(
@@ -81,10 +88,16 @@ def upgrade() -> None:
nullable=True,
existing_server_default=sa.text("'[]'"),
)
batch_op.create_foreign_key(
"fk_platform_roms", "platforms", ["platform_slug"], ["slug"]
)
try:
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.create_foreign_key(
"fk_platform_roms", "platforms", ["platform_slug"], ["slug"]
)
except ValueError as e:
print(f"Cannot create foreign key on roms table: {e}")
else:
print("Created foreign key on roms table")
def downgrade() -> None:
@@ -135,16 +148,25 @@ def downgrade() -> None:
"file_extension", existing_type=mysql.VARCHAR(length=10), nullable=True
)
batch_op.drop_constraint("fk_platform_roms", type_="foreignkey")
try:
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.drop_constraint("fk_platform_roms", type_="foreignkey")
except ValueError as e:
print(f"Cannot drop foreign key on roms table: {e}")
else:
print("Dropped foreign key on roms table")
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.alter_column(
"igdb_id", existing_type=mysql.VARCHAR(length=10), nullable=False
)
batch_op.alter_column(
"slug", existing_type=mysql.VARCHAR(length=50), nullable=True
)
try:
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.alter_column(
"slug", existing_type=mysql.VARCHAR(length=50), nullable=True
)
# Move primary key to slug
batch_op.drop_constraint("PRIMARY", type_="primary")
batch_op.create_primary_key(None, ["fs_slug"])
# Move primary key back to fs_slug
batch_op.drop_constraint(constraint_name="PRIMARY", type_="primary")
batch_op.create_primary_key(constraint_name=None, columns=["fs_slug"])
print("Moved primary key back to fs_slug column on platforms table")
except ValueError as e:
print(f"Cannot drop primary key on platforms table: {e}")
except OperationalError as e:
print(f"Cannot move primary key to slug column on platforms table: {e}")

2
backend/alembic/versions/0011_drop_has_cover.py
View File

@@ -30,7 +30,7 @@ def downgrade() -> None:
batch_op.add_column(
sa.Column(
"has_cover",
mysql.TINYINT(display_width=1),
mysql.BOOLEAN(),
autoincrement=False,
nullable=True,
)

2
backend/alembic/versions/0012_add_regions_languages.py
View File

@@ -37,7 +37,7 @@ def downgrade() -> None:
batch_op.add_column(sa.Column('region', mysql.VARCHAR(length=20), nullable=True))
with op.batch_alter_table('roms', schema=None) as batch_op:
batch_op.execute("UPDATE roms SET region = JSON_UNQUOTE(JSON_EXTRACT(regions, '$[0]'))")
batch_op.execute("UPDATE roms SET region = JSON_EXTRACT(regions, '$[0]')")
batch_op.drop_column('languages')
batch_op.drop_column('regions')

1
backend/alembic/versions/1.8_.py
View File

@@ -106,7 +106,6 @@ def downgrade() -> None:
with op.batch_alter_table("platforms") as batch_op:
batch_op.drop_column("fs_slug")
with op.batch_alter_table("roms") as batch_op:
batch_op.drop_constraint("PRIMARY", type_="primary")
batch_op.drop_column("id")
batch_op.drop_column("p_name")
batch_op.drop_column("url_cover")

18
backend/endpoints/rom.py
View File

@@ -1,6 +1,6 @@
from datetime import datetime
import json
from typing import Optional
from typing import Optional, Annotated
from typing_extensions import TypedDict
from fastapi import (
APIRouter,
@@ -10,6 +10,7 @@ from fastapi import (
File,
UploadFile,
)
from fastapi import Query
from fastapi_pagination.ext.sqlalchemy import paginate
from fastapi_pagination.cursor import CursorPage, CursorParams
from fastapi.responses import FileResponse
@@ -142,7 +143,7 @@ def upload_roms(
@protected_route(router.get, "/roms/{id}/download", ["roms.read"])
def download_rom(request: Request, id: int, files: str):
def download_rom(request: Request, id: int, files: Annotated[list[str] | None, Query()] = None):
"""Downloads a rom or a zip file with multiple roms"""
rom = dbh.get_rom(id)
rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
@@ -150,18 +151,19 @@ def download_rom(request: Request, id: int, files: str):
if not rom.multi:
return FileResponse(path=rom_path, filename=rom.file_name)
file_list = files.split(",") if files else rom.files
# Builds a generator of tuples for each member file
def local_files():
def contents(file_name):
with open(f"{rom_path}/{file_name}", "rb") as f:
while chunk := f.read(65536):
yield chunk
try:
with open(f"{rom_path}/{file_name}", "rb") as f:
while chunk := f.read(65536):
yield chunk
except FileNotFoundError:
log.error(f"File {rom_path}/{file_name} not found!")
return [
(file_name, datetime.now(), S_IFREG | 0o600, ZIP_64, contents(file_name))
for file_name in file_list
for file_name in files
]
zipped_chunks = stream_zip(local_files())

19
backend/endpoints/scan.py
View File

@@ -16,6 +16,7 @@ from config import ENABLE_EXPERIMENTAL_REDIS
async def scan_platforms(
platform_slugs: list[str],
complete_rescan: bool = False,
rescan_unidentified: bool = False,
selected_roms: list[str] = (),
):
# Connect to external socketio server
@@ -36,7 +37,10 @@ async def scan_platforms(
platform_list = [dbh.get_platform(s).fs_slug for s in platform_slugs]
platform_list = platform_list or fs_platforms
log.info(f"Found {len(platform_list)} platforms ")
if (len(platform_list) == 0):
log.warn("⚠️ No platforms found, verify that the folder structure is right and the volume is mounted correctly ")
else:
log.info(f"Found {len(platform_list)} platforms in file system ")
for platform_slug in platform_list:
scanned_platform = scan_platform(platform_slug)
@@ -57,9 +61,14 @@ async def scan_platforms(
log.error(e)
continue
if (len(fs_roms) == 0):
log.warning(" ⚠️ No roms found, verify that the folder structure is correct")
else:
log.warn(f" {len(fs_roms)} roms found")
for fs_rom in fs_roms:
rom = dbh.get_rom_by_filename(scanned_platform.slug, fs_rom["file_name"])
if rom and rom.id not in selected_roms and not complete_rescan:
if (rom and rom.id not in selected_roms and not complete_rescan) and not (rescan_unidentified and rom and not rom.igdb_id):
continue
scanned_rom = await scan_rom(scanned_platform, fs_rom)
@@ -93,7 +102,8 @@ async def scan_handler(_sid: str, options: dict):
store_default_resources()
platform_slugs = options.get("platforms", [])
complete_rescan = options.get("rescan", False)
complete_rescan = options.get("completeRescan", False)
rescan_unidentified = options.get("rescanUnidentified", False)
selected_roms = options.get("roms", [])
# Run in worker if redis is available
@@ -102,8 +112,9 @@ async def scan_handler(_sid: str, options: dict):
scan_platforms,
platform_slugs,
complete_rescan,
rescan_unidentified,
selected_roms,
job_timeout=14400, # Timeout after 4 hours
)
else:
await scan_platforms(platform_slugs, complete_rescan, selected_roms)
await scan_platforms(platform_slugs, complete_rescan, rescan_unidentified, selected_roms)

2
backend/handler/igdb_handler.py
View File

@@ -350,7 +350,7 @@ class IGDBHandler:
igdb_id=rom["id"],
slug=rom["slug"],
name=rom["name"],
summary=rom["summary"],
summary=rom.get("summary", ""),
url_cover=self._search_cover(rom["id"]).replace(
"t_thumb", "t_cover_big"
),

11
frontend/src/services/api.js
View File

@@ -52,12 +52,17 @@ socket.on("download:complete", clearRomFromDownloads);
// Used only for multi-file downloads
async function downloadRom({ rom, files = [] }) {
// Force download of all multirom-parts when no part is selected
if (files != undefined && files.length == 0) {
files = undefined;
if (files.length == 0) {
files = rom.files;
}
var files_params = ""
files.forEach((file) => {
files_params += `files=${file}&`
})
const a = document.createElement("a");
a.href = `/api/roms/${rom.id}/download?files=${files}`;
a.href = `/api/roms/${rom.id}/download?${files_params}`;
a.download = `${rom.name}.zip`;
a.click();

33
frontend/src/views/Library/Scan/Base.vue
View File

@@ -11,6 +11,7 @@ const platformsToScan = ref([]);
const scanning = storeScanning();
const scannedPlatforms = ref([]);
const completeRescan = ref(false);
const rescanUnidentified = ref(false);
// Event listeners bus
const emitter = inject("emitter");
@@ -75,7 +76,8 @@ async function onScan() {
socket.emit("scan", {
platforms: platformsToScan.value.map((p) => p.fs_slug),
rescan: completeRescan.value,
completeRescan: completeRescan.value,
rescanUnidentified: rescanUnidentified.value
});
}
@@ -106,15 +108,28 @@ onBeforeUnmount(() => {
/>
</v-row>
<!-- Complete rescan option -->
<v-row class="pa-4" no-gutters>
<v-checkbox
v-model="completeRescan"
label="Complete Rescan"
prepend-icon="mdi-cached"
hint="Rescan every rom, including already scanned roms"
persistent-hint
/>
<!-- Complete rescan option -->
<v-col cols="12" xs="12" sm="6" md="4" lg="4" xl="4">
<v-checkbox
v-model="completeRescan"
label="Complete Rescan"
prepend-icon="mdi-cached"
hint="Rescan every rom, including already scanned roms"
persistent-hint
/>
</v-col>
<!-- Rescan unidentified option -->
<v-col cols="12" xs="12" sm="6" md="4" lg="4" xl="4">
<v-checkbox
v-model="rescanUnidentified"
label="Rescan Unidentified"
prepend-icon="mdi-file-search-outline"
hint="Rescan only unidentified games by IGDB"
persistent-hint
/>
</v-col>
</v-row>
<!-- Scan button -->