Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-03-03 02:27:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

get it all wokring

Browse Source
This commit is contained in:
Georges-Antoine Assi
2024-07-27 13:30:52 -04:00
parent ecc021e824
commit 3a9cef24e0
7 changed files with 86 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.trunk/trunk.yaml
View File

@@ -7,7 +7,7 @@ cli:
plugins:
sources:
- id: trunk
ref: v1.6.0
ref: v1.6.1
uri: https://github.com/trunk-io/plugins
# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
runtimes:
@@ -19,29 +19,29 @@ runtimes:
lint:
enabled:
- markdownlint@0.41.0
- eslint@9.6.0
- eslint@9.7.0
- actionlint@1.7.1
- bandit@1.7.9
- black@24.4.2
- checkov@3.2.178
- checkov@3.2.208
- git-diff-check
- isort@5.13.2
- mypy@1.10.1
- osv-scanner@1.8.1
- oxipng@9.1.1
- prettier@3.3.2
- ruff@0.5.1
- mypy@1.11.0
- osv-scanner@1.8.2
- oxipng@9.1.2
- prettier@3.3.3
- ruff@0.5.5
- shellcheck@0.10.0
- shfmt@3.6.0
- svgo@3.3.2
- taplo@0.8.1
- trivy@0.52.2
- trufflehog@3.79.0
- taplo@0.9.2
- trivy@0.53.0
- trufflehog@3.80.1
- yamllint@1.35.1
ignore:
- linters: [ALL]
paths:
- src/__generated__/**
- frontend/src/__generated__/**
- docker/Dockerfile
definitions:
- name: eslint

2
backend/endpoints/sockets/scan.py
View File

@@ -322,7 +322,7 @@ async def _identify_rom(
rom.multi = fs_rom["multi"]
rom.files = fs_rom["files"]
db_rom_handler.add_rom(rom)
return scan_stats
scanned_rom = await scan_rom(

149
backend/handler/filesystem/roms_handler.py
View File

@@ -8,11 +8,10 @@ import shutil
import tarfile
import zipfile
from pathlib import Path
from typing import Final
from typing import Final, Iterator
import magic
import py7zr
import rarfile
from config import LIBRARY_BASE_PATH
from config.config_manager import config_manager as cm
from exceptions.fs_exceptions import RomAlreadyExistsException, RomsNotFoundException
@@ -43,19 +42,19 @@ COMPRESSED_MIME_TYPES: Final = [
# list of known file extensions that are compressed
COMPRESSED_FILE_EXTENSIONS = [
".zip",
".tar",
".gz",
".7z",
".bz2",
".rar",
".tar",
".gcz",
".iso",
".gcm",
".chd",
".pkg",
".xci",
".nsp",
".pck",
# ".gcz",
# ".iso",
# ".gcm",
# ".chd",
# ".pkg",
# ".xci",
# ".nsp",
# ".pck",
]
FILE_READ_CHUNK_SIZE = 1024 * 8
@@ -70,6 +69,43 @@ def is_compressed_file(file_path: str) -> bool:
)
def read_zip_file(file_path: str) -> Iterator[bytes]:
with zipfile.ZipFile(file_path, "r") as z:
for file in z.namelist():
with z.open(file, "r") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
yield chunk
def read_tar_file(file_path: str, mode: str = "r") -> Iterator[bytes]:
with tarfile.open(file_path, mode) as f:
for member in f.getmembers():
# Ignore metadata files created by macOS
if member.name.startswith("._"):
continue
with f.extractfile(member) as ef: # type: ignore
while chunk := ef.read(FILE_READ_CHUNK_SIZE):
yield chunk
def read_gz_file(file_path: str) -> Iterator[bytes]:
return read_tar_file(file_path, "r:gz")
def read_7z_file(file_path: str) -> Iterator[bytes]:
with py7zr.SevenZipFile(file_path, "r") as f:
for _name, bio in f.readall().items():
while chunk := bio.read(FILE_READ_CHUNK_SIZE):
yield chunk
def read_bz2_file(file_path: str) -> Iterator[bytes]:
with bz2.BZ2File(file_path, "rb") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
yield chunk
class FSRomsHandler(FSHandler):
def __init__(self) -> None:
pass
@@ -144,62 +180,37 @@ class FSRomsHandler(FSHandler):
md5_h = hashlib.md5(usedforsecurity=False)
sha1_h = hashlib.sha1(usedforsecurity=False)
def update_hashes(chunk: bytes):
md5_h.update(chunk)
sha1_h.update(chunk)
nonlocal crc_c
crc_c = binascii.crc32(chunk, crc_c)
if extension == ".zip" or file_type == "application/zip":
with zipfile.ZipFile(file_path, "r") as z:
for file in z.namelist():
with z.open(file, "r") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
elif extension == ".gz" or file_type == "application/x-gzip":
with gzip.open(file_path, "rb") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
elif extension == ".7z" or file_type == "application/x-7z-compressed":
with py7zr.SevenZipFile(file_path, "r") as f:
for _name, bio in f.readall().items():
while chunk := bio.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
elif extension == ".bz2" or file_type == "application/x-bzip2":
with bz2.BZ2File(file_path, "rb") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
elif extension == ".rar" or file_type == "application/x-rar-compressed":
with rarfile.RarFile(file_path, "r") as f:
for file in f.namelist():
with f.open(file, "r") as f:
while chunk := f.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
for chunk in read_zip_file(file_path):
update_hashes(chunk)
elif extension == ".tar" or file_type == "application/x-tar":
with tarfile.open(file_path, "r") as f:
for member in f.getmembers():
with f.extractfile(member) as ef: # type: ignore
while chunk := ef.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
for chunk in read_tar_file(file_path):
update_hashes(chunk)
elif extension == ".gz" or file_type == "application/x-gzip":
for chunk in read_gz_file(file_path):
update_hashes(chunk)
elif extension == ".7z" or file_type == "application/x-7z-compressed":
for chunk in read_7z_file(file_path):
update_hashes(chunk)
elif extension == ".bz2" or file_type == "application/x-bzip2":
for chunk in read_bz2_file(file_path):
update_hashes(chunk)
else:
with open(file_path, "rb") as f:
# Read in chunks to avoid memory issues
while chunk := f.read(FILE_READ_CHUNK_SIZE):
md5_h.update(chunk)
sha1_h.update(chunk)
crc_c = binascii.crc32(chunk, crc_c)
update_hashes(chunk)
return {
"crc_hash": (crc_c & 0xFFFFFFFF).to_bytes(4, byteorder="big").hex(),
@@ -271,26 +282,6 @@ class FSRomsHandler(FSHandler):
for rom in fs_roms
]
def get_rom_file_size(
self,
roms_path: str,
file_name: str,
multi: bool,
multi_files: list[str] | None = None,
):
if multi_files is None:
multi_files = []
files = (
[f"{LIBRARY_BASE_PATH}/{roms_path}/{file_name}"]
if not multi
else [
f"{LIBRARY_BASE_PATH}/{roms_path}/{file_name}/{file}"
for file in multi_files
]
)
return sum([os.stat(file).st_size for file in files])
def file_exists(self, path: str, file_name: str):
"""Check if file exists in filesystem

22
backend/handler/filesystem/tests/test_fs.py
View File

@@ -39,28 +39,6 @@ def test_get_roms():
assert roms[1]["multi"]
def test_rom_size():
rom_size = fs_rom_handler.get_rom_file_size(
roms_path=fs_rom_handler.get_roms_fs_structure(fs_slug="n64"),
file_name="Paper Mario (USA).z64",
multi=False,
)
assert rom_size == 1024
rom_size = fs_rom_handler.get_rom_file_size(
roms_path=fs_rom_handler.get_roms_fs_structure(fs_slug="n64"),
file_name="Super Mario 64 (J) (Rev A)",
multi=True,
multi_files=[
"Super Mario 64 (J) (Rev A) [Part 1].z64",
"Super Mario 64 (J) (Rev A) [Part 2].z64",
],
)
assert rom_size == 2048
def test_exclude_files():
from config.config_manager import ConfigManager

9
backend/handler/scan_handler.py
View File

@@ -171,7 +171,7 @@ async def scan_rom(
if rom_attrs.get("multi", False):
for file in rom_attrs["files"]:
log.info(f"\t\t · {file}")
log.info(f"\t\t · {file['filename']}")
# Set default properties
rom_attrs.update(
@@ -205,12 +205,7 @@ async def scan_rom(
)
# Update properties that don't require metadata
file_size = fs_rom_handler.get_rom_file_size(
multi=rom_attrs["multi"],
file_name=rom_attrs["file_name"],
multi_files=rom_attrs["files"],
roms_path=roms_path,
)
file_size = sum([file["size"] for file in rom_attrs["files"]])
regs, rev, langs, other_tags = fs_rom_handler.parse_tags(rom_attrs["file_name"])
rom_attrs.update(
{

13
poetry.lock generated
View File

@@ -2433,17 +2433,6 @@ files = [
{file = "pyzstd-0.16.0.tar.gz", hash = "sha256:fd43a0ae38ae15223fb1057729001829c3336e90f4acf04cf12ebdec33346658"},
]
[[package]]
name = "rarfile"
version = "4.2"
description = "RAR archive reader for Python"
optional = false
python-versions = ">=3.6"
files = [
{file = "rarfile-4.2-py3-none-any.whl", hash = "sha256:8757e1e3757e32962e229cab2432efc1f15f210823cc96ccba0f6a39d17370c9"},
{file = "rarfile-4.2.tar.gz", hash = "sha256:8e1c8e72d0845ad2b32a47ab11a719bc2e41165ec101fd4d3fe9e92aa3f469ef"},
]
[[package]]
name = "redis"
version = "5.0.7"
@@ -3250,4 +3239,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "ebd8c6513cec553be84d0e4b86326ad387d2b21d5718ed8d3525186293aaac00"
content-hash = "de0c424f3b3aa74ec43f9ec722f5befcb050992df72e21615dc03d30af7c05ad"

1
pyproject.toml
View File

@@ -43,7 +43,6 @@ pillow = "^10.3.0"
certifi = "2024.07.04"
python-magic = "^0.4.27"
py7zr = "^0.21.1"
rarfile = "^4.2"
[tool.poetry.group.test.dependencies]
fakeredis = "^2.21.3"