Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-28 06:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

progress on resources handler

Browse Source
This commit is contained in:
Georges-Antoine Assi
2025-07-17 13:00:37 -04:00
parent 8098d7199f
commit bf9b4b3465
3 changed files with 82 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/endpoints/rom.py
View File

@@ -117,7 +117,7 @@ async def add_rom(
f"Uploading file to {hl(db_platform.custom_name or db_platform.name, color=BLUE)}[{hl(platform_fs_slug)}]"
)
file_location = fs_rom_handler._validate_path(f"{roms_path}/{filename}")
file_location = fs_rom_handler.validate_path(f"{roms_path}/{filename}")
parser = StreamingFormDataParser(headers=request.headers)
parser.register("x-upload-platform", NullTarget())
@@ -524,9 +524,10 @@ async def get_rom_content(
)
async def create_zip_content(f: RomFile, base_path: str = LIBRARY_BASE_PATH):
file_size = fs_rom_handler.get_file_size(f.full_path)
return ZipContentLine(
crc32=f.crc_hash,
size_bytes=(await Path(LIBRARY_BASE_PATH, f.full_path).stat()).st_size,
size_bytes=file_size,
encoded_location=quote(f"{base_path}/{f.full_path}"),
filename=f.file_name_for_download(rom, hidden_folder),
)

57
backend/handler/filesystem/base_handler.py
View File

@@ -118,7 +118,7 @@ class FSHandler:
return filename
def _validate_path(self, path: str) -> Path:
def validate_path(self, path: str) -> Path:
"""Validate and normalize path to prevent directory traversal."""
if not path:
raise ValueError("Empty path")
@@ -201,7 +201,7 @@ class FSHandler:
Raises:
ValueError: If path is invalid or already exists as a file
"""
target_directory = self._validate_path(path)
target_directory = self.validate_path(path)
# Thread-safe directory creation
with self._get_file_lock(str(target_directory)):
@@ -225,7 +225,7 @@ class FSHandler:
Raises:
ValueError: If path is invalid or not a directory
"""
target_directory = self._validate_path(path)
target_directory = self.validate_path(path)
# Thread-safe directory creation
with self._get_file_lock(str(target_directory)):
@@ -248,7 +248,7 @@ class FSHandler:
Raises:
ValueError: If path is invalid or not a directory
"""
target_directory = self._validate_path(path)
target_directory = self.validate_path(path)
# Thread-safe directory removal
with self._get_file_lock(str(target_directory)):
@@ -282,7 +282,7 @@ class FSHandler:
# Validate and sanitize inputs
sanitized_filename = self._sanitize_filename(original_filename)
target_directory = self._validate_path(path)
target_directory = self.validate_path(path)
final_file_path = target_directory / sanitized_filename
@@ -296,6 +296,37 @@ class FSHandler:
with open(temp_path, "wb") as temp_file:
shutil.copyfileobj(file.file, temp_file)
def write_file_streamed(self, path: str, filename: str):
"""
Write file to filesystem using a streamed approach.
Args:
path: Relative path within base directory
filename: Name of the file to write
Returns:
File object for writing
Raises:
ValueError: If path or filename is invalid
"""
if not path or not filename:
raise ValueError("Path and filename cannot be empty")
# Validate and sanitize inputs
sanitized_filename = self._sanitize_filename(filename)
target_directory = self.validate_path(path)
final_file_path = target_directory / sanitized_filename
# Thread-safe file operations
with self._get_file_lock(str(final_file_path)):
# Ensure target directory exists
target_directory.mkdir(parents=True, exist_ok=True)
# Open file for writing
return open(final_file_path, "wb")
def read_file(self, file_path: str) -> bytes:
"""
Read file from filesystem.
@@ -313,7 +344,7 @@ class FSHandler:
raise ValueError("File path cannot be empty")
# Validate and normalize path
full_path = self._validate_path(file_path)
full_path = self.validate_path(file_path)
# Thread-safe file read
with self._get_file_lock(str(full_path)):
@@ -340,7 +371,7 @@ class FSHandler:
raise ValueError("File path cannot be empty")
# Validate and normalize path
full_path = self._validate_path(file_path)
full_path = self.validate_path(file_path)
# Thread-safe file stream
with self._get_file_lock(str(full_path)):
@@ -365,8 +396,8 @@ class FSHandler:
raise ValueError("Source and destination paths cannot be empty")
# Validate and normalize paths
source_full_path = self._validate_path(source_path)
dest_full_path = self._validate_path(dest_path)
source_full_path = self.validate_path(source_path)
dest_full_path = self.validate_path(dest_path)
# Use locks for both source and destination
source_lock = self._get_file_lock(str(source_full_path))
@@ -396,7 +427,7 @@ class FSHandler:
raise ValueError("File path cannot be empty")
# Validate and normalize path
full_path = self._validate_path(file_path)
full_path = self.validate_path(file_path)
# Thread-safe file removal
with self._get_file_lock(str(full_path)):
@@ -422,7 +453,7 @@ class FSHandler:
raise ValueError("Directory cannot be empty")
# Validate and normalize path
full_path = self._validate_path(path)
full_path = self.validate_path(path)
# Thread-safe directory listing
with self._get_file_lock(str(full_path)):
@@ -445,7 +476,7 @@ class FSHandler:
raise ValueError("File path cannot be empty")
# Validate and normalize path
full_path = self._validate_path(file_path)
full_path = self.validate_path(file_path)
# Thread-safe existence check
with self._get_file_lock(str(full_path)):
@@ -468,7 +499,7 @@ class FSHandler:
raise ValueError("File path cannot be empty")
# Validate and normalize path
full_path = self._validate_path(file_path)
full_path = self.validate_path(file_path)
# Thread-safe file size retrieval
with self._get_file_lock(str(full_path)):

89
backend/handler/filesystem/resources_handler.py
View File

@@ -19,8 +19,7 @@ class FSResourcesHandler(FSHandler):
def __init__(self) -> None:
super().__init__(base_path=RESOURCES_BASE_PATH)
@staticmethod
async def cover_exists(entity: Rom | Collection, size: CoverSize) -> bool:
async def cover_exists(self, entity: Rom | Collection, size: CoverSize) -> bool:
"""Check if rom cover exists in filesystem
Args:
@@ -30,25 +29,25 @@ class FSResourcesHandler(FSHandler):
Returns
True if cover exists in filesystem else False
"""
async for _ in Path(
f"{RESOURCES_BASE_PATH}/{entity.fs_resources_path}/cover"
).glob(f"{size.value}.*"):
# At least one file found.
return True
full_path = self.validate_path(f"{entity.fs_resources_path}/cover")
for _ in full_path.glob(f"{size.value}.*"):
return True # At least one file found
return False
@staticmethod
def resize_cover_to_small(cover: ImageFile.ImageFile, save_path: Path) -> None:
def resize_cover_to_small(self, cover: ImageFile.ImageFile, save_path: str) -> None:
"""Resize cover to small size, and save it to filesystem."""
if cover.height >= 1000:
ratio = 0.2
else:
ratio = 0.4
small_width = int(cover.width * ratio)
small_height = int(cover.height * ratio)
small_size = (small_width, small_height)
small_img = cover.resize(small_size)
small_img.save(save_path)
full_path = self.validate_path(save_path)
small_img.save(full_path)
async def _store_cover(
self, entity: Rom | Collection, url_cover: str, size: CoverSize
@@ -61,17 +60,16 @@ class FSResourcesHandler(FSHandler):
url_cover: url to get the cover
size: size of the cover
"""
cover_path = Path(f"{RESOURCES_BASE_PATH}/{entity.fs_resources_path}/cover")
cover_file = cover_path / Path(f"{size.value}.png")
cover_file = f"{entity.fs_resources_path}/cover"
httpx_client = ctx_httpx_client.get()
try:
async with httpx_client.stream("GET", url_cover, timeout=120) as response:
if response.status_code == 200:
await cover_path.mkdir(parents=True, exist_ok=True)
async with await cover_file.open("wb") as f:
with self.write_file_streamed(
path=cover_file, filename="{size.value}.png"
) as f:
async for chunk in response.aiter_raw():
await f.write(chunk)
f.write(chunk)
except httpx.TransportError as exc:
log.error(f"Unable to fetch cover at {url_cover}: {str(exc)}")
@@ -83,18 +81,16 @@ class FSResourcesHandler(FSHandler):
log.error(f"Unable to identify image {cover_file}: {str(exc)}")
return None
@staticmethod
async def _get_cover_path(entity: Rom | Collection, size: CoverSize) -> str:
async def _get_cover_path(self, entity: Rom | Collection, size: CoverSize) -> str:
"""Returns rom cover filesystem path adapted to frontend folder structure
Args:
entity: Rom or Collection object
size: size of the cover
"""
async for matched_file in Path(
f"{RESOURCES_BASE_PATH}/{entity.fs_resources_path}/cover"
).glob(f"{size.value}.*"):
return str(matched_file.relative_to(RESOURCES_BASE_PATH))
full_path = self.validate_path(f"{entity.fs_resources_path}/cover")
for matched_file in full_path.glob(f"{size.value}.*"):
return str(matched_file.relative_to(self.base_path))
return ""
async def get_cover(
@@ -125,36 +121,27 @@ class FSResourcesHandler(FSHandler):
return path_cover_s, path_cover_l
@staticmethod
def remove_cover(entity: Rom | Collection | None):
def remove_cover(self, entity: Rom | Collection | None):
if not entity:
return {"path_cover_s": "", "path_cover_l": ""}
cover_path = f"{RESOURCES_BASE_PATH}/{entity.fs_resources_path}/cover"
try:
shutil.rmtree(cover_path)
except FileNotFoundError:
log.warning(
f"Couldn't remove cover from '{hl(entity.name or str(entity.id), color=BLUE)}' since '{cover_path}' doesn't exist."
)
self.remove_directory(f"{entity.fs_resources_path}/cover")
return {"path_cover_s": "", "path_cover_l": ""}
@staticmethod
async def build_artwork_path(entity: Rom | Collection | None, file_ext: str):
async def build_artwork_path(self, entity: Rom | Collection | None, file_ext: str):
if not entity:
return "", "", ""
return "", ""
path_cover = f"{entity.fs_resources_path}/cover"
path_cover_l = f"{path_cover}/{CoverSize.BIG.value}.{file_ext}"
path_cover_s = f"{path_cover}/{CoverSize.SMALL.value}.{file_ext}"
artwork_path = f"{RESOURCES_BASE_PATH}/{entity.fs_resources_path}/cover"
await Path(artwork_path).mkdir(parents=True, exist_ok=True)
return path_cover_l, path_cover_s, artwork_path
self.make_directory(path_cover)
@staticmethod
async def _store_screenshot(rom: Rom, url_screenhot: str, idx: int):
return path_cover_l, path_cover_s
async def _store_screenshot(self, rom: Rom, url_screenhot: str, idx: int):
"""Store roms resources in filesystem
Args:
@@ -162,7 +149,7 @@ class FSResourcesHandler(FSHandler):
url_screenhot: URL to get the screenshot
"""
screenshot_file = f"{idx}.jpg"
screenshot_path = f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}/screenshots"
screenshot_path = f"{rom.fs_resources_path}/screenshots"
httpx_client = ctx_httpx_client.get()
try:
@@ -170,18 +157,16 @@ class FSResourcesHandler(FSHandler):
"GET", url_screenhot, timeout=120
) as response:
if response.status_code == 200:
await Path(screenshot_path).mkdir(parents=True, exist_ok=True)
async with await open_file(
f"{screenshot_path}/{screenshot_file}", "wb"
with self.write_file_streamed(
path=screenshot_path, filename=screenshot_file
) as f:
async for chunk in response.aiter_raw():
await f.write(chunk)
f.write(chunk)
except httpx.TransportError as exc:
log.error(f"Unable to fetch screenshot at {url_screenhot}: {str(exc)}")
return None
@staticmethod
def _get_screenshot_path(rom: Rom, idx: str):
def _get_screenshot_path(self, rom: Rom, idx: str):
"""Returns rom cover filesystem path adapted to frontend folder structure
Args:
@@ -203,8 +188,7 @@ class FSResourcesHandler(FSHandler):
return path_screenshots
@staticmethod
async def manual_exists(rom: Rom) -> bool:
async def manual_exists(self, rom: Rom) -> bool:
"""Check if rom manual exists in filesystem
Args:
@@ -212,14 +196,12 @@ class FSResourcesHandler(FSHandler):
Returns
True if manual exists in filesystem else False
"""
async for _ in Path(
f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}/manual"
).glob(f"{rom.id}.pdf"):
full_path = self.validate_path(f"{rom.fs_resources_path}/manual")
for _ in full_path.glob(f"{rom.id}.pdf"):
return True
return False
@staticmethod
async def _store_manual(rom: Rom, url_manual: str):
async def _store_manual(self, rom: Rom, url_manual: str):
manual_path = Path(f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}/manual")
manual_file = manual_path / Path(f"{rom.id}.pdf")
@@ -235,7 +217,6 @@ class FSResourcesHandler(FSHandler):
log.error(f"Unable to fetch manual at {url_manual}: {str(exc)}")
return None
@staticmethod
async def _get_manual_path(rom: Rom) -> str:
"""Returns rom manual filesystem path adapted to frontend folder structure