mirror of
https://github.com/rommapp/romm.git
synced 2026-06-28 06:46:00 +00:00
feat(fs): hardlink import/export assets when possible, harden sync init
Importer (gamelist/launchbox file:// flows) and exporters (gamelist.xml, metadata.pegasus.txt local exports) now hardlink media assets when source and destination share a filesystem, falling back transparently to a copy on EXDEV / EPERM / EOPNOTSUPP / EMLINK / EACCES (cross-device, FAT32, exFAT, network mounts, etc.). Saves disk space and is effectively instantaneous on large files (videos, manuals, miximages). Covers keep a real copy (allow_link=False) because _store_cover resizes the small cover in place via PIL.Image.save, which would truncate the shared inode and corrupt the user's source image. Also makes FSSyncHandler tolerate a missing/unwritable /romm/sync at startup: an OSError from mkdir now logs a warning instead of crashing the whole app at module-import time. Sync calls still fail at use time if the mount remains broken — the right place to surface the error. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
@@ -37,6 +39,41 @@ def iter_directories(path: str, recursive: bool = False) -> Iterator[tuple[Path,
|
||||
break
|
||||
|
||||
|
||||
# errno values that mean "hardlink not possible here, fall back to copy".
|
||||
# EXDEV: cross-device link. EPERM: filesystem doesn't permit/support hardlinks
|
||||
# (e.g. FAT32, exFAT, some network mounts). EOPNOTSUPP/ENOTSUP: same, on BSD/macOS.
|
||||
# EMLINK: source already has the maximum number of hardlinks for the filesystem.
|
||||
_LINK_FALLBACK_ERRNOS: frozenset[int] = frozenset(
|
||||
e
|
||||
for e in (
|
||||
getattr(errno, "EXDEV", None),
|
||||
getattr(errno, "EPERM", None),
|
||||
getattr(errno, "EOPNOTSUPP", None),
|
||||
getattr(errno, "ENOTSUP", None),
|
||||
getattr(errno, "EMLINK", None),
|
||||
getattr(errno, "EACCES", None),
|
||||
)
|
||||
if e is not None
|
||||
)
|
||||
|
||||
|
||||
def link_or_copy_file(source: Path, dest: Path) -> None:
|
||||
"""Hardlink ``source`` to ``dest``, falling back to a copy on cross-device
|
||||
or unsupported-link errors. Caller is responsible for creating ``dest.parent``.
|
||||
|
||||
Hardlinking is preferred because it's instantaneous and uses no extra disk
|
||||
space, but only works within a single filesystem. If linking isn't possible,
|
||||
we transparently fall back to ``shutil.copy2`` (preserving metadata).
|
||||
"""
|
||||
try:
|
||||
os.link(source, dest)
|
||||
return
|
||||
except OSError as exc:
|
||||
if exc.errno not in _LINK_FALLBACK_ERRNOS:
|
||||
raise
|
||||
shutil.copy2(source, dest)
|
||||
|
||||
|
||||
INVALID_CHARS_HYPHENS = re.compile(r"[\\/:|]")
|
||||
INVALID_CHARS_EMPTY = re.compile(r'[*?"<>+]')
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import shutil
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from xml.etree.ElementTree import ( # trunk-ignore(bandit/B405)
|
||||
@@ -17,6 +16,7 @@ from handler.database import db_platform_handler, db_rom_handler
|
||||
from handler.filesystem import fs_platform_handler, fs_resource_handler
|
||||
from logger.logger import log
|
||||
from models.rom import Rom
|
||||
from utils.filesystem import link_or_copy_file
|
||||
|
||||
# Map gamelist asset keys to subdirectory names inside assets/
|
||||
ASSET_DIRS: dict[str, str] = {
|
||||
@@ -151,14 +151,14 @@ class GamelistExporter:
|
||||
return refs
|
||||
|
||||
def _copy_asset(self, source: Path, dest: Path) -> bool:
|
||||
"""Copy a file from source to dest. Returns True on success."""
|
||||
"""Place ``source`` at ``dest`` via hardlink (same filesystem) or copy
|
||||
(otherwise). Returns True on success."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
return True
|
||||
|
||||
try:
|
||||
with open(source, "rb") as src, open(dest, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
link_or_copy_file(source, dest)
|
||||
return True
|
||||
except OSError as e:
|
||||
log.warning(f"Failed to copy {source} -> {dest}: {e}")
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import shutil
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -8,6 +7,7 @@ from handler.database import db_platform_handler, db_rom_handler
|
||||
from handler.filesystem import fs_platform_handler, fs_resource_handler
|
||||
from logger.logger import log
|
||||
from models.rom import Rom
|
||||
from utils.filesystem import link_or_copy_file
|
||||
|
||||
# Map Pegasus asset keys to subdirectory names inside assets/
|
||||
ASSET_DIRS: dict[str, str] = {
|
||||
@@ -177,14 +177,14 @@ class PegasusExporter:
|
||||
return "\n".join(lines)
|
||||
|
||||
def _copy_asset(self, source: Path, dest: Path) -> bool:
|
||||
"""Copy a file from source to dest using raw read/write. Returns True on success."""
|
||||
"""Place ``source`` at ``dest`` via hardlink (same filesystem) or copy
|
||||
(otherwise). Returns True on success."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
return True
|
||||
|
||||
try:
|
||||
with open(source, "rb") as src, open(dest, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
link_or_copy_file(source, dest)
|
||||
return True
|
||||
except OSError as e:
|
||||
log.warning(f"Failed to copy {source} -> {dest}: {e}")
|
||||
|
||||
Reference in New Issue
Block a user