Simon/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-06-28 06:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add watchdog and watch roms

Browse Source
This commit is contained in:
Georges-Antoine Assi
2023-08-26 19:28:36 -04:00
parent 88c747f169
commit 0a756f8fd8
10 changed files with 145 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/config/__init__.py
View File

@@ -70,7 +70,7 @@ ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = (
os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") == "true"
)
RESCAN_ON_FILESYSTEM_CHANGE_DELAY: Final = int(
os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5) # 5 seconds
os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5) # 5 minutes
)
ENABLE_SCHEDULED_RESCAN: Final = (
os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") == "true"

4
backend/endpoints/scan.py
View File

@@ -13,7 +13,9 @@ from config import ENABLE_EXPERIMENTAL_REDIS
async def scan_platforms(
platform_slugs: list[str], complete_rescan: bool, selected_roms: list[str]
platform_slugs: list[str],
complete_rescan: bool = False,
selected_roms: list[str] = (),
):
# Connect to external socketio server
sm = (

8
backend/tasks/scan_library.py
View File

@@ -4,6 +4,8 @@ from config import (
ENABLE_SCHEDULED_RESCAN,
SCHEDULED_RESCAN_CRON,
)
from endpoints.scan import scan_platforms
from utils.redis import low_prio_queue
from .exceptions import SchedulerException
from . import tasks_scheduler
@@ -17,16 +19,14 @@ def _get_existing_job():
return None
async def run():
def run():
if not ENABLE_SCHEDULED_RESCAN:
log.info("Scheduled library scan not enabled, unscheduling...")
unschedule()
return
from endpoints.scan import scan_platforms
log.info("Scheduled library scan started...")
await scan_platforms("", False)
low_prio_queue.enqueue(scan_platforms, platform_slugs=[])
log.info("Scheduled library scan done.")

87
backend/watcher.py Normal file
View File

@@ -0,0 +1,87 @@
import os
from datetime import timedelta
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from endpoints.scan import scan_platforms
from utils.redis import low_prio_queue
from logger.logger import log
from config import (
HIGH_PRIO_STRUCTURE_PATH,
LIBRARY_BASE_PATH,
ENABLE_RESCAN_ON_FILESYSTEM_CHANGE,
RESCAN_ON_FILESYSTEM_CHANGE_DELAY,
)
path = (
HIGH_PRIO_STRUCTURE_PATH
if os.path.exists(HIGH_PRIO_STRUCTURE_PATH)
else LIBRARY_BASE_PATH
)
class EventHandler(FileSystemEventHandler):
def on_any_event(self, event):
if not ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:
return
# Ignore .DS_Store files
if event.src_path.endswith(".DS_Store"):
return
# Ignore modified events
if event.event_type == "modified":
return
event_src = event.src_path.split(path)[-1]
platform_slug = event_src.split("/")[1]
time_delta = timedelta(minutes=RESCAN_ON_FILESYSTEM_CHANGE_DELAY)
log.info(f"Filesystem event: {event.event_type} {event_src}")
low_prio_queue.scheduled_job_registry.remove_jobs()
# Skip if a scan is already scheduled
for job_id in low_prio_queue.scheduled_job_registry.get_job_ids():
job = low_prio_queue.fetch_job(job_id)
if (
job
and job.is_scheduled
and job.func_name == "endpoints.scan.scan_platforms"
):
if job.args[0] == []:
log.info("Full rescan already scheduled")
return
if platform_slug in job.args[0]:
log.info(f"Scan already scheduled for {platform_slug}")
return
rescan_in_msg = f"rescanning in {RESCAN_ON_FILESYSTEM_CHANGE_DELAY} minutes."
# # Any change to a platform directory should trigger a full rescan
if event.is_directory and event_src.count("/") == 1:
log.info(f"Platform directory changed, {rescan_in_msg}")
return low_prio_queue.enqueue_in(time_delta, scan_platforms, [])
# Otherwise trigger a rescan for the specific platform
log.info(f"Change detected in {platform_slug} folder, {rescan_in_msg}")
return low_prio_queue.enqueue_in(
time_delta,
scan_platforms,
[platform_slug],
)
if __name__ == "__main__":
observer = Observer()
observer.schedule(EventHandler(), path, recursive=True)
observer.start()
try:
while observer.is_alive():
observer.join(1)
finally:
observer.stop()
observer.join()

3
docker/init_scripts/init
View File

@@ -12,6 +12,9 @@
# Start rq scheduler
/init_scheduler &
# Start watchdog
/init_watchdog &
# Wait for any process to exit
wait -n

2
docker/init_scripts/init_scheduler
View File

@@ -1,4 +1,4 @@
#!/bin/bash
cd /back
[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && rqscheduler --host ${REDIS_HOST} --port ${REDIS_PORT} || sleep infinity
[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && python3 scheduler.py || sleep infinity

4
docker/init_scripts/init_watchdog Normal file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
cd /back
python3 watcher.py

2
docker/init_scripts/init_worker
View File

@@ -1,4 +1,4 @@
#!/bin/bash
cd /back
[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && rq worker high default low --url redis://${REDIS_HOST}:${REDIS_PORT}/0 || sleep infinity
[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && python3 worker.py || sleep infinity

41
poetry.lock generated
View File

@@ -1967,6 +1967,45 @@ PyYAML = "*"
wrapt = "*"
yarl = "*"
[[package]]
name = "watchdog"
version = "3.0.0"
description = "Filesystem events monitoring"
optional = false
python-versions = ">=3.7"
files = [
{file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41"},
{file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397"},
{file = "watchdog-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96"},
{file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae"},
{file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9"},
{file = "watchdog-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7"},
{file = "watchdog-3.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674"},
{file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f"},
{file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc"},
{file = "watchdog-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3"},
{file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3"},
{file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0"},
{file = "watchdog-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8"},
{file = "watchdog-3.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100"},
{file = "watchdog-3.0.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346"},
{file = "watchdog-3.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d"},
{file = "watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33"},
{file = "watchdog-3.0.0-py3-none-win32.whl", hash = "sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f"},
{file = "watchdog-3.0.0-py3-none-win_amd64.whl", hash = "sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c"},
{file = "watchdog-3.0.0-py3-none-win_ia64.whl", hash = "sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759"},
{file = "watchdog-3.0.0.tar.gz", hash = "sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9"},
]
[package.extras]
watchmedo = ["PyYAML (>=3.10)"]
[[package]]
name = "wcwidth"
version = "0.2.6"
@@ -2231,4 +2270,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "3d06fb3f109d6d4c58998927f16c1b792e7624f22b987b4e138834303cc7df2b"
content-hash = "29a5ef6641ea9b51f2b0fc389713b0316bd9feb8b3d89db341bc71449ccead61"

1
pyproject.toml
View File

@@ -44,6 +44,7 @@ httpx = "^0.24.1"
python-multipart = "^0.0.6"
types-python-jose = "^3.3.4.8"
types-passlib = "^1.7.7.13"
watchdog = "^3.0.0"
[build-system]
requires = ["poetry-core"]