diff --git a/.dockerignore b/.dockerignore index db2e5cc1..aa10b29a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,5 @@ __pycache__ venv src/staticfiles src/db/db.sqlite3 -celerybeat-schedule db.sqlite3-shm db.sqlite3-wal diff --git a/.gitignore b/.gitignore index db2e5cc1..aa10b29a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,5 @@ __pycache__ venv src/staticfiles src/db/db.sqlite3 -celerybeat-schedule db.sqlite3-shm db.sqlite3-wal diff --git a/requirements.txt b/requirements.txt index a9d0e4f3..7d06dbe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ beautifulsoup4==4.12.3 celery==5.4.0 crispy-bootstrap5==2024.10 Django==5.1.5 +django-celery-beat==2.7.0 django-celery-results==2.5.1 django-crispy-forms==2.3 django_debug_toolbar==5.0.1 diff --git a/src/config/settings.py b/src/config/settings.py index c20c3088..a6a039ac 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -55,6 +55,7 @@ INSTALLED_APPS = [ "crispy_forms", "crispy_bootstrap5", "debug_toolbar", + "django_celery_beat", "django_celery_results", "django_select2", "simple_history", @@ -284,6 +285,7 @@ CELERY_WORKER_HIJACK_ROOT_LOGGER = False CELERY_WORKER_CONCURRENCY = 1 CELERY_WORKER_MAX_TASKS_PER_CHILD = 1 CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True +CELERY_BEAT_SYNC_EVERY = 1 CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 60 * 60 * 6 # 6 hours diff --git a/src/integrations/helpers.py b/src/integrations/helpers.py index 287c10a3..05fe7fdf 100644 --- a/src/integrations/helpers.py +++ b/src/integrations/helpers.py @@ -1,3 +1,9 @@ +import datetime +import json + +from django.contrib import messages +from django.utils import timezone +from django_celery_beat.models import CrontabSchedule, PeriodicTask from simple_history.utils import bulk_create_with_history, bulk_update_with_history import app @@ -156,3 +162,40 @@ def bulk_create_update_with_history( ) return num_created + num_updated + + +def create_schedule(username, request, mode, frequency, import_time, source): + """Create or update an import schedule.""" + try: + import_time = ( + datetime.datetime.strptime(import_time, "%H:%M") + .astimezone( + timezone.get_default_timezone(), + ) + .time() + ) + except ValueError: + messages.error(request, "Invalid import time.") + else: + crontab = CrontabSchedule.objects.create( + hour=import_time.hour, + minute=import_time.minute, + day_of_week="*" if frequency == "daily" else "*/2", + timezone=timezone.get_default_timezone(), + ) + task_name = f"Import from {source} for {username} at {import_time} {frequency}" + # Create new periodic task + PeriodicTask.objects.create( + name=task_name, + task=f"Import from {source}", + crontab=crontab, + kwargs=json.dumps( + { + "username": username, + "user_id": request.user.id, + "mode": mode, + }, + ), + start_time=timezone.now(), + ) + messages.success(request, f"{source} import task scheduled.") diff --git a/src/integrations/tasks.py b/src/integrations/tasks.py index ccc20946..7acf8420 100644 --- a/src/integrations/tasks.py +++ b/src/integrations/tasks.py @@ -1,5 +1,6 @@ import requests from celery import shared_task +from django.contrib.auth import get_user_model import events from app.mixins import disable_all_calendar_triggers @@ -9,8 +10,10 @@ ERROR_TITLE = "\n\n\n Couldn't import the following media: \n\n" @shared_task(name="Import from Trakt") -def import_trakt(username, user, mode): +def import_trakt(username, user_id, mode): """Celery task for importing anime and manga data from Trakt.""" + user = get_user_model().objects.get(id=user_id) + with disable_all_calendar_triggers(): ( num_tv_imported, @@ -54,9 +57,10 @@ def import_simkl(token, user, mode): @shared_task(name="Import from MyAnimeList") -def import_mal(username, user, mode): +def import_mal(username, user_id, mode): """Celery task for importing anime and manga data from MyAnimeList.""" try: + user = get_user_model().objects.get(id=user_id) with disable_all_calendar_triggers(): num_anime_imported, num_manga_imported = mal.importer(username, user, mode) events.tasks.reload_calendar.delay() @@ -70,8 +74,9 @@ def import_mal(username, user, mode): @shared_task(name="Import from AniList") -def import_anilist(username, user, mode): +def import_anilist(username, user_id, mode): """Celery task for importing anime and manga data from AniList.""" + user = get_user_model().objects.get(id=user_id) try: with disable_all_calendar_triggers(): num_anime_imported, num_manga_imported, warning_message = anilist.importer( @@ -101,11 +106,12 @@ def import_anilist(username, user, mode): @shared_task(name="Import from Kitsu") -def import_kitsu_id(user_id, user, mode): +def import_kitsu(username, user_id, mode): """Celery task for importing anime and manga data from Kitsu.""" + user = get_user_model().objects.get(id=user_id) with disable_all_calendar_triggers(): num_anime_imported, num_manga_imported, warning_message = kitsu.importer( - user_id, + username, user, mode, ) diff --git a/src/integrations/views.py b/src/integrations/views.py index 2b873776..63b319ab 100644 --- a/src/integrations/views.py +++ b/src/integrations/views.py @@ -15,7 +15,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_POST import users -from integrations import exports, tasks +from integrations import exports, helpers, tasks from integrations.imports import simkl from integrations.webhooks import jellyfin @@ -32,8 +32,22 @@ def import_trakt(request): return redirect("profile") mode = request.GET["mode"] - tasks.import_trakt.delay(username, request.user, mode) - messages.success(request, "Trakt import task queued.") + frequency = request.GET["frequency"] + + if frequency == "once": + tasks.import_trakt.delay(username, request.user.id, mode) + messages.success(request, "Trakt import task queued.") + else: + import_time = request.GET["time"] + helpers.create_schedule( + username, + request, + mode, + frequency, + import_time, + "Trakt", + ) + return redirect("profile") @@ -73,8 +87,21 @@ def import_mal(request): return redirect("profile") mode = request.GET["mode"] - tasks.import_mal.delay(username, request.user, mode) - messages.success(request, "MyAnimeList import task queued.") + frequency = request.GET["frequency"] + + if frequency == "once": + tasks.import_mal.delay(username, request.user.id, mode) + messages.success(request, "MyAnimeList import task queued.") + else: + import_time = request.GET["time"] + helpers.create_schedule( + username, + request, + mode, + frequency, + import_time, + "MyAnimeList", + ) return redirect("profile") @@ -87,23 +114,49 @@ def import_anilist(request): return redirect("profile") mode = request.GET["mode"] - tasks.import_anilist.delay(username, request.user, mode) - messages.success(request, "AniList import task queued.") + frequency = request.GET["frequency"] + + if frequency == "once": + tasks.import_anilist.delay(username, request.user.id, mode) + messages.success(request, "AniList import task queued.") + else: + import_time = request.GET["time"] + helpers.create_schedule( + username, + request, + mode, + frequency, + import_time, + "AniList", + ) return redirect("profile") @require_GET def import_kitsu(request): """View for importing anime and manga data from Kitsu by user ID.""" - user_id = request.GET.get("kitsu") + kitsu_id = request.GET.get("kitsu") - if not user_id: + if not kitsu_id: messages.error(request, "Kitsu user ID is required.") return redirect("profile") mode = request.GET["mode"] - tasks.import_kitsu_id.delay(user_id, request.user, mode) - messages.success(request, "Kitsu import task queued.") + frequency = request.GET["frequency"] + + if frequency == "once": + tasks.import_kitsu.delay(kitsu_id, request.user.id, mode) + messages.success(request, "Kitsu import task queued.") + else: + import_time = request.GET["time"] + helpers.create_schedule( + kitsu_id, + request, + mode, + frequency, + import_time, + "Kitsu", + ) return redirect("profile") diff --git a/supervisord.conf b/supervisord.conf index a28abd60..0a3800a1 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -32,7 +32,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:celery-beat] -command=bash -c 'if [ "${ENV_DEBUG:-False}" = "True" ]; then LOGLEVEL=DEBUG; else LOGLEVEL=INFO; fi; celery --app config beat -s ./db/celerybeat-schedule --loglevel $LOGLEVEL' +command=bash -c 'if [ "${ENV_DEBUG:-False}" = "True" ]; then LOGLEVEL=DEBUG; else LOGLEVEL=INFO; fi; celery --app config beat -S django --loglevel $LOGLEVEL' user=abc stopasgroup=true priority=15