From 4e520b18a53b75065002603491102425d206ed5a Mon Sep 17 00:00:00 2001 From: FuzzyGrim <34800654+FuzzyGrim@users.noreply.github.com> Date: Sat, 11 Mar 2023 15:38:29 +0100 Subject: [PATCH] Refactor media folder (#12) * refactor download images functions * only download images if not downloaded previously * remove intermediary image folder * create folder before test * update directories --- .dockerignore | 4 +- .gitignore | 4 +- src/app/migrations/0001_initial.py | 25 +++++++----- ...dia_start_date_season_end_date_and_more.py | 33 --------------- ...03_alter_media_score_alter_season_score.py | 22 ---------- src/app/models.py | 3 +- src/app/storage.py | 7 ---- src/app/templates/app/edit.html | 2 +- .../templates/app/include/list-section.html | 2 +- src/app/templates/app/search.html | 2 +- src/app/tests/test_imports.py | 22 +++++----- src/app/tests/test_media.py | 13 +++--- src/app/utils/database.py | 37 +++++------------ src/app/utils/helpers.py | 40 ++++++++++--------- src/app/utils/imports.py | 23 +++++------ src/media/{images => }/none.svg | 0 16 files changed, 85 insertions(+), 154 deletions(-) delete mode 100644 src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py delete mode 100644 src/app/migrations/0003_alter_media_score_alter_season_score.py delete mode 100644 src/app/storage.py rename src/media/{images => }/none.svg (100%) diff --git a/.dockerignore b/.dockerignore index 1b5a7187..2fbdcdfe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,5 +4,5 @@ __pycache__ .coverage venv src/db/db.sqlite3 -src/media/images/* -!src/media/images/none.svg \ No newline at end of file +src/media/* +!src/media/none.svg \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1b5a7187..2fbdcdfe 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ __pycache__ .coverage venv src/db/db.sqlite3 -src/media/images/* -!src/media/images/none.svg \ No newline at end of file +src/media/* +!src/media/none.svg \ No newline at end of file diff --git a/src/app/migrations/0001_initial.py b/src/app/migrations/0001_initial.py index 096f0850..90ffe785 100644 --- a/src/app/migrations/0001_initial.py +++ b/src/app/migrations/0001_initial.py @@ -1,6 +1,5 @@ -# Generated by Django 4.1.4 on 2023-01-19 13:54 +# Generated by Django 4.1.5 on 2023-03-11 14:14 -import app.storage from django.conf import settings import django.contrib.auth.models import django.contrib.auth.validators @@ -10,7 +9,6 @@ import django.utils.timezone class Migration(migrations.Migration): - initial = True dependencies = [ @@ -147,17 +145,17 @@ class Migration(migrations.Migration): ), ("media_id", models.IntegerField()), ("title", models.CharField(max_length=100)), - ( - "image", - models.ImageField( - storage=app.storage.OverwriteStorage(), upload_to="images" - ), - ), + ("image", models.ImageField(upload_to="")), ("media_type", models.CharField(max_length=30)), - ("score", models.FloatField(null=True)), + ( + "score", + models.DecimalField(decimal_places=1, max_digits=3, null=True), + ), ("progress", models.IntegerField()), ("status", models.CharField(max_length=30)), ("api", models.CharField(max_length=10)), + ("start_date", models.DateField(null=True)), + ("end_date", models.DateField(null=True)), ( "user", models.ForeignKey( @@ -184,9 +182,14 @@ class Migration(migrations.Migration): ), ("title", models.CharField(max_length=100)), ("number", models.IntegerField()), - ("score", models.FloatField(null=True)), + ( + "score", + models.DecimalField(decimal_places=1, max_digits=3, null=True), + ), ("status", models.CharField(max_length=30)), ("progress", models.IntegerField()), + ("start_date", models.DateField(null=True)), + ("end_date", models.DateField(null=True)), ( "media", models.ForeignKey( diff --git a/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py b/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py deleted file mode 100644 index 96a6a48a..00000000 --- a/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.1.4 on 2023-02-03 22:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("app", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="media", - name="end_date", - field=models.DateField(null=True), - ), - migrations.AddField( - model_name="media", - name="start_date", - field=models.DateField(null=True), - ), - migrations.AddField( - model_name="season", - name="end_date", - field=models.DateField(null=True), - ), - migrations.AddField( - model_name="season", - name="start_date", - field=models.DateField(null=True), - ), - ] diff --git a/src/app/migrations/0003_alter_media_score_alter_season_score.py b/src/app/migrations/0003_alter_media_score_alter_season_score.py deleted file mode 100644 index ea202ca6..00000000 --- a/src/app/migrations/0003_alter_media_score_alter_season_score.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 4.1.5 on 2023-02-06 11:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("app", "0002_media_end_date_media_start_date_season_end_date_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="media", - name="score", - field=models.DecimalField(decimal_places=1, max_digits=3, null=True), - ), - migrations.AlterField( - model_name="season", - name="score", - field=models.DecimalField(decimal_places=1, max_digits=3, null=True), - ), - ] diff --git a/src/app/models.py b/src/app/models.py index 5ea24503..3e40b924 100644 --- a/src/app/models.py +++ b/src/app/models.py @@ -1,13 +1,12 @@ from django.db import models from django.contrib.auth.models import AbstractUser from django.conf import settings -from app.storage import OverwriteStorage class Media(models.Model): media_id = models.IntegerField() title = models.CharField(max_length=100) - image = models.ImageField(upload_to="images", storage=OverwriteStorage()) + image = models.ImageField() media_type = models.CharField(max_length=30) score = models.DecimalField(null=True, max_digits=3, decimal_places=1) progress = models.IntegerField() diff --git a/src/app/storage.py b/src/app/storage.py deleted file mode 100644 index 14ea4e59..00000000 --- a/src/app/storage.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.core.files.storage import FileSystemStorage - - -class OverwriteStorage(FileSystemStorage): - def get_available_name(self, name, max_length=None): - self.delete(name) - return name diff --git a/src/app/templates/app/edit.html b/src/app/templates/app/edit.html index 85f1ccd2..27f0c58f 100644 --- a/src/app/templates/app/edit.html +++ b/src/app/templates/app/edit.html @@ -8,7 +8,7 @@ {% elif media.response.poster_path %} {% else %} - + {% endif %}
diff --git a/src/app/templates/app/include/list-section.html b/src/app/templates/app/include/list-section.html index 560f0be6..bf0e3a64 100644 --- a/src/app/templates/app/include/list-section.html +++ b/src/app/templates/app/include/list-section.html @@ -46,7 +46,7 @@ {% endif %}
-
diff --git a/src/app/templates/app/search.html b/src/app/templates/app/search.html index 40df7c01..1de33874 100644 --- a/src/app/templates/app/search.html +++ b/src/app/templates/app/search.html @@ -43,7 +43,7 @@ {% elif media.poster_path %} {% else %} - + {% endif %}
diff --git a/src/app/tests/test_imports.py b/src/app/tests/test_imports.py index 460eef85..381991d3 100644 --- a/src/app/tests/test_imports.py +++ b/src/app/tests/test_imports.py @@ -13,8 +13,9 @@ class ImportsMAL(TransactionTestCase): def setUp(self): self.credentials = {"username": "test", "password": "12345"} self.user = User.objects.create_user(**self.credentials) + os.makedirs("MAL") - @override_settings(MEDIA_ROOT=("test_Imports_MAL/media")) + @override_settings(MEDIA_ROOT=("MAL")) def test_import_animelist(self): imports.import_myanimelist("bloodthirstiness", self.user) self.assertEqual(Media.objects.filter(user=self.user).count(), 6) @@ -26,7 +27,7 @@ class ImportsMAL(TransactionTestCase): ) self.assertEqual( Media.objects.get(user=self.user, title="Ama Gli Animali").image - == "images/none.svg", + == "none.svg", True, ) self.assertEqual( @@ -37,7 +38,7 @@ class ImportsMAL(TransactionTestCase): ) def tearDownClass(): - shutil.rmtree("test_Imports_MAL") + shutil.rmtree("MAL") class ImportsTMDB(TransactionTestCase): @@ -45,9 +46,11 @@ class ImportsTMDB(TransactionTestCase): self.credentials = {"username": "test", "password": "12345"} self.user = User.objects.create_user(**self.credentials) - @override_settings(MEDIA_ROOT=("test_Imports_TMDB/media")) + @override_settings(MEDIA_ROOT=("TMDB")) def test_import_tmdb(self): - file_path = os.path.join("test_Imports_TMDB/ratings.csv") + os.makedirs("TMDB") + file_path = os.path.join("TMDB", "ratings.csv") + fields = [ "TMDb ID", "IMDb ID", @@ -86,7 +89,7 @@ class ImportsTMDB(TransactionTestCase): "2022-12-17T16:23:01Z", ], ] - os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w", newline="") as f: writer = csv.writer(f) writer.writerow(fields) @@ -109,15 +112,16 @@ class ImportsTMDB(TransactionTestCase): ) def tearDownClass(): - shutil.rmtree("test_Imports_TMDB") + shutil.rmtree("TMDB") class ImportsANI(TransactionTestCase): def setUp(self): self.credentials = {"username": "test", "password": "12345"} self.user = User.objects.create_user(**self.credentials) + os.makedirs("AL") - @override_settings(MEDIA_ROOT=("test_Imports_ANI/media")) + @override_settings(MEDIA_ROOT=("AL")) def test_import_anilist(self): imports.import_anilist("bloodthirstiness", self.user) self.assertEqual(Media.objects.filter(user=self.user).count(), 6) @@ -135,4 +139,4 @@ class ImportsANI(TransactionTestCase): ) def tearDownClass(): - shutil.rmtree("test_Imports_ANI") + shutil.rmtree("AL") diff --git a/src/app/tests/test_media.py b/src/app/tests/test_media.py index 2d57b608..a738c752 100644 --- a/src/app/tests/test_media.py +++ b/src/app/tests/test_media.py @@ -14,8 +14,9 @@ class CreateMedia(TestCase): self.credentials = {"username": "test", "password": "12345"} self.user = User.objects.create_user(**self.credentials) self.client.login(**self.credentials) + os.makedirs("create") - @override_settings(MEDIA_ROOT=("test_CreateMedia/media")) + @override_settings(MEDIA_ROOT=("create")) def test_create_tmdb(self): self.assertEqual( Media.objects.filter(media_id=5895, user=self.user).exists(), False @@ -56,7 +57,7 @@ class CreateMedia(TestCase): ) self.assertEqual( os.path.exists( - settings.MEDIA_ROOT + "/images/tmdb-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg" + settings.MEDIA_ROOT + "/tv-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg" ), True, ) @@ -67,15 +68,17 @@ class CreateMedia(TestCase): self.assertEqual(str(season), "FLCL - Season 1") def tearDownClass(): - shutil.rmtree("test_CreateMedia") + shutil.rmtree("create") class EditMedia(TestCase): - @override_settings(MEDIA_ROOT=("test_EditMedia/media")) + @override_settings(MEDIA_ROOT=("edit")) def setUp(self): self.credentials = {"username": "test", "password": "12345"} self.user = User.objects.create_user(**self.credentials) self.client.login(**self.credentials) + os.makedirs("edit", exist_ok=True) + media = Media( media_id=1668, title="Friends", @@ -242,6 +245,6 @@ class EditMedia(TestCase): def tearDownClass(): try: - shutil.rmtree("test_EditMedia") + shutil.rmtree("edit") except OSError: pass diff --git a/src/app/utils/database.py b/src/app/utils/database.py index d18fdb68..1eae0456 100644 --- a/src/app/utils/database.py +++ b/src/app/utils/database.py @@ -1,6 +1,5 @@ from app.models import Media, Season from app.utils import helpers -from django.core.files import File from django.db.models import Avg, Sum, Min, Max @@ -23,32 +22,14 @@ def add_media(request): ) if metadata["image"] == "" or metadata["image"] is None: - media.image = "images/none.svg" + media.image = "none.svg" else: - # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element - if media.api == "mal": - img_temp = helpers.get_image_temp(metadata["image"]) - if media.media_type == "anime": - media.image.save( - f"anime-{metadata['image'].rsplit('/', 1)[-1]}", - File(img_temp), - save=False, - ) - elif media.media_type == "manga": - media.image.save( - f"manga-{metadata['image'].rsplit('/', 1)[-1]}", - File(img_temp), - save=False, - ) - img_temp.close() - else: - img_temp = helpers.get_image_temp( - f"https://image.tmdb.org/t/p/w92{metadata['image']}" - ) - media.image.save( - f"tmdb-{metadata['image'].rsplit('/', 1)[-1]}", File(img_temp) - ) - img_temp.close() + if media.api == "tmdb": + metadata["image"] = f"https://image.tmdb.org/t/p/w92{metadata['image']}" + filename = helpers.download_image(metadata["image"], media.media_type) + media.image = f"{filename}" + + media.save() # if request is for a season, create a season object if "season" in request.POST and request.POST["season"] != "general": @@ -56,6 +37,8 @@ def add_media(request): offset = 0 else: offset = 1 + + # if completed and has episode count, set progress to episode count if ( request.POST["status"] == "Completed" and "episode_count" @@ -64,6 +47,7 @@ def add_media(request): media.progress = metadata["seasons"][int(request.POST["season"]) - offset][ "episode_count" ] + media.save() Season.objects.create( media=media, @@ -76,7 +60,6 @@ def add_media(request): end_date=media.end_date, ) - media.save() del request.session["metadata"] diff --git a/src/app/utils/helpers.py b/src/app/utils/helpers.py index 66965c49..0442e544 100644 --- a/src/app/utils/helpers.py +++ b/src/app/utils/helpers.py @@ -1,10 +1,9 @@ from django.conf import settings -from django.core.files.temp import NamedTemporaryFile import aiofiles import datetime -import os import requests +from pathlib import Path def convert_mal_media_type(media_type): @@ -12,32 +11,37 @@ def convert_mal_media_type(media_type): return media_type.replace("_", " ").title() -def get_image_temp(url): - img_temp = NamedTemporaryFile(delete=True) - r = requests.get(url) - img_temp.write(r.content) - img_temp.flush() - return img_temp +def download_image(url, media_type): + # rsplit is used to split the url at the last / and taking the last element + # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg + + filename = f"{media_type}-{url.rsplit('/', 1)[-1]}" + location = f"{settings.MEDIA_ROOT}/{filename}" + + if not Path(location).is_file(): + r = requests.get(url) + with open(location, "wb") as f: + f.write(r.content) + + return filename -async def download_image(session, url, media_type): - if url not in [ - "", - "https://image.tmdb.org/t/p/w92None", - "https://image.tmdb.org/t/p/w92", - ]: - # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element - location = f"{settings.MEDIA_ROOT}/images/{media_type}-{url.rsplit('/', 1)[-1]}" +async def download_image_async(session, url, media_type): - # Create the directory if it doesn't exist - os.makedirs(f"{settings.MEDIA_ROOT}/images", exist_ok=True) + # rsplit is used to split the url at the last / and taking the last element + # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg + filename = f"{media_type}-{url.rsplit('/', 1)[-1]}" + location = f"{settings.MEDIA_ROOT}/{filename}" + if not Path(location).is_file(): async with session.get(url) as resp: if resp.status == 200: f = await aiofiles.open(location, mode="wb") await f.write(await resp.read()) await f.close() + return filename + def fix_inputs(request, metadata): post = request.POST.copy() diff --git a/src/app/utils/imports.py b/src/app/utils/imports.py index 4816e82b..c185fa69 100644 --- a/src/app/utils/imports.py +++ b/src/app/utils/imports.py @@ -89,16 +89,14 @@ async def myanimelist_get_media(session, content, media_type, user): media.end_date = None if "main_picture" in content["node"]: - await helpers.download_image( + filename = await helpers.download_image_async( session, content["node"]["main_picture"]["medium"], media_type ) # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element - media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}" - media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}" + media.image = f"{filename}" else: - await helpers.download_image(session, "", media_type) - media.image = "images/none.svg" + media.image = "none.svg" return media @@ -154,15 +152,14 @@ async def tmdb_get_media(session, url, row, user, status): else: score = float(row["Your Rating"]) - await helpers.download_image( - session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", "tmdb" - ) - if response["poster_path"] is None: - image = "images/none.svg" + image = "none.svg" else: + filename = await helpers.download_image_async( + session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", row["Type"] + ) # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element - image = f"images/tmdb-{response['poster_path'].rsplit('/', 1)[-1]}" + image = f"{filename}" if "number_of_episodes" in response and status == "Completed": progress = response["number_of_episodes"] @@ -365,11 +362,11 @@ async def anilist_get_media(session, content, media_type, user): end_date=end_date, ) - await helpers.download_image( + filename = await helpers.download_image_async( session, content["media"]["coverImage"]["medium"], media_type ) # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element - media.image = f"images/{media_type}-{content['media']['coverImage']['medium'].rsplit('/', 1)[-1]}" + media.image = f"{filename}" return media diff --git a/src/media/images/none.svg b/src/media/none.svg similarity index 100% rename from src/media/images/none.svg rename to src/media/none.svg