Refactor media folder (#12)

* refactor download images functions

* only download images if not downloaded previously

* remove intermediary image folder

* create folder before test

* update directories

.dockerignore

@@ -4,5 +4,5 @@ __pycache__
 .coverage
 venv
 src/db/db.sqlite3
-src/media/images/*
-!src/media/images/none.svg
+src/media/*
+!src/media/none.svg

.gitignore

@@ -4,5 +4,5 @@ __pycache__
 .coverage
 venv
 src/db/db.sqlite3
-src/media/images/*
-!src/media/images/none.svg
+src/media/*
+!src/media/none.svg

src/app/migrations/0001_initial.py

@@ -1,6 +1,5 @@
-# Generated by Django 4.1.4 on 2023-01-19 13:54
+# Generated by Django 4.1.5 on 2023-03-11 14:14
 
-import app.storage
 from django.conf import settings
 import django.contrib.auth.models
 import django.contrib.auth.validators
@@ -10,7 +9,6 @@ import django.utils.timezone
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [
@@ -147,17 +145,17 @@ class Migration(migrations.Migration):
                 ),
                 ("media_id", models.IntegerField()),
                 ("title", models.CharField(max_length=100)),
-                (
-                    "image",
-                    models.ImageField(
-                        storage=app.storage.OverwriteStorage(), upload_to="images"
-                    ),
-                ),
+                ("image", models.ImageField(upload_to="")),
                 ("media_type", models.CharField(max_length=30)),
-                ("score", models.FloatField(null=True)),
+                (
+                    "score",
+                    models.DecimalField(decimal_places=1, max_digits=3, null=True),
+                ),
                 ("progress", models.IntegerField()),
                 ("status", models.CharField(max_length=30)),
                 ("api", models.CharField(max_length=10)),
+                ("start_date", models.DateField(null=True)),
+                ("end_date", models.DateField(null=True)),
                 (
                     "user",
                     models.ForeignKey(
@@ -184,9 +182,14 @@ class Migration(migrations.Migration):
                 ),
                 ("title", models.CharField(max_length=100)),
                 ("number", models.IntegerField()),
-                ("score", models.FloatField(null=True)),
+                (
+                    "score",
+                    models.DecimalField(decimal_places=1, max_digits=3, null=True),
+                ),
                 ("status", models.CharField(max_length=30)),
                 ("progress", models.IntegerField()),
+                ("start_date", models.DateField(null=True)),
+                ("end_date", models.DateField(null=True)),
                 (
                     "media",
                     models.ForeignKey(

src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py

@@ -1,33 +0,0 @@
-# Generated by Django 4.1.4 on 2023-02-03 22:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("app", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="media",
-            name="end_date",
-            field=models.DateField(null=True),
-        ),
-        migrations.AddField(
-            model_name="media",
-            name="start_date",
-            field=models.DateField(null=True),
-        ),
-        migrations.AddField(
-            model_name="season",
-            name="end_date",
-            field=models.DateField(null=True),
-        ),
-        migrations.AddField(
-            model_name="season",
-            name="start_date",
-            field=models.DateField(null=True),
-        ),
-    ]

src/app/migrations/0003_alter_media_score_alter_season_score.py

@@ -1,22 +0,0 @@
-# Generated by Django 4.1.5 on 2023-02-06 11:28
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("app", "0002_media_end_date_media_start_date_season_end_date_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="media",
-            name="score",
-            field=models.DecimalField(decimal_places=1, max_digits=3, null=True),
-        ),
-        migrations.AlterField(
-            model_name="season",
-            name="score",
-            field=models.DecimalField(decimal_places=1, max_digits=3, null=True),
-        ),
-    ]

src/app/models.py

@@ -1,13 +1,12 @@
 from django.db import models
 from django.contrib.auth.models import AbstractUser
 from django.conf import settings
-from app.storage import OverwriteStorage
 
 
 class Media(models.Model):
     media_id = models.IntegerField()
     title = models.CharField(max_length=100)
-    image = models.ImageField(upload_to="images", storage=OverwriteStorage())
+    image = models.ImageField()
     media_type = models.CharField(max_length=30)
     score = models.DecimalField(null=True, max_digits=3, decimal_places=1)
     progress = models.IntegerField()

src/app/storage.py

@@ -1,7 +0,0 @@
-from django.core.files.storage import FileSystemStorage
-
-
-class OverwriteStorage(FileSystemStorage):
-    def get_available_name(self, name, max_length=None):
-        self.delete(name)
-        return name

src/app/templates/app/edit.html

@@ -8,7 +8,7 @@
     {% elif media.response.poster_path %}
       <img class="poster" src="https://image.tmdb.org/t/p/w500{{media.response.poster_path}}"/>
     {% else %}
-      <img class="image-not-found" src="{% get_media_prefix %}/images/none.svg"/>
+      <img class="image-not-found" src="{% get_media_prefix %}/none.svg"/>
     {% endif %}
 
     <div class="content">

src/app/templates/app/include/list-section.html

@@ -46,7 +46,7 @@
   {% endif %}
   
   <div class="edit p-2">
-    <button type="button" class="btn open-modal-button" data-bs-toggle="modal" data-bs-target="#modal-{{media.media_type}}_{{media.media_id}}" data-url="{{media.media_type}}_{{media.id}}">
+    <button type="button" class="btn open-modal-button" data-bs-toggle="modal" data-bs-target="#modal-{{media.media_type}}_{{media.media_id}}" data-url="{{media.media_type}}_{{media.media_id}}">
       <i class="bi bi-pen-fill"></i>
     </button>
   </div>

src/app/templates/app/search.html

@@ -43,7 +43,7 @@
                 {% elif media.poster_path %}
                   <img class="poster flex-shrink-0 card-img-top" src="https://image.tmdb.org/t/p/w780{{media.poster_path}}"/>  
                 {% else %}
-                  <img class="image-not-found flex-shrink-0 card-img-top" src="{% get_media_prefix %}/images/none.svg"/>
+                  <img class="image-not-found flex-shrink-0 card-img-top" src="{% get_media_prefix %}/none.svg"/>
                 {% endif %}
 
                 <div class="card-body">

src/app/tests/test_imports.py

@@ -13,8 +13,9 @@ class ImportsMAL(TransactionTestCase):
     def setUp(self):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
+        os.makedirs("MAL")
 
-    @override_settings(MEDIA_ROOT=("test_Imports_MAL/media"))
+    @override_settings(MEDIA_ROOT=("MAL"))
     def test_import_animelist(self):
         imports.import_myanimelist("bloodthirstiness", self.user)
         self.assertEqual(Media.objects.filter(user=self.user).count(), 6)
@@ -26,7 +27,7 @@ class ImportsMAL(TransactionTestCase):
         )
         self.assertEqual(
             Media.objects.get(user=self.user, title="Ama Gli Animali").image
-            == "images/none.svg",
+            == "none.svg",
             True,
         )
         self.assertEqual(
@@ -37,7 +38,7 @@ class ImportsMAL(TransactionTestCase):
         )
 
     def tearDownClass():
-        shutil.rmtree("test_Imports_MAL")
+        shutil.rmtree("MAL")
 
 
 class ImportsTMDB(TransactionTestCase):
@@ -45,9 +46,11 @@ class ImportsTMDB(TransactionTestCase):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
 
-    @override_settings(MEDIA_ROOT=("test_Imports_TMDB/media"))
+    @override_settings(MEDIA_ROOT=("TMDB"))
     def test_import_tmdb(self):
-        file_path = os.path.join("test_Imports_TMDB/ratings.csv")
+        os.makedirs("TMDB")
+        file_path = os.path.join("TMDB", "ratings.csv")
+
         fields = [
             "TMDb ID",
             "IMDb ID",
@@ -86,7 +89,7 @@ class ImportsTMDB(TransactionTestCase):
                 "2022-12-17T16:23:01Z",
             ],
         ]
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
         with open(file_path, "w", newline="") as f:
             writer = csv.writer(f)
             writer.writerow(fields)
@@ -109,15 +112,16 @@ class ImportsTMDB(TransactionTestCase):
             )
 
     def tearDownClass():
-        shutil.rmtree("test_Imports_TMDB")
+        shutil.rmtree("TMDB")
 
 
 class ImportsANI(TransactionTestCase):
     def setUp(self):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
+        os.makedirs("AL")
 
-    @override_settings(MEDIA_ROOT=("test_Imports_ANI/media"))
+    @override_settings(MEDIA_ROOT=("AL"))
     def test_import_anilist(self):
         imports.import_anilist("bloodthirstiness", self.user)
         self.assertEqual(Media.objects.filter(user=self.user).count(), 6)
@@ -135,4 +139,4 @@ class ImportsANI(TransactionTestCase):
         )
 
     def tearDownClass():
-        shutil.rmtree("test_Imports_ANI")
+        shutil.rmtree("AL")

src/app/tests/test_media.py

@@ -14,8 +14,9 @@ class CreateMedia(TestCase):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
         self.client.login(**self.credentials)
+        os.makedirs("create")
 
-    @override_settings(MEDIA_ROOT=("test_CreateMedia/media"))
+    @override_settings(MEDIA_ROOT=("create"))
     def test_create_tmdb(self):
         self.assertEqual(
             Media.objects.filter(media_id=5895, user=self.user).exists(), False
@@ -56,7 +57,7 @@ class CreateMedia(TestCase):
         )
         self.assertEqual(
             os.path.exists(
-                settings.MEDIA_ROOT + "/images/tmdb-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg"
+                settings.MEDIA_ROOT + "/tv-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg"
             ),
             True,
         )
@@ -67,15 +68,17 @@ class CreateMedia(TestCase):
         self.assertEqual(str(season), "FLCL - Season 1")
 
     def tearDownClass():
-        shutil.rmtree("test_CreateMedia")
+        shutil.rmtree("create")
 
 
 class EditMedia(TestCase):
-    @override_settings(MEDIA_ROOT=("test_EditMedia/media"))
+    @override_settings(MEDIA_ROOT=("edit"))
     def setUp(self):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
         self.client.login(**self.credentials)
+        os.makedirs("edit", exist_ok=True)
+
         media = Media(
             media_id=1668,
             title="Friends",
@@ -242,6 +245,6 @@ class EditMedia(TestCase):
 
     def tearDownClass():
         try:
-            shutil.rmtree("test_EditMedia")
+            shutil.rmtree("edit")
         except OSError:
             pass

src/app/utils/database.py

@@ -1,6 +1,5 @@
 from app.models import Media, Season
 from app.utils import helpers
-from django.core.files import File
 from django.db.models import Avg, Sum, Min, Max
 
 
@@ -23,32 +22,14 @@ def add_media(request):
     )
 
     if metadata["image"] == "" or metadata["image"] is None:
-        media.image = "images/none.svg"
+        media.image = "none.svg"
     else:
-        # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
-        if media.api == "mal":
-            img_temp = helpers.get_image_temp(metadata["image"])
-            if media.media_type == "anime":
-                media.image.save(
-                    f"anime-{metadata['image'].rsplit('/', 1)[-1]}",
-                    File(img_temp),
-                    save=False,
-                )
-            elif media.media_type == "manga":
-                media.image.save(
-                    f"manga-{metadata['image'].rsplit('/', 1)[-1]}",
-                    File(img_temp),
-                    save=False,
-                )
-            img_temp.close()
-        else:
-            img_temp = helpers.get_image_temp(
-                f"https://image.tmdb.org/t/p/w92{metadata['image']}"
-            )
-            media.image.save(
-                f"tmdb-{metadata['image'].rsplit('/', 1)[-1]}", File(img_temp)
-            )
-            img_temp.close()
+        if media.api == "tmdb":
+            metadata["image"] = f"https://image.tmdb.org/t/p/w92{metadata['image']}"
+        filename = helpers.download_image(metadata["image"], media.media_type)
+        media.image = f"{filename}"
+
+    media.save()
 
     # if request is for a season, create a season object
     if "season" in request.POST and request.POST["season"] != "general":
@@ -56,6 +37,8 @@ def add_media(request):
             offset = 0
         else:
             offset = 1
+
+        # if completed and has episode count, set progress to episode count
         if (
             request.POST["status"] == "Completed"
             and "episode_count"
@@ -64,6 +47,7 @@ def add_media(request):
             media.progress = metadata["seasons"][int(request.POST["season"]) - offset][
                 "episode_count"
             ]
+            media.save()
 
         Season.objects.create(
             media=media,
@@ -76,7 +60,6 @@ def add_media(request):
             end_date=media.end_date,
         )
 
-    media.save()
     del request.session["metadata"]
 
 

src/app/utils/helpers.py

@@ -1,10 +1,9 @@
 from django.conf import settings
-from django.core.files.temp import NamedTemporaryFile
 
 import aiofiles
 import datetime
-import os
 import requests
+from pathlib import Path
 
 
 def convert_mal_media_type(media_type):
@@ -12,32 +11,37 @@ def convert_mal_media_type(media_type):
     return media_type.replace("_", " ").title()
 
 
-def get_image_temp(url):
-    img_temp = NamedTemporaryFile(delete=True)
-    r = requests.get(url)
-    img_temp.write(r.content)
-    img_temp.flush()
-    return img_temp
+def download_image(url, media_type):
+    # rsplit is used to split the url at the last / and taking the last element
+    # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+
+    filename = f"{media_type}-{url.rsplit('/', 1)[-1]}"
+    location = f"{settings.MEDIA_ROOT}/{filename}"
+
+    if not Path(location).is_file():
+        r = requests.get(url)
+        with open(location, "wb") as f:
+            f.write(r.content)
+
+    return filename
 
 
-async def download_image(session, url, media_type):
-    if url not in [
-        "",
-        "https://image.tmdb.org/t/p/w92None",
-        "https://image.tmdb.org/t/p/w92",
-    ]:
-        # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
-        location = f"{settings.MEDIA_ROOT}/images/{media_type}-{url.rsplit('/', 1)[-1]}"
+async def download_image_async(session, url, media_type):
 
-        # Create the directory if it doesn't exist
-        os.makedirs(f"{settings.MEDIA_ROOT}/images", exist_ok=True)
+    # rsplit is used to split the url at the last / and taking the last element
+    # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+    filename = f"{media_type}-{url.rsplit('/', 1)[-1]}"
+    location = f"{settings.MEDIA_ROOT}/{filename}"
 
+    if not Path(location).is_file():
         async with session.get(url) as resp:
             if resp.status == 200:
                 f = await aiofiles.open(location, mode="wb")
                 await f.write(await resp.read())
                 await f.close()
 
+    return filename
+
 
 def fix_inputs(request, metadata):
     post = request.POST.copy()

src/app/utils/imports.py

@@ -89,16 +89,14 @@ async def myanimelist_get_media(session, content, media_type, user):
         media.end_date = None
 
     if "main_picture" in content["node"]:
-        await helpers.download_image(
+        filename = await helpers.download_image_async(
             session, content["node"]["main_picture"]["medium"], media_type
         )
         # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
-        media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}"
-        media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}"
+        media.image = f"{filename}"
 
     else:
-        await helpers.download_image(session, "", media_type)
-        media.image = "images/none.svg"
+        media.image = "none.svg"
 
     return media
 
@@ -154,15 +152,14 @@ async def tmdb_get_media(session, url, row, user, status):
         else:
             score = float(row["Your Rating"])
 
-        await helpers.download_image(
-            session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", "tmdb"
-        )
-
         if response["poster_path"] is None:
-            image = "images/none.svg"
+            image = "none.svg"
         else:
+            filename = await helpers.download_image_async(
+                session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", row["Type"]
+            )
             # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
-            image = f"images/tmdb-{response['poster_path'].rsplit('/', 1)[-1]}"
+            image = f"{filename}"
 
         if "number_of_episodes" in response and status == "Completed":
             progress = response["number_of_episodes"]
@@ -365,11 +362,11 @@ async def anilist_get_media(session, content, media_type, user):
         end_date=end_date,
     )
 
-    await helpers.download_image(
+    filename = await helpers.download_image_async(
         session, content["media"]["coverImage"]["medium"], media_type
     )
 
     # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
-    media.image = f"images/{media_type}-{content['media']['coverImage']['medium'].rsplit('/', 1)[-1]}"
+    media.image = f"{filename}"
 
     return media