From 4e520b18a53b75065002603491102425d206ed5a Mon Sep 17 00:00:00 2001
From: FuzzyGrim <34800654+FuzzyGrim@users.noreply.github.com>
Date: Sat, 11 Mar 2023 15:38:29 +0100
Subject: [PATCH] Refactor media folder (#12)
* refactor download images functions
* only download images if not downloaded previously
* remove intermediary image folder
* create folder before test
* update directories
---
.dockerignore | 4 +-
.gitignore | 4 +-
src/app/migrations/0001_initial.py | 25 +++++++-----
...dia_start_date_season_end_date_and_more.py | 33 ---------------
...03_alter_media_score_alter_season_score.py | 22 ----------
src/app/models.py | 3 +-
src/app/storage.py | 7 ----
src/app/templates/app/edit.html | 2 +-
.../templates/app/include/list-section.html | 2 +-
src/app/templates/app/search.html | 2 +-
src/app/tests/test_imports.py | 22 +++++-----
src/app/tests/test_media.py | 13 +++---
src/app/utils/database.py | 37 +++++------------
src/app/utils/helpers.py | 40 ++++++++++---------
src/app/utils/imports.py | 23 +++++------
src/media/{images => }/none.svg | 0
16 files changed, 85 insertions(+), 154 deletions(-)
delete mode 100644 src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py
delete mode 100644 src/app/migrations/0003_alter_media_score_alter_season_score.py
delete mode 100644 src/app/storage.py
rename src/media/{images => }/none.svg (100%)
diff --git a/.dockerignore b/.dockerignore
index 1b5a7187..2fbdcdfe 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -4,5 +4,5 @@ __pycache__
.coverage
venv
src/db/db.sqlite3
-src/media/images/*
-!src/media/images/none.svg
\ No newline at end of file
+src/media/*
+!src/media/none.svg
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 1b5a7187..2fbdcdfe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,5 +4,5 @@ __pycache__
.coverage
venv
src/db/db.sqlite3
-src/media/images/*
-!src/media/images/none.svg
\ No newline at end of file
+src/media/*
+!src/media/none.svg
\ No newline at end of file
diff --git a/src/app/migrations/0001_initial.py b/src/app/migrations/0001_initial.py
index 096f0850..90ffe785 100644
--- a/src/app/migrations/0001_initial.py
+++ b/src/app/migrations/0001_initial.py
@@ -1,6 +1,5 @@
-# Generated by Django 4.1.4 on 2023-01-19 13:54
+# Generated by Django 4.1.5 on 2023-03-11 14:14
-import app.storage
from django.conf import settings
import django.contrib.auth.models
import django.contrib.auth.validators
@@ -10,7 +9,6 @@ import django.utils.timezone
class Migration(migrations.Migration):
-
initial = True
dependencies = [
@@ -147,17 +145,17 @@ class Migration(migrations.Migration):
),
("media_id", models.IntegerField()),
("title", models.CharField(max_length=100)),
- (
- "image",
- models.ImageField(
- storage=app.storage.OverwriteStorage(), upload_to="images"
- ),
- ),
+ ("image", models.ImageField(upload_to="")),
("media_type", models.CharField(max_length=30)),
- ("score", models.FloatField(null=True)),
+ (
+ "score",
+ models.DecimalField(decimal_places=1, max_digits=3, null=True),
+ ),
("progress", models.IntegerField()),
("status", models.CharField(max_length=30)),
("api", models.CharField(max_length=10)),
+ ("start_date", models.DateField(null=True)),
+ ("end_date", models.DateField(null=True)),
(
"user",
models.ForeignKey(
@@ -184,9 +182,14 @@ class Migration(migrations.Migration):
),
("title", models.CharField(max_length=100)),
("number", models.IntegerField()),
- ("score", models.FloatField(null=True)),
+ (
+ "score",
+ models.DecimalField(decimal_places=1, max_digits=3, null=True),
+ ),
("status", models.CharField(max_length=30)),
("progress", models.IntegerField()),
+ ("start_date", models.DateField(null=True)),
+ ("end_date", models.DateField(null=True)),
(
"media",
models.ForeignKey(
diff --git a/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py b/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py
deleted file mode 100644
index 96a6a48a..00000000
--- a/src/app/migrations/0002_media_end_date_media_start_date_season_end_date_and_more.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by Django 4.1.4 on 2023-02-03 22:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("app", "0001_initial"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="media",
- name="end_date",
- field=models.DateField(null=True),
- ),
- migrations.AddField(
- model_name="media",
- name="start_date",
- field=models.DateField(null=True),
- ),
- migrations.AddField(
- model_name="season",
- name="end_date",
- field=models.DateField(null=True),
- ),
- migrations.AddField(
- model_name="season",
- name="start_date",
- field=models.DateField(null=True),
- ),
- ]
diff --git a/src/app/migrations/0003_alter_media_score_alter_season_score.py b/src/app/migrations/0003_alter_media_score_alter_season_score.py
deleted file mode 100644
index ea202ca6..00000000
--- a/src/app/migrations/0003_alter_media_score_alter_season_score.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Generated by Django 4.1.5 on 2023-02-06 11:28
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
- dependencies = [
- ("app", "0002_media_end_date_media_start_date_season_end_date_and_more"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="media",
- name="score",
- field=models.DecimalField(decimal_places=1, max_digits=3, null=True),
- ),
- migrations.AlterField(
- model_name="season",
- name="score",
- field=models.DecimalField(decimal_places=1, max_digits=3, null=True),
- ),
- ]
diff --git a/src/app/models.py b/src/app/models.py
index 5ea24503..3e40b924 100644
--- a/src/app/models.py
+++ b/src/app/models.py
@@ -1,13 +1,12 @@
from django.db import models
from django.contrib.auth.models import AbstractUser
from django.conf import settings
-from app.storage import OverwriteStorage
class Media(models.Model):
media_id = models.IntegerField()
title = models.CharField(max_length=100)
- image = models.ImageField(upload_to="images", storage=OverwriteStorage())
+ image = models.ImageField()
media_type = models.CharField(max_length=30)
score = models.DecimalField(null=True, max_digits=3, decimal_places=1)
progress = models.IntegerField()
diff --git a/src/app/storage.py b/src/app/storage.py
deleted file mode 100644
index 14ea4e59..00000000
--- a/src/app/storage.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from django.core.files.storage import FileSystemStorage
-
-
-class OverwriteStorage(FileSystemStorage):
- def get_available_name(self, name, max_length=None):
- self.delete(name)
- return name
diff --git a/src/app/templates/app/edit.html b/src/app/templates/app/edit.html
index 85f1ccd2..27f0c58f 100644
--- a/src/app/templates/app/edit.html
+++ b/src/app/templates/app/edit.html
@@ -8,7 +8,7 @@
{% elif media.response.poster_path %}
{% else %}
-
+
{% endif %}
diff --git a/src/app/templates/app/include/list-section.html b/src/app/templates/app/include/list-section.html
index 560f0be6..bf0e3a64 100644
--- a/src/app/templates/app/include/list-section.html
+++ b/src/app/templates/app/include/list-section.html
@@ -46,7 +46,7 @@
{% endif %}
-
diff --git a/src/app/templates/app/search.html b/src/app/templates/app/search.html
index 40df7c01..1de33874 100644
--- a/src/app/templates/app/search.html
+++ b/src/app/templates/app/search.html
@@ -43,7 +43,7 @@
{% elif media.poster_path %}

{% else %}
-

+

{% endif %}
diff --git a/src/app/tests/test_imports.py b/src/app/tests/test_imports.py
index 460eef85..381991d3 100644
--- a/src/app/tests/test_imports.py
+++ b/src/app/tests/test_imports.py
@@ -13,8 +13,9 @@ class ImportsMAL(TransactionTestCase):
def setUp(self):
self.credentials = {"username": "test", "password": "12345"}
self.user = User.objects.create_user(**self.credentials)
+ os.makedirs("MAL")
- @override_settings(MEDIA_ROOT=("test_Imports_MAL/media"))
+ @override_settings(MEDIA_ROOT=("MAL"))
def test_import_animelist(self):
imports.import_myanimelist("bloodthirstiness", self.user)
self.assertEqual(Media.objects.filter(user=self.user).count(), 6)
@@ -26,7 +27,7 @@ class ImportsMAL(TransactionTestCase):
)
self.assertEqual(
Media.objects.get(user=self.user, title="Ama Gli Animali").image
- == "images/none.svg",
+ == "none.svg",
True,
)
self.assertEqual(
@@ -37,7 +38,7 @@ class ImportsMAL(TransactionTestCase):
)
def tearDownClass():
- shutil.rmtree("test_Imports_MAL")
+ shutil.rmtree("MAL")
class ImportsTMDB(TransactionTestCase):
@@ -45,9 +46,11 @@ class ImportsTMDB(TransactionTestCase):
self.credentials = {"username": "test", "password": "12345"}
self.user = User.objects.create_user(**self.credentials)
- @override_settings(MEDIA_ROOT=("test_Imports_TMDB/media"))
+ @override_settings(MEDIA_ROOT=("TMDB"))
def test_import_tmdb(self):
- file_path = os.path.join("test_Imports_TMDB/ratings.csv")
+ os.makedirs("TMDB")
+ file_path = os.path.join("TMDB", "ratings.csv")
+
fields = [
"TMDb ID",
"IMDb ID",
@@ -86,7 +89,7 @@ class ImportsTMDB(TransactionTestCase):
"2022-12-17T16:23:01Z",
],
]
- os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
with open(file_path, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(fields)
@@ -109,15 +112,16 @@ class ImportsTMDB(TransactionTestCase):
)
def tearDownClass():
- shutil.rmtree("test_Imports_TMDB")
+ shutil.rmtree("TMDB")
class ImportsANI(TransactionTestCase):
def setUp(self):
self.credentials = {"username": "test", "password": "12345"}
self.user = User.objects.create_user(**self.credentials)
+ os.makedirs("AL")
- @override_settings(MEDIA_ROOT=("test_Imports_ANI/media"))
+ @override_settings(MEDIA_ROOT=("AL"))
def test_import_anilist(self):
imports.import_anilist("bloodthirstiness", self.user)
self.assertEqual(Media.objects.filter(user=self.user).count(), 6)
@@ -135,4 +139,4 @@ class ImportsANI(TransactionTestCase):
)
def tearDownClass():
- shutil.rmtree("test_Imports_ANI")
+ shutil.rmtree("AL")
diff --git a/src/app/tests/test_media.py b/src/app/tests/test_media.py
index 2d57b608..a738c752 100644
--- a/src/app/tests/test_media.py
+++ b/src/app/tests/test_media.py
@@ -14,8 +14,9 @@ class CreateMedia(TestCase):
self.credentials = {"username": "test", "password": "12345"}
self.user = User.objects.create_user(**self.credentials)
self.client.login(**self.credentials)
+ os.makedirs("create")
- @override_settings(MEDIA_ROOT=("test_CreateMedia/media"))
+ @override_settings(MEDIA_ROOT=("create"))
def test_create_tmdb(self):
self.assertEqual(
Media.objects.filter(media_id=5895, user=self.user).exists(), False
@@ -56,7 +57,7 @@ class CreateMedia(TestCase):
)
self.assertEqual(
os.path.exists(
- settings.MEDIA_ROOT + "/images/tmdb-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg"
+ settings.MEDIA_ROOT + "/tv-FkgA8CcmiLJGVCRYRQ2g2UfVtF.jpg"
),
True,
)
@@ -67,15 +68,17 @@ class CreateMedia(TestCase):
self.assertEqual(str(season), "FLCL - Season 1")
def tearDownClass():
- shutil.rmtree("test_CreateMedia")
+ shutil.rmtree("create")
class EditMedia(TestCase):
- @override_settings(MEDIA_ROOT=("test_EditMedia/media"))
+ @override_settings(MEDIA_ROOT=("edit"))
def setUp(self):
self.credentials = {"username": "test", "password": "12345"}
self.user = User.objects.create_user(**self.credentials)
self.client.login(**self.credentials)
+ os.makedirs("edit", exist_ok=True)
+
media = Media(
media_id=1668,
title="Friends",
@@ -242,6 +245,6 @@ class EditMedia(TestCase):
def tearDownClass():
try:
- shutil.rmtree("test_EditMedia")
+ shutil.rmtree("edit")
except OSError:
pass
diff --git a/src/app/utils/database.py b/src/app/utils/database.py
index d18fdb68..1eae0456 100644
--- a/src/app/utils/database.py
+++ b/src/app/utils/database.py
@@ -1,6 +1,5 @@
from app.models import Media, Season
from app.utils import helpers
-from django.core.files import File
from django.db.models import Avg, Sum, Min, Max
@@ -23,32 +22,14 @@ def add_media(request):
)
if metadata["image"] == "" or metadata["image"] is None:
- media.image = "images/none.svg"
+ media.image = "none.svg"
else:
- # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
- if media.api == "mal":
- img_temp = helpers.get_image_temp(metadata["image"])
- if media.media_type == "anime":
- media.image.save(
- f"anime-{metadata['image'].rsplit('/', 1)[-1]}",
- File(img_temp),
- save=False,
- )
- elif media.media_type == "manga":
- media.image.save(
- f"manga-{metadata['image'].rsplit('/', 1)[-1]}",
- File(img_temp),
- save=False,
- )
- img_temp.close()
- else:
- img_temp = helpers.get_image_temp(
- f"https://image.tmdb.org/t/p/w92{metadata['image']}"
- )
- media.image.save(
- f"tmdb-{metadata['image'].rsplit('/', 1)[-1]}", File(img_temp)
- )
- img_temp.close()
+ if media.api == "tmdb":
+ metadata["image"] = f"https://image.tmdb.org/t/p/w92{metadata['image']}"
+ filename = helpers.download_image(metadata["image"], media.media_type)
+ media.image = f"{filename}"
+
+ media.save()
# if request is for a season, create a season object
if "season" in request.POST and request.POST["season"] != "general":
@@ -56,6 +37,8 @@ def add_media(request):
offset = 0
else:
offset = 1
+
+ # if completed and has episode count, set progress to episode count
if (
request.POST["status"] == "Completed"
and "episode_count"
@@ -64,6 +47,7 @@ def add_media(request):
media.progress = metadata["seasons"][int(request.POST["season"]) - offset][
"episode_count"
]
+ media.save()
Season.objects.create(
media=media,
@@ -76,7 +60,6 @@ def add_media(request):
end_date=media.end_date,
)
- media.save()
del request.session["metadata"]
diff --git a/src/app/utils/helpers.py b/src/app/utils/helpers.py
index 66965c49..0442e544 100644
--- a/src/app/utils/helpers.py
+++ b/src/app/utils/helpers.py
@@ -1,10 +1,9 @@
from django.conf import settings
-from django.core.files.temp import NamedTemporaryFile
import aiofiles
import datetime
-import os
import requests
+from pathlib import Path
def convert_mal_media_type(media_type):
@@ -12,32 +11,37 @@ def convert_mal_media_type(media_type):
return media_type.replace("_", " ").title()
-def get_image_temp(url):
- img_temp = NamedTemporaryFile(delete=True)
- r = requests.get(url)
- img_temp.write(r.content)
- img_temp.flush()
- return img_temp
+def download_image(url, media_type):
+ # rsplit is used to split the url at the last / and taking the last element
+ # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+
+ filename = f"{media_type}-{url.rsplit('/', 1)[-1]}"
+ location = f"{settings.MEDIA_ROOT}/{filename}"
+
+ if not Path(location).is_file():
+ r = requests.get(url)
+ with open(location, "wb") as f:
+ f.write(r.content)
+
+ return filename
-async def download_image(session, url, media_type):
- if url not in [
- "",
- "https://image.tmdb.org/t/p/w92None",
- "https://image.tmdb.org/t/p/w92",
- ]:
- # rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
- location = f"{settings.MEDIA_ROOT}/images/{media_type}-{url.rsplit('/', 1)[-1]}"
+async def download_image_async(session, url, media_type):
- # Create the directory if it doesn't exist
- os.makedirs(f"{settings.MEDIA_ROOT}/images", exist_ok=True)
+ # rsplit is used to split the url at the last / and taking the last element
+ # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+ filename = f"{media_type}-{url.rsplit('/', 1)[-1]}"
+ location = f"{settings.MEDIA_ROOT}/{filename}"
+ if not Path(location).is_file():
async with session.get(url) as resp:
if resp.status == 200:
f = await aiofiles.open(location, mode="wb")
await f.write(await resp.read())
await f.close()
+ return filename
+
def fix_inputs(request, metadata):
post = request.POST.copy()
diff --git a/src/app/utils/imports.py b/src/app/utils/imports.py
index 4816e82b..c185fa69 100644
--- a/src/app/utils/imports.py
+++ b/src/app/utils/imports.py
@@ -89,16 +89,14 @@ async def myanimelist_get_media(session, content, media_type, user):
media.end_date = None
if "main_picture" in content["node"]:
- await helpers.download_image(
+ filename = await helpers.download_image_async(
session, content["node"]["main_picture"]["medium"], media_type
)
# rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
- media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}"
- media.image = f"images/{media_type}-{content['node']['main_picture']['medium'].rsplit('/', 1)[-1]}"
+ media.image = f"{filename}"
else:
- await helpers.download_image(session, "", media_type)
- media.image = "images/none.svg"
+ media.image = "none.svg"
return media
@@ -154,15 +152,14 @@ async def tmdb_get_media(session, url, row, user, status):
else:
score = float(row["Your Rating"])
- await helpers.download_image(
- session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", "tmdb"
- )
-
if response["poster_path"] is None:
- image = "images/none.svg"
+ image = "none.svg"
else:
+ filename = await helpers.download_image_async(
+ session, f"https://image.tmdb.org/t/p/w92{response['poster_path']}", row["Type"]
+ )
# rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
- image = f"images/tmdb-{response['poster_path'].rsplit('/', 1)[-1]}"
+ image = f"{filename}"
if "number_of_episodes" in response and status == "Completed":
progress = response["number_of_episodes"]
@@ -365,11 +362,11 @@ async def anilist_get_media(session, content, media_type, user):
end_date=end_date,
)
- await helpers.download_image(
+ filename = await helpers.download_image_async(
session, content["media"]["coverImage"]["medium"], media_type
)
# rspilt is used to get the filename from the url by splitting the url at the last / and taking the last element
- media.image = f"images/{media_type}-{content['media']['coverImage']['medium'].rsplit('/', 1)[-1]}"
+ media.image = f"{filename}"
return media
diff --git a/src/media/images/none.svg b/src/media/none.svg
similarity index 100%
rename from src/media/images/none.svg
rename to src/media/none.svg