From a0bd5750199e7eecdda7299934ce4f8fc23b27cb Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Thu, 26 Dec 2024 23:59:50 -0300 Subject: [PATCH] fix: MySQL migration and driver support Trying to run MySQL as database (instead of MariaDB) was failing due to incompatible migrations and lack of support for the MySQL driver. These small changes fix the issue and recover the support for MySQL as database driver. Support for MySQL is on a best-effort basis, as the project's main database is MariaDB. --- .../alembic/versions/0009_models_refactor.py | 8 +-- backend/alembic/versions/0014_asset_files.py | 4 +- backend/alembic/versions/1.8.1_.py | 12 +++- backend/config/__init__.py | 6 +- backend/config/config_manager.py | 40 ++++++----- poetry.lock | 72 +++++++++++++++---- pyproject.toml | 6 +- 7 files changed, 102 insertions(+), 46 deletions(-) diff --git a/backend/alembic/versions/0009_models_refactor.py b/backend/alembic/versions/0009_models_refactor.py index 21472bb8f..11b8f901c 100644 --- a/backend/alembic/versions/0009_models_refactor.py +++ b/backend/alembic/versions/0009_models_refactor.py @@ -80,13 +80,13 @@ def upgrade() -> None: "url_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=True, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "path_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=True, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) try: @@ -127,13 +127,13 @@ def downgrade() -> None: "path_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=False, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "url_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=False, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "file_size_units", existing_type=mysql.VARCHAR(length=10), nullable=True diff --git a/backend/alembic/versions/0014_asset_files.py b/backend/alembic/versions/0014_asset_files.py index c2690ed58..c0114b4fc 100644 --- a/backend/alembic/versions/0014_asset_files.py +++ b/backend/alembic/versions/0014_asset_files.py @@ -33,8 +33,8 @@ SIZE_UNIT_TO_BYTES = { def migrate_to_mysql() -> None: - if ROMM_DB_DRIVER != "mariadb": - raise Exception("Version 3.0 requires MariaDB as database driver!") + if ROMM_DB_DRIVER not in ("mariadb", "mysql"): + raise Exception("Version 3.0 requires MariaDB or MySQL as database driver!") # Skip if sqlite database is not mounted if not os.path.exists(f"{SQLITE_DB_BASE_PATH}/romm.db"): diff --git a/backend/alembic/versions/1.8.1_.py b/backend/alembic/versions/1.8.1_.py index 374d1a482..29e24ff3c 100644 --- a/backend/alembic/versions/1.8.1_.py +++ b/backend/alembic/versions/1.8.1_.py @@ -20,11 +20,19 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table("roms") as batch_op: batch_op.add_column( - sa.Column("url_screenshots", sa.JSON(), nullable=False, server_default="[]") + sa.Column( + "url_screenshots", + sa.JSON(), + nullable=False, + server_default=sa.text("(JSON_ARRAY())"), + ) ) batch_op.add_column( sa.Column( - "path_screenshots", sa.JSON(), nullable=False, server_default="[]" + "path_screenshots", + sa.JSON(), + nullable=False, + server_default=sa.text("(JSON_ARRAY())"), ) ) # ### end Alembic commands ### diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 3c0568ee9..0ef19fab8 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -25,12 +25,13 @@ RESOURCES_BASE_PATH: Final = f"{ROMM_BASE_PATH}/resources" ASSETS_BASE_PATH: Final = f"{ROMM_BASE_PATH}/assets" FRONTEND_RESOURCES_PATH: Final = "/assets/romm/resources" -# MARIADB +# DATABASE DB_HOST: Final = os.environ.get("DB_HOST", "127.0.0.1") DB_PORT: Final = int(os.environ.get("DB_PORT", 3306)) DB_USER: Final = os.environ.get("DB_USER") DB_PASSWD: Final = os.environ.get("DB_PASSWD") DB_NAME: Final = os.environ.get("DB_NAME", "romm") +ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb") # REDIS REDIS_HOST: Final = os.environ.get("REDIS_HOST", "127.0.0.1") @@ -62,9 +63,6 @@ STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "") # MOBYGAMES MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "") -# DB DRIVERS -ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb") - # AUTH ROMM_AUTH_SECRET_KEY: Final = os.environ.get( "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 428400e62..3b7499aac 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -83,30 +83,34 @@ class ConfigManager: str: database connection string """ - if ROMM_DB_DRIVER == "mariadb": - if not DB_USER or not DB_PASSWD: - log.critical( - "Missing database credentials, check your environment variables!" - ) - sys.exit(3) - - return URL.create( - drivername="mariadb+mariadbconnector", - username=DB_USER, - password=DB_PASSWD, - host=DB_HOST, - port=DB_PORT, - database=DB_NAME, - ) - # DEPRECATED if ROMM_DB_DRIVER == "sqlite": log.critical("Sqlite is not supported anymore, migrate to mariaDB") sys.exit(6) # DEPRECATED - log.critical(f"{ROMM_DB_DRIVER} database not supported") - sys.exit(3) + if ROMM_DB_DRIVER == "mariadb": + driver = "mariadb+mariadbconnector" + elif ROMM_DB_DRIVER == "mysql": + driver = "mysql+mysqlconnector" + else: + log.critical(f"{ROMM_DB_DRIVER} database not supported") + sys.exit(3) + + if not DB_USER or not DB_PASSWD: + log.critical( + "Missing database credentials, check your environment variables!" + ) + sys.exit(3) + + return URL.create( + drivername=driver, + username=DB_USER, + password=DB_PASSWD, + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + ) def _parse_config(self): """Parses each entry in the config.yml""" diff --git a/poetry.lock b/poetry.lock index 7b40c3691..ff22dadf9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "alembic" @@ -1044,22 +1044,22 @@ testing = ["pytest"] [[package]] name = "mariadb" -version = "1.1.10" +version = "1.1.11" description = "Python MariaDB extension" optional = false python-versions = ">=3.8" files = [ - {file = "mariadb-1.1.10-cp310-cp310-win32.whl", hash = "sha256:1d81b22efbaaf4c5bc5e4cc4e2ef3c459538c1a939371089d8c5591d6f26a62e"}, - {file = "mariadb-1.1.10-cp310-cp310-win_amd64.whl", hash = "sha256:dff8b28ce4044574870d7bdd2d9f9f5da8e5f95a7ff6d226185db733060d1a93"}, - {file = "mariadb-1.1.10-cp311-cp311-win32.whl", hash = "sha256:1ce87971c02375236ff8933e6c593c748e7b2f2950b86eabfab4289fd250ea63"}, - {file = "mariadb-1.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:4521aa721f926946bd71491f872e8babc78fa97755ed2114f5684b77363107cb"}, - {file = "mariadb-1.1.10-cp312-cp312-win32.whl", hash = "sha256:03d6284ef713d1cad40146576a4cc2d6cbc1662060f2a0e59b174e1694521698"}, - {file = "mariadb-1.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:8c8c6b27486b0e1772a23002c702b5fd244eecf9f05633dd6cb345fc26755a20"}, - {file = "mariadb-1.1.10-cp38-cp38-win32.whl", hash = "sha256:5d652117e2fdf12b9723e7452a05fce9e6ccbae6ea48871b21a3a8fde259dc48"}, - {file = "mariadb-1.1.10-cp38-cp38-win_amd64.whl", hash = "sha256:49200378c614984f5ec875481662a49d7c97c2be27970b01b32fa4b7520d4e22"}, - {file = "mariadb-1.1.10-cp39-cp39-win32.whl", hash = "sha256:d7b09ec4abd02ed235257feb769f90cd4066e8f536b55b92f5166103d5b66a63"}, - {file = "mariadb-1.1.10-cp39-cp39-win_amd64.whl", hash = "sha256:29040e426f877ddc45f337c6eb381b6bbab63cc6bf8431a28effe30162142513"}, - {file = "mariadb-1.1.10.tar.gz", hash = "sha256:a332893e3ef7ceb7970ab4bd7c844bcb4bd68a051ca51313566f9808d7411f2d"}, + {file = "mariadb-1.1.11-cp310-cp310-win32.whl", hash = "sha256:dbc4cf0e302ca82d46f9431a0b04f048e9c21ee56d6f3162c29605f84d63b40c"}, + {file = "mariadb-1.1.11-cp310-cp310-win_amd64.whl", hash = "sha256:579420293fa790d5ae0a6cb4bdb7e8be8facc2ceefb6123c2b0e8042b3fa725d"}, + {file = "mariadb-1.1.11-cp311-cp311-win32.whl", hash = "sha256:0f8de8d66ca71bd102f34a970a331b7d75bdf7f8050d80e37cdcc6ff3c85cf7a"}, + {file = "mariadb-1.1.11-cp311-cp311-win_amd64.whl", hash = "sha256:3f64b520089cb60c4f8302f365ed0ae057c4c859ab70fc8b1c4358192c3c8f27"}, + {file = "mariadb-1.1.11-cp312-cp312-win32.whl", hash = "sha256:f6dfdc954edf02b6519419a054798cda6034dc459d1d482e3329e37aa27d34f0"}, + {file = "mariadb-1.1.11-cp312-cp312-win_amd64.whl", hash = "sha256:2e72ea65f1d7d8563ee84e172f2a583193092bdb6ff83c470ca9722873273ecc"}, + {file = "mariadb-1.1.11-cp313-cp313-win32.whl", hash = "sha256:d7302ccd15f0beee7b286885cbf6ac71ddc240374691d669784d99f89ba34d79"}, + {file = "mariadb-1.1.11-cp313-cp313-win_amd64.whl", hash = "sha256:c1992ebf9c6f012ac158e33fef9f2c4ba899f721064c4ae3a3489233793296c0"}, + {file = "mariadb-1.1.11-cp39-cp39-win32.whl", hash = "sha256:6f28d8ccc597a3a1368be14078110f743900dbb3b0c7f1cce3072d83bec59c8a"}, + {file = "mariadb-1.1.11-cp39-cp39-win_amd64.whl", hash = "sha256:e94f1738bec09c97b601ddbb1908eb24524ba4630f507a775d82ffdb6c5794b3"}, + {file = "mariadb-1.1.11.tar.gz", hash = "sha256:cf6647cee081e21d0994b409ba8c8fa2077f3972f1de3627c5502fb31d14f806"}, ] [package.dependencies] @@ -1329,6 +1329,48 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "mysql-connector-python" +version = "9.1.0" +description = "A self-contained Python driver for communicating with MySQL servers, using an API that is compliant with the Python Database API Specification v2.0 (PEP 249)." +optional = false +python-versions = ">=3.9" +files = [ + {file = "mysql-connector-python-9.1.0.tar.gz", hash = "sha256:346261a2aeb743a39cf66ba8bde5e45931d313b76ce0946a69a6d1187ec7d279"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:dcdcf380d07b9ca6f18a95e9516a6185f2ab31a53d290d5e698e77e59c043c9e"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:948ef0c7da87901176d4320e0f40a3277ee06fe6f58ce151c1e60d8d50fdeaf4"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:abf16fc1155ebeba5558e5702dd7210d634ac8da484eca05a640b68a548dc7cf"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aceaab679b852c0a2ec0eed9eb2a490171b3493484f1881b605cbf2f9c5fde6d"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:72dcce5f2e4f5910d65f02eb318c1e4622464da007a3ae5e9ccd64169d8efac3"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:9b23a8e2acee91b5120febe00c53e7f472b9b6d49618e39fa1af86cdc1f0ade8"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:e15153cb8ab5fcec00b99077de536489d22d4809fc28f633850398fef0560b1f"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fec943d333851c4b5e57cd0b04dde36e6817f0d4d62b2a58ce028a82be444866"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c36a9b9ebf9587aaa5d7928468fefe8faf6fc993a03cb242bb160ede9cf75b2d"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:7b2eb48518b8c2bc9636883d264b291e5c93824fc6b61823ca9cf396a09474ad"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:f67b22e3eaf5b03ffac97232d3dd67b56abcacad907ad4391c847bad5ba58f0e"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:c75f674a52b8820c90d466183b2bb59f89bcf09d17ebe9b391313d89565c8896"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e75ecb3df2c2cbe4d92d5dd58a318fa708edebc0fa2d850fc2a9d42481dbb808"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7d99c0a841a2c2a0e4d5b28376c1bfac794ec3821b66eb6fa2f7702cec820ee8"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:30a8f0ba84f8adf15a4877e80b3f97f786ce35616d918b9310578a2bd22952d5"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:d627ebafc0327b935d8783454e7a4b5c32324ed39a2a1589239490ab850bf7d7"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:e26a08a9500407fa8f4a6504f7077d1312bec4fa52cb0a58c1ad324ca1f3eeaa"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:109e17a4ada1442e3881a51e2bbabcb336ad229a619ac61e9ad24bd6b9b117bd"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4f102452c64332b7e042fa37b84d4f15332bd639e479d15035f2a005fb9fbb34"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25e261f3260ec798c48cb910862a299e565548a1b5421dec84315ddbc9ef28c4"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:ec4386b2426bfb07f83455bf895d8a7e2d6c067343ac05be5511083ca2424991"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:28fd99ee464ac3b02d1e2a71a63ca4f25c6110e4414a46a5b64631e6d2096899"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:e2f0876e1efd76e05853cb0a623dba2746ee70686c043019d811737dd5c3d871"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:6d7d5d458d0d600bbbebd9f2bce551e386b359bcce6026f7369b57922d26f13a"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:c350b1aaf257b1b778f44b8bfaeda07751f55e150f5a7464342f36e4aac8e805"}, + {file = "mysql_connector_python-9.1.0-py2.py3-none-any.whl", hash = "sha256:dacf1aa84dc7dd8ae908626c3ae50fce956d0105130c7465fd248a4f035d50b1"}, +] + +[package.extras] +dns-srv = ["dnspython (==2.6.1)"] +fido2 = ["fido2 (==1.1.2)"] +gssapi = ["gssapi (==1.8.3)"] +telemetry = ["opentelemetry-api (==1.18.0)", "opentelemetry-exporter-otlp-proto-http (==1.18.0)", "opentelemetry-sdk (==1.18.0)"] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -2761,6 +2803,8 @@ files = [ [package.dependencies] greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +mariadb = {version = ">=1.0.1,<1.1.2 || >1.1.2,<1.1.5 || >1.1.5,<1.1.10 || >1.1.10", optional = true, markers = "extra == \"mariadb-connector\""} +mysql-connector-python = {version = "*", optional = true, markers = "extra == \"mysql-connector\""} typing-extensions = ">=4.6.0" [package.extras] @@ -3468,4 +3512,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "bdd25954db1482642634d4460b0320fdcbaf77b00c7f7606a4c06b7c86f38463" +content-hash = "0f6f275d52781259c3780482aab4270849bba5f4e09bcd1c90bea8ead769dc72" diff --git a/pyproject.toml b/pyproject.toml index a83577f2b..e94071fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,10 @@ uvicorn = "0.29.0" gunicorn = "22.0.0" websockets = "12.0" python-socketio = "5.11.1" -SQLAlchemy = "^2.0.30" +SQLAlchemy = { version = "^2.0.30", extras = [ + "mariadb-connector", + "mysql-connector", +] } alembic = "1.13.1" PyYAML = "6.0.1" Unidecode = "1.3.8" @@ -23,7 +26,6 @@ emoji = "2.10.1" python-dotenv = "1.0.1" sqlakeyset = "^2.0.1708907391" pydash = "^7.0.7" -mariadb = "1.1.10" rq = "^1.16.1" redis = "^5.0" passlib = { extras = ["bcrypt"], version = "^1.7.4" }