Complete rewrite of backend

DEVELOPER-SETUP.md

@@ -25,10 +25,17 @@ poetry install
 docker-compose up -d
 ```
 
-### Run the backend
+### Run the migrations
 
 ```sh
-python3 backend/main.py
+cd backend
+alembic upgrade head
+```
+
+## And finally run the backend
+
+```sh
+python main.py
 ```
 
 ## Setting up the frontend

README-DEV.md

@@ -1,89 +0,0 @@
-# romm
-
-## Project Setup for development
-
-### Create python virtualenv
-
-```sh
-python3 -m venv backend/venv/romm
-```
-
-### Activate romm virtualenv
-
-```sh
-source backend/venv/romm/bin/activate
-```
-
-### Install venv dependencies
-
-```sh
-pip install -r backend/dependencies/requirements.txt
-```
-
-### Create environment variables file with the following variables
-
-```sh
-touch envs.env
-
-# IGDB auth
-CLIENT_ID=""
-CLIENT_SECRET=""
-
-# STEAMGRIDDB API KEY
-STEAMGRIDDB_API_KEY=""
-
-# Library path
-LIBRARY_PATH="library_path/"
-
-# DB related config
-ROMM_DB_HOST=""
-ROMM_DB_PORT=
-ROMM_DB_ROOT_PASSWD=""
-ROMM_DB_USER=""
-ROMM_DB_PASSWD=""
-ROMM_DB_CONFIG_PATH=""
-```
-
-### Export environment variables
-
-```sh
-export $(cat envs.env | xargs)
-```
-
-### Create mariadb docker container
-
-```sh
-docker-compose up -d (to your docker-compose file for run mariadb)
-```
-
-### Install node.js dependencies
-
-```sh
-npm install
-```
-
-## Project Testing
-
-### Activate romm virtualenv
-
-```sh
-source backend/venv/romm/bin/activate
-```
-
-### Export environment variables
-
-```sh
-export $(cat envs.env | xargs)
-```
-
-### Start backend API
-
-```sh
-python3 backend/src/main.py
-```
-
-### Start frontend (compile and hot-reload)
-
-```sh
-npm run dev
-```

backend/alembic/env.py

@@ -1,9 +1,11 @@
 import sys
 from pathlib import Path
-
 from logging.config import fileConfig
-
 from sqlalchemy import create_engine
+from dotenv import find_dotenv, load_dotenv
+
+load_dotenv(find_dotenv())
+
 from config.config_loader import ConfigLoader
 cl = ConfigLoader()
 

backend/handler/db_handler.py

@@ -128,4 +128,4 @@ class DBHandler:
                 rom = s.scalar(select(Rom).filter_by(p_slug=platform, file_name=file_name))
                 return rom.id if rom else None
         except ProgrammingError as e:
-            self.raise_error(e)
+            self.raise_error(e)

backend/main.py

@@ -1,8 +1,10 @@
+import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-import uvicorn
+from dotenv import find_dotenv, load_dotenv
+
+load_dotenv(find_dotenv())
 
-from config import DEV_PORT, DEV_HOST
 from endpoints import scan, search, platform, rom
 
 app = FastAPI()
@@ -18,12 +20,12 @@ app.include_router(search.router)
 app.include_router(platform.router)
 app.include_router(rom.router)
 
-
 @app.on_event("startup")
 def startup() -> None:
     """Startup application."""
     pass
 
-
 if __name__ == '__main__':
+    from config import DEV_PORT, DEV_HOST
+
     uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=True)

config/custom.cnf

@@ -0,0 +1,196 @@
+## custom configuration file based on https://github.com/just-containers/mariadb/blob/master/rootfs/etc/mysql/my.cnf
+## please be aware that changing options here may break things
+#
+# The MySQL database server configuration file.
+#
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# especially if they contain "#" chars...
+[client]
+port            = 3306
+socket          = /var/run/mysqld/mysqld.sock
+
+default-character-set = utf8mb4
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket          = /var/run/mysqld/mysqld.sock
+nice            = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+user            = abc
+pid-file        = /var/run/mysqld/mysqld.pid
+socket          = /var/run/mysqld/mysqld.sock
+port            = 3306
+basedir         = /usr
+datadir         = /var/lib/mysql
+tmpdir          = /tmp
+lc_messages_dir = /usr/share/mysql
+lc_messages     = en_US
+skip-external-locking
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+#bind-address           = 127.0.0.1
+#
+# * Fine Tuning
+#
+key_buffer_size         = 128M
+max_connections         = 100
+connect_timeout         = 5
+wait_timeout            = 600
+max_allowed_packet      = 16M
+thread_cache_size       = 128
+thread_stack            = 192K
+sort_buffer_size        = 4M
+bulk_insert_buffer_size = 16M
+tmp_table_size          = 32M
+max_heap_table_size     = 32M
+
+#performance_schema = on
+character_set_server    = utf8mb4
+collation_server        = utf8mb4_general_ci
+transaction_isolation   = READ-COMMITTED
+binlog_format           = MIXED
+
+#
+# * MyISAM
+#
+# This replaces the startup script and checks MyISAM tables if needed
+# the first time they are touched. On error, make copy and try a repair.
+myisam_recover         = BACKUP
+#open-files-limit       = 2000
+table_open_cache        = 400
+#table_cache            = 64
+#thread_concurrency     = 10
+myisam_sort_buffer_size = 512M
+concurrent_insert       = 2
+read_buffer_size        = 2M
+read_rnd_buffer_size    = 1M
+#
+# * Query Cache Configuration
+#
+# Cache only tiny result sets, so we can fit more in the query cache.
+query_cache_limit               = 128K
+query_cache_size                = 64M
+# for more write intensive setups, set to DEMAND or OFF
+query_cache_type               = DEMAND
+#
+# * Logging and Replication
+#
+console                 = 1
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+# As of 5.1 you can enable the log at runtime!
+#general_log             = 1
+#general_log_file        = /config/log/mysql/mysql.log
+#
+# Error log - should be very few entries.
+#
+log_warnings            = 2
+# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf
+#log_error               = /config/log/mysql/mysql.log
+#
+# Enable the slow query log to see queries with especially long duration
+slow_query_log          = 1
+slow_query_log_file     = /config/log/mysql/mariadb-slow.log
+long_query_time         = 5
+#log_slow_rate_limit    = 1000
+#log-queries-not-using-indexes
+#log_slow_admin_statements
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see
+#       https://mariadb.com/kb/en/setting-up-replication/
+#       about other settings you may need to change.
+#server-id              = 1
+#report_host            = master1
+#auto_increment_increment = 2
+#auto_increment_offset  = 1
+log_bin                 = /config/log/mysql/mariadb-bin
+log_bin_index           = /config/log/mysql/mariadb-bin.index
+# not fab for performance, but safer
+#sync_binlog             = 1
+#binlog_do_db            = include_database_name
+#binlog_ignore_db        = include_database_name
+expire_logs_days        = 10
+max_binlog_size         = 100M
+# slaves
+#relay_log              = /config/log/mysql/relay-bin
+#relay_log_index        = /config/log/mysql/relay-bin.index
+#relay_log_info_file    = /config/log/mysql/relay-bin.info
+#log_slave_updates
+#read_only
+#
+# If applications support it, this stricter sql_mode prevents some
+# mistakes like inserting invalid dates etc.
+#sql_mode               = NO_ENGINE_SUBSTITUTION,TRADITIONAL
+#
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+default_storage_engine  = InnoDB
+# you can't just change log file size, requires special procedure
+#innodb_log_file_size   = 50M
+innodb_buffer_pool_size = 256M
+innodb_log_buffer_size  = 8M
+innodb_file_per_table   = 1
+innodb_open_files       = 400
+innodb_io_capacity      = 400
+innodb_flush_method     = O_DIRECT
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet      = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completion
+
+[isamchk]
+key_buffer         = 16M
+
+#
+# * Galera-related settings
+#
+[galera]
+# Mandatory settings
+#wsrep_on=ON
+#wsrep_provider=
+#wsrep_cluster_address=
+#binlog_format=MIXED
+#default_storage_engine=InnoDB
+#innodb_autoinc_lock_mode=2
+#
+# Allow server to accept connections on all interfaces.
+#
+#bind-address=0.0.0.0
+#
+# Optional setting
+#wsrep_slave_threads=1
+#innodb_flush_log_at_trx_commit=0

config/databases/2f77f6b12fd4.err

@@ -0,0 +1,27 @@
+230509 13:41:40 mysqld_safe Starting mariadbd daemon with databases from /config/databases
+2023-05-09 13:41:40 0 [Warning] Setting lower_case_table_names=2 because file system for /config/databases/ is case insensitive
+2023-05-09 13:41:40 0 [Note] Starting MariaDB 10.6.12-MariaDB-log source revision 4c79e15cc3716f69c044d4287ad2160da8101cdc as process 1126
+2023-05-09 13:41:40 0 [Note] Using unique option prefix 'myisam_recover' is error-prone and can break in the future. Please use the full name 'myisam-recover-options' instead.
+2023-05-09 13:41:40 0 [Note] InnoDB: Compressed tables use zlib 1.2.13
+2023-05-09 13:41:40 0 [Note] InnoDB: Number of pools: 1
+2023-05-09 13:41:40 0 [Note] InnoDB: Using crc32 + pclmulqdq instructions
+2023-05-09 13:41:40 0 [Note] mariadbd: O_TMPFILE is not supported on /tmp (disabling future attempts)
+2023-05-09 13:41:40 0 [Warning] InnoDB: Linux Native AIO disabled.
+2023-05-09 13:41:40 0 [Note] InnoDB: Initializing buffer pool, total size = 268435456, chunk size = 134217728
+2023-05-09 13:41:40 0 [Note] InnoDB: Completed initialization of buffer pool
+2023-05-09 13:41:40 0 [Note] InnoDB: 128 rollback segments are active.
+2023-05-09 13:41:40 0 [Note] InnoDB: Creating shared tablespace for temporary tables
+2023-05-09 13:41:40 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
+2023-05-09 13:41:40 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
+2023-05-09 13:41:40 0 [Note] InnoDB: 10.6.12 started; log sequence number 42338; transaction id 14
+2023-05-09 13:41:40 0 [Note] InnoDB: Loading buffer pool(s) from /config/databases/ib_buffer_pool
+2023-05-09 13:41:40 0 [Note] Plugin 'FEEDBACK' is disabled.
+2023-05-09 13:41:40 0 [Note] InnoDB: Buffer pool(s) load completed at 230509 13:41:40
+2023-05-09 13:41:40 0 [Note] Server socket created on IP: '0.0.0.0'.
+2023-05-09 13:41:40 0 [Note] Server socket created on IP: '::'.
+2023-05-09 13:41:40 0 [Note] /usr/bin/mariadbd: ready for connections.
+Version: '10.6.12-MariaDB-log'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  MariaDB Server
+2023-05-09 13:41:41 4 [Warning] Aborted connection 4 to db: 'unconnected' user: 'unauthenticated' host: 'localhost' (This connection closed normally without authentication)
+2023-05-09 13:44:15 6 [Warning] IP address '172.22.0.1' could not be resolved: Name does not resolve
+2023-05-09 13:45:19 6 [Warning] Aborted connection 6 to db: 'romm' user: 'romm' host: '172.22.0.1' (Got an error reading communication packets)
+2023-05-09 13:49:13 7 [Warning] Aborted connection 7 to db: 'romm' user: 'romm' host: '172.22.0.1' (Got an error reading communication packets)

config/databases/4be8de1623a7.err

@@ -0,0 +1,37 @@
+230509 03:06:55 mysqld_safe Starting mariadbd daemon with databases from /config/databases
+2023-05-09  3:06:55 0 [Warning] Setting lower_case_table_names=2 because file system for /config/databases/ is case insensitive
+2023-05-09  3:06:55 0 [Note] Starting MariaDB 10.6.12-MariaDB-log source revision 4c79e15cc3716f69c044d4287ad2160da8101cdc as process 1126
+2023-05-09  3:06:55 0 [Note] Using unique option prefix 'myisam_recover' is error-prone and can break in the future. Please use the full name 'myisam-recover-options' instead.
+2023-05-09  3:06:55 0 [Note] InnoDB: Compressed tables use zlib 1.2.13
+2023-05-09  3:06:55 0 [Note] InnoDB: Number of pools: 1
+2023-05-09  3:06:55 0 [Note] InnoDB: Using crc32 + pclmulqdq instructions
+2023-05-09  3:06:55 0 [Note] mariadbd: O_TMPFILE is not supported on /tmp (disabling future attempts)
+2023-05-09  3:06:55 0 [Warning] InnoDB: Linux Native AIO disabled.
+2023-05-09  3:06:55 0 [Note] InnoDB: Initializing buffer pool, total size = 268435456, chunk size = 134217728
+2023-05-09  3:06:55 0 [Note] InnoDB: Completed initialization of buffer pool
+2023-05-09  3:06:55 0 [Note] InnoDB: 128 rollback segments are active.
+2023-05-09  3:06:55 0 [Note] InnoDB: Creating shared tablespace for temporary tables
+2023-05-09  3:06:55 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
+2023-05-09  3:06:55 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
+2023-05-09  3:06:55 0 [Note] InnoDB: 10.6.12 started; log sequence number 42326; transaction id 14
+2023-05-09  3:06:55 0 [Note] InnoDB: Loading buffer pool(s) from /config/databases/ib_buffer_pool
+2023-05-09  3:06:55 0 [Note] Plugin 'FEEDBACK' is disabled.
+2023-05-09  3:06:55 0 [Note] InnoDB: Buffer pool(s) load completed at 230509  3:06:55
+2023-05-09  3:06:55 0 [Note] Server socket created on IP: '0.0.0.0'.
+2023-05-09  3:06:55 0 [Note] Server socket created on IP: '::'.
+2023-05-09  3:06:55 0 [Note] /usr/bin/mariadbd: ready for connections.
+Version: '10.6.12-MariaDB-log'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  MariaDB Server
+2023-05-09  3:06:55 4 [Warning] Aborted connection 4 to db: 'unconnected' user: 'unauthenticated' host: 'localhost' (This connection closed normally without authentication)
+2023-05-09  3:07:35 6 [Warning] IP address '172.22.0.1' could not be resolved: Name does not resolve
+2023-05-09  3:07:36 6 [Warning] Aborted connection 6 to db: 'unconnected' user: 'romm' host: '172.22.0.1' (Got an error reading communication packets)
+2023-05-09  3:07:50 7 [Warning] Aborted connection 7 to db: 'romm' user: 'romm' host: '172.22.0.1' (Got an error reading communication packets)
+2023-05-09  3:08:44 0 [Note] /usr/bin/mariadbd (initiated by: unknown): Normal shutdown
+2023-05-09  3:08:44 0 [Note] InnoDB: FTS optimize thread exiting.
+2023-05-09  3:08:44 0 [Note] InnoDB: Starting shutdown...
+2023-05-09  3:08:44 0 [Note] InnoDB: Dumping buffer pool(s) to /config/databases/ib_buffer_pool
+2023-05-09  3:08:44 0 [Note] InnoDB: Buffer pool(s) dump completed at 230509  3:08:44
+2023-05-09  3:08:44 0 [Note] InnoDB: Removed temporary tablespace data file: "./ibtmp1"
+2023-05-09  3:08:44 0 [Note] InnoDB: Shutdown completed; log sequence number 42338; transaction id 15
+2023-05-09  3:08:44 0 [Note] /usr/bin/mariadbd: Shutdown complete
+
+230509 03:08:44 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended

config/databases/a8bcfc48dd9b.err

@@ -0,0 +1,34 @@
+230509 03:05:34 mysqld_safe Starting mariadbd daemon with databases from /config/databases
+2023-05-09  3:05:34 0 [Warning] Setting lower_case_table_names=2 because file system for /config/databases/ is case insensitive
+2023-05-09  3:05:34 0 [Note] Starting MariaDB 10.6.12-MariaDB-log source revision 4c79e15cc3716f69c044d4287ad2160da8101cdc as process 1301
+2023-05-09  3:05:34 0 [Note] Using unique option prefix 'myisam_recover' is error-prone and can break in the future. Please use the full name 'myisam-recover-options' instead.
+2023-05-09  3:05:34 0 [Note] InnoDB: Compressed tables use zlib 1.2.13
+2023-05-09  3:05:34 0 [Note] InnoDB: Number of pools: 1
+2023-05-09  3:05:34 0 [Note] InnoDB: Using crc32 + pclmulqdq instructions
+2023-05-09  3:05:34 0 [Note] mariadbd: O_TMPFILE is not supported on /tmp (disabling future attempts)
+2023-05-09  3:05:34 0 [Warning] InnoDB: Linux Native AIO disabled.
+2023-05-09  3:05:34 0 [Note] InnoDB: Initializing buffer pool, total size = 268435456, chunk size = 134217728
+2023-05-09  3:05:34 0 [Note] InnoDB: Completed initialization of buffer pool
+2023-05-09  3:05:34 0 [Note] InnoDB: 128 rollback segments are active.
+2023-05-09  3:05:34 0 [Note] InnoDB: Creating shared tablespace for temporary tables
+2023-05-09  3:05:34 0 [Note] InnoDB: Setting file './ibtmp1' size to 12 MB. Physically writing the file full; Please wait ...
+2023-05-09  3:05:34 0 [Note] InnoDB: File './ibtmp1' size is now 12 MB.
+2023-05-09  3:05:34 0 [Note] InnoDB: 10.6.12 started; log sequence number 42314; transaction id 14
+2023-05-09  3:05:34 0 [Note] InnoDB: Loading buffer pool(s) from /config/databases/ib_buffer_pool
+2023-05-09  3:05:34 0 [Note] Plugin 'FEEDBACK' is disabled.
+2023-05-09  3:05:34 0 [Note] InnoDB: Buffer pool(s) load completed at 230509  3:05:34
+2023-05-09  3:05:34 0 [Note] Server socket created on IP: '0.0.0.0'.
+2023-05-09  3:05:34 0 [Note] Server socket created on IP: '::'.
+2023-05-09  3:05:34 0 [Note] /usr/bin/mariadbd: ready for connections.
+Version: '10.6.12-MariaDB-log'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  MariaDB Server
+2023-05-09  3:05:35 4 [Warning] Aborted connection 4 to db: 'unconnected' user: 'unauthenticated' host: 'localhost' (This connection closed normally without authentication)
+2023-05-09  3:05:47 0 [Note] /usr/bin/mariadbd (initiated by: unknown): Normal shutdown
+2023-05-09  3:05:47 0 [Note] InnoDB: FTS optimize thread exiting.
+2023-05-09  3:05:47 0 [Note] InnoDB: Starting shutdown...
+2023-05-09  3:05:47 0 [Note] InnoDB: Dumping buffer pool(s) to /config/databases/ib_buffer_pool
+2023-05-09  3:05:47 0 [Note] InnoDB: Buffer pool(s) dump completed at 230509  3:05:47
+2023-05-09  3:05:47 0 [Note] InnoDB: Removed temporary tablespace data file: "./ibtmp1"
+2023-05-09  3:05:47 0 [Note] InnoDB: Shutdown completed; log sequence number 42326; transaction id 15
+2023-05-09  3:05:47 0 [Note] /usr/bin/mariadbd: Shutdown complete
+
+230509 03:05:47 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended

config/databases/ib_buffer_pool

@@ -0,0 +1,171 @@
+4,3
+4,2
+4,1
+4,0
+3,6
+3,5
+3,4
+3,3
+3,2
+3,1
+3,0
+2,3
+2,2
+2,1
+2,0
+1,3
+1,2
+1,1
+1,0
+0,243
+0,9
+0,0
+0,1
+0,300
+0,299
+0,298
+0,297
+0,296
+0,295
+0,294
+0,293
+0,292
+0,291
+0,290
+0,289
+0,288
+0,287
+0,286
+0,285
+0,284
+0,283
+0,282
+0,281
+0,280
+0,279
+0,278
+0,277
+0,276
+0,275
+0,274
+0,273
+0,272
+0,271
+0,270
+0,269
+0,268
+0,267
+0,266
+0,265
+0,264
+0,263
+0,262
+0,261
+0,260
+0,259
+0,258
+0,257
+0,256
+0,255
+0,254
+0,253
+0,252
+0,251
+0,250
+0,249
+0,248
+0,247
+0,246
+0,245
+0,244
+0,242
+0,241
+0,240
+0,239
+0,238
+0,237
+0,236
+0,235
+0,234
+0,233
+0,232
+0,231
+0,230
+0,229
+0,228
+0,227
+0,226
+0,225
+0,224
+0,223
+0,222
+0,221
+0,220
+0,219
+0,218
+0,217
+0,216
+0,215
+0,214
+0,213
+0,212
+0,211
+0,210
+0,209
+0,208
+0,207
+0,206
+0,205
+0,204
+0,203
+0,202
+0,201
+0,200
+0,199
+0,198
+0,197
+0,196
+0,195
+0,194
+0,193
+0,192
+0,63
+0,62
+0,61
+0,60
+0,59
+0,58
+0,57
+0,56
+0,55
+0,54
+0,53
+0,52
+0,51
+0,50
+0,311
+0,49
+0,310
+0,48
+0,309
+0,47
+0,308
+0,46
+0,307
+0,45
+0,301
+0,6
+0,5
+0,304
+0,303
+0,306
+0,305
+0,302
+0,12
+0,10
+0,8
+0,11
+0,4
+0,2
+0,3
+0,7

config/databases/mysql/db.opt

@@ -0,0 +1,2 @@
+default-character-set=utf8mb4
+default-collation=utf8mb4_general_ci