[WIP] build redis into container

expose port right

backend/config/__init__.py

@@ -26,9 +26,8 @@ DB_PASSWD: Final = os.environ.get("DB_PASSWD")
 DB_NAME: Final = os.environ.get("DB_NAME", "romm")
 
 # REDIS
-REDIS_HOST: Final = os.environ.get("REDIS_HOST", "localhost")
-REDIS_PORT: Final = int(os.environ.get("REDIS_PORT", 6379))
-REDIS_PASSWORD: Final = os.environ.get("REDIS_PASSWORD")
+REDIS_HOST: Final = "localhost"
+REDIS_PORT: Final = 6379
 
 # IGDB
 IGDB_CLIENT_ID: Final = os.environ.get(

backend/handler/redis_handler.py

@@ -1,7 +1,7 @@
 import sys
 from enum import Enum
 
-from config import REDIS_HOST, REDIS_PASSWORD, REDIS_PORT
+from config import REDIS_HOST, REDIS_PORT
 from logger.logger import log
 from redis import Redis
 from rq import Queue
@@ -39,15 +39,8 @@ class FallbackCache:
         return repr(self)
 
 
-redis_client = Redis(
-    host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=0
-)
-
-redis_url = (
-    f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
-    if REDIS_PASSWORD
-    else f"redis://{REDIS_HOST}:{REDIS_PORT}"
-)
+redis_client = Redis(host=REDIS_HOST, port=REDIS_PORT, db=0)
+redis_url = f"redis://{REDIS_HOST}:{REDIS_PORT}"
 
 high_prio_queue = Queue(name=QueuePrio.HIGH.value, connection=redis_client)
 default_queue = Queue(name=QueuePrio.DEFAULT.value, connection=redis_client)
@@ -61,7 +54,6 @@ else:
     cache = Redis(
         host=REDIS_HOST,
         port=REDIS_PORT,
-        password=REDIS_PASSWORD,
         db=0,
         decode_responses=True,
     )

docker/Dockerfile

@@ -23,7 +23,8 @@ RUN apk add --upgrade \
         libffi \
         mariadb-connector-c \
         netcat-openbsd \
-        python3
+        python3 \
+        tzdata
 
 # Install additional build dependencies
 RUN apk add --upgrade \
@@ -33,7 +34,13 @@ RUN apk add --upgrade \
         musl-dev \
         python3-dev \
         py3-pip \
-        git
+        git \
+        wget \
+        coreutils \
+        dpkg-dev dpkg \
+        linux-headers \
+        make \
+        openssl-dev
 
 # Create python venv to not clash with OS python packages
 RUN python3 -m venv /backend/
@@ -56,7 +63,7 @@ RUN . /backend/bin/activate && \
 # cleanup python dependencies that are not needed anymore
 RUN . /backend/bin/activate && \
     grep -v -x -f /installed_poetry_requirements.txt /installed_pip_requirements.txt > /build_requirements.txt && \
-    pip uninstall -y -r /build_requirements.txt 
+    pip uninstall -y -r /build_requirements.txt
 
 COPY ./backend /backend
 
@@ -64,6 +71,55 @@ COPY ./backend /backend
 COPY ./docker/init_scripts/* /
 COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf
 
+# Install redis
+ENV REDIS_VERSION 7.2.4
+ENV REDIS_DOWNLOAD_URL http://download.redis.io/releases/redis-7.2.4.tar.gz
+ENV REDIS_DOWNLOAD_SHA 8d104c26a154b29fd67d6568b4f375212212ad41e0c2caa3d66480e78dbd3b59
+
+RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \
+	echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \
+	mkdir -p /usr/src/redis; \
+	tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \
+	rm redis.tar.gz; \
+	grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' /usr/src/redis/src/config.c; \
+	sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' /usr/src/redis/src/config.c; \
+	grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' /usr/src/redis/src/config.c; \
+	gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	extraJemallocConfigureFlags="--build=$gnuArch"; \
+	dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in (amd64 | i386 | x32) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=12";; (*) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=16";; esac; \
+	extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-hugepage=21"; \
+	grep -F 'cd jemalloc && ./configure ' /usr/src/redis/deps/Makefile; \
+	sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \
+	grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \
+	export BUILD_TLS=yes; \
+	make -C /usr/src/redis -j "$(nproc)" all; \
+	make -C /usr/src/redis install; \
+	serverMd5="$(md5sum /usr/local/bin/redis-server | cut -d' ' -f1)"; export serverMd5; \
+	find /usr/local/bin/redis* -maxdepth 0 \
+		-type f -not -name redis-server \
+		-exec sh -eux -c ' \
+			md5="$(md5sum "$1" | cut -d" " -f1)"; \
+			test "$md5" = "$serverMd5"; \
+		' -- '{}' ';' \
+		-exec ln -svfT 'redis-server' '{}' ';' \
+	; \
+	rm -r /usr/src/redis; \
+	runDeps="$( \
+		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
+			| tr ',' '\n' \
+			| sort -u \
+			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+	)"; \
+	apk add --no-network --virtual .redis-rundeps $runDeps; \
+	apk del --no-network .build-deps; \
+	redis-cli --version; \
+	redis-server --version; \
+	echo '{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","name":"redis-server-sbom","packages":[{"name":"redis-server","versionInfo":"7.2.4","SPDXID":"SPDXRef-Package--redis-server","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:generic/redis-server@7.2.4?os_name=alpine&os_version=3.19"}],"licenseDeclared":"BSD-3-Clause"}]}' > /usr/local/redis.spdx.json
+
+RUN mkdir /redis-data && chown 1000:1000 /redis-data
+VOLUME /redis-data
+
 # cleanup additional build dependencies
 RUN apk del \
         gcc \
@@ -72,7 +128,13 @@ RUN apk del \
         musl-dev \
         python3-dev \
         py3-pip \
-        git
+        git \
+        wget \
+        coreutils \
+        dpkg-dev dpkg \
+        linux-headers \
+        make \
+        openssl-dev
 
 # cleanup leftover files that are not needed at runtime
 RUN rm -r \
@@ -92,5 +154,6 @@ RUN mkdir /romm && chown 1000:1000 /romm -R
 
 # Expose ports and start
 EXPOSE 8080
+EXPOSE 6379/tcp
 WORKDIR /romm
 CMD ["/init"]

docker/init_scripts/init

@@ -56,6 +56,19 @@ start_bin_nginx () {
     fi
 }
 
+start_bin_redis-server () {
+    info_log "starting redis-server"
+
+    # Check if /usr/local/etc/redis/redis.conf exists and use it if so
+    if [ -f /usr/local/etc/redis/redis.conf ]; then
+        redis-server /usr/local/etc/redis/redis.conf &
+    else
+        redis-server &
+    fi
+    REDIS_PID=$!
+    echo $REDIS_PID > /tmp/redis-server.pid
+}
+
 # function that runs our independent python scripts and creates corresponding PID files,
 start_python () {
     SCRIPT="${1}"
@@ -93,6 +106,9 @@ cd /backend || { error_log "/backend directory doesn't seem to exist"; }
 
 # function definition done, lets start our main loop
 while true; do
+    # Start redis server if we dont have a corresponding PID file
+    watchdog_process_pid bin redis-server
+
     # Run needed database migrations on startup,
     # but only if it was not successful since the last full docker container start
     if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then

examples/docker-compose.example.yml

@@ -19,7 +19,6 @@ services:
       - ROMM_AUTH_SECRET_KEY= # Generate a key with `openssl rand -hex 32`
       - ROMM_AUTH_USERNAME=admin
       - ROMM_AUTH_PASSWORD= # default: admin
-      - REDIS_HOST=romm-redis
     volumes:
       - romm_resources:/romm/resources" # Resources fetched from IGDB (covers, screenshots, etc.)
       - "/path/to/library:/romm/library" # Your game library
@@ -30,7 +29,6 @@ services:
       - 80:8080
     depends_on:
       - romm-db
-      - romm-redis
 
   romm-db:
     image: mariadb:latest
@@ -43,8 +41,3 @@ services:
       - MYSQL_PASSWORD=
     volumes:
       - mysql_data:/var/lib/mysql
-
-  romm-redis:
-    image: redis:alpine
-    container_name: romm-redis
-    restart: unless-stopped

unraid_template/romm.xml

@@ -41,7 +41,6 @@
    <Config Name="DB_PASSWD" Target="DB_PASSWD" Default="" Mode="" Description="Database password for DB_USER" Type="Variable" Display="advanced" Required="true" Mask="true"/>
    <Config Name="REDIS_HOST" Target="REDIS_HOST" Default="127.0.0.1" Mode="" Description="Redis host" Type="Variable" Display="advanced" Required="true" Mask="false"/>
    <Config Name="REDIS_PORT" Target="REDIS_PORT" Default="6379" Mode="" Description="Redis port" Type="Variable" Display="advanced" Required="true" Mask="false"/>
-   <Config Name="REDIS_PASSWORD" Target="REDIS_PASSWORD" Default="" Mode="" Description="Redis password" Type="Variable" Display="advanced" Required="false" Mask="false"/>
    <Config Name="ROMM_AUTH_USERNAME" Target="ROMM_AUTH_USERNAME" Default="admin" Mode="" Description="Default admin username" Type="Variable" Display="advanced" Required="false" Mask="false"/>
    <Config Name="ROMM_AUTH_PASSWORD" Target="ROMM_AUTH_PASSWORD" Default="" Mode="" Description="Default admin password" Type="Variable" Display="advanced" Required="false" Mask="true"/>
    <Config Name="ROMM_AUTH_SECRET_KEY" Target="ROMM_AUTH_SECRET_KEY" Default="" Mode="" Description="Generate a key with `openssl rand -hex 32`" Type="Variable" Display="advanced" Required="false" Mask="true"/>