diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 16eadd2c5..061715b18 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -26,9 +26,8 @@ DB_PASSWD: Final = os.environ.get("DB_PASSWD") DB_NAME: Final = os.environ.get("DB_NAME", "romm") # REDIS -REDIS_HOST: Final = os.environ.get("REDIS_HOST", "localhost") -REDIS_PORT: Final = int(os.environ.get("REDIS_PORT", 6379)) -REDIS_PASSWORD: Final = os.environ.get("REDIS_PASSWORD") +REDIS_HOST: Final = "localhost" +REDIS_PORT: Final = 6379 # IGDB IGDB_CLIENT_ID: Final = os.environ.get( diff --git a/backend/handler/redis_handler.py b/backend/handler/redis_handler.py index 6fe3156b0..1a5250cde 100644 --- a/backend/handler/redis_handler.py +++ b/backend/handler/redis_handler.py @@ -1,7 +1,7 @@ import sys from enum import Enum -from config import REDIS_HOST, REDIS_PASSWORD, REDIS_PORT +from config import REDIS_HOST, REDIS_PORT from logger.logger import log from redis import Redis from rq import Queue @@ -39,15 +39,8 @@ class FallbackCache: return repr(self) -redis_client = Redis( - host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=0 -) - -redis_url = ( - f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}" - if REDIS_PASSWORD - else f"redis://{REDIS_HOST}:{REDIS_PORT}" -) +redis_client = Redis(host=REDIS_HOST, port=REDIS_PORT, db=0) +redis_url = f"redis://{REDIS_HOST}:{REDIS_PORT}" high_prio_queue = Queue(name=QueuePrio.HIGH.value, connection=redis_client) default_queue = Queue(name=QueuePrio.DEFAULT.value, connection=redis_client) @@ -61,7 +54,6 @@ else: cache = Redis( host=REDIS_HOST, port=REDIS_PORT, - password=REDIS_PASSWORD, db=0, decode_responses=True, ) diff --git a/docker/Dockerfile b/docker/Dockerfile index 27de486f1..c00373ca8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,7 +23,8 @@ RUN apk add --upgrade \ libffi \ mariadb-connector-c \ netcat-openbsd \ - python3 + python3 \ + tzdata # Install additional build dependencies RUN apk add --upgrade \ @@ -33,7 +34,13 @@ RUN apk add --upgrade \ musl-dev \ python3-dev \ py3-pip \ - git + git \ + wget \ + coreutils \ + dpkg-dev dpkg \ + linux-headers \ + make \ + openssl-dev # Create python venv to not clash with OS python packages RUN python3 -m venv /backend/ @@ -56,7 +63,7 @@ RUN . /backend/bin/activate && \ # cleanup python dependencies that are not needed anymore RUN . /backend/bin/activate && \ grep -v -x -f /installed_poetry_requirements.txt /installed_pip_requirements.txt > /build_requirements.txt && \ - pip uninstall -y -r /build_requirements.txt + pip uninstall -y -r /build_requirements.txt COPY ./backend /backend @@ -64,6 +71,55 @@ COPY ./backend /backend COPY ./docker/init_scripts/* / COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf +# Install redis +ENV REDIS_VERSION 7.2.4 +ENV REDIS_DOWNLOAD_URL http://download.redis.io/releases/redis-7.2.4.tar.gz +ENV REDIS_DOWNLOAD_SHA 8d104c26a154b29fd67d6568b4f375212212ad41e0c2caa3d66480e78dbd3b59 + +RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ + echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ + mkdir -p /usr/src/redis; \ + tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \ + rm redis.tar.gz; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' /usr/src/redis/src/config.c; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' /usr/src/redis/src/config.c; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' /usr/src/redis/src/config.c; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + extraJemallocConfigureFlags="--build=$gnuArch"; \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in (amd64 | i386 | x32) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=12";; (*) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=16";; esac; \ + extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-hugepage=21"; \ + grep -F 'cd jemalloc && ./configure ' /usr/src/redis/deps/Makefile; \ + sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \ + grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ + export BUILD_TLS=yes; \ + make -C /usr/src/redis -j "$(nproc)" all; \ + make -C /usr/src/redis install; \ + serverMd5="$(md5sum /usr/local/bin/redis-server | cut -d' ' -f1)"; export serverMd5; \ + find /usr/local/bin/redis* -maxdepth 0 \ + -type f -not -name redis-server \ + -exec sh -eux -c ' \ + md5="$(md5sum "$1" | cut -d" " -f1)"; \ + test "$md5" = "$serverMd5"; \ + ' -- '{}' ';' \ + -exec ln -svfT 'redis-server' '{}' ';' \ + ; \ + rm -r /usr/src/redis; \ + runDeps="$( \ + scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \ + | tr ',' '\n' \ + | sort -u \ + | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ + )"; \ + apk add --no-network --virtual .redis-rundeps $runDeps; \ + apk del --no-network .build-deps; \ + redis-cli --version; \ + redis-server --version; \ + echo '{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","name":"redis-server-sbom","packages":[{"name":"redis-server","versionInfo":"7.2.4","SPDXID":"SPDXRef-Package--redis-server","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:generic/redis-server@7.2.4?os_name=alpine&os_version=3.19"}],"licenseDeclared":"BSD-3-Clause"}]}' > /usr/local/redis.spdx.json + +RUN mkdir /redis-data && chown 1000:1000 /redis-data +VOLUME /redis-data + # cleanup additional build dependencies RUN apk del \ gcc \ @@ -72,7 +128,13 @@ RUN apk del \ musl-dev \ python3-dev \ py3-pip \ - git + git \ + wget \ + coreutils \ + dpkg-dev dpkg \ + linux-headers \ + make \ + openssl-dev # cleanup leftover files that are not needed at runtime RUN rm -r \ @@ -92,5 +154,6 @@ RUN mkdir /romm && chown 1000:1000 /romm -R # Expose ports and start EXPOSE 8080 +EXPOSE 6379/tcp WORKDIR /romm CMD ["/init"] diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 9d57157de..9db5c9f58 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -56,6 +56,19 @@ start_bin_nginx () { fi } +start_bin_redis-server () { + info_log "starting redis-server" + + # Check if /usr/local/etc/redis/redis.conf exists and use it if so + if [ -f /usr/local/etc/redis/redis.conf ]; then + redis-server /usr/local/etc/redis/redis.conf & + else + redis-server & + fi + REDIS_PID=$! + echo $REDIS_PID > /tmp/redis-server.pid +} + # function that runs our independent python scripts and creates corresponding PID files, start_python () { SCRIPT="${1}" @@ -93,6 +106,9 @@ cd /backend || { error_log "/backend directory doesn't seem to exist"; } # function definition done, lets start our main loop while true; do + # Start redis server if we dont have a corresponding PID file + watchdog_process_pid bin redis-server + # Run needed database migrations on startup, # but only if it was not successful since the last full docker container start if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then diff --git a/examples/docker-compose.example.yml b/examples/docker-compose.example.yml index ca05f56b8..508546bc3 100644 --- a/examples/docker-compose.example.yml +++ b/examples/docker-compose.example.yml @@ -19,7 +19,6 @@ services: - ROMM_AUTH_SECRET_KEY= # Generate a key with `openssl rand -hex 32` - ROMM_AUTH_USERNAME=admin - ROMM_AUTH_PASSWORD= # default: admin - - REDIS_HOST=romm-redis volumes: - romm_resources:/romm/resources" # Resources fetched from IGDB (covers, screenshots, etc.) - "/path/to/library:/romm/library" # Your game library @@ -30,7 +29,6 @@ services: - 80:8080 depends_on: - romm-db - - romm-redis romm-db: image: mariadb:latest @@ -43,8 +41,3 @@ services: - MYSQL_PASSWORD= volumes: - mysql_data:/var/lib/mysql - - romm-redis: - image: redis:alpine - container_name: romm-redis - restart: unless-stopped diff --git a/unraid_template/romm.xml b/unraid_template/romm.xml index be711d4dc..74ec6e4ab 100644 --- a/unraid_template/romm.xml +++ b/unraid_template/romm.xml @@ -41,7 +41,6 @@ -