scan websocket migrated to socket.io

2026-06-28 14:56:01 +00:00 · 2023-05-11 14:17:30 +02:00
parent 3a11b96653
commit 302b8cf073
9 changed files with 243 additions and 81 deletions
--- a/backend/endpoints/scan.py
+++ b/backend/endpoints/scan.py
@@ -1,60 +0,0 @@
-from fastapi import APIRouter, WebSocket, status, HTTPException
-import emoji
-import json
-
-from logger.logger import log, COLORS
-from utils import fs, fastapi
-from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException
-from handler import dbh
-from models.platform import Platform
-from models.rom import Rom
-
-router = APIRouter()
-
-
-@router.websocket("/scan")
-async def scan(websocket: WebSocket, platforms: str, complete_rescan: bool=False) -> dict:
-    """Scan platforms and roms and write them in database."""
-
-    await websocket.accept()
-    log.info(emoji.emojize(":magnifying_glass_tilted_right: Scanning "))
-    fs.store_default_resources()
-
-    # Scanning platforms
-    try:
-        fs_platforms: list[str] = fs.get_platforms()
-    except PlatformsNotFoundException as e:
-        error: str = f"{e}" 
-        log.warning(error)
-        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error)
-    
-    platforms: list[str] = json.loads(platforms) if len(json.loads(platforms)) > 0 else fs_platforms
-    log.info(f"Platforms to be scanned: {', '.join(platforms)}")
-    for platform in platforms:
-        log.info(emoji.emojize(f":video_game: {platform} {COLORS['reset']}"))
-        scanned_platform: Platform = fastapi.scan_platform(platform)
-        if platform != str(scanned_platform): log.info(f"Identified as {COLORS['blue']}{scanned_platform}{COLORS['reset']}")
-        dbh.add_platform(scanned_platform)
-
-        # Scanning roms
-        try:
-            fs_roms: list[str] = fs.get_roms(scanned_platform.fs_slug)
-        except RomsNotFoundException as e:
-            error: str = f"{e}" 
-            log.warning(error)
-            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error)
-        for rom in fs_roms:
-            rom_id: int = dbh.rom_exists(scanned_platform.slug, rom['file_name'])
-            if rom_id and not complete_rescan: continue
-            await websocket.send_text(f"Scanning: {scanned_platform} - {rom['file_name']}")
-            log.info(f"Scanning {COLORS['orange']}{rom['file_name']}{COLORS['reset']}")
-            import time
-            time.sleep(10)
-            if rom['multi']: [log.info(f"\t - {COLORS['orange_i']}{file}{COLORS['reset']}") for file in rom['files']]
-            scanned_rom: Rom = fastapi.scan_rom(scanned_platform, rom)
-            if rom_id: scanned_rom.id = rom_id
-            dbh.add_rom(scanned_rom)
-        dbh.purge_roms(scanned_platform.slug, [rom['file_name'] for rom in fs_roms])
-    dbh.purge_platforms(fs_platforms)
-    await websocket.send_text("")
-    await websocket.close()
--- a/backend/main.py
+++ b/backend/main.py
@@ -1,11 +1,21 @@
 import uvicorn
-from fastapi import FastAPI
+import emoji
+import json
+from fastapi import FastAPI, status, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi_socketio import SocketManager

 from config import DEV_PORT, DEV_HOST
-from endpoints import scan, search, platform, rom
+from logger.logger import log, COLORS
+from utils import fs, fastapi
+from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException
+from handler import dbh
+from models.platform import Platform
+from models.rom import Rom
+from endpoints import search, platform, rom

 app = FastAPI()
+socket_manager = SocketManager(app=app)
 app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
@@ -13,15 +23,60 @@ app.add_middleware(
    allow_methods=["*"],
    allow_headers=["*"],
 )
-app.include_router(scan.router)
 app.include_router(search.router)
 app.include_router(platform.router)
 app.include_router(rom.router)

+
+@app.sio.on("scan")
+async def scan(sid, platforms: str, complete_rescan: bool=True):
+    """Scan platforms and roms and write them in database."""
+
+    log.info(emoji.emojize(":magnifying_glass_tilted_right: Scanning "))
+    fs.store_default_resources()
+
+    try: # Scanning platforms
+        fs_platforms: list[str] = fs.get_platforms()
+    except PlatformsNotFoundException as e:
+        error: str = f"{e}" 
+        log.warning(error)
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error)
+    
+    platforms: list[str] = json.loads(platforms) if len(json.loads(platforms)) > 0 else fs_platforms
+    log.info(f"Platforms to be scanned: {', '.join(platforms)}")
+    for platform in platforms:
+        log.info(emoji.emojize(f":video_game: {platform} {COLORS['reset']}"))
+        scanned_platform: Platform = fastapi.scan_platform(platform)
+        if platform != str(scanned_platform): log.info(f"Identified as {COLORS['blue']}{scanned_platform}{COLORS['reset']}")
+        dbh.add_platform(scanned_platform)
+
+        try: # Scanning roms
+            fs_roms: list[str] = fs.get_roms(scanned_platform.fs_slug)
+        except RomsNotFoundException as e:
+            error: str = f"{e}" 
+            log.warning(error)
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error)
+        for rom in fs_roms:
+            rom_id: int = dbh.rom_exists(scanned_platform.slug, rom['file_name'])
+            if rom_id and not complete_rescan: continue
+            await app.sio.emit('scanning', {'platform': scanned_platform.name, 'rom': rom['file_name']})
+            await app.sio.emit('') # Workaround to emit in real-time
+            log.info(f"Scanning {COLORS['orange']}{rom['file_name']}{COLORS['reset']}")
+            if rom['multi']: [log.info(f"\t - {COLORS['orange_i']}{file}{COLORS['reset']}") for file in rom['files']]
+            scanned_rom: Rom = fastapi.scan_rom(scanned_platform, rom)
+            if rom_id: scanned_rom.id = rom_id
+            dbh.add_rom(scanned_rom)
+        dbh.purge_roms(scanned_platform.slug, [rom['file_name'] for rom in fs_roms])
+    dbh.purge_platforms(fs_platforms)
+    await app.sio.emit('done', 'Scan completed!')
+
+
@app.on_event("startup")
 def startup() -> None:
    """Startup application."""
    pass

+
 if __name__ == '__main__':
    uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=True)
+    # uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=False, workers=2)
--- a/docker/nginx/default.conf
+++ b/docker/nginx/default.conf
@@ -47,5 +47,11 @@ http {
                rewrite /api/(.*) /$1 break;
                proxy_pass http://localhost:5000/;
            }
+            location /ws {
+                proxy_pass http://localhost:5000;
+                proxy_http_version 1.1;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection "upgrade";
+            }
        }
 }
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -16,6 +16,7 @@
        "mitt": "^3.0.0",
        "pinia": "^2.0.35",
        "roboto-fontface": "*",
+        "socket.io-client": "^4.6.1",
        "vue": "^3.2.13",
        "vue-router": "^4.0.0",
        "vuetify": "^3.1.15",
@@ -2120,6 +2121,11 @@
        }
      }
    },
+    "node_modules/@socket.io/component-emitter": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz",
+      "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg=="
+    },
    "node_modules/@surma/rollup-plugin-off-main-thread": {
      "version": "2.2.3",
      "resolved": "https://registry.npmjs.org/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz",
@@ -2811,7 +2817,6 @@
      "version": "4.3.4",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "devOptional": true,
      "dependencies": {
        "ms": "2.1.2"
      },
@@ -2896,6 +2901,26 @@
      "integrity": "sha512-FRHZO+1tUNO4TOPXmlxetkoaIY8uwHzd1kKopK/Gx2SKn1L47wJXWD44wxP5CGRyyP98z/c8e1eBzJrgPeiBOg==",
      "dev": true
    },
+    "node_modules/engine.io-client": {
+      "version": "6.4.0",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.4.0.tgz",
+      "integrity": "sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g==",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1",
+        "engine.io-parser": "~5.0.3",
+        "ws": "~8.11.0",
+        "xmlhttprequest-ssl": "~2.0.0"
+      }
+    },
+    "node_modules/engine.io-parser": {
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.6.tgz",
+      "integrity": "sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
    "node_modules/es-abstract": {
      "version": "1.21.2",
      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.2.tgz",
@@ -4699,8 +4724,7 @@
    "node_modules/ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-      "devOptional": true
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
    "node_modules/nanoid": {
      "version": "3.3.4",
@@ -5478,6 +5502,32 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/socket.io-client": {
+      "version": "4.6.1",
+      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.6.1.tgz",
+      "integrity": "sha512-5UswCV6hpaRsNg5kkEHVcbBIXEYoVbMQaHJBXJCyEQ+CiFPV1NIOY0XOFWG4XR4GZcB8Kn6AsRs/9cy9TbqVMQ==",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.2",
+        "engine.io-client": "~6.4.0",
+        "socket.io-parser": "~4.2.1"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/socket.io-parser": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.2.tgz",
+      "integrity": "sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw==",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
    "node_modules/source-map": {
      "version": "0.6.1",
      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -6508,6 +6558,26 @@
      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
      "dev": true
    },
+    "node_modules/ws": {
+      "version": "8.11.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
+      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/xml-name-validator": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
@@ -6517,6 +6587,14 @@
        "node": ">=12"
      }
    },
+    "node_modules/xmlhttprequest-ssl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz",
+      "integrity": "sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
    "node_modules/yallist": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -17,6 +17,7 @@
    "mitt": "^3.0.0",
    "pinia": "^2.0.35",
    "roboto-fontface": "*",
+    "socket.io-client": "^4.6.1",
    "vue": "^3.2.13",
    "vue-router": "^4.0.0",
    "vuetify": "^3.1.15",
--- a/frontend/src/views/library/Scan.vue
+++ b/frontend/src/views/library/Scan.vue
@@ -1,6 +1,6 @@
 <script setup>
-import axios from "axios"
 import { ref, inject } from "vue"
+import { io } from "socket.io-client";
 import { storePlatforms } from '@/stores/platforms.js'
 import { storeScanning } from '@/stores/scanning.js'

@@ -16,14 +16,18 @@ const emitter = inject('emitter')

 // Functions
 async function scan() {
-    scanning.set(true)
-    const socket = new WebSocket('ws://localhost:5000/scan?platforms='+JSON.stringify(platformsToScan.value.map(p => p.fs_slug))+'&complete_rescan='+completeRescan.value)
-    socket.onmessage = function(e){ wsMsg.value = e.data }
-    socket.onclose = function(){ 
-        scanning.set(false)
+    scanning.set(true);
+    wsMsg.value = 'Scanning...'
+    const socket = io({ path: '/ws/socket.io/', transports: ['websocket', 'polling'] })
+    socket.on("connect", () => {console.log("ws connected")})
+    socket.on('disconnect', () => {console.log('ws disconnected');});
+    socket.on("scanning", (params) => {wsMsg.value = 'Scanning > '+params['platform']+' - '+params['rom']})
+    socket.on("done", (msg) => { 
+        scanning.set(false);
        emitter.emit('snackbarScan', {'msg': "Scan completed successfully!", 'icon': 'mdi-check-bold', 'color': 'green'})
-        emitter.emit('refresh')
-    }
+        wsMsg.value = msg
+    })
+    socket.emit("scan", JSON.stringify(platformsToScan.value.map(p => p.fs_slug)), completeRescan.value)
 }
 </script>

--- a/frontend/vite.config.js
+++ b/frontend/vite.config.js
@@ -52,12 +52,16 @@ export default defineConfig({
  server: {
    proxy: {
        '/api': {
-            target: 'http://localhost:5000',
-            changeOrigin: false,
-            secure: false,      
-            ws: true,
-            rewrite: (path) => path.replace(/^\/api/, ''),
-        }
+          target: 'http://localhost:5000',
+          changeOrigin: false,
+          secure: false,
+          rewrite: (path) => path.replace(/^\/api/, ''),
+        },
+        '/ws': {
+          target: 'http://localhost:5000',
+          changeOrigin: false,
+          ws: true,
+        },
    },
    port: 3000,
  },
--- a/poetry.lock
+++ b/poetry.lock
@@ -41,6 +41,23 @@ doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
 test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"]
 trio = ["trio (>=0.16,<0.22)"]

+[[package]]
+name = "bidict"
+version = "0.22.1"
+description = "The bidirectional mapping library for Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "bidict-0.22.1-py3-none-any.whl", hash = "sha256:6ef212238eb884b664f28da76f33f1d28b260f665fc737b413b287d5487d1e7b"},
+    {file = "bidict-0.22.1.tar.gz", hash = "sha256:1e0f7f74e4860e6d0943a05d4134c63a2fad86f3d4732fb265bd79e4e856d81d"},
+]
+
+[package.extras]
+docs = ["furo", "sphinx", "sphinx-copybutton"]
+lint = ["pre-commit"]
+test = ["hypothesis", "pytest", "pytest-benchmark[histogram]", "pytest-cov", "pytest-xdist", "sortedcollections", "sortedcontainers", "sphinx"]
+
 [[package]]
 name = "certifi"
 version = "2023.5.7"
@@ -201,6 +218,25 @@ dev = ["pre-commit (>=2.17.0,<3.0.0)", "ruff (==0.0.138)", "uvicorn[standard] (>
 doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer-cli (>=0.0.13,<0.0.14)", "typer[all] (>=0.6.1,<0.8.0)"]
 test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6.5.0,<8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.7)", "pyyaml (>=5.3.1,<7.0.0)", "ruff (==0.0.138)", "sqlalchemy (>=1.3.18,<1.4.43)", "types-orjson (==3.6.2)", "types-ujson (==5.7.0.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"]

+[[package]]
+name = "fastapi-socketio"
+version = "0.0.10"
+description = "Easily integrate socket.io with your FastAPI app."
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "fastapi-socketio-0.0.10.tar.gz", hash = "sha256:202f9b319f010001cbd1114ec92a0d9eb5f5ca9316eae5fd41a6088da0812727"},
+    {file = "fastapi_socketio-0.0.10-py3-none-any.whl", hash = "sha256:11c2bfa3f25d786bd860ed13c892472e86bfeba85e7a0bec4f922ae5e4d8650f"},
+]
+
+[package.dependencies]
+fastapi = ">=0.61.1"
+python-socketio = ">=4.6.0"
+
+[package.extras]
+test = ["pytest"]
+
 [[package]]
 name = "greenlet"
 version = "2.0.2"
@@ -483,6 +519,42 @@ files = [
 [package.extras]
 cli = ["click (>=5.0)"]

+[[package]]
+name = "python-engineio"
+version = "4.4.1"
+description = "Engine.IO server and client for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "python-engineio-4.4.1.tar.gz", hash = "sha256:eb3663ecb300195926b526386f712dff84cd092c818fb7b62eeeda9160120c29"},
+    {file = "python_engineio-4.4.1-py3-none-any.whl", hash = "sha256:28ab67f94cba2e5f598cbb04428138fd6bb8b06d3478c939412da445f24f0773"},
+]
+
+[package.extras]
+asyncio-client = ["aiohttp (>=3.4)"]
+client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"]
+
+[[package]]
+name = "python-socketio"
+version = "5.8.0"
+description = "Socket.IO server and client for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "python-socketio-5.8.0.tar.gz", hash = "sha256:e714f4dddfaaa0cb0e37a1e2deef2bb60590a5b9fea9c343dd8ca5e688416fd9"},
+    {file = "python_socketio-5.8.0-py3-none-any.whl", hash = "sha256:7adb8867aac1c2929b9c1429f1c02e12ca4c36b67c807967393e367dfbb01441"},
+]
+
+[package.dependencies]
+bidict = ">=0.21.0"
+python-engineio = ">=4.3.0"
+
+[package.extras]
+asyncio-client = ["aiohttp (>=3.4)"]
+client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"]
+
 [[package]]
 name = "pyyaml"
 version = "6.0"
@@ -807,4 +879,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "346cb1da6a2a7a8a34abcf4fb8ea7e4f98b73bf2859656fa0f88fa6d9eabe305"
+content-hash = "be87b7c005a212de6e70dacc2cc478fbe8b1b1d5270ae78e25e85cacebabdb0c"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,8 @@ requests = "2.30.0"
 fastapi = "0.95.1"
 uvicorn = "0.22.0"
 websockets = "11.0.3"
+python-socketio = "5.8.0"
+fastapi-socketio = "0.0.10"
 mariadb = "1.1.6"
 SQLAlchemy = "2.0.12"
 alembic = "1.10.4"