refactor: improve dynamic RDAP DNS data (#6971)

.github/workflows/autofix.yml

@@ -30,10 +30,6 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
-      - name: Update RDAP DNS data from IANA
-        run: wget -O server/model/rdap-dns.json https://data.iana.org/rdap/dns.json
-        continue-on-error: true
-
       - name: Auto-fix JavaScript/Vue linting issues
         run: npm run lint-fix:js
         continue-on-error: true

db/knex_migrations/2026-02-07-0000-disable-domain-expiry-unsupported-tlds.js

@@ -1,5 +1,13 @@
 const { parse: parseTld } = require("tldts");
-const rdapDnsData = require("../../server/model/rdap-dns.json");
+
+/*
+ * TODO:
+ *  This migration file is scary, because the json file is dynamically updated.
+ *  Problem 1: Migration files should ideally be stateless.
+ *  Problem 2: This migration only runs once, what happens if rdp-dns.json is updated after this migration has run?
+ *  Have to investigate later.
+ */
+const rdapDnsData = require("../../extra/rdap-dns.json");
 
 const TYPES_WITH_DOMAIN_EXPIRY_SUPPORT_VIA_FIELD = {
     http: "url",

server/model/domain_expiry.js

@@ -6,16 +6,19 @@ const { setting, setSetting } = require("../util-server");
 const { Notification } = require("../notification");
 const TranslatableError = require("../translatable-error");
 const dayjs = require("dayjs");
+const { Settings } = require("../settings");
 
-// Load static RDAP DNS data from local file (auto-updated by CI)
-const rdapDnsData = require("./rdap-dns.json");
+let cacheRdapDnsData = null;
+let nextChecking = 0;
+let running = false;
 
 /**
  * Find the RDAP server for a given TLD
  * @param {string} tld TLD
  * @returns {string|null} First RDAP server found
  */
-function getRdapServer(tld) {
+async function getRdapServer(tld) {
+    const rdapDnsData = await getRdapDnsData();
     const services = rdapDnsData["services"] ?? [];
     const rootTld = tld?.split(".").pop();
     if (rootTld) {
@@ -29,6 +32,76 @@ function getRdapServer(tld) {
     return null;
 }
 
+/**
+ * Get RDAP DNS data from IANA and save to Setting
+ * @returns {Promise<{}>} RDAP DNS data
+ */
+async function getRdapDnsData() {
+    // Cache for one week
+    if (cacheRdapDnsData && Date.now() < nextChecking) {
+        return cacheRdapDnsData;
+    }
+
+    let data = null;
+
+    // Avoid multiple simultaneous updates
+    // Use older data first if another update is in progress
+    if (running) {
+        return await getOfflineRdapDnsData();
+    }
+
+    try {
+        running = true;
+        log.info("rdap", "Updating RDAP DNS data from IANA...");
+        const response = await fetch("https://data.iana.org/rdap/dns.json");
+        if (!response.ok) {
+            throw new Error(`HTTP error: ${response.status}`);
+        }
+
+        data = await response.json();
+
+        // Simple validation
+        if (!data.services || !Array.isArray(data.services)) {
+            throw new Error("Invalid RDAP DNS data structure");
+        }
+
+        // Next week
+        nextChecking = Date.now() + 7 * 24 * 60 * 60 * 1000;
+        await Settings.set("rdapDnsData", data);
+        log.info("rdap", "RDAP DNS data updated successfully. Number of services: " + data.services.length);
+    } catch (error) {
+        log.info("rdap", `Uable to update RDAP DNS data from source: ${error.message}`);
+        data = await getOfflineRdapDnsData();
+
+        // Check again next day
+        nextChecking = Date.now() + 24 * 60 * 60 * 1000;
+    }
+
+    running = false;
+    return data;
+}
+
+/**
+ * Get RDAP DNS data from Setting or hardcoded file as fallback
+ * Fail safe
+ * @returns {Promise<{}>} RDAP DNS data
+ */
+async function getOfflineRdapDnsData() {
+    let data = null;
+    try {
+        data = await Settings.get("rdapDnsData");
+
+        // Simple validation
+        if (!data.services || !Array.isArray(data.services)) {
+            throw new Error("Invalid RDAP DNS data structure");
+        }
+    } catch (e) {
+        // If not downloaded previously, use the hardcoded data
+        data = require("../../extra/rdap-dns.json");
+    }
+    return data;
+}
+
 /**
  * Request RDAP server to retrieve the expiry date of a domain
  * @param {string} domain Domain to retrieve the expiry date from
@@ -36,7 +109,7 @@ function getRdapServer(tld) {
  */
 async function getRdapDomainExpiryDate(domain) {
     const tld = DomainExpiry.parseTld(domain).publicSuffix;
-    const rdapServer = getRdapServer(tld);
+    const rdapServer = await getRdapServer(tld);
     if (rdapServer === null) {
         log.warn("rdap", `No RDAP server found, TLD ${tld} not supported.`);
         return null;
@@ -164,7 +237,7 @@ class DomainExpiry extends BeanModel {
 
         const publicSuffix = tld.publicSuffix;
         const rootTld = publicSuffix.split(".").pop();
-        const rdap = getRdapServer(publicSuffix);
+        const rdap = await getRdapServer(publicSuffix);
         if (!rdap) {
             throw new TranslatableError("domain_expiry_unsupported_unsupported_tld_no_rdap_endpoint", {
                 publicSuffix,