From 3a46279b25615147f121bbc2f9193a429d2d8fdc Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Thu, 19 Feb 2026 12:55:23 +0800 Subject: [PATCH] refactor: improve dynamic RDAP DNS data (#6971) --- .github/workflows/autofix.yml | 4 - ...-disable-domain-expiry-unsupported-tlds.js | 10 ++- {server/model => extra}/rdap-dns.json | 0 server/model/domain_expiry.js | 83 +++++++++++++++++-- 4 files changed, 87 insertions(+), 10 deletions(-) rename {server/model => extra}/rdap-dns.json (100%) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 17a2275c5..a6844df60 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -30,10 +30,6 @@ jobs: - name: Install dependencies run: npm ci - - name: Update RDAP DNS data from IANA - run: wget -O server/model/rdap-dns.json https://data.iana.org/rdap/dns.json - continue-on-error: true - - name: Auto-fix JavaScript/Vue linting issues run: npm run lint-fix:js continue-on-error: true diff --git a/db/knex_migrations/2026-02-07-0000-disable-domain-expiry-unsupported-tlds.js b/db/knex_migrations/2026-02-07-0000-disable-domain-expiry-unsupported-tlds.js index 9685d46cc..995b8ea02 100644 --- a/db/knex_migrations/2026-02-07-0000-disable-domain-expiry-unsupported-tlds.js +++ b/db/knex_migrations/2026-02-07-0000-disable-domain-expiry-unsupported-tlds.js @@ -1,5 +1,13 @@ const { parse: parseTld } = require("tldts"); -const rdapDnsData = require("../../server/model/rdap-dns.json"); + +/* + * TODO: + * This migration file is scary, because the json file is dynamically updated. + * Problem 1: Migration files should ideally be stateless. + * Problem 2: This migration only runs once, what happens if rdp-dns.json is updated after this migration has run? + * Have to investigate later. + */ +const rdapDnsData = require("../../extra/rdap-dns.json"); const TYPES_WITH_DOMAIN_EXPIRY_SUPPORT_VIA_FIELD = { http: "url", diff --git a/server/model/rdap-dns.json b/extra/rdap-dns.json similarity index 100% rename from server/model/rdap-dns.json rename to extra/rdap-dns.json diff --git a/server/model/domain_expiry.js b/server/model/domain_expiry.js index 20236f640..73fc020b9 100644 --- a/server/model/domain_expiry.js +++ b/server/model/domain_expiry.js @@ -6,16 +6,19 @@ const { setting, setSetting } = require("../util-server"); const { Notification } = require("../notification"); const TranslatableError = require("../translatable-error"); const dayjs = require("dayjs"); +const { Settings } = require("../settings"); -// Load static RDAP DNS data from local file (auto-updated by CI) -const rdapDnsData = require("./rdap-dns.json"); +let cacheRdapDnsData = null; +let nextChecking = 0; +let running = false; /** * Find the RDAP server for a given TLD * @param {string} tld TLD * @returns {string|null} First RDAP server found */ -function getRdapServer(tld) { +async function getRdapServer(tld) { + const rdapDnsData = await getRdapDnsData(); const services = rdapDnsData["services"] ?? []; const rootTld = tld?.split(".").pop(); if (rootTld) { @@ -29,6 +32,76 @@ function getRdapServer(tld) { return null; } +/** + * Get RDAP DNS data from IANA and save to Setting + * @returns {Promise<{}>} RDAP DNS data + */ +async function getRdapDnsData() { + // Cache for one week + if (cacheRdapDnsData && Date.now() < nextChecking) { + return cacheRdapDnsData; + } + + let data = null; + + // Avoid multiple simultaneous updates + // Use older data first if another update is in progress + if (running) { + return await getOfflineRdapDnsData(); + } + + try { + running = true; + log.info("rdap", "Updating RDAP DNS data from IANA..."); + const response = await fetch("https://data.iana.org/rdap/dns.json"); + if (!response.ok) { + throw new Error(`HTTP error: ${response.status}`); + } + + data = await response.json(); + + // Simple validation + if (!data.services || !Array.isArray(data.services)) { + throw new Error("Invalid RDAP DNS data structure"); + } + + // Next week + nextChecking = Date.now() + 7 * 24 * 60 * 60 * 1000; + await Settings.set("rdapDnsData", data); + log.info("rdap", "RDAP DNS data updated successfully. Number of services: " + data.services.length); + } catch (error) { + log.info("rdap", `Uable to update RDAP DNS data from source: ${error.message}`); + data = await getOfflineRdapDnsData(); + + // Check again next day + nextChecking = Date.now() + 24 * 60 * 60 * 1000; + } + + running = false; + return data; +} + +/** + * Get RDAP DNS data from Setting or hardcoded file as fallback + * Fail safe + * @returns {Promise<{}>} RDAP DNS data + */ +async function getOfflineRdapDnsData() { + let data = null; + try { + data = await Settings.get("rdapDnsData"); + + // Simple validation + if (!data.services || !Array.isArray(data.services)) { + throw new Error("Invalid RDAP DNS data structure"); + } + } catch (e) { + // If not downloaded previously, use the hardcoded data + data = require("../../extra/rdap-dns.json"); + } + return data; +} + /** * Request RDAP server to retrieve the expiry date of a domain * @param {string} domain Domain to retrieve the expiry date from @@ -36,7 +109,7 @@ function getRdapServer(tld) { */ async function getRdapDomainExpiryDate(domain) { const tld = DomainExpiry.parseTld(domain).publicSuffix; - const rdapServer = getRdapServer(tld); + const rdapServer = await getRdapServer(tld); if (rdapServer === null) { log.warn("rdap", `No RDAP server found, TLD ${tld} not supported.`); return null; @@ -164,7 +237,7 @@ class DomainExpiry extends BeanModel { const publicSuffix = tld.publicSuffix; const rootTld = publicSuffix.split(".").pop(); - const rdap = getRdapServer(publicSuffix); + const rdap = await getRdapServer(publicSuffix); if (!rdap) { throw new TranslatableError("domain_expiry_unsupported_unsupported_tld_no_rdap_endpoint", { publicSuffix,