domainSyncWorker.js

Go to the documentation of this file.

/**
 * Domain Sync Worker
 * Syncs domains from Cloudflare, Moniker, and eNom to local database every 5 minutes
 * Uses Redis for caching domain data (10 minute TTL)
 * Provides faster page loads and reduces API calls to registrars
 */
 
const Redis = require('ioredis');
const pool = require('./db');
const { listZones, getZone } = require('./services/cloudflare');
const monikerService = require('./services/moniker');
const enomService = require('./services/enom');
 
const redisConfig = require('./config/redis');
let redisClient = null;
 
const SYNC_INTERVAL = 5 * 60 * 1000; // 5 minutes
const REDIS_CACHE_TTL = 10 * 60; // 10 minutes in seconds
const REDIS_KEY_PREFIX = 'domains:sync:';
const API_DELAY = 500; // Delay between API calls to avoid rate limiting
 
/**
 * Initialize Redis connection
 */
function initRedis() {
  if (!redisClient) {
    redisClient = new Redis({
      ...redisConfig,
      maxRetriesPerRequest: null,
      lazyConnect: true
    });
 
    redisClient.on('error', (err) => {
      console.error('[Domain Sync] Redis error:', err.message);
    });
 
    redisClient.on('connect', () => {
      console.log('[Domain Sync] Redis connected');
    });
  }
  return redisClient.connect().catch(err => {
    console.error('[Domain Sync] Redis connection failed:', err.message);
  });
}
 
/**
 * Sync domains from Cloudflare (DNS zones)
 * Fetches all zones from Cloudflare API
 * Returns a Map indexed by domain name for easy enrichment
 */
async function syncCloudflare() {
  const startTime = Date.now();
  console.log('[Domain Sync] Starting Cloudflare sync...');
  
  try {
    const zones = await listZones();
    
    if (!zones || zones.length === 0) {
      console.log('[Domain Sync] No Cloudflare zones found');
      return new Map();
    }
 
    // Create a Map indexed by domain name for fast lookup
    const zoneMap = new Map();
    
    zones.forEach(zone => {
      zoneMap.set(zone.name.toLowerCase(), {
        zone_id: zone.id,
        account_id: zone.account?.id,
        status: zone.status,
        nameservers: zone.name_servers || [],
        development_mode: zone.development_mode,
        plan: zone.plan?.name,
        created_on: zone.created_on
      });
    });
 
    const elapsed = Date.now() - startTime;
    console.log(`[Domain Sync] Cloudflare sync complete: ${zoneMap.size} zones (${elapsed}ms)`);
    return zoneMap;
  } catch (error) {
    console.error('[Domain Sync] Cloudflare sync error:', error.message);
    return new Map();
  }
}
 
/**
 * Check if nameservers are Cloudflare nameservers
 * @param {Array} nameservers - Array of nameserver strings
 * @returns {boolean} - True if Cloudflare nameservers detected
 */
function isCloudflareNameservers(nameservers) {
  if (!Array.isArray(nameservers) || nameservers.length === 0) {
    return false;
  }
  
  // Check if any nameserver contains cloudflare
  return nameservers.some(ns => 
    ns && ns.toLowerCase().includes('cloudflare')
  );
}
 
/**
 * Sync domains from Moniker registrar
 * Fetches ALL domains from Moniker API
 * @param cloudflareZones
 */
async function syncMoniker(cloudflareZones) {
  const startTime = Date.now();
  console.log('[Domain Sync] Starting Moniker sync...');
  
  try {
    // Fetch ALL domains from Moniker API (not from database)
    const monikerDomains = await monikerService.getAllDomains();
    
    if (monikerDomains.length === 0) {
      console.log('[Domain Sync] No Moniker domains found in account');
      return [];
    }
 
    console.log(`[Domain Sync] Found ${monikerDomains.length} Moniker domains from API`);
    const domains = [];
 
    // Process each domain from Moniker
    for (const domain of monikerDomains) {
      try {
        // Rate limiting delay
        await new Promise(resolve => setTimeout(resolve, API_DELAY));
 
        // Get full domain details
        const domainInfo = await monikerService.getDomainInfo(domain.name);
        
        if (!domainInfo) {
          console.log(`[Domain Sync] Could not get info for ${domain.name}`);
          continue;
        }
 
        // Build base domain object with Moniker registration data
        const autoRenew = domain.auto_renew || domainInfo.raw?.AutoRenew === 'true';
        console.log(`[Domain Sync] ${domain.name} - auto_renew from API: ${domain.auto_renew}, from raw: ${domainInfo.raw?.AutoRenew}, final: ${autoRenew}`);
        
        const domainObj = {
          name: domain.name,
          registrar: 'moniker',
          registrar_domain_id: domainInfo.raw?.DomainID || domain.name,
          status: domainInfo.status || domain.status || 'unknown',
          expiration_date: domainInfo.expiration_date,
          registration_date: domainInfo.registration_date || domain.registration_date,
          nameservers: domainInfo.nameservers || [],
          metadata: {
            domain_type: 'registered', // Moniker is a registrar - full domain management
            auto_renew: autoRenew,
            locked: domain.locked || domainInfo.raw?.Locked === 'true',
            privacy_enabled: domainInfo.raw?.PrivacyEnabled === 'true'
          }
        };
 
        // Check if domain uses Cloudflare nameservers
        if (isCloudflareNameservers(domainObj.nameservers)) {
          const cloudflareZone = cloudflareZones.get(domain.name.toLowerCase());
          
          if (cloudflareZone) {
            console.log(`[Domain Sync] Enriching ${domain.name} with Cloudflare zone data`);
            
            // Enrich with Cloudflare zone data
            domainObj.metadata.cloudflare_zone_id = cloudflareZone.zone_id;
            domainObj.metadata.cloudflare_account_id = cloudflareZone.account_id;
            domainObj.metadata.cloudflare_status = cloudflareZone.status;
            domainObj.metadata.cloudflare_plan = cloudflareZone.plan;
            domainObj.metadata.has_cloudflare_dns = true;
            
            // Update nameservers from Cloudflare (more current)
            if (cloudflareZone.nameservers && cloudflareZone.nameservers.length > 0) {
              domainObj.nameservers = cloudflareZone.nameservers;
            }
          } else {
            // Uses Cloudflare NS but no zone found (might be pending or external)
            domainObj.metadata.has_cloudflare_dns = false;
            console.log(`[Domain Sync] ${domain.name} uses Cloudflare NS but no zone found`);
          }
        }
 
        domains.push(domainObj);
      } catch (err) {
        console.error(`[Domain Sync] Error processing Moniker domain ${domain.name}:`, err.message);
      }
    }
 
    const elapsed = Date.now() - startTime;
    console.log(`[Domain Sync] Moniker sync complete: ${domains.length}/${monikerDomains.length} domains (${elapsed}ms)`);
    return domains;
  } catch (error) {
    console.error('[Domain Sync] Moniker sync error:', error.message);
    return [];
  }
}
 
/**
 * Sync domains from eNom registrar
 * Fetches ALL domains from eNom API
 * @param cloudflareZones
 */
async function syncEnom(cloudflareZones) {
  const startTime = Date.now();
  console.log('[Domain Sync] Starting eNom sync...');
  
  try {
    // Fetch ALL domains from eNom API (not from database)
    const enomDomains = await enomService.getAllDomains();
    
    if (enomDomains.length === 0) {
      console.log('[Domain Sync] No eNom domains found in account');
      return [];
    }
 
    console.log(`[Domain Sync] Found ${enomDomains.length} eNom domains from API`);
    const domains = [];
 
    // Process each domain from eNom
    for (const domain of enomDomains) {
      try {
        // Rate limiting delay
        await new Promise(resolve => setTimeout(resolve, API_DELAY));
 
        // Get full domain details
        const domainInfo = await enomService.getDomainInfo(domain.name);
        
        if (!domainInfo) {
          console.log(`[Domain Sync] Could not get info for ${domain.name}`);
          continue;
        }
 
        // Build base domain object with eNom registration data
        const autoRenew = domain.auto_renew || domainInfo.auto_renew;
        console.log(`[Domain Sync] ${domain.name} - auto_renew from API: ${domain.auto_renew}, from info: ${domainInfo.auto_renew}, final: ${autoRenew}`);
        
        const domainObj = {
          name: domain.name,
          registrar: 'enom',
          registrar_domain_id: domainInfo.raw?.domainnameid || domain.name,
          status: domainInfo.status || domain.status || 'unknown',
          expiration_date: domainInfo.expiration_date || domain.expiration_date,
          registration_date: domainInfo.registration_date || domain.registration_date,
          nameservers: domainInfo.nameservers || [],
          metadata: {
            domain_type: 'registered', // eNom is a registrar - full domain management
            auto_renew: autoRenew,
            locked: domain.locked || domainInfo.raw?.['registrar-lock'] === '1',
            privacy_enabled: domainInfo.raw?.['whois-protection'] === 'Enabled'
          }
        };
 
        // Check if domain uses Cloudflare nameservers
        if (isCloudflareNameservers(domainObj.nameservers)) {
          const cloudflareZone = cloudflareZones.get(domain.name.toLowerCase());
          
          if (cloudflareZone) {
            console.log(`[Domain Sync] Enriching ${domain.name} with Cloudflare zone data`);
            
            // Enrich with Cloudflare zone data
            domainObj.metadata.cloudflare_zone_id = cloudflareZone.zone_id;
            domainObj.metadata.cloudflare_account_id = cloudflareZone.account_id;
            domainObj.metadata.cloudflare_status = cloudflareZone.status;
            domainObj.metadata.cloudflare_plan = cloudflareZone.plan;
            domainObj.metadata.has_cloudflare_dns = true;
            
            // Update nameservers from Cloudflare (more current)
            if (cloudflareZone.nameservers && cloudflareZone.nameservers.length > 0) {
              domainObj.nameservers = cloudflareZone.nameservers;
            }
          } else {
            // Uses Cloudflare NS but no zone found
            domainObj.metadata.has_cloudflare_dns = false;
            console.log(`[Domain Sync] ${domain.name} uses Cloudflare NS but no zone found`);
          }
        }
 
        domains.push(domainObj);
      } catch (err) {
        console.error(`[Domain Sync] Error processing eNom domain ${domain.name}:`, err.message);
      }
    }
 
    const elapsed = Date.now() - startTime;
    console.log(`[Domain Sync] eNom sync complete: ${domains.length}/${enomDomains.length} domains (${elapsed}ms)`);
    return domains;
  } catch (error) {
    console.error('[Domain Sync] eNom sync error:', error.message);
    return [];
  }
}
 
/**
 * Process DNS-only Cloudflare zones
 * Creates domain entries for Cloudflare zones that are not registered with Moniker/eNom
 * @param cloudflareZones
 * @param monikerDomains
 * @param enomDomains
 */
function processDnsOnlyZones(cloudflareZones, monikerDomains, enomDomains) {
  const registeredDomains = new Set();
  
  // Collect all registered domain names
  monikerDomains.forEach(d => registeredDomains.add(d.name.toLowerCase()));
  enomDomains.forEach(d => registeredDomains.add(d.name.toLowerCase()));
  
  const dnsOnlyDomains = [];
  
  // Find Cloudflare zones that are not registered with our registrars
  cloudflareZones.forEach((zoneData, domainName) => {
    if (!registeredDomains.has(domainName)) {
      // This is a DNS-only zone (not registered through our system)
      dnsOnlyDomains.push({
        name: domainName,
        registrar: 'cloudflare',
        registrar_domain_id: zoneData.zone_id,
        status: zoneData.status === 'active' ? 'active' : 'inactive',
        expiration_date: null, // DNS-only zones don't have expiration dates
        nameservers: zoneData.nameservers || [],
        created_date: zoneData.created_on,
        metadata: {
          domain_type: 'dns_only', // DNS management only, not registered through our platform
          zone_id: zoneData.zone_id,
          account_id: zoneData.account_id,
          development_mode: zoneData.development_mode,
          plan: zoneData.plan
        }
      });
    }
  });
  
  console.log(`[Domain Sync] Found ${dnsOnlyDomains.length} DNS-only Cloudflare zones`);
  return dnsOnlyDomains;
}
 
/**
 * Update database with synced domain data
 * @param registrar
 * @param domains
 */
async function updateDatabase(registrar, domains) {
  if (domains.length === 0) return;
 
  console.log(`[Domain Sync] Updating database with ${domains.length} ${registrar} domains...`);
  let updated = 0;
  let inserted = 0;
 
  for (const domain of domains) {
    try {
      // Check if domain exists
      const existingDomain = await pool.query(
        `SELECT domain_id FROM domains WHERE domain_name = $1`,
        [domain.name]
      );
 
      if (existingDomain.rows.length > 0) {
        // Update existing domain
        const metadataJson = JSON.stringify(domain.metadata);
        console.log(`[Domain Sync] Updating ${domain.name} with metadata:`, metadataJson);
        
        await pool.query(
          `UPDATE domains SET 
            status = $1,
            registrar_domain_id = $2,
            expiration_date = $3,
            nameservers = $4,
            metadata = $5,
            updated_at = NOW()
          WHERE domain_name = $6`,
          [
            domain.status,
            domain.registrar_domain_id,
            domain.expiration_date,
            JSON.stringify(domain.nameservers),
            metadataJson,
            domain.name
          ]
        );
        updated++;
      } else {
        // Insert new domain (assign to root tenant)
        const tenantResult = await pool.query(
          `SELECT tenant_id FROM tenants WHERE subdomain IN ('admin', 'everydaytech', 'root') OR is_msp = true ORDER BY tenant_id LIMIT 1`
        );
 
        if (tenantResult.rows.length > 0) {
          const tenantId = tenantResult.rows[0].tenant_id;
          
          await pool.query(
            `INSERT INTO domains (
              tenant_id, domain_name, registrar, registrar_domain_id,
              status, expiration_date, nameservers, metadata,
              registration_date, created_at, updated_at
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW(), NOW())`,
            [
              tenantId,
              domain.name,
              registrar,
              domain.registrar_domain_id,
              domain.status,
              domain.expiration_date,
              JSON.stringify(domain.nameservers),
              JSON.stringify(domain.metadata),
              domain.created_date || null
            ]
          );
          inserted++;
        }
      }
    } catch (err) {
      console.error(`[Domain Sync] Error updating domain ${domain.name}:`, err.message);
    }
  }
 
  console.log(`[Domain Sync] Database updated: ${updated} updated, ${inserted} inserted`);
}
 
/**
 * Main sync orchestrator
 * New architecture: Two-way sync with enrichment
 * 1. Fetch ALL domains from each registrar API
 * 2. Fetch ALL Cloudflare zones
 * 3. Enrich registered domains with Cloudflare DNS data if they use Cloudflare NS
 * 4. Process DNS-only Cloudflare zones
 * 5. Update database and cache
 */
async function syncAllDomains() {
  const startTime = Date.now();
  console.log('[Domain Sync] ========================================');
  console.log('[Domain Sync] Starting two-way domain sync');
  console.log('[Domain Sync] ========================================');
 
  try {
    // Ensure Redis is connected
    if (!redisClient || redisClient.status !== 'ready') {
      await initRedis();
    }
 
    // Step 1: Fetch Cloudflare zones first (needed for enrichment)
    console.log('[Domain Sync] Step 1: Fetching Cloudflare zones...');
    const cloudflareZones = await syncCloudflare();
 
    // Step 2: Sync from registrars and enrich with Cloudflare data
    console.log('[Domain Sync] Step 2: Syncing registrars with Cloudflare enrichment...');
    const [monikerDomains, enomDomains] = await Promise.all([
      syncMoniker(cloudflareZones),
      syncEnom(cloudflareZones)
    ]);
 
    // Step 3: Process DNS-only Cloudflare zones (not registered with our registrars)
    console.log('[Domain Sync] Step 3: Processing DNS-only Cloudflare zones...');
    const dnsOnlyDomains = processDnsOnlyZones(cloudflareZones, monikerDomains, enomDomains);
 
    // Step 4: Update database for all domain types
    console.log('[Domain Sync] Step 4: Updating database...');
    await Promise.all([
      updateDatabase('moniker', monikerDomains),
      updateDatabase('enom', enomDomains),
      updateDatabase('cloudflare', dnsOnlyDomains)
    ]);
 
    // Step 5: Cache results in Redis with TTL
    if (redisClient && redisClient.status === 'ready') {
      console.log('[Domain Sync] Step 5: Caching results in Redis...');
      await Promise.all([
        redisClient.setex(
          `${REDIS_KEY_PREFIX}moniker`,
          REDIS_CACHE_TTL,
          JSON.stringify(monikerDomains)
        ),
        redisClient.setex(
          `${REDIS_KEY_PREFIX}enom`,
          REDIS_CACHE_TTL,
          JSON.stringify(enomDomains)
        ),
        redisClient.setex(
          `${REDIS_KEY_PREFIX}cloudflare`,
          REDIS_CACHE_TTL,
          JSON.stringify(dnsOnlyDomains)
        )
      ]);
 
      // Store sync summary with enrichment stats
      const enrichedMoniker = monikerDomains.filter(d => d.metadata.has_cloudflare_dns).length;
      const enrichedEnom = enomDomains.filter(d => d.metadata.has_cloudflare_dns).length;
      
      const summary = {
        lastSync: new Date().toISOString(),
        moniker: { 
          count: monikerDomains.length,
          cloudflareEnriched: enrichedMoniker
        },
        enom: { 
          count: enomDomains.length,
          cloudflareEnriched: enrichedEnom
        },
        cloudflare: { 
          count: dnsOnlyDomains.length,
          dnsOnly: dnsOnlyDomains.length
        },
        totals: {
          registered: monikerDomains.length + enomDomains.length,
          dnsOnly: dnsOnlyDomains.length,
          all: monikerDomains.length + enomDomains.length + dnsOnlyDomains.length,
          enriched: enrichedMoniker + enrichedEnom
        }
      };
 
      await redisClient.setex(
        `${REDIS_KEY_PREFIX}summary`,
        REDIS_CACHE_TTL,
        JSON.stringify(summary)
      );
 
      console.log('[Domain Sync] Results cached in Redis');
    }
 
    const elapsed = Date.now() - startTime;
    const totalDomains = monikerDomains.length + enomDomains.length + dnsOnlyDomains.length;
    const enrichedCount = monikerDomains.filter(d => d.metadata.has_cloudflare_dns).length + 
                          enomDomains.filter(d => d.metadata.has_cloudflare_dns).length;
    
    console.log('[Domain Sync] ========================================');
    console.log(`[Domain Sync] Two-way sync complete: ${totalDomains} total domains (${elapsed}ms)`);
    console.log(`[Domain Sync] - Moniker (registered): ${monikerDomains.length}`);
    console.log(`[Domain Sync] - eNom (registered): ${enomDomains.length}`);
    console.log(`[Domain Sync] - Cloudflare (DNS-only): ${dnsOnlyDomains.length}`);
    console.log(`[Domain Sync] - Enriched with Cloudflare DNS: ${enrichedCount}`);
    console.log('[Domain Sync] ========================================');
 
  } catch (error) {
    console.error('[Domain Sync] Error during sync:', error);
  }
}
 
/**
 * Start the sync worker
 */
function startSyncWorker() {
  console.log(`[Domain Sync] Worker started - two-way sync every ${SYNC_INTERVAL / 1000 / 60} minutes`);
  console.log('[Domain Sync] Architecture:');
  console.log('[Domain Sync]   1. Fetch domains from Moniker, eNom, Cloudflare APIs');
  console.log('[Domain Sync]   2. Enrich registered domains with Cloudflare DNS data');
  console.log('[Domain Sync]   3. Track DNS-only Cloudflare zones separately');
  console.log('[Domain Sync] Redis caching enabled (10 min TTL)');
  
  // Initialize Redis and run first sync
  initRedis()
    .then(() => {
      console.log('[Domain Sync] Running initial two-way sync...');
      return syncAllDomains();
    })
    .then(() => {
      // Schedule periodic sync
      setInterval(syncAllDomains, SYNC_INTERVAL);
      console.log('[Domain Sync] Periodic two-way sync scheduled');
    })
    .catch(err => {
      console.error('[Domain Sync] Startup error:', err);
    });
}
 
// Start the worker if this file is run directly
if (require.main === module) {
  startSyncWorker();
}
 
module.exports = { startSyncWorker, syncAllDomains };