agentMonitor.js

Go to the documentation of this file.

/**
 * Agent Monitoring Worker
 * Monitors agent status (online/offline) and creates notifications for status changes
 */
 
const pool = require('../services/db');
const { notifyAgentStatusChange, notifyMonitoringAlert } = require('../utils/notificationHelper');
 
// Configuration
const CHECK_INTERVAL = 2 * 60 * 1000; // Check every 2 minutes
const OFFLINE_THRESHOLD = 5 * 60 * 1000; // 5 minutes without heartbeat = offline
const ALERT_CPU_THRESHOLD = 90; // CPU % threshold
const ALERT_MEMORY_THRESHOLD = 90; // Memory % threshold
const ALERT_DISK_THRESHOLD = 90; // Disk % threshold
 
// Track which agents we've already notified about being offline
const notifiedOfflineAgents = new Set();
 
/**
 * Check for agents that have gone offline and send notifications
 */
async function checkOfflineAgents() {
  try {
    console.log('[AgentMonitor] Checking for offline agents...');
 
    // Check if agents table exists with required columns (Agent v2 system)
    // MeshCentral uses a different monitoring approach
    const tableCheck = await pool.query(`
      SELECT column_name 
      FROM information_schema.columns 
      WHERE table_name = 'agents' 
        AND column_name IN ('last_seen', 'status', 'agent_uuid')
    `);
    
    if (tableCheck.rows.length < 3) {
      console.log('[AgentMonitor] agents table not available (using MeshCentral), skipping offline checks');
      return;
    }
 
    // Find agents that haven't sent heartbeat in OFFLINE_THRESHOLD and were previously online
    const result = await pool.query(`
      SELECT 
        a.agent_id,
        a.agent_uuid,
        a.hostname,
        a.status,
        a.last_seen,
        a.customer_id,
        a.tenant_id,
        EXTRACT(EPOCH FROM (NOW() - a.last_seen)) AS seconds_since_last_seen
      FROM agents a
      WHERE 
        a.last_seen < NOW() - INTERVAL '${OFFLINE_THRESHOLD / 1000} seconds'
        AND a.status != 'offline'
        AND a.status != 'uninstalled'
      ORDER BY a.last_seen DESC
    `);
 
    if (result.rows.length > 0) {
      console.log(`[AgentMonitor] Found ${result.rows.length} agents to mark offline`);
    }
 
    for (const agent of result.rows) {
      // Only notify once per agent going offline
      if (!notifiedOfflineAgents.has(agent.agent_uuid)) {
        console.log(`[AgentMonitor] Agent ${agent.hostname} (${agent.agent_uuid}) is offline (last seen: ${Math.floor(agent.seconds_since_last_seen / 60)} minutes ago)`);
 
        // Update agent status to offline
        await pool.query(
          'UPDATE agents SET status = $1 WHERE agent_uuid = $2',
          ['offline', agent.agent_uuid]
        );
 
        // Send notification
        try {
          await notifyAgentStatusChange({
            agent_id: agent.agent_id,
            hostname: agent.hostname,
            status: 'offline',
            customer_id: agent.customer_id,
            tenant_id: agent.tenant_id
          });
 
          notifiedOfflineAgents.add(agent.agent_uuid);
          console.log(`[AgentMonitor] Sent offline notification for ${agent.hostname}`);
        } catch (notifError) {
          console.error(`[AgentMonitor] Error sending notification for ${agent.hostname}:`, notifError);
        }
      }
    }
 
    // Check for agents that have come back online
    const onlineAgents = await pool.query(`
      SELECT agent_uuid, hostname
      FROM agents
      WHERE 
        last_seen >= NOW() - INTERVAL '${OFFLINE_THRESHOLD / 1000} seconds'
        AND status = 'online'
    `);
 
    // Clear notifications for agents that are back online
    for (const agent of onlineAgents.rows) {
      if (notifiedOfflineAgents.has(agent.agent_uuid)) {
        notifiedOfflineAgents.delete(agent.agent_uuid);
        console.log(`[AgentMonitor] Agent ${agent.hostname} is back online`);
      }
    }
 
  } catch (err) {
    console.error('[AgentMonitor] Error checking offline agents:', err);
  }
}
 
/**
 * Check for monitoring alerts (CPU, Memory, Disk thresholds)
 */
async function checkMonitoringAlerts() {
  try {
    console.log('[AgentMonitor] Checking for monitoring alerts...');
 
    // Check if agent_metrics table exists and has required columns
    // This is graceful handling for legacy Agent v2 systems or missing migrations
    const tableCheck = await pool.query(`
      SELECT column_name 
      FROM information_schema.columns 
      WHERE table_name = 'agent_metrics' 
        AND column_name IN ('cpu_usage', 'memory_usage', 'disk_usage')
    `);
    
    if (tableCheck.rows.length < 3) {
      // agent_metrics table doesn't exist or missing required columns
      // This is expected if using MeshCentral instead of Agent v2
      console.log('[AgentMonitor] agent_metrics table not available (using MeshCentral), skipping monitoring alerts');
      return;
    }
 
    // Get latest metrics for all online agents
    const result = await pool.query(`
      SELECT 
        a.agent_id,
        a.agent_uuid,
        a.hostname,
        a.customer_id,
        a.tenant_id,
        am.cpu_usage,
        am.memory_usage,
        am.disk_usage,
        am.timestamp
      FROM agents a
      LEFT JOIN LATERAL (
        SELECT cpu_usage, memory_usage, disk_usage, timestamp
        FROM agent_metrics
        WHERE agent_id = a.agent_id
        ORDER BY timestamp DESC
        LIMIT 1
      ) am ON true
      WHERE 
        a.status = 'online'
        AND am.timestamp IS NOT NULL
        AND am.timestamp > NOW() - INTERVAL '10 minutes'
    `);
 
    for (const agent of result.rows) {
      // Check CPU threshold
      if (agent.cpu_usage && agent.cpu_usage > ALERT_CPU_THRESHOLD) {
        await notifyMonitoringAlert({
          agent_id: agent.agent_id,
          hostname: agent.hostname,
          alertType: 'cpu',
          message: `CPU usage on ${agent.hostname} is at ${agent.cpu_usage.toFixed(1)}%, exceeding threshold of ${ALERT_CPU_THRESHOLD}%`,
          value: agent.cpu_usage,
          threshold: ALERT_CPU_THRESHOLD,
          customer_id: agent.customer_id,
          tenant_id: agent.tenant_id
        });
        console.log(`[AgentMonitor] CPU alert for ${agent.hostname}: ${agent.cpu_usage.toFixed(1)}%`);
      }
 
      // Check Memory threshold
      if (agent.memory_usage && agent.memory_usage > ALERT_MEMORY_THRESHOLD) {
        await notifyMonitoringAlert({
          agent_id: agent.agent_id,
          hostname: agent.hostname,
          alertType: 'memory',
          message: `Memory usage on ${agent.hostname} is at ${agent.memory_usage.toFixed(1)}%, exceeding threshold of ${ALERT_MEMORY_THRESHOLD}%`,
          value: agent.memory_usage,
          threshold: ALERT_MEMORY_THRESHOLD,
          customer_id: agent.customer_id,
          tenant_id: agent.tenant_id
        });
        console.log(`[AgentMonitor] Memory alert for ${agent.hostname}: ${agent.memory_usage.toFixed(1)}%`);
      }
 
      // Check Disk threshold
      if (agent.disk_usage && agent.disk_usage > ALERT_DISK_THRESHOLD) {
        await notifyMonitoringAlert({
          agent_id: agent.agent_id,
          hostname: agent.hostname,
          alertType: 'disk',
          message: `Disk usage on ${agent.hostname} is at ${agent.disk_usage.toFixed(1)}%, exceeding threshold of ${ALERT_DISK_THRESHOLD}%`,
          value: agent.disk_usage,
          threshold: ALERT_DISK_THRESHOLD,
          customer_id: agent.customer_id,
          tenant_id: agent.tenant_id
        });
        console.log(`[AgentMonitor] Disk alert for ${agent.hostname}: ${agent.disk_usage.toFixed(1)}%`);
      }
    }
 
  } catch (err) {
    console.error('[AgentMonitor] Error checking monitoring alerts:', err);
  }
}
 
/**
 * Start the monitoring worker
 */
function startAgentMonitoring() {
  console.log('[AgentMonitor] Starting agent monitoring worker...');
  console.log(`[AgentMonitor] Check interval: ${CHECK_INTERVAL / 1000}s`);
  console.log(`[AgentMonitor] Offline threshold: ${OFFLINE_THRESHOLD / 1000}s`);
 
  // Run initial check
  checkOfflineAgents();
  checkMonitoringAlerts();
 
  // Schedule periodic checks
  setInterval(checkOfflineAgents, CHECK_INTERVAL);
  setInterval(checkMonitoringAlerts, CHECK_INTERVAL);
 
  console.log('[AgentMonitor] Agent monitoring worker started');
}
 
// Start monitoring if this file is run directly
if (require.main === module) {
  startAgentMonitoring();
 
  // Keep process alive
  process.on('SIGINT', () => {
    console.log('[AgentMonitor] Shutting down gracefully...');
    process.exit(0);
  });
}
 
module.exports = {
  startAgentMonitoring,
  checkOfflineAgents,
  checkMonitoringAlerts
};