package servermonitoring import ( "fmt" "strconv" "sync" "time" "service-operation/pocketbase" ) // StatusAlert tracks alert state for server status changes type StatusAlert struct { IsActive bool Status string LastAlerted time.Time RetryCount int } // ServerMonitor tracks server metrics and sends notifications type ServerMonitor struct { pbClient *ServerPocketBaseClient notificationService *ServerNotificationService thresholdMonitor *ThresholdMonitor lastStatuses map[string]string // Track last known status for each server activeStatusAlerts map[string]*StatusAlert // Track status alerts with retry counts mu sync.RWMutex stopChan chan bool isRunning bool } // NewServerMonitor creates a new server monitor func NewServerMonitor(pbClient *pocketbase.PocketBaseClient) *ServerMonitor { // log.Println("πŸ”§ Creating new ServerMonitor instance") serverPBClient := NewServerPocketBaseClient(pbClient) notificationService := NewServerNotificationService(pbClient) thresholdMonitor := NewThresholdMonitor(serverPBClient, notificationService) return &ServerMonitor{ pbClient: serverPBClient, notificationService: notificationService, thresholdMonitor: thresholdMonitor, lastStatuses: make(map[string]string), activeStatusAlerts: make(map[string]*StatusAlert), stopChan: make(chan bool), isRunning: false, } } // Start begins monitoring servers func (sm *ServerMonitor) Start() { sm.mu.Lock() defer sm.mu.Unlock() if sm.isRunning { // log.Println("⚠️ Server monitor is already running") return } sm.isRunning = true // log.Println("πŸš€ Starting server monitoring service...") go sm.monitoringLoop() } // Stop stops monitoring servers func (sm *ServerMonitor) Stop() { sm.mu.Lock() defer sm.mu.Unlock() if !sm.isRunning { // log.Println("⚠️ Server monitor is not running") return } // log.Println("πŸ›‘ Stopping server monitoring service...") sm.isRunning = false sm.stopChan <- true } // monitoringLoop runs the main monitoring loop func (sm *ServerMonitor) monitoringLoop() { // log.Println("πŸ”„ Starting monitoring loop with 30-second intervals") ticker := time.NewTicker(30 * time.Second) // Check every 30 seconds for more responsive monitoring defer ticker.Stop() // Initial check // log.Println("πŸ” Performing initial server check...") sm.checkAllServers() for { select { case <-ticker.C: // log.Println("⏰ Timer triggered - checking all servers") sm.checkAllServers() case <-sm.stopChan: // log.Println("πŸ›‘ Stop signal received - exiting monitoring loop") return } } } // checkAllServers checks all servers for missing metrics func (sm *ServerMonitor) checkAllServers() { // log.Println("πŸ“‹ Fetching all servers from database...") servers, err := sm.pbClient.GetAllServers() if err != nil { // log.Printf("❌ CRITICAL ERROR: Failed to fetch servers: %v", err) _ = err return } // log.Printf("βœ… Successfully fetched %d servers", len(servers)) if len(servers) == 0 { // log.Println("⚠️ No servers found in database") return } for i, server := range servers { // log.Printf("πŸ” [%d/%d] Processing server: %s (ID: %s)", i+1, len(servers), server.Name, server.ServerID) sm.checkServerMetrics(server) // log.Printf("βœ… [%d/%d] Completed processing server: %s", i+1, len(servers), server.Name) _ = i } // log.Println("πŸ“‹ Completed checking all servers") } // checkServerMetrics checks if a server has recent metrics and agent status func (sm *ServerMonitor) checkServerMetrics(server Server) { // log.Printf("πŸ” === DETAILED SERVER CHECK FOR: %s ===", server.Name) // Debug server fields BEFORE any operations sm.pbClient.DebugServerFields(server.ID, "BEFORE_CHECK") // log.Printf("πŸ“Š Server Details:") // log.Printf(" - ID: %s", server.ID) // log.Printf(" - Server ID: %s", server.ServerID) // log.Printf(" - Name: %s", server.Name) // log.Printf(" - Current Status: %s", server.Status) // log.Printf(" - Agent Status: %s", server.AgentStatus) // log.Printf(" - Check Interval: %d minutes", server.CheckInterval) // log.Printf(" - Notification ID: '%s'", server.NotificationID) // log.Printf(" - Template ID: '%s'", server.TemplateID) // log.Printf(" - Threshold ID: '%s'", server.ThresholdID) // log.Printf(" - IP Address: %s", server.IPAddress) // log.Printf(" - OS Type: %s", server.OSType) // log.Printf(" - Max Retries: %s", server.MaxRetries) // Track the notification configuration BEFORE any updates beforeNotificationID := server.NotificationID beforeTemplateID := server.TemplateID _ = beforeNotificationID _ = beforeTemplateID // Skip paused servers if server.Status == "paused" { // log.Printf("⏸️ Server %s is paused - skipping monitoring", server.Name) return } sm.mu.Lock() lastStatus := sm.lastStatuses[server.ServerID] sm.mu.Unlock() // log.Printf("πŸ“ˆ Status Tracking:") // log.Printf(" - Last Known Status: '%s'", lastStatus) // log.Printf(" - Current Database Status: '%s'", server.Status) currentStatus := "up" var message string var statusReason string // Check agent status first - if agent is stopped, server is definitely down if server.AgentStatus == "stopped" { currentStatus = "down" statusReason = "agent is stopped" message = fmt.Sprintf("πŸ”΄ Server %s is DOWN - Agent has stopped running", server.Name) // log.Printf("❌ SERVER DOWN DETECTED: %s - Agent is stopped", server.Name) } else if server.AgentStatus == "running" { // log.Printf("βœ… Agent is running for server %s - checking metrics...", server.Name) // If agent is running, check metrics checkIntervalMinutes := time.Duration(server.CheckInterval) * time.Minute gracePeriod := 30 * time.Second metricsTimeout := checkIntervalMinutes + gracePeriod // Minimum timeout of 2 minutes to avoid false positives if metricsTimeout < 2*time.Minute { metricsTimeout = 2 * time.Minute } // log.Printf("πŸ• Metrics Check Configuration:") // log.Printf(" - Check Interval: %v", checkIntervalMinutes) // log.Printf(" - Grace Period: %v", gracePeriod) // log.Printf(" - Total Timeout: %v", metricsTimeout) // Get recent metrics for this server // log.Printf("πŸ” Fetching recent metrics for server %s...", server.Name) metrics, err := sm.pbClient.GetLatestServerMetrics(server.ServerID, metricsTimeout) if err != nil { // log.Printf("❌ ERROR: Failed to fetch metrics for server %s: %v", server.Name, err) _ = err return } // log.Printf("πŸ“Š Metrics Query Results:") // log.Printf(" - Found %d metrics within timeout period", len(metrics)) if len(metrics) == 0 { // No recent metrics found - server is down currentStatus = "down" statusReason = fmt.Sprintf("no metrics received in last %v", metricsTimeout) message = fmt.Sprintf("πŸ”΄ Server %s is DOWN - No metrics received in the last %v (check interval: %d minutes)", server.Name, metricsTimeout, server.CheckInterval) // log.Printf("❌ SERVER DOWN DETECTED: %s - No recent metrics", server.Name) } else { // Recent metrics found - server is up currentStatus = "up" latestMetric := metrics[0] age := time.Since(latestMetric.CreatedTime) statusReason = fmt.Sprintf("metrics received %v ago", age.Round(time.Second)) message = fmt.Sprintf("βœ… Server %s is operational - Latest metrics received %v ago (check interval: %d minutes)", server.Name, age.Round(time.Second), server.CheckInterval) // log.Printf("βœ… SERVER UP: %s - Latest metric %v ago", server.Name, age.Round(time.Second)) // log.Printf("πŸ“Š Latest Metric Details:") // log.Printf(" - Metric ID: %s", latestMetric.ID) // log.Printf(" - Created: %v", latestMetric.CreatedTime) // log.Printf(" - Age: %v", age.Round(time.Second)) // log.Printf(" - Status: %s", latestMetric.Status) // Check thresholds if server is up and has metrics // log.Printf("🎯 Checking thresholds for server %s...", server.Name) sm.thresholdMonitor.CheckServerThresholds(server, metrics) } } else { // log.Printf("⚠️ Unknown agent status '%s' for server %s", server.AgentStatus, server.Name) currentStatus = "up" // Default to up for unknown status statusReason = fmt.Sprintf("unknown agent status: %s", server.AgentStatus) message = fmt.Sprintf("⚠️ Server %s has unknown agent status: %s", server.Name, server.AgentStatus) } // log.Printf("πŸ“Š Status Determination:") // log.Printf(" - Determined Status: %s", currentStatus) // log.Printf(" - Reason: %s", statusReason) // log.Printf(" - Message: %s", message) // Check if status has changed statusChanged := lastStatus != currentStatus && lastStatus != "" isFirstCheck := lastStatus == "" _ = statusChanged _ = isFirstCheck // log.Printf("πŸ”„ Status Change Analysis:") // log.Printf(" - Status Changed: %t", statusChanged) // log.Printf(" - First Check: %t", isFirstCheck) // log.Printf(" - Previous Status: '%s'", lastStatus) // log.Printf(" - New Status: '%s'", currentStatus) // Update last known status sm.mu.Lock() sm.lastStatuses[server.ServerID] = currentStatus sm.mu.Unlock() // Update server status in database if it changed if server.Status != currentStatus { // log.Printf("πŸ’Ύ === DATABASE UPDATE FOR: %s ===", server.Name) // log.Printf(" - BEFORE UPDATE:") // log.Printf(" * Status: %s β†’ %s", server.Status, currentStatus) // log.Printf(" * Notification ID: '%s'", beforeNotificationID) // log.Printf(" * Template ID: '%s'", beforeTemplateID) // Log the update attempt LogServerUpdateAttempt(server.ID, "UpdateServerStatus", fmt.Sprintf("Status change from %s to %s", server.Status, currentStatus)) // Debug fields before update sm.pbClient.DebugServerFields(server.ID, "BEFORE_UPDATE") if err := sm.pbClient.UpdateServerStatus(server.ID, currentStatus); err != nil { // log.Printf("❌ CRITICAL ERROR: Failed to update server status for %s: %v", server.Name, err) _ = err } else { // log.Printf("βœ… Successfully updated database status for server %s", server.Name) // Debug fields after update sm.pbClient.DebugServerFields(server.ID, "AFTER_UPDATE") } } else { // log.Printf("ℹ️ Database status already matches current status (%s) for server %s", currentStatus, server.Name) } // Handle status notifications with retry logic sm.handleStatusNotifications(server, currentStatus, message, statusChanged, isFirstCheck) // Final status summary // log.Printf("πŸ“Š === FINAL STATUS SUMMARY FOR: %s ===", server.Name) // log.Printf(" - Final Status: %s (%s)", currentStatus, statusReason) // log.Printf(" - Database Updated: %t", server.Status != currentStatus) // log.Printf(" - Notification Config Preserved: ID='%s', Template='%s'", server.NotificationID, server.TemplateID) // Final debug after all operations sm.pbClient.DebugServerFields(server.ID, "FINAL_CHECK") // log.Printf("=== END SERVER CHECK FOR: %s ===\n", server.Name) _ = statusReason } // handleStatusNotifications handles sending notifications based on status changes and retry logic func (sm *ServerMonitor) handleStatusNotifications(server Server, currentStatus, message string, statusChanged, isFirstCheck bool) { // log.Printf("πŸ“± === NOTIFICATION ANALYSIS FOR: %s ===", server.Name) // log.Printf("πŸ”§ Notification Configuration:") // log.Printf(" - Notification ID: '%s'", server.NotificationID) // log.Printf(" - Template ID: '%s'", server.TemplateID) // log.Printf(" - Has Notification Config: %t", server.NotificationID != "") // log.Printf(" - Max Retries: %s", server.MaxRetries) if server.NotificationID == "" { // log.Printf("πŸ”• NOTIFICATION SKIPPED: No notification ID configured for server %s", server.Name) return } // Parse max retries maxRetries, err := strconv.Atoi(server.MaxRetries) if err != nil { // log.Printf("Invalid max_retries value '%s' for server %s, using default 3", server.MaxRetries, server.Name) maxRetries = 3 } // Determine if notification should be sent shouldSendNotification := false notificationReason := "" if currentStatus == "down" { if sm.shouldSendStatusAlert(server.ServerID, "down", maxRetries) { shouldSendNotification = true retryCount := sm.getStatusRetryCount(server.ServerID) notificationReason = fmt.Sprintf("server down - retry %d/%d", retryCount+1, maxRetries) } else { notificationReason = "server down but max retries reached" } } else if currentStatus == "up" && sm.shouldSendRecoveryStatusAlert(server.ServerID) { shouldSendNotification = true notificationReason = "server recovery notification" } // log.Printf("🎯 Notification Decision:") // log.Printf(" - Should Send: %t", shouldSendNotification) // log.Printf(" - Reason: %s", notificationReason) if shouldSendNotification { // log.Printf("πŸ“€ SENDING NOTIFICATION for server %s", server.Name) // log.Printf("πŸ“§ Notification Details:") // log.Printf(" - Server Name: %s", server.Name) // log.Printf(" - Status: %s", currentStatus) // log.Printf(" - Message: %s", message) // log.Printf(" - Notification ID: %s", server.NotificationID) // log.Printf(" - Template ID: %s", server.TemplateID) // log.Printf(" - Reason: %s", notificationReason) err := sm.notificationService.SendServerNotification(server, currentStatus, message) if err != nil { // log.Printf("❌ NOTIFICATION FAILED for server %s: %v", server.Name, err) _ = err } else { // log.Printf("βœ… NOTIFICATION SENT SUCCESSFULLY for server %s", server.Name) // Update alert tracking based on status if currentStatus == "down" { sm.setStatusAlert(server.ServerID, "down") } else if currentStatus == "up" { sm.clearStatusAlert(server.ServerID) } } } else { // log.Printf("πŸ“± NOTIFICATION SKIPPED for server %s: %s", server.Name, notificationReason) } _ = statusChanged _ = isFirstCheck _ = notificationReason } // shouldSendStatusAlert determines if a status alert should be sent based on retry count func (sm *ServerMonitor) shouldSendStatusAlert(serverID, status string, maxRetries int) bool { sm.mu.RLock() alert, exists := sm.activeStatusAlerts[serverID] sm.mu.RUnlock() if !exists { return true // No existing alert, send new one } // Check if we've reached max retries if alert.RetryCount >= maxRetries { // log.Printf("πŸ”‡ Status alert for server %s has reached max retries (%d), not sending notification", serverID, maxRetries) return false } // Check if enough time has passed since last alert (resend every 5 minutes) timeSinceLastAlert := time.Since(alert.LastAlerted) return timeSinceLastAlert > 5*time.Minute } // getStatusRetryCount returns the current retry count for a status alert func (sm *ServerMonitor) getStatusRetryCount(serverID string) int { sm.mu.RLock() defer sm.mu.RUnlock() if alert, exists := sm.activeStatusAlerts[serverID]; exists { return alert.RetryCount } return 0 } // shouldSendRecoveryStatusAlert determines if a recovery alert should be sent func (sm *ServerMonitor) shouldSendRecoveryStatusAlert(serverID string) bool { sm.mu.RLock() defer sm.mu.RUnlock() _, exists := sm.activeStatusAlerts[serverID] return exists // Send recovery only if there was an active alert } // setStatusAlert creates or updates a status alert with incremented retry count func (sm *ServerMonitor) setStatusAlert(serverID, status string) { sm.mu.Lock() defer sm.mu.Unlock() alert, exists := sm.activeStatusAlerts[serverID] if exists { // Update existing alert and increment retry count alert.Status = status alert.LastAlerted = time.Now() alert.RetryCount++ // log.Printf("πŸ“Š Updated status alert for server %s - Retry count: %d", serverID, alert.RetryCount) } else { // Create new alert sm.activeStatusAlerts[serverID] = &StatusAlert{ IsActive: true, Status: status, LastAlerted: time.Now(), RetryCount: 1, // Start with 1 since we're sending the first notification } // log.Printf("πŸ†• Created new status alert for server %s - Retry count: 1", serverID) } } // clearStatusAlert removes a status alert func (sm *ServerMonitor) clearStatusAlert(serverID string) { sm.mu.Lock() defer sm.mu.Unlock() if _, exists := sm.activeStatusAlerts[serverID]; exists { // log.Printf("πŸ—‘οΈ Cleared status alert for server %s", serverID) delete(sm.activeStatusAlerts, serverID) } }