feat: Implement comprehensive multi-channel notification system with templating and SSL monitoring

- Add robust notification system supporting 7+ communication channels with intelligent message templating, resource monitoring, and SSL certificate alerts. Notification Channels: (Telegram, Discord, Slack, Signal, Email, Google Chat, Webhooks).

- This notification system provides enterprise-grade alerting capabilities with extensive customization options and multi-channel redundancy for critical service monitoring.
This commit is contained in:
Tola Leng
2025-08-14 20:50:31 +07:00
parent 58242c33f8
commit 933e70a3f6
40 changed files with 7159 additions and 97 deletions
@@ -0,0 +1,113 @@
package uptimemonitoring
import (
"encoding/json"
"fmt"
"net/http"
"time"
"service-operation/pocketbase"
)
// UptimeClient handles uptime service operations
type UptimeClient struct {
pbClient *pocketbase.PocketBaseClient
}
// MetricRecord represents a metric record from any collection
type MetricRecord struct {
ID string `json:"id"`
ServiceID string `json:"service_id"`
Status string `json:"status"`
ResponseTime int `json:"response_time"`
ErrorMessage string `json:"error_message"` // Added missing ErrorMessage field
Timestamp time.Time `json:"timestamp"`
Created time.Time `json:"created"`
}
// NewUptimeClient creates a new uptime client
func NewUptimeClient(pbClient *pocketbase.PocketBaseClient) *UptimeClient {
return &UptimeClient{
pbClient: pbClient,
}
}
// GetServices retrieves all services for monitoring
func (uc *UptimeClient) GetServices() ([]UptimeService, error) {
url := fmt.Sprintf("%s/api/collections/services/records", uc.pbClient.GetBaseURL())
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch services: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to fetch services, status: %d", resp.StatusCode)
}
var response struct {
Items []UptimeService `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to decode services: %v", err)
}
// log.Printf("✅ Fetched %d services for monitoring", len(response.Items))
return response.Items, nil
}
// GetLatestMetricRecord gets the latest metric record for a service from specified collection
func (uc *UptimeClient) GetLatestMetricRecord(serviceID, collection string) ([]MetricRecord, error) {
url := fmt.Sprintf("%s/api/collections/%s/records?filter=service_id='%s'&sort=-timestamp&perPage=1",
uc.pbClient.GetBaseURL(), collection, serviceID)
// log.Printf("🔍 [METRICS-QUERY] Getting latest record for service %s from %s", serviceID, collection)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch metric record: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to fetch metric record, status: %d", resp.StatusCode)
}
var response struct {
Items []struct {
ID string `json:"id"`
ServiceID string `json:"service_id"`
Status string `json:"status"`
ResponseTime int `json:"response_time"`
ErrorMessage string `json:"error_message"` // Added missing ErrorMessage field
Timestamp string `json:"timestamp"`
Created string `json:"created"`
} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to decode metric record: %v", err)
}
var records []MetricRecord
for _, item := range response.Items {
timestamp, _ := time.Parse(time.RFC3339, item.Timestamp)
created, _ := time.Parse(time.RFC3339, item.Created)
records = append(records, MetricRecord{
ID: item.ID,
ServiceID: item.ServiceID,
Status: item.Status,
ResponseTime: item.ResponseTime,
ErrorMessage: item.ErrorMessage, // Include ErrorMessage in the record
Timestamp: timestamp,
Created: created,
})
}
// log.Printf("📊 [METRICS-RESULT] Found %d records for service %s in %s", len(records), serviceID, collection)
return records, nil
}
@@ -0,0 +1,106 @@
package uptimemonitoring
import (
"time"
"service-operation/pocketbase"
)
// UptimeMonitor monitors uptime services and sends notifications
type UptimeMonitor struct {
client *UptimeClient
notificationService *UptimeNotificationService
checkInterval time.Duration
stopChan chan bool
}
// NewUptimeMonitor creates a new uptime monitor
func NewUptimeMonitor(pbClient *pocketbase.PocketBaseClient) *UptimeMonitor {
// Initialize uptime client
uptimeClient := NewUptimeClient(pbClient)
// Initialize notification service
notificationService := NewUptimeNotificationService(uptimeClient, pbClient)
// Set check interval to 30 seconds to match server monitoring
checkInterval := 30 * time.Second
return &UptimeMonitor{
client: uptimeClient,
notificationService: notificationService,
checkInterval: checkInterval,
stopChan: make(chan bool, 1),
}
}
// Start begins monitoring uptime services
func (um *UptimeMonitor) Start() {
// log.Printf("🚀 [UPTIME-MONITOR] Starting with check interval: %v", um.checkInterval)
// log.Printf("⏰ [UPTIME-MONITOR] Startup grace period: 2 minutes (no notifications for UP services)")
// Run initial check
um.checkServices()
// Set up periodic checking every 30 seconds
ticker := time.NewTicker(um.checkInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
um.checkServices()
case <-um.stopChan:
// log.Println("🛑 [UPTIME-MONITOR] Monitoring stopped")
return
}
}
}
// Stop gracefully stops the uptime monitoring
func (um *UptimeMonitor) Stop() {
// log.Printf("🛑 [UPTIME-MONITOR] Stopping uptime monitoring...")
select {
case um.stopChan <- true:
default:
}
}
// checkServices fetches all services and checks their notification requirements
func (um *UptimeMonitor) checkServices() {
// log.Printf("🔍 [UPTIME-CHECK] Checking uptime services for status changes...")
services, err := um.client.GetServices()
if err != nil {
// log.Printf("❌ [UPTIME-ERROR] Failed to fetch services: %v", err)
_ = err
return
}
// log.Printf("📊 [UPTIME-SERVICES] Found %d services to check", len(services))
successCount := 0
errorCount := 0
for _, service := range services {
// Skip paused services
if service.Status == "paused" {
// log.Printf("⏸️ [UPTIME-SKIP] Service %s is paused - skipping", service.Name)
continue
}
// Check each service for notification requirements
if err := um.notificationService.CheckAndNotifyService(service); err != nil {
// log.Printf("❌ [UPTIME-FAILED] Failed to process service %s: %v", service.Name, err)
_ = err
errorCount++
} else {
successCount++
}
}
// log.Printf("✅ [UPTIME-SUMMARY] Processed %d services (success: %d, errors: %d)",
// len(services), successCount, errorCount)
_ = successCount
_ = errorCount
}
@@ -0,0 +1,272 @@
package uptimemonitoring
import (
"fmt"
"strings"
"time"
"service-operation/notification"
"service-operation/pocketbase"
)
// UptimeNotificationService handles notifications for uptime services
type UptimeNotificationService struct {
client *UptimeClient
notificationMgr *notification.NotificationManager
statusTracker *ServiceStatusTracker
startupTime time.Time
gracePeriod time.Duration
}
// NewUptimeNotificationService creates a new uptime notification service
func NewUptimeNotificationService(client *UptimeClient, pbClient *pocketbase.PocketBaseClient) *UptimeNotificationService {
return &UptimeNotificationService{
client: client,
notificationMgr: notification.NewNotificationManager(pbClient),
statusTracker: NewServiceStatusTracker(),
startupTime: time.Now(),
gracePeriod: 2 * time.Minute, // 2 minute grace period after startup
}
}
// isInGracePeriod checks if we're still in the startup grace period
func (uns *UptimeNotificationService) isInGracePeriod() bool {
return time.Since(uns.startupTime) < uns.gracePeriod
}
// CheckAndNotifyService checks a service and sends notifications for status changes
func (uns *UptimeNotificationService) CheckAndNotifyService(service UptimeService) error {
// log.Printf("🔍 [UPTIME-CHECK] Checking service: %s (ID: %s, Type: %s)", service.Name, service.ID, service.ServiceType)
// Check if notifications are enabled for this service
if !uns.isNotificationEnabled(service) {
// log.Printf("⚠️ [SKIP] Notifications disabled for service %s", service.Name)
return nil
}
// Get ACTUAL current status from PocketBase metrics (including error message)
actualCurrentStatus, responseTime, errorMessage, err := uns.getActualServiceStatusWithError(service)
if err != nil {
// log.Printf("❌ [ERROR] Failed to get actual status for %s: %v", service.Name, err)
// If we can't get status, assume it's down
actualCurrentStatus = "down"
responseTime = 0
errorMessage = fmt.Sprintf("Failed to retrieve service metrics: %v", err)
}
// Get previously tracked status
previousStatus := uns.statusTracker.GetLastStatus(service.ID)
// log.Printf("📊 [STATUS-COMPARE] Service %s: PREVIOUS='%s' -> ACTUAL_CURRENT='%s', Error: '%s'", service.Name, previousStatus, actualCurrentStatus, errorMessage)
// Initialize tracking for new services
if previousStatus == "" {
// log.Printf("🆕 [INIT] First time tracking service %s - setting initial status: %s", service.Name, actualCurrentStatus)
// Check if we're in startup grace period
if uns.isInGracePeriod() {
// log.Printf("⏰ [GRACE-PERIOD] Service %s - within startup grace period, skipping initial notification", service.Name)
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
return nil
}
// Only send initial notification if service is DOWN (critical status)
if actualCurrentStatus == "down" {
// log.Printf("🚨 [CRITICAL-INIT] Service %s is DOWN on first check - sending notification", service.Name)
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, "INITIAL_DOWN_DETECTED")
if err != nil {
// log.Printf("❌ [FAILED] Initial DOWN notification failed for %s: %v", service.Name, err)
_ = err
} else {
// log.Printf("✅ [SUCCESS] Initial DOWN notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
}
} else {
// log.Printf("️ [INIT-SKIP] Service %s is %s on first check - skipping notification (not critical)", service.Name, actualCurrentStatus)
}
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
return nil
}
// Check if there's a REAL status change
if previousStatus != actualCurrentStatus {
// log.Printf("🚨 [REAL-CHANGE] STATUS CHANGE for %s: %s -> %s", service.Name, previousStatus, actualCurrentStatus)
// Send notification immediately for real status change
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, fmt.Sprintf("STATUS_CHANGE_%s_TO_%s", previousStatus, actualCurrentStatus))
if err != nil {
// log.Printf("❌ [FAILED] Status change notification failed for %s: %v", service.Name, err)
return err
}
// log.Printf("✅ [SUCCESS] Status change notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
} else if actualCurrentStatus == "down" {
// For services that remain down, send periodic notifications (every 5 minutes)
lastNotified := uns.statusTracker.GetLastNotificationTime(service.ID)
if time.Since(lastNotified) >= 5*time.Minute {
// log.Printf("⏰ [PERIODIC] Service %s still down for 5+ minutes - sending reminder", service.Name)
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, "PERIODIC_DOWN_REMINDER")
if err != nil {
// log.Printf("❌ [FAILED] Periodic down notification failed for %s: %v", service.Name, err)
return err
}
// log.Printf("✅ [SUCCESS] Periodic down notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
}
} else {
// log.Printf("️ [NO-CHANGE] No status change for %s (both %s) - no notification needed", service.Name, actualCurrentStatus)
}
return nil
}
// getActualServiceStatusWithError gets the real current status from PocketBase metrics including error message
func (uns *UptimeNotificationService) getActualServiceStatusWithError(service UptimeService) (string, int64, string, error) {
// Get the appropriate collection based on service type
collection := uns.getCollectionForServiceType(service.ServiceType)
// log.Printf("🔍 [METRICS-CHECK] Getting actual status for %s from collection: %s", service.Name, collection)
// Get the latest metric record for this service
records, err := uns.client.GetLatestMetricRecord(service.ID, collection)
if err != nil {
// log.Printf("❌ [METRICS-ERROR] Failed to get metrics for %s: %v", service.Name, err)
return "unknown", 0, fmt.Sprintf("Failed to retrieve metrics: %v", err), err
}
if len(records) == 0 {
// log.Printf("⚠️ [NO-METRICS] No metrics found for service %s", service.Name)
return "down", 0, "No metrics data available", fmt.Errorf("no metrics found for service %s", service.Name)
}
// Get the latest record
latestRecord := records[0]
actualStatus := latestRecord.Status
responseTime := int64(latestRecord.ResponseTime)
errorMessage := latestRecord.ErrorMessage // Get the error message from the record
// log.Printf("📈 [METRICS-RESULT] Latest metric for %s: Status=%s, ResponseTime=%dms, Error='%s', Timestamp=%s",
// service.Name, actualStatus, responseTime, errorMessage, latestRecord.Timestamp)
return actualStatus, responseTime, errorMessage, nil
}
// getCollectionForServiceType maps service types to their corresponding collections
func (uns *UptimeNotificationService) getCollectionForServiceType(serviceType string) string {
switch strings.ToLower(serviceType) {
case "ping", "icmp":
return "ping_data"
case "dns":
return "dns_data"
case "tcp":
return "tcp_data"
case "http", "https":
return "uptime_data"
default:
return "uptime_data"
}
}
// sendImmediateNotification sends notification with zero delay for status changes
func (uns *UptimeNotificationService) sendImmediateNotification(service UptimeService, status string, responseTime int64, errorMessage, reason string) error {
// log.Printf("⚡ [IMMEDIATE-SEND] Sending notification for %s (status: %s, error: '%s', reason: %s)", service.Name, status, errorMessage, reason)
// Build notification payload with actual current status and error message
payload := &notification.NotificationPayload{
ServiceName: service.Name,
Status: status,
Host: service.Host,
Port: service.Port,
ServiceType: service.ServiceType,
ResponseTime: responseTime,
ErrorMessage: errorMessage, // Include the error message from metrics
Timestamp: time.Now(),
Message: uns.generateMessage(service, status, reason),
}
// Add optional fields
if service.Domain != "" {
payload.Domain = service.Domain
}
if service.URL != "" {
payload.URL = service.URL
}
if service.RegionName != "" {
payload.RegionName = service.RegionName
}
if service.AgentID != "" {
payload.AgentID = service.AgentID
}
payload.Uptime = service.Uptime
// log.Printf("⚡ [TELEGRAM-SEND] Sending uptime notification for %s via NotificationManager with error: '%s'", service.Name, errorMessage)
// Send notification using the uptime manager
return uns.notificationMgr.SendUptimeServiceNotification(payload, service.NotificationID, service.TemplateID)
}
// generateMessage creates a status message for notifications
func (uns *UptimeNotificationService) generateMessage(service UptimeService, status, reason string) string {
statusEmoji := "✅"
action := "is operational"
switch strings.ToLower(status) {
case "down":
statusEmoji = "❌"
action = "is DOWN"
case "warning":
statusEmoji = "⚠️"
action = "has issues"
case "up":
statusEmoji = "✅"
previousStatus := uns.statusTracker.GetLastStatus(service.ID)
if previousStatus == "down" {
action = "has RECOVERED"
statusEmoji = "🔄"
} else {
action = "is operational"
}
}
message := fmt.Sprintf("%s [UPTIME] %s %s", statusEmoji, service.Name, action)
if service.Host != "" {
message += fmt.Sprintf(" | Host: %s", service.Host)
}
// Add reason for debugging (only in logs, not in user message)
// log.Printf("📝 [MSG-DEBUG] Generated uptime message for %s: %s [%s]", service.Name, message, reason)
return message
}
// isNotificationEnabled checks if notifications should be sent
func (uns *UptimeNotificationService) isNotificationEnabled(service UptimeService) bool {
// Check notification_status boolean field
if !service.NotificationStatus {
// log.Printf("📵 Service %s notification_status: false", service.Name)
return false
}
// Check alerts field for muted status
if service.Alerts == "muted" {
// log.Printf("🔇 Service %s alerts muted", service.Name)
return false
}
// Check if notification_id is configured
if service.NotificationID == "" {
// log.Printf("📵 Service %s has no notification_id", service.Name)
return false
}
// log.Printf("✅ Service %s notifications enabled (ID: %s)", service.Name, service.NotificationID)
return true
}
@@ -0,0 +1,73 @@
package uptimemonitoring
import (
"sync"
"time"
)
// ServiceStatusTracker tracks service status changes and notification timing
type ServiceStatusTracker struct {
lastStatus map[string]string
lastNotificationTime map[string]time.Time
mu sync.RWMutex
}
// NewServiceStatusTracker creates a new service status tracker
func NewServiceStatusTracker() *ServiceStatusTracker {
return &ServiceStatusTracker{
lastStatus: make(map[string]string),
lastNotificationTime: make(map[string]time.Time),
}
}
// GetLastStatus returns the last known status for a service
func (sst *ServiceStatusTracker) GetLastStatus(serviceID string) string {
sst.mu.RLock()
defer sst.mu.RUnlock()
status := sst.lastStatus[serviceID]
// log.Printf("🔍 GetLastStatus for %s: %s", serviceID, status)
return status
}
// UpdateStatus updates the last known status for a service
func (sst *ServiceStatusTracker) UpdateStatus(serviceID, status string) {
sst.mu.Lock()
defer sst.mu.Unlock()
oldStatus := sst.lastStatus[serviceID]
sst.lastStatus[serviceID] = status
// log.Printf("📝 UpdateStatus for %s: %s -> %s", serviceID, oldStatus, status)
_ = oldStatus
}
// GetLastNotificationTime returns the last time a notification was sent for a service
func (sst *ServiceStatusTracker) GetLastNotificationTime(serviceID string) time.Time {
sst.mu.RLock()
defer sst.mu.RUnlock()
lastTime := sst.lastNotificationTime[serviceID]
// log.Printf("⏰ GetLastNotificationTime for %s: %v", serviceID, lastTime)
return lastTime
}
// SetLastNotificationTime sets the last notification time for a service
func (sst *ServiceStatusTracker) SetLastNotificationTime(serviceID string, t time.Time) {
sst.mu.Lock()
defer sst.mu.Unlock()
sst.lastNotificationTime[serviceID] = t
// log.Printf("⏰ SetLastNotificationTime for %s: %v", serviceID, t)
}
// ShouldNotify determines if a notification should be sent (legacy method for compatibility)
func (sst *ServiceStatusTracker) ShouldNotify(serviceID, currentStatus string) bool {
sst.mu.RLock()
lastStatus := sst.lastStatus[serviceID]
sst.mu.RUnlock()
shouldNotify := lastStatus == "" || lastStatus != currentStatus
// log.Printf("🤔 ShouldNotify for %s: lastStatus='%s', currentStatus='%s', result=%t", serviceID, lastStatus, currentStatus, shouldNotify)
// Send notification if status changed or if it's the first check
return shouldNotify
}
@@ -0,0 +1,41 @@
package uptimemonitoring
// UptimeService represents a service from PocketBase
type UptimeService struct {
ID string `json:"id"`
Name string `json:"name"`
Host string `json:"host"`
Uptime int `json:"uptime"`
ResponseTime int64 `json:"response_time"`
LastChecked string `json:"last_checked"`
Port int `json:"port"`
Domain string `json:"domain"`
HeartbeatInterval int `json:"heartbeat_interval"`
MaxRetries int `json:"max_retries"`
NotificationID string `json:"notification_id"`
TemplateID string `json:"template_id"`
ServiceType string `json:"service_type"`
Status string `json:"status"`
URL string `json:"url"`
Alerts string `json:"alerts"`
StatusCodes string `json:"status_codes"`
Keyword string `json:"keyword"`
RegionName string `json:"region_name"`
AgentID string `json:"agent_id"`
RegionalStatus string `json:"regional_status"`
NotificationStatus bool `json:"notification_status"` // Changed from string to bool
GroupID string `json:"group_id"`
WebhookID string `json:"webhook_id"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// UptimeServicesResponse represents the response from PocketBase services API
type UptimeServicesResponse struct {
Page int `json:"page"`
PerPage int `json:"perPage"`
TotalItems int `json:"totalItems"`
TotalPages int `json:"totalPages"`
Items []UptimeService `json:"items"`
}