feat: Implement comprehensive multi-channel notification system with templating and SSL monitoring

- Add robust notification system supporting 7+ communication channels with intelligent message templating, resource monitoring, and SSL certificate alerts. Notification Channels: (Telegram, Discord, Slack, Signal, Email, Google Chat, Webhooks).

- This notification system provides enterprise-grade alerting capabilities with extensive customization options and multi-channel redundancy for critical service monitoring.
This commit is contained in:
Tola Leng
2025-08-14 20:50:31 +07:00
parent 58242c33f8
commit 933e70a3f6
40 changed files with 7159 additions and 97 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ PORT=8091
# Operation defaults
DEFAULT_COUNT=4
DEFAULT_TIMEOUT=10s
DEFAULT_TIMEOUT=15s
MAX_COUNT=20
MAX_TIMEOUT=30s
+1 -1
View File
@@ -23,7 +23,7 @@ func Load() *Config {
cfg := &Config{
Port: getEnv("PORT", "8091"),
DefaultCount: getEnvInt("DEFAULT_COUNT", 4),
DefaultTimeout: getEnvDuration("DEFAULT_TIMEOUT", 10*time.Second),
DefaultTimeout: getEnvDuration("DEFAULT_TIMEOUT", 15*time.Second),
MaxCount: getEnvInt("MAX_COUNT", 20),
MaxTimeout: getEnvDuration("MAX_TIMEOUT", 30*time.Second),
EnableLogging: getEnvBool("ENABLE_LOGGING", true),
+85 -21
View File
@@ -12,38 +12,79 @@ import (
"service-operation/handlers"
"service-operation/monitoring"
"service-operation/pocketbase"
servermonitoring "service-operation/server-monitoring"
sslmonitoring "service-operation/ssl-monitoring"
uptimemonitoring "service-operation/uptime-monitoring"
)
func main() {
//log.Println("🚀 === STARTING SERVICE OPERATION SERVER ===")
cfg := config.Load()
//log.Printf("📋 Configuration loaded:")
//log.Printf(" - Port: %s", cfg.Port)
//log.Printf(" - Backend PB Enabled: %t", cfg.PocketBaseEnabled)
if cfg.PocketBaseEnabled {
//log.Printf(" - PocketBase URL: %s", cfg.PocketBaseURL)
}
// Initialize PocketBase client (no credentials required)
var pbClient *pocketbase.PocketBaseClient
var monitoringService *monitoring.MonitoringService
var sslMonitoringService *monitoring.SSLMonitoringService
var sslNotificationService *sslmonitoring.SSLMonitor
var serverMonitoringService *servermonitoring.ServerMonitoringService
var uptimeMonitoringService *uptimemonitoring.UptimeMonitor
if cfg.PocketBaseEnabled {
//log.Println("🔧 Initializing PocketBase client...")
var err error
pbClient, err = pocketbase.NewPocketBaseClient(cfg.PocketBaseURL)
if err != nil {
log.Printf("Warning: Failed to initialize PocketBase client: %v", err)
//log.Printf("⚠️ WARNING: Failed to initialize PocketBase client: %v", err)
} else {
//log.Println("✅ PocketBase client initialized successfully")
//log.Println("🔍 Testing PocketBase connection...")
if err := pbClient.TestConnection(); err != nil {
log.Printf("Warning: Backend connection test failed: %v", err)
//log.Printf("⚠️ WARNING: PocketBase connection test failed: %v", err)
} else {
// Initialize and start monitoring service with regional support
//log.Println("✅ PocketBase connection test successful")
// Initialize and start service monitoring with regional support
//log.Println("🔧 Initializing service monitoring...")
monitoringService = monitoring.NewMonitoringService(pbClient)
go monitoringService.Start()
log.Println("Uptime Monitoring service started with Default System agent support")
//log.Println("✅ Service monitoring started with regional agent support")
// Initialize and start SSL monitoring service (unchanged - no regional support)
// Initialize and start SSL monitoring service (original)
//log.Println("🔧 Initializing SSL monitoring (original)...")
sslMonitoringService = monitoring.NewSSLMonitoringService(pbClient)
go sslMonitoringService.Start()
log.Println("SSL monitoring service started (independent of Default System Agent)")
//log.Println("✅ SSL monitoring started (independent of regional agents)")
// Initialize and start SSL notification service (new)
//log.Println("🔧 Initializing SSL notification monitoring...")
sslNotificationService = sslmonitoring.NewSSLMonitor(pbClient)
go sslNotificationService.Start()
//log.Println("✅ SSL notification monitoring started with Telegram support")
// Initialize and start server monitoring service
//log.Println("🔧 Initializing server monitoring...")
serverMonitoringService = servermonitoring.NewServerMonitoringService(pbClient)
serverMonitoringService.Start()
//log.Println("✅ Server monitoring started with notification support")
// Initialize and start uptime monitoring service
//log.Println("🔧 Initializing uptime monitoring...")
uptimeMonitoringService = uptimemonitoring.NewUptimeMonitor(pbClient)
go uptimeMonitoringService.Start()
//log.Println("✅ Uptime monitoring started with notification support")
}
}
}
//log.Println("🔧 Initializing HTTP handlers...")
handler := handlers.NewOperationHandler(cfg, pbClient)
router := mux.NewRouter()
@@ -61,24 +102,28 @@ func main() {
// Health check
router.HandleFunc("/health", handler.HandleHealth).Methods("GET")
log.Printf("Service Operation starting on port %s", cfg.Port)
log.Printf("=== 🌐 CHECKCLE SERVICE OPERATION SERVER READY ===")
log.Printf("🚀 Starting on port %s", cfg.Port)
if pbClient != nil {
log.Printf("Backend Server integration enabled at %s", pbClient.GetBaseURL())
log.Printf("Backend integration enabled at %s ", pbClient.GetBaseURL())
}
if monitoringService != nil {
log.Printf("Automatic service monitoring enabled with regional agent support")
log.Printf("✓Service monitoring enabled with regional agent support")
}
if sslMonitoringService != nil {
log.Printf("SSL certificate monitoring enabled (independent)")
//log.Printf("🔒 SSL certificate monitoring enabled (independent)")
}
log.Printf("Endpoints:")
log.Printf(" POST /operation - Full operation test (ping, dns, tcp, http, ssl)")
log.Printf(" GET /operation/quick?type=<type>&host=<host> - Quick operation test")
log.Printf(" POST /ping - Legacy ping endpoint")
log.Printf(" GET /ping/quick?host=<host> - Legacy quick ping test")
log.Printf(" GET /health - Health check")
log.Printf("Default SystemSupported operations: ping, dns, tcp, http, ssl")
log.Printf("Default System: Tracks 'Default' region connection status")
if sslNotificationService != nil {
log.Printf("✓SSL notification monitoring enabled with Telegram support")
}
if serverMonitoringService != nil {
log.Printf("✓Server monitoring enabled with notification support")
}
if uptimeMonitoringService != nil {
log.Printf("✓Uptime monitoring enabled with notification support")
}
log.Printf("✓Supported operations: ping, dns, tcp, http, ssl")
// Setup graceful shutdown
c := make(chan os.Signal, 1)
@@ -86,18 +131,37 @@ func main() {
go func() {
<-c
log.Println("Shutting down monitoring services...")
log.Println("🛑 === GRACEFUL SHUTDOWN INITIATED ===")
log.Println("🛑 Shutting down monitoring services...")
if monitoringService != nil {
log.Println("🛑 Stopping service monitoring...")
monitoringService.Stop()
}
if sslMonitoringService != nil {
log.Println("🛑 Stopping SSL monitoring...")
sslMonitoringService.Stop()
}
log.Println("Service stopped")
if sslNotificationService != nil {
log.Println("🛑 Stopping SSL notification monitoring...")
sslNotificationService.Stop()
}
if serverMonitoringService != nil {
log.Println("🛑 Stopping server monitoring...")
serverMonitoringService.Stop()
}
if uptimeMonitoringService != nil {
log.Println("🛑 Stopping uptime monitoring...")
uptimeMonitoringService.Stop()
}
log.Println("✅ All services stopped gracefully")
log.Println("🛑 === SERVICE OPERATION SERVER STOPPED ===")
os.Exit(0)
}()
//log.Println("🌐 HTTP server starting...")
if err := http.ListenAndServe(":"+cfg.Port, router); err != nil {
log.Fatal("Failed to start server:", err)
log.Fatal("❌ FATAL: Failed to start HTTP server:", err)
}
}
@@ -0,0 +1,71 @@
package monitoring
import (
"log"
"service-operation/notification"
"service-operation/pocketbase"
)
// NotificationMonitor handles notification logic for the monitoring service
type NotificationMonitor struct {
notifier *notification.ServiceNotifier
lastStatuses map[string]string // Track last known status for each service
}
// NewNotificationMonitor creates a new notification monitor
func NewNotificationMonitor(pbClient *pocketbase.PocketBaseClient) *NotificationMonitor {
return &NotificationMonitor{
notifier: notification.NewServiceNotifier(pbClient),
lastStatuses: make(map[string]string),
}
}
// CheckAndNotify checks if notification should be sent and sends it
func (nm *NotificationMonitor) CheckAndNotify(service pocketbase.Service, currentStatus string) {
// Skip if no notification configured
if service.NotificationID == "" {
return
}
// Get last known status
lastStatus, exists := nm.lastStatuses[service.ID]
// Send notification if:
// 1. This is the first check (no previous status)
// 2. Status has changed
// 3. Service is currently down (always notify for down status)
shouldNotify := !exists || lastStatus != currentStatus || currentStatus == "down"
if shouldNotify {
log.Printf("Sending notification for service %s: %s -> %s", service.Name, lastStatus, currentStatus)
// Send notification using custom method
if err := nm.notifier.NotifyCustom(
service.NotificationID,
service.TemplateID,
service.Name,
currentStatus,
nm.formatStatusMessage(service, currentStatus),
); err != nil {
log.Printf("Failed to send notification: %v", err)
}
}
// Update last known status
nm.lastStatuses[service.ID] = currentStatus
}
// formatStatusMessage creates a formatted message for the status change
func (nm *NotificationMonitor) formatStatusMessage(service pocketbase.Service, status string) string {
switch status {
case "up":
return "Service is now operational"
case "down":
return "Service is currently unavailable"
case "warning":
return "Service is experiencing issues"
default:
return "Service status has changed"
}
}
@@ -49,7 +49,7 @@ func (rm *RegionalMonitor) Start() {
rm.updateConnectionStatus("online")
rm.isOnline = true
log.Printf("Regional monitor started for region: %s (Agent ID: %s)",
log.Printf("✓Default Regional monitor started for region: %s (Agent ID: %s)",
service.RegionName, service.AgentID)
go func() {
@@ -37,7 +37,7 @@ func (ms *MonitoringService) Start() {
}
ms.isRunning = true
log.Println("Starting monitoring service...")
//log.Println("Starting monitoring service...")
// Start regional monitoring
ms.regionalMonitor.Start()
@@ -1,8 +1,8 @@
package monitoring
import (
"fmt"
"log"
"time"
"service-operation/operations"
@@ -30,14 +30,14 @@ func (s *SSLMonitoringService) Start() {
ticker := time.NewTicker(1 * time.Minute) // Check every minute for scheduling
defer ticker.Stop()
log.Println("SSL monitoring service started with interval and check_at scheduling")
// log.Println("SSL monitoring service started with interval and check_at scheduling")
for {
select {
case <-ticker.C:
s.checkSSLCertificates()
case <-s.stopChan:
log.Println("SSL monitoring service stopped")
// log.Println("SSL monitoring service stopped")
return
}
}
@@ -48,21 +48,24 @@ func (s *SSLMonitoringService) Stop() {
}
func (s *SSLMonitoringService) checkSSLCertificates() {
//log.Println("Fetching SSL certificates from PocketBase...")
// log.Println("Fetching SSL certificates from PocketBase...")
certificates, err := s.pbClient.GetSSLCertificates()
if err != nil {
log.Printf("Failed to fetch SSL certificates: %v", err)
// log.Printf("Failed to fetch SSL certificates: %v", err)
_ = err
return
}
//log.Printf("Found %d SSL certificates to check", len(certificates))
// log.Printf("Found %d SSL certificates to check", len(certificates))
for _, cert := range certificates {
if s.shouldCheckCertificate(cert) {
go s.checkSingleCertificateWithRetry(cert)
}
}
_ = certificates
}
func (s *SSLMonitoringService) shouldCheckCertificate(cert types.SSLCertificate) bool {
@@ -72,29 +75,33 @@ func (s *SSLMonitoringService) shouldCheckCertificate(cert types.SSLCertificate)
if cert.CheckAt != "" {
if checkAt, err := s.parseFlexibleTime(cert.CheckAt); err == nil {
if now.After(checkAt) || now.Equal(checkAt) {
log.Printf("Certificate %s is due for manual check (check_at: %s)",
cert.Domain, checkAt.Format("2006-01-02 15:04:05"))
// log.Printf("Certificate %s is due for manual check (check_at: %s)",
// cert.Domain, checkAt.Format("2006-01-02 15:04:05"))
_ = checkAt
return true
} else {
//log.Printf("Certificate %s scheduled for later check (check_at: %s)",
// log.Printf("Certificate %s scheduled for later check (check_at: %s)",
// cert.Domain, checkAt.Format("2006-01-02 15:04:05"))
//return false
_ = checkAt
return false
}
} else {
log.Printf("Error parsing check_at for %s: %v", cert.Domain, err)
// log.Printf("Error parsing check_at for %s: %v", cert.Domain, err)
_ = err
}
}
// Priority 2: Check based on check_interval (in days) from last update
if cert.Updated == "" {
log.Printf("Certificate %s has never been checked, scheduling check", cert.Domain)
// log.Printf("Certificate %s has never been checked, scheduling check", cert.Domain)
return true
}
// Parse last check time from updated field
lastCheck, err := s.parseFlexibleTime(cert.Updated)
if err != nil {
log.Printf("Error parsing last check time for %s, scheduling check: %v", cert.Domain, err)
// log.Printf("Error parsing last check time for %s, scheduling check: %v", cert.Domain, err)
_ = err
return true
}
@@ -112,11 +119,15 @@ func (s *SSLMonitoringService) shouldCheckCertificate(cert types.SSLCertificate)
shouldCheck := now.After(nextCheck)
if shouldCheck {
log.Printf("Certificate %s is due for interval check (last: %s, interval: %d days)",
cert.Domain, lastCheck.Format("2006-01-02 15:04:05"), adjustedIntervalDays)
// log.Printf("Certificate %s is due for interval check (last: %s, interval: %d days)",
// cert.Domain, lastCheck.Format("2006-01-02 15:04:05"), adjustedIntervalDays)
_ = lastCheck
_ = adjustedIntervalDays
} else {
//log.Printf("Certificate %s not due yet (next check: %s, interval: %d days)",
//cert.Domain, nextCheck.Format("2006-01-02 15:04:05"), adjustedIntervalDays)
// log.Printf("Certificate %s not due yet (next check: %s, interval: %d days)",
// cert.Domain, nextCheck.Format("2006-01-02 15:04:05"), adjustedIntervalDays)
_ = nextCheck
_ = adjustedIntervalDays
}
return shouldCheck
@@ -171,16 +182,16 @@ func (s *SSLMonitoringService) adjustCheckIntervalDays(cert types.SSLCertificate
func (s *SSLMonitoringService) checkSingleCertificateWithRetry(cert types.SSLCertificate) {
retryCount := s.retryQueue[cert.ID]
log.Printf("🔍 Checking SSL certificate for domain: %s (attempt %d/%d)",
cert.Domain, retryCount+1, s.maxRetries+1)
// log.Printf("🔍 Checking SSL certificate for domain: %s (attempt %d/%d)",
// cert.Domain, retryCount+1, s.maxRetries+1)
result, err := s.performSSLCheck(cert.Domain)
if err != nil && retryCount < s.maxRetries {
// Increment retry count and schedule retry
s.retryQueue[cert.ID] = retryCount + 1
log.Printf("SSL check failed for %s, will retry (%d/%d): %v",
cert.Domain, retryCount+1, s.maxRetries, err)
// log.Printf("SSL check failed for %s, will retry (%d/%d): %v",
// cert.Domain, retryCount+1, s.maxRetries, err)
// Schedule retry with exponential backoff
go func() {
@@ -188,6 +199,9 @@ func (s *SSLMonitoringService) checkSingleCertificateWithRetry(cert types.SSLCer
time.Sleep(backoffDuration)
s.checkSingleCertificateWithRetry(cert)
}()
_ = retryCount
_ = err
return
}
@@ -195,8 +209,8 @@ func (s *SSLMonitoringService) checkSingleCertificateWithRetry(cert types.SSLCer
delete(s.retryQueue, cert.ID)
if err != nil {
log.Printf("❌ SSL check failed for domain %s after %d attempts: %v",
cert.Domain, s.maxRetries+1, err)
// log.Printf("❌ SSL check failed for domain %s after %d attempts: %v",
// cert.Domain, s.maxRetries+1, err)
s.updateCertificateWithError(cert, err)
return
}
@@ -206,28 +220,31 @@ func (s *SSLMonitoringService) checkSingleCertificateWithRetry(cert types.SSLCer
}
func (s *SSLMonitoringService) performSSLCheck(domain string) (*types.OperationResult, error) {
log.Printf("Performing SSL check for domain: %s", domain)
// log.Printf("Performing SSL check for domain: %s", domain)
sslOp := operations.NewSSLOperation(30 * time.Second)
result, err := sslOp.Execute(domain)
if err != nil {
log.Printf("SSL operation failed for %s: %v", domain, err)
// log.Printf("SSL operation failed for %s: %v", domain, err)
_ = err
return nil, err
}
if result == nil {
log.Printf("SSL operation returned nil result for %s", domain)
// log.Printf("SSL operation returned nil result for %s", domain)
return nil, fmt.Errorf("SSL check returned nil result")
}
log.Printf("SSL check completed for %s: success=%v, days_left=%d",
domain, result.Success, result.SSLDaysLeft)
// log.Printf("SSL check completed for %s: success=%v, days_left=%d",
// domain, result.Success, result.SSLDaysLeft)
_ = domain
_ = result
return result, nil
}
func (s *SSLMonitoringService) updateCertificateWithError(cert types.SSLCertificate, err error) {
log.Printf("Updating certificate %s with error status", cert.Domain)
// log.Printf("Updating certificate %s with error status", cert.Domain)
updateData := map[string]interface{}{
"status": "error",
@@ -250,18 +267,24 @@ func (s *SSLMonitoringService) updateCertificateWithError(cert types.SSLCertific
updateData["check_at"] = nextCheck.Format(time.RFC3339)
if updateErr := s.pbClient.UpdateSSLCertificate(cert.ID, updateData); updateErr != nil {
log.Printf("Failed to update SSL certificate %s with error status: %v", cert.ID, updateErr)
// log.Printf("Failed to update SSL certificate %s with error status: %v", cert.ID, updateErr)
_ = updateErr
} else {
log.Printf("📝 Updated certificate %s with error status (next check in %d days)",
cert.Domain, errorIntervalDays)
// log.Printf("📝 Updated certificate %s with error status (next check in %d days)",
// cert.Domain, errorIntervalDays)
_ = errorIntervalDays
}
_ = err
_ = updateData
_ = nextCheck
}
func (s *SSLMonitoringService) updateCertificateWithResults(cert types.SSLCertificate, result *types.OperationResult) {
status := getSSLStatus(result)
log.Printf("Updating certificate %s with results: status=%s, days_left=%d, issuer=%s",
cert.Domain, status, result.SSLDaysLeft, result.SSLIssuer)
// log.Printf("Updating certificate %s with results: status=%s, days_left=%d, issuer=%s",
// cert.Domain, status, result.SSLDaysLeft, result.SSLIssuer)
updateData := map[string]interface{}{
"status": status,
@@ -290,11 +313,18 @@ func (s *SSLMonitoringService) updateCertificateWithResults(cert types.SSLCertif
updateData["check_at"] = nextCheck.Format(time.RFC3339)
if err := s.pbClient.UpdateSSLCertificate(cert.ID, updateData); err != nil {
log.Printf("Failed to update SSL certificate %s: %v", cert.ID, err)
// log.Printf("Failed to update SSL certificate %s: %v", cert.ID, err)
_ = err
} else {
log.Printf("✅ SSL certificate updated for %s: %s (%d days left, issuer: %s, next check in %d days)",
cert.Domain, status, result.SSLDaysLeft, result.SSLIssuer, adjustedIntervalDays)
// log.Printf("✅ SSL certificate updated for %s: %s (%d days left, issuer: %s, next check in %d days)",
// cert.Domain, status, result.SSLDaysLeft, result.SSLIssuer, adjustedIntervalDays)
_ = status
_ = result
_ = adjustedIntervalDays
}
_ = updateData
_ = nextCheck
}
func getSSLStatus(result *types.OperationResult) string {
@@ -0,0 +1,214 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
)
// DiscordService handles Discord notifications
type DiscordService struct{}
// NewDiscordService creates a new Discord notification service
func NewDiscordService() *DiscordService {
return &DiscordService{}
}
// DiscordPayload represents the payload for Discord webhook
type DiscordPayload struct {
Content string `json:"content,omitempty"`
Embeds []DiscordEmbed `json:"embeds,omitempty"`
}
// DiscordEmbed represents a Discord embed
type DiscordEmbed struct {
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Color int `json:"color,omitempty"`
Timestamp string `json:"timestamp,omitempty"`
Fields []DiscordEmbedField `json:"fields,omitempty"`
Footer *DiscordEmbedFooter `json:"footer,omitempty"`
}
// DiscordEmbedField represents a field in Discord embed
type DiscordEmbedField struct {
Name string `json:"name"`
Value string `json:"value"`
Inline bool `json:"inline,omitempty"`
}
// DiscordEmbedFooter represents footer in Discord embed
type DiscordEmbedFooter struct {
Text string `json:"text"`
}
// SendNotification sends a notification via Discord webhook
func (ds *DiscordService) SendNotification(config *AlertConfiguration, message string) error {
if config.DiscordWebhookURL == "" {
return fmt.Errorf("discord webhook URL is required")
}
// Create rich embed for better formatting
embed := ds.createDiscordEmbed(message)
payload := DiscordPayload{
Embeds: []DiscordEmbed{embed},
}
jsonData, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal Discord payload: %v", err)
}
resp, err := http.Post(config.DiscordWebhookURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("discord webhook request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("discord webhook error, status: %d", resp.StatusCode)
}
return nil
}
// createDiscordEmbed creates a formatted Discord embed from the message
func (ds *DiscordService) createDiscordEmbed(message string) DiscordEmbed {
// Parse the message to extract structured information
lines := strings.Split(message, "\n")
if len(lines) == 0 {
return DiscordEmbed{
Description: message,
Color: ds.getDefaultColor(),
Timestamp: time.Now().Format(time.RFC3339),
}
}
// Extract title from first line and determine status
title := strings.TrimSpace(lines[0])
color := ds.determineColorFromMessage(title)
// Create fields from the remaining lines
var fields []DiscordEmbedField
for i, line := range lines {
if i == 0 {
continue // Skip title line
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Parse field from "• Key: Value" format
if strings.HasPrefix(line, "•") || strings.HasPrefix(line, "-") {
line = strings.TrimPrefix(line, "•")
line = strings.TrimPrefix(line, "-")
line = strings.TrimSpace(line)
parts := strings.SplitN(line, ":", 2)
if len(parts) == 2 {
fieldName := strings.TrimSpace(parts[0])
fieldValue := strings.TrimSpace(parts[1])
if fieldName != "" && fieldValue != "" {
fields = append(fields, DiscordEmbedField{
Name: fieldName,
Value: fieldValue,
Inline: true,
})
}
} else {
// If not in key:value format, add as description field
fields = append(fields, DiscordEmbedField{
Name: "Details",
Value: line,
Inline: false,
})
}
}
}
// If no structured fields found, use the entire message as description
if len(fields) == 0 {
// Remove the first line from description since it's used as title
description := ""
if len(lines) > 1 {
description = strings.Join(lines[1:], "\n")
}
return DiscordEmbed{
Title: title,
Description: description,
Color: color,
Timestamp: time.Now().Format(time.RFC3339),
Footer: &DiscordEmbedFooter{
Text: "CheckCle System Alert",
},
}
}
return DiscordEmbed{
Title: title,
Color: color,
Timestamp: time.Now().Format(time.RFC3339),
Fields: fields,
Footer: &DiscordEmbedFooter{
Text: "CheckCle System Alert",
},
}
}
// determineColorFromMessage determines Discord embed color based on message content
func (ds *DiscordService) determineColorFromMessage(message string) int {
messageLower := strings.ToLower(message)
// Red for critical/error states
if strings.Contains(messageLower, "expired") ||
strings.Contains(messageLower, "down") ||
strings.Contains(messageLower, "failed") ||
strings.Contains(messageLower, "error") ||
strings.Contains(messageLower, "critical") ||
strings.Contains(messageLower, "🚨") ||
strings.Contains(messageLower, "🔴") {
return 15158332 // Red (#E74C3C)
}
// Orange for warnings
if strings.Contains(messageLower, "expiring_soon") ||
strings.Contains(messageLower, "expiring soon") ||
strings.Contains(messageLower, "warning") ||
strings.Contains(messageLower, "maintenance") ||
strings.Contains(messageLower, "paused") ||
strings.Contains(messageLower, "⚠️") ||
strings.Contains(messageLower, "🟡") ||
strings.Contains(messageLower, "🟠") {
return 15105570 // Orange (#E67E22)
}
// Green for success/up states
if strings.Contains(messageLower, "up") ||
strings.Contains(messageLower, "resolved") ||
strings.Contains(messageLower, "success") ||
strings.Contains(messageLower, "restored") ||
strings.Contains(messageLower, "valid") ||
strings.Contains(messageLower, "🟢") ||
strings.Contains(messageLower, "✅") {
return 3066993 // Green (#2ECC71)
}
// Blue for info/default
return 3447003 // Blue (#3498DB)
}
// getDefaultColor returns the default Discord embed color (blue)
func (ds *DiscordService) getDefaultColor() int {
return 3447003 // Blue (#3498DB)
}
@@ -0,0 +1,532 @@
package notification
import (
"crypto/tls"
"fmt"
"net/smtp"
"strconv"
"strings"
"time"
)
// EmailService handles email notifications
type EmailService struct{}
// NewEmailService creates a new email notification service
func NewEmailService() *EmailService {
// log.Printf("✅ Email notification service initialized")
return &EmailService{}
}
// SendNotification sends a notification via email
func (es *EmailService) SendNotification(config *AlertConfiguration, message string) error {
// log.Printf("📧 === SENDING EMAIL NOTIFICATION ===")
// log.Printf("🔔 Email notification request received")
// log.Printf("📊 Email Configuration:")
// log.Printf(" - Email Address: %s", config.EmailAddress)
// log.Printf(" - Sender Name: %s", config.EmailSenderName)
// log.Printf(" - SMTP Server: %s", config.SMTPServer)
// log.Printf(" - SMTP Port: %s", config.SMTPPort)
// log.Printf(" - SMTP Password present: %t", config.SMTPPassword != "")
// log.Printf(" - Message Length: %d chars", len(message))
// Validate email configuration
if config.EmailAddress == "" || config.SMTPServer == "" || config.SMTPPort == "" {
// log.Printf("❌ EMAIL CONFIGURATION ERROR: Missing required fields")
// log.Printf(" - Email Address present: %t", config.EmailAddress != "")
// log.Printf(" - SMTP Server present: %t", config.SMTPServer != "")
// log.Printf(" - SMTP Port present: %t", config.SMTPPort != "")
return fmt.Errorf("email configuration is incomplete")
}
if config.SMTPPassword == "" {
// log.Printf("⚠️ WARNING: SMTP password not provided - authentication may fail")
}
port, err := strconv.Atoi(config.SMTPPort)
if err != nil {
// log.Printf("❌ SMTP PORT ERROR: Invalid port '%s': %v", config.SMTPPort, err)
return fmt.Errorf("invalid SMTP port: %v", err)
}
// log.Printf("✅ Email configuration validation passed")
// log.Printf("🔧 Parsed SMTP Port: %d", port)
// Determine alert severity and get appropriate emoji/color
severity, emoji := es.parseMessageSeverity(message)
// log.Printf("📋 Message Analysis:")
// log.Printf(" - Detected Severity: %s", severity)
// log.Printf(" - Emoji: %s", emoji)
// Create enhanced email content
subject := es.createEmailSubject(config.EmailSenderName, severity)
htmlBody := es.createHTMLEmailBody(message, severity, emoji)
plainBody := es.createPlainEmailBody(message, severity, emoji)
// log.Printf("📝 Email Content Created:")
// log.Printf(" - Subject: %s", subject)
// log.Printf(" - HTML Body Length: %d chars", len(htmlBody))
// log.Printf(" - Plain Body Length: %d chars", len(plainBody))
// Prepare email message with both HTML and plain text
emailMessage := es.createMIMEMessage(config.EmailSenderName, config.EmailAddress, subject, htmlBody, plainBody)
// log.Printf("📤 Preparing to send email...")
// log.Printf(" - SMTP Server: %s:%d", config.SMTPServer, port)
// log.Printf(" - From: %s", config.EmailSenderName)
// log.Printf(" - To: %s", config.EmailAddress)
// Send email with enhanced SMTP handling
err = es.sendSMTPEmail(config.SMTPServer, port, config.EmailSenderName, config.EmailAddress, config.SMTPPassword, emailMessage)
if err != nil {
// log.Printf("❌ EMAIL SENDING FAILED: %v", err)
// log.Printf("❌ Error Details:")
// log.Printf(" - SMTP Server: %s:%d", config.SMTPServer, port)
// log.Printf(" - Error Type: %T", err)
// log.Printf(" - Error Message: %s", err.Error())
return fmt.Errorf("failed to send email: %v", err)
}
// log.Printf("✅ EMAIL SENT SUCCESSFULLY")
// log.Printf("✅ Email Details:")
// log.Printf(" - Recipient: %s", config.EmailAddress)
// log.Printf(" - Subject: %s", subject)
// log.Printf(" - Severity: %s", severity)
// log.Printf(" - Timestamp: %s", time.Now().Format("2006-01-02 15:04:05"))
// log.Printf("=== EMAIL NOTIFICATION COMPLETE ===")
return nil
}
// parseMessageSeverity analyzes the message to determine severity level
func (es *EmailService) parseMessageSeverity(message string) (string, string) {
messageUpper := strings.ToUpper(message)
// Check for critical indicators
if strings.Contains(messageUpper, "🚨") || strings.Contains(messageUpper, "CRITICAL") ||
strings.Contains(messageUpper, "DOWN") || strings.Contains(messageUpper, "ERROR") ||
strings.Contains(messageUpper, "FAILED") || strings.Contains(messageUpper, "🔴") {
return "CRITICAL", "🚨"
}
// Check for warning indicators
if strings.Contains(messageUpper, "⚠️") || strings.Contains(messageUpper, "WARNING") ||
strings.Contains(messageUpper, "🟡") || strings.Contains(messageUpper, "INCIDENT") {
return "WARNING", "⚠️"
}
// Check for success indicators
if strings.Contains(messageUpper, "🟢") || strings.Contains(messageUpper, "UP") ||
strings.Contains(messageUpper, "SUCCESS") || strings.Contains(messageUpper, "RESOLVED") ||
strings.Contains(messageUpper, "RESTORED") {
return "SUCCESS", "✅"
}
// Check for maintenance indicators
if strings.Contains(messageUpper, "🟠") || strings.Contains(messageUpper, "MAINTENANCE") ||
strings.Contains(messageUpper, "PAUSED") {
return "MAINTENANCE", "🔧"
}
// Default to info
return "INFO", "️"
}
// createEmailSubject creates an appropriate email subject
func (es *EmailService) createEmailSubject(senderName, severity string) string {
prefix := "Service Alert"
if senderName != "" {
prefix = fmt.Sprintf("%s - Service Alert", senderName)
}
switch severity {
case "CRITICAL":
return fmt.Sprintf("🚨 [CRITICAL] %s", prefix)
case "WARNING":
return fmt.Sprintf("⚠️ [WARNING] %s", prefix)
case "SUCCESS":
return fmt.Sprintf("✅ [RESOLVED] %s", prefix)
case "MAINTENANCE":
return fmt.Sprintf("🔧 [MAINTENANCE] %s", prefix)
default:
return fmt.Sprintf("️ [INFO] %s", prefix)
}
}
// createHTMLEmailBody creates a rich HTML email body
func (es *EmailService) createHTMLEmailBody(message, severity, emoji string) string {
// Get color scheme based on severity
var bgColor, borderColor, textColor string
switch severity {
case "CRITICAL":
bgColor = "#fee2e2"
borderColor = "#dc2626"
textColor = "#991b1b"
case "WARNING":
bgColor = "#fef3c7"
borderColor = "#d97706"
textColor = "#92400e"
case "SUCCESS":
bgColor = "#dcfce7"
borderColor = "#16a34a"
textColor = "#15803d"
case "MAINTENANCE":
bgColor = "#fed7aa"
borderColor = "#ea580c"
textColor = "#c2410c"
default:
bgColor = "#dbeafe"
borderColor = "#2563eb"
textColor = "#1d4ed8"
}
// Format message content for HTML
htmlMessage := strings.ReplaceAll(message, "\n", "<br>")
// Enhance bullet points and formatting
htmlMessage = strings.ReplaceAll(htmlMessage, " - ", "<br>&nbsp;&nbsp;• ")
htmlMessage = strings.ReplaceAll(htmlMessage, "Service:", "<strong>Service:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Status:", "<strong>Status:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Host:", "<strong>Host:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Type:", "<strong>Type:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Response time:", "<strong>Response time:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Time:", "<strong>Time:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Domain:", "<strong>Domain:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Days Remaining:", "<strong>Days Remaining:</strong>")
htmlMessage = strings.ReplaceAll(htmlMessage, "Expiration Date:", "<strong>Expiration Date:</strong>")
return fmt.Sprintf(`
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Service Alert Notification</title>
</head>
<body style="margin: 0; padding: 20px; font-family: Arial, sans-serif; background-color: #f5f5f5;">
<div style="max-width: 600px; margin: 0 auto; background-color: white; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1);">
<!-- Header -->
<div style="background-color: %s; color: %s; padding: 20px; border-radius: 8px 8px 0 0; border-left: 4px solid %s;">
<h2 style="margin: 0; font-size: 18px;">
%s Service Alert Notification
</h2>
</div>
<!-- Content -->
<div style="padding: 20px;">
<div style="background-color: #f8f9fa; padding: 15px; border-radius: 6px; border-left: 3px solid %s;">
<p style="margin: 0; font-size: 14px; line-height: 1.6; color: #333;">
%s
</p>
</div>
<!-- Footer -->
<div style="margin-top: 20px; padding-top: 15px; border-top: 1px solid #e5e7eb;">
<p style="margin: 0; font-size: 12px; color: #6b7280; text-align: center;">
This is an automated notification from your monitoring system.<br>
Generated at: %s
</p>
</div>
</div>
</div>
</body>
</html>`,
bgColor, textColor, borderColor, emoji, borderColor, htmlMessage, time.Now().Format("2006-01-02 15:04:05 MST"))
}
// createPlainEmailBody creates a plain text email body
func (es *EmailService) createPlainEmailBody(message, severity, emoji string) string {
separator := strings.Repeat("=", 50)
return fmt.Sprintf(`%s
%s SERVICE ALERT NOTIFICATION
%s
%s
%s
This is an automated notification from your monitoring system.
Generated at: %s
%s`,
separator,
emoji+" "+severity,
separator,
message,
separator,
time.Now().Format("2006-01-02 15:04:05 MST"),
separator)
}
// createMIMEMessage creates a MIME message with both HTML and plain text
func (es *EmailService) createMIMEMessage(fromName, toEmail, subject, htmlBody, plainBody string) string {
boundary := fmt.Sprintf("boundary_%d", time.Now().Unix())
headers := fmt.Sprintf(`From: %s
To: %s
Subject: %s
MIME-Version: 1.0
Content-Type: multipart/alternative; boundary="%s"
`, fromName, toEmail, subject, boundary)
body := fmt.Sprintf(`--%s
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 7bit
%s
--%s
Content-Type: text/html; charset=UTF-8
Content-Transfer-Encoding: 7bit
%s
--%s--
`, boundary, plainBody, boundary, htmlBody, boundary)
return headers + body
}
// sendSMTPEmail sends the email using SMTP with proper authentication
func (es *EmailService) sendSMTPEmail(smtpServer string, port int, fromEmail, toEmail, password, message string) error {
addr := fmt.Sprintf("%s:%d", smtpServer, port)
// log.Printf("🔌 Connecting to SMTP server: %s", addr)
// Extract hostname from SMTP server for proper HELO
hostname := smtpServer
if strings.Contains(hostname, ".") {
// Use the SMTP server hostname for HELO
hostname = smtpServer
}
// log.Printf("🔧 Using hostname for HELO: %s", hostname)
// For port 587 (STARTTLS) - most common for authenticated SMTP
if port == 587 {
// log.Printf("🔐 Attempting STARTTLS connection with authentication...")
return es.sendWithSTARTTLSAuth(addr, hostname, fromEmail, toEmail, password, message)
}
// For port 465 (SSL/TLS)
if port == 465 {
// log.Printf("🔒 Attempting SSL connection with authentication...")
return es.sendWithSSLAuth(addr, hostname, fromEmail, toEmail, password, message)
}
// For port 25 (Plain SMTP with optional STARTTLS)
if port == 25 {
// log.Printf("📧 Attempting plain SMTP with optional STARTTLS...")
return es.sendWithSTARTTLSAuth(addr, hostname, fromEmail, toEmail, password, message)
}
// Fallback to STARTTLS for any other port
// log.Printf("📧 Using STARTTLS with auth fallback for port %d...", port)
return es.sendWithSTARTTLSAuth(addr, hostname, fromEmail, toEmail, password, message)
}
// sendWithSTARTTLSAuth sends email with STARTTLS and authentication
func (es *EmailService) sendWithSTARTTLSAuth(addr, hostname, fromEmail, toEmail, password, message string) error {
// log.Printf("🔐 Establishing STARTTLS connection to %s", addr)
// Connect to server
client, err := smtp.Dial(addr)
if err != nil {
// log.Printf("❌ Failed to connect to SMTP server: %v", err)
return fmt.Errorf("failed to connect to SMTP server: %v", err)
}
defer client.Close()
// Send EHLO with proper hostname
// log.Printf("👋 Sending EHLO with hostname: %s", hostname)
if err = client.Hello(hostname); err != nil {
// log.Printf("❌ EHLO failed: %v", err)
return fmt.Errorf("EHLO failed: %v", err)
}
// Check if STARTTLS is supported and use it
if ok, _ := client.Extension("STARTTLS"); ok {
// log.Printf("🔐 STARTTLS supported, initiating TLS...")
tlsConfig := &tls.Config{
ServerName: strings.Split(addr, ":")[0],
InsecureSkipVerify: false,
}
if err = client.StartTLS(tlsConfig); err != nil {
// log.Printf("❌ STARTTLS failed: %v", err)
return fmt.Errorf("STARTTLS failed: %v", err)
}
// log.Printf("✅ TLS connection established")
} else {
// log.Printf("⚠️ STARTTLS not supported by server, continuing with plain connection")
}
// Check for AUTH support and authenticate if available
if ok, mechanisms := client.Extension("AUTH"); ok {
// Suppress unused variable warning
_ = mechanisms
// log.Printf("🔑 AUTH extension supported with mechanisms: %s", mechanisms)
// Use the provided credentials for authentication
username := fromEmail
// log.Printf("🔐 Attempting authentication for user: %s", username)
// log.Printf("🔑 Password provided: %t", password != "")
if password != "" {
// Create auth mechanism - try PLAIN first as it's most common
auth := smtp.PlainAuth("", username, password, strings.Split(addr, ":")[0])
if err := client.Auth(auth); err != nil {
// log.Printf("❌ Authentication failed: %v", err)
return fmt.Errorf("SMTP authentication failed: %v", err)
} else {
// log.Printf("✅ Authentication successful")
}
} else {
// log.Printf("⚠️ No password provided, skipping authentication")
return fmt.Errorf("SMTP password is required for authentication")
}
} else {
// log.Printf("️ AUTH extension not available, proceeding without authentication")
}
// Set sender
// log.Printf("📤 Setting sender: %s", fromEmail)
if err = client.Mail(fromEmail); err != nil {
// log.Printf("❌ Failed to set sender: %v", err)
return fmt.Errorf("failed to set sender: %v", err)
}
// Set recipient
// log.Printf("📥 Setting recipient: %s", toEmail)
if err = client.Rcpt(toEmail); err != nil {
// log.Printf("❌ Failed to set recipient: %v", err)
return fmt.Errorf("failed to set recipient: %v", err)
}
// Send message
// log.Printf("📝 Sending message data...")
w, err := client.Data()
if err != nil {
// log.Printf("❌ Failed to initiate data transfer: %v", err)
return fmt.Errorf("failed to initiate data transfer: %v", err)
}
_, err = w.Write([]byte(message))
if err != nil {
// log.Printf("❌ Failed to write message data: %v", err)
return fmt.Errorf("failed to write message data: %v", err)
}
err = w.Close()
if err != nil {
// log.Printf("❌ Failed to close data writer: %v", err)
return fmt.Errorf("failed to close data writer: %v", err)
}
// Quit gracefully
err = client.Quit()
if err != nil {
// log.Printf("⚠️ Warning during QUIT: %v", err)
// Don't return error for QUIT issues as email might have been sent
}
// log.Printf("✅ Email sent successfully via STARTTLS")
return nil
}
// sendWithSSLAuth sends email with SSL/TLS and authentication (for port 465)
func (es *EmailService) sendWithSSLAuth(addr, hostname, fromEmail, toEmail, password, message string) error {
// log.Printf("🔒 Establishing SSL/TLS connection to %s", addr)
// Create TLS configuration
tlsConfig := &tls.Config{
InsecureSkipVerify: false,
ServerName: strings.Split(addr, ":")[0],
}
// Connect with TLS
conn, err := tls.Dial("tcp", addr, tlsConfig)
if err != nil {
// log.Printf("❌ Failed to establish TLS connection: %v", err)
return fmt.Errorf("failed to establish TLS connection: %v", err)
}
defer conn.Close()
// Create SMTP client
client, err := smtp.NewClient(conn, strings.Split(addr, ":")[0])
if err != nil {
// log.Printf("❌ Failed to create SMTP client: %v", err)
return fmt.Errorf("failed to create SMTP client: %v", err)
}
defer client.Quit()
// Send EHLO with proper hostname
// log.Printf("👋 Sending EHLO with hostname: %s", hostname)
if err = client.Hello(hostname); err != nil {
// log.Printf("❌ EHLO failed: %v", err)
return fmt.Errorf("EHLO failed: %v", err)
}
// Authenticate if AUTH is supported and password is provided
if ok, mechanisms := client.Extension("AUTH"); ok {
// Suppress unused variable warning
_ = mechanisms
// log.Printf("🔑 AUTH extension supported with mechanisms: %s", mechanisms)
username := fromEmail
// log.Printf("🔐 Attempting authentication for user: %s", username)
// log.Printf("🔑 Password provided: %t", password != "")
if password != "" {
auth := smtp.PlainAuth("", username, password, strings.Split(addr, ":")[0])
if err := client.Auth(auth); err != nil {
// log.Printf("❌ Authentication failed: %v", err)
return fmt.Errorf("SMTP authentication failed: %v", err)
} else {
// log.Printf("✅ Authentication successful")
}
} else {
// log.Printf("⚠️ No password provided, skipping authentication")
return fmt.Errorf("SMTP password is required for authentication")
}
}
// Set sender and recipient
// log.Printf("📤 Setting sender: %s", fromEmail)
if err := client.Mail(fromEmail); err != nil {
// log.Printf("❌ Failed to set sender: %v", err)
return fmt.Errorf("failed to set sender: %v", err)
}
// log.Printf("📥 Setting recipient: %s", toEmail)
if err := client.Rcpt(toEmail); err != nil {
// log.Printf("❌ Failed to set recipient: %v", err)
return fmt.Errorf("failed to set recipient: %v", err)
}
// Send message
// log.Printf("📝 Sending message data...")
w, err := client.Data()
if err != nil {
// log.Printf("❌ Failed to initiate data transfer: %v", err)
return fmt.Errorf("failed to initiate data transfer: %v", err)
}
_, err = w.Write([]byte(message))
if err != nil {
// log.Printf("❌ Failed to write message data: %v", err)
return fmt.Errorf("failed to write message data: %v", err)
}
err = w.Close()
if err != nil {
// log.Printf("❌ Failed to close data writer: %v", err)
return fmt.Errorf("failed to close data writer: %v", err)
}
// log.Printf("✅ Email sent successfully via SSL/TLS")
return nil
}
@@ -0,0 +1,336 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// GoogleChatService handles Google Chat notifications
type GoogleChatService struct{}
// NewGoogleChatService creates a new Google Chat notification service
func NewGoogleChatService() *GoogleChatService {
return &GoogleChatService{}
}
// GoogleChatPayload represents the payload for Google Chat webhook
type GoogleChatPayload struct {
Text string `json:"text"`
}
// SendNotification sends a notification via Google Chat webhook
func (gcs *GoogleChatService) SendNotification(config *AlertConfiguration, message string) error {
// fmt.Printf("💬 [GOOGLE_CHAT] Attempting to send notification...\n")
// fmt.Printf("💬 [GOOGLE_CHAT] Config - Webhook URL present: %v\n", config.GoogleChatWebhookURL != "")
// fmt.Printf("💬 [GOOGLE_CHAT] Message: %s\n", message)
if config.GoogleChatWebhookURL == "" {
return fmt.Errorf("google chat webhook URL is required")
}
payload := GoogleChatPayload{
Text: message,
}
jsonData, err := json.Marshal(payload)
if err != nil {
// fmt.Printf("❌ [GOOGLE_CHAT] JSON marshal error: %v\n", err)
return err
}
// fmt.Printf("💬 [GOOGLE_CHAT] Sending POST request to webhook...\n")
resp, err := http.Post(config.GoogleChatWebhookURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
// fmt.Printf("❌ [GOOGLE_CHAT] HTTP error: %v\n", err)
return err
}
defer resp.Body.Close()
// fmt.Printf("💬 [GOOGLE_CHAT] Response status: %d\n", resp.StatusCode)
if resp.StatusCode != http.StatusOK {
// fmt.Printf("❌ [GOOGLE_CHAT] Webhook error, status: %d\n", resp.StatusCode)
return fmt.Errorf("google chat webhook error, status: %d", resp.StatusCode)
}
// fmt.Printf("✅ [GOOGLE_CHAT] Message sent successfully!\n")
return nil
}
// SendServerNotification sends a server-specific notification via Google Chat
func (gcs *GoogleChatService) SendServerNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) error {
message := gcs.generateServerMessage(payload, template, resourceType)
return gcs.SendNotification(config, message)
}
// SendServiceNotification sends a service-specific notification via Google Chat
func (gcs *GoogleChatService) SendServiceNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServiceNotificationTemplate) error {
message := gcs.generateServiceMessage(payload, template)
return gcs.SendNotification(config, message)
}
// generateServerMessage creates a message for server notifications using server template
func (gcs *GoogleChatService) generateServerMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var templateMessage string
// Select appropriate template message based on status and resource type
switch strings.ToLower(payload.Status) {
case "down":
templateMessage = template.DownMessage
case "up":
templateMessage = template.UpMessage
case "warning":
templateMessage = template.WarningMessage
case "paused":
templateMessage = template.PausedMessage
default:
// Handle resource-specific messages
switch resourceType {
case "cpu":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUMessage
} else {
templateMessage = template.CPUMessage
}
case "ram", "memory":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreRAMMessage
} else {
templateMessage = template.RAMMessage
}
case "disk":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskMessage
} else {
templateMessage = template.DiskMessage
}
case "network":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreNetworkMessage
} else {
templateMessage = template.NetworkMessage
}
case "cpu_temp", "cpu_temperature":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUTempMessage
} else {
templateMessage = template.CPUTempMessage
}
case "disk_io":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskIOMessage
} else {
templateMessage = template.DiskIOMessage
}
default:
templateMessage = template.WarningMessage
}
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = gcs.generateDefaultServerMessage(payload, resourceType)
}
return gcs.replacePlaceholders(templateMessage, payload)
}
// generateServiceMessage creates a message for service notifications using service template
func (gcs *GoogleChatService) generateServiceMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var templateMessage string
// Select appropriate template message based on status
switch strings.ToLower(payload.Status) {
case "up":
templateMessage = template.UpMessage
case "down":
templateMessage = template.DownMessage
case "maintenance":
templateMessage = template.MaintenanceMessage
case "incident":
templateMessage = template.IncidentMessage
case "resolved":
templateMessage = template.ResolvedMessage
case "warning":
templateMessage = template.WarningMessage
default:
templateMessage = template.WarningMessage
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = gcs.generateDefaultUptimeMessage(payload)
}
return gcs.replacePlaceholders(templateMessage, payload)
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (gcs *GoogleChatService) replacePlaceholders(message string, payload *NotificationPayload) string {
// Replace basic placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", gcs.safeString(payload.Host))
message = strings.ReplaceAll(message, "${hostname}", gcs.safeString(payload.Hostname))
// Replace URL with fallback to host
url := gcs.safeString(payload.URL)
if url == "N/A" && payload.Host != "" {
url = payload.Host
}
message = strings.ReplaceAll(message, "${url}", url)
// Replace domain
message = strings.ReplaceAll(message, "${domain}", gcs.safeString(payload.Domain))
// Replace service type
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
// Replace region and agent info
message = strings.ReplaceAll(message, "${region_name}", gcs.safeString(payload.RegionName))
message = strings.ReplaceAll(message, "${agent_id}", gcs.safeString(payload.AgentID))
// Handle numeric fields safely
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
// Replace server monitoring fields
message = strings.ReplaceAll(message, "${cpu_usage}", gcs.safeString(payload.CPUUsage))
message = strings.ReplaceAll(message, "${ram_usage}", gcs.safeString(payload.RAMUsage))
message = strings.ReplaceAll(message, "${disk_usage}", gcs.safeString(payload.DiskUsage))
message = strings.ReplaceAll(message, "${network_usage}", gcs.safeString(payload.NetworkUsage))
message = strings.ReplaceAll(message, "${cpu_temp}", gcs.safeString(payload.CPUTemp))
message = strings.ReplaceAll(message, "${disk_io}", gcs.safeString(payload.DiskIO))
message = strings.ReplaceAll(message, "${threshold}", gcs.safeString(payload.Threshold))
// Replace error message - important for uptime services
message = strings.ReplaceAll(message, "${error_message}", gcs.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${error}", gcs.safeString(payload.ErrorMessage))
// Replace time placeholders
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
return message
}
// safeString returns the string value or "N/A" if empty
func (gcs *GoogleChatService) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// generateDefaultUptimeMessage creates a default uptime message with proper formatting
func (gcs *GoogleChatService) generateDefaultUptimeMessage(payload *NotificationPayload) string {
// Status emoji mapping for Google Chat
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
case "maintenance", "paused":
statusEmoji = "🟠"
}
message := fmt.Sprintf("%s Service %s is %s.", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
// Build formatted details
details := []string{}
// Add URL or host
if payload.URL != "" {
details = append(details, fmt.Sprintf(" - Host URL: %s", payload.URL))
} else if payload.Host != "" {
details = append(details, fmt.Sprintf(" - Host: %s", payload.Host))
}
// Add service type
if payload.ServiceType != "" {
details = append(details, fmt.Sprintf(" - Type: %s", strings.ToUpper(payload.ServiceType)))
}
// Add port if available
if payload.Port > 0 {
details = append(details, fmt.Sprintf(" - Port: %d", payload.Port))
}
// Add domain if available
if payload.Domain != "" {
details = append(details, fmt.Sprintf(" - Domain: %s", payload.Domain))
}
// Add response time
if payload.ResponseTime > 0 {
details = append(details, fmt.Sprintf(" - Response time: %dms", payload.ResponseTime))
} else {
details = append(details, " - Response time: N/A")
}
// Add region info
if payload.RegionName != "" {
details = append(details, fmt.Sprintf(" - Region: %s", payload.RegionName))
}
// Add agent info
if payload.AgentID != "" {
details = append(details, fmt.Sprintf(" - Agent: %s", payload.AgentID))
}
// Add uptime if available
if payload.Uptime > 0 {
details = append(details, fmt.Sprintf(" - Uptime: %d%%", payload.Uptime))
}
// Add timestamp
details = append(details, fmt.Sprintf(" - Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05")))
// Combine message with details
if len(details) > 0 {
message += "\n" + strings.Join(details, "\n")
}
return message
}
// generateDefaultServerMessage creates a default server message
func (gcs *GoogleChatService) generateDefaultServerMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
}
return fmt.Sprintf("%s🖥️ Server %s (%s) status: %s", statusEmoji, payload.ServiceName, payload.Hostname, strings.ToUpper(payload.Status))
}
@@ -0,0 +1,61 @@
package notification
import (
"log"
"time"
"service-operation/pocketbase"
"service-operation/types"
)
// ServiceNotifier provides an easy interface to send notifications for service events
type ServiceNotifier struct {
manager *NotificationManager
}
// NewServiceNotifier creates a new service notifier
func NewServiceNotifier(pbClient *pocketbase.PocketBaseClient) *ServiceNotifier {
return &ServiceNotifier{
manager: NewNotificationManager(pbClient),
}
}
// NotifyServiceStatus sends a notification for service status change
func (sn *ServiceNotifier) NotifyServiceStatus(service pocketbase.Service, result *types.OperationResult) {
// Check if service has notification configured
if service.NotificationID == "" {
return
}
// Create notification payload
payload := &NotificationPayload{
ServiceName: service.Name,
Status: service.Status,
Host: service.Host,
Port: service.Port,
ServiceType: service.ServiceType,
ResponseTime: result.ResponseTime.Milliseconds(),
Timestamp: time.Now(),
ErrorMessage: result.Error,
}
// Send notification
if err := sn.manager.SendServiceNotification(payload, service.NotificationID, service.TemplateID); err != nil {
log.Printf("Failed to send notification for service %s: %v", service.Name, err)
} else {
log.Printf("Notification sent successfully for service %s", service.Name)
}
}
// NotifyCustom sends a custom notification
func (sn *ServiceNotifier) NotifyCustom(notificationID, templateID, serviceName, status, message string) error {
payload := &NotificationPayload{
ServiceName: serviceName,
Status: status,
Message: message,
Timestamp: time.Now(),
}
return sn.manager.SendServiceNotification(payload, notificationID, templateID)
}
@@ -0,0 +1,67 @@
package notification
import (
// "log"
"service-operation/pocketbase"
)
// NotificationManager handles all notification operations
type NotificationManager struct {
pbClient *pocketbase.PocketBaseClient
services map[string]NotificationService
serverManager *ServerNotificationManager
uptimeManager *UptimeNotificationManager
sslManager *SSLNotificationManager
}
// NewNotificationManager creates a new notification manager
func NewNotificationManager(pbClient *pocketbase.PocketBaseClient) *NotificationManager {
// Initialize notification services
services := make(map[string]NotificationService)
// log.Printf("🔧 Initializing notification services...")
services["telegram"] = NewTelegramService()
services["signal"] = NewSignalService()
services["discord"] = NewDiscordService()
services["slack"] = NewSlackService()
services["google_chat"] = NewGoogleChatService()
services["email"] = NewEmailService()
services["webhook"] = NewWebhookService()
// log.Printf("✅ Notification services initialized: %v", getKeys(services))
// Create specialized managers
serverManager := NewServerNotificationManager(pbClient, services)
uptimeManager := NewUptimeNotificationManager(pbClient, services)
sslManager := NewSSLNotificationManager(pbClient, services)
return &NotificationManager{
pbClient: pbClient,
services: services,
serverManager: serverManager,
uptimeManager: uptimeManager,
sslManager: sslManager,
}
}
// SendServiceNotification sends notification for a service based on its configuration
func (nm *NotificationManager) SendServiceNotification(payload *NotificationPayload, notificationID, templateID string) error {
return nm.serverManager.SendServiceNotification(payload, notificationID, templateID)
}
// SendResourceNotification sends notification for specific resource alerts (CPU, RAM, Disk, etc.)
func (nm *NotificationManager) SendResourceNotification(payload *NotificationPayload, notificationID, templateID, resourceType string) error {
return nm.serverManager.SendResourceNotification(payload, notificationID, templateID, resourceType)
}
// SendUptimeServiceNotification sends notification for uptime services using service templates
func (nm *NotificationManager) SendUptimeServiceNotification(payload *NotificationPayload, notificationID, templateID string) error {
return nm.uptimeManager.SendUptimeServiceNotification(payload, notificationID, templateID)
}
// SendSSLNotification sends notification for SSL certificates using SSL templates
func (nm *NotificationManager) SendSSLNotification(payload *NotificationPayload, notificationID, templateID string) error {
return nm.sslManager.SendSSLNotification(payload, notificationID, templateID)
}
@@ -0,0 +1,480 @@
package notification
import (
"fmt"
"strings"
"service-operation/pocketbase"
)
// ServerNotificationManager handles server-specific notifications
type ServerNotificationManager struct {
pbClient *pocketbase.PocketBaseClient
services map[string]NotificationService
}
// NewServerNotificationManager creates a new server notification manager
func NewServerNotificationManager(pbClient *pocketbase.PocketBaseClient, services map[string]NotificationService) *ServerNotificationManager {
return &ServerNotificationManager{
pbClient: pbClient,
services: services,
}
}
// SendServiceNotification sends notification for a service based on its configuration
func (snm *ServerNotificationManager) SendServiceNotification(payload *NotificationPayload, notificationID, templateID string) error {
// log.Printf("📨 SendServiceNotification called with notification_id: %s, template_id: %s", notificationID, templateID)
// log.Printf("📨 Payload: %+v", payload)
if notificationID == "" {
// log.Printf("❌ Notification ID is required but was empty")
return fmt.Errorf("notification ID is required")
}
// Parse multiple notification IDs
notificationIDs := parseNotificationIDs(notificationID)
if len(notificationIDs) == 0 {
// log.Printf("❌ No valid notification IDs found")
return fmt.Errorf("no valid notification IDs found")
}
var errors []string
successCount := 0
// Send notification to each channel
for _, id := range notificationIDs {
// log.Printf("📤 Processing notification ID: %s", id)
// Check if notification is enabled
if !isNotificationEnabled(snm.pbClient, id) {
// log.Printf("⚠️ Notification %s is disabled, skipping", id)
continue
}
// Get alert configuration
alertConfig, err := getAlertConfiguration(snm.pbClient, id)
if err != nil {
// log.Printf("❌ Failed to get alert configuration for %s: %v", id, err)
errors = append(errors, fmt.Sprintf("failed to get config for %s: %v", id, err))
continue
}
// Get template if provided
var template *ServerNotificationTemplate
if templateID != "" {
template, err = getNotificationTemplate(snm.pbClient, templateID)
if err != nil {
// log.Printf("⚠️ Warning: failed to get template %s for notification %s: %v", templateID, id, err)
}
}
// Generate message from template or use default
message := snm.generateMessage(payload, template)
// log.Printf("📝 Generated message for %s: %s", id, message)
// Send notification using appropriate service
service, exists := snm.services[alertConfig.NotificationType]
if !exists {
// log.Printf("❌ Unsupported notification type for %s: %s", id, alertConfig.NotificationType)
errors = append(errors, fmt.Sprintf("unsupported notification type for %s: %s", id, alertConfig.NotificationType))
continue
}
err = service.SendNotification(alertConfig, message)
if err != nil {
// log.Printf("❌ Failed to send notification via %s for %s: %v", alertConfig.NotificationType, id, err)
errors = append(errors, fmt.Sprintf("failed to send via %s for %s: %v", alertConfig.NotificationType, id, err))
} else {
// log.Printf("✅ Successfully sent notification via %s for %s", alertConfig.NotificationType, id)
successCount++
}
}
// Report results
if successCount > 0 {
// log.Printf("✅ Successfully sent %d out of %d notifications", successCount, len(notificationIDs))
}
if len(errors) > 0 {
// log.Printf("❌ Errors occurred: %v", errors)
if successCount == 0 {
return fmt.Errorf("all notifications failed: %v", errors)
}
// If some succeeded, just log errors but don't return error
// log.Printf("⚠️ Some notifications failed but %d succeeded", successCount)
}
return nil
}
// SendResourceNotification sends notification for specific resource alerts (CPU, RAM, Disk, etc.)
func (snm *ServerNotificationManager) SendResourceNotification(payload *NotificationPayload, notificationID, templateID, resourceType string) error {
// log.Printf("📨 SendResourceNotification called for resource: %s", resourceType)
// log.Printf("📨 Notification ID: %s, Template ID: %s", notificationID, templateID)
// log.Printf("📨 Payload: %+v", payload)
if notificationID == "" {
// log.Printf("❌ Notification ID is required but was empty")
return fmt.Errorf("notification ID is required")
}
// Parse multiple notification IDs
notificationIDs := parseNotificationIDs(notificationID)
if len(notificationIDs) == 0 {
// log.Printf("❌ No valid notification IDs found")
return fmt.Errorf("no valid notification IDs found")
}
var errors []string
successCount := 0
// Send notification to each channel
for _, id := range notificationIDs {
// log.Printf("📤 Processing resource notification ID: %s", id)
// Check if notification is enabled
if !isNotificationEnabled(snm.pbClient, id) {
// log.Printf("⚠️ Notification %s is disabled, skipping", id)
continue
}
// Get alert configuration
alertConfig, err := getAlertConfiguration(snm.pbClient, id)
if err != nil {
// log.Printf("❌ Failed to get alert configuration for %s: %v", id, err)
errors = append(errors, fmt.Sprintf("failed to get config for %s: %v", id, err))
continue
}
// Get template if provided
var template *ServerNotificationTemplate
if templateID != "" {
template, err = getNotificationTemplate(snm.pbClient, templateID)
if err != nil {
// log.Printf("⚠️ Warning: failed to get template %s for notification %s: %v", templateID, id, err)
}
}
// Generate resource-specific message
message := snm.generateResourceMessage(payload, template, resourceType)
// log.Printf("📝 Generated resource message for %s: %s", id, message)
// Send notification
service, exists := snm.services[alertConfig.NotificationType]
if !exists {
// log.Printf("❌ Unsupported notification type for %s: %s", id, alertConfig.NotificationType)
errors = append(errors, fmt.Sprintf("unsupported notification type for %s: %s", id, alertConfig.NotificationType))
continue
}
err = service.SendNotification(alertConfig, message)
if err != nil {
// log.Printf("❌ Failed to send resource notification via %s for %s: %v", alertConfig.NotificationType, id, err)
errors = append(errors, fmt.Sprintf("failed to send via %s for %s: %v", alertConfig.NotificationType, id, err))
} else {
// log.Printf("✅ Successfully sent resource notification via %s for %s", alertConfig.NotificationType, id)
successCount++
}
}
// Report results
if successCount > 0 {
// log.Printf("✅ Successfully sent %d out of %d resource notifications", successCount, len(notificationIDs))
}
if len(errors) > 0 {
// log.Printf("❌ Resource notification errors occurred: %v", errors)
if successCount == 0 {
return fmt.Errorf("all resource notifications failed: %v", errors)
}
// If some succeeded, just log errors but don't return error
// log.Printf("⚠️ Some resource notifications failed but %d succeeded", successCount)
}
return nil
}
// generateResourceMessage creates notification message for specific resource alerts
func (snm *ServerNotificationManager) generateResourceMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var baseMessage string
// Select appropriate resource message from template based on status
if template != nil {
// log.Printf("🔧 Using template for resource type: %s with status: %s", resourceType, payload.Status)
// Check if this is a recovery/restore notification
if payload.Status == "up" {
// Use restore messages for recovery notifications
switch strings.ToLower(resourceType) {
case "cpu":
baseMessage = template.RestoreCPUMessage
case "ram", "memory":
baseMessage = template.RestoreRAMMessage
case "disk":
baseMessage = template.RestoreDiskMessage
case "network":
baseMessage = template.RestoreNetworkMessage
case "cpu_temp", "cpu_temperature":
baseMessage = template.RestoreCPUTempMessage
case "disk_io":
baseMessage = template.RestoreDiskIOMessage
default:
baseMessage = template.UpMessage // fallback to general up message
}
} else {
// Use regular alert messages for warning/down notifications
switch strings.ToLower(resourceType) {
case "cpu":
baseMessage = template.CPUMessage
case "ram", "memory":
baseMessage = template.RAMMessage
case "disk":
baseMessage = template.DiskMessage
case "network":
baseMessage = template.NetworkMessage
case "cpu_temp", "cpu_temperature":
baseMessage = template.CPUTempMessage
case "disk_io":
baseMessage = template.DiskIOMessage
default:
baseMessage = template.WarningMessage // fallback to warning message
}
}
// log.Printf("📝 Template message selected for %s (status: %s): %s", resourceType, payload.Status, baseMessage)
}
// If no template or empty message, use default
if baseMessage == "" {
// log.Printf("🔧 Using default message for resource %s", resourceType)
baseMessage = snm.getDefaultResourceMessage(payload, resourceType)
}
// Replace placeholders with actual values
message := snm.replacePlaceholders(baseMessage, payload)
// log.Printf("📝 Final resource message after placeholder replacement: %s", message)
return message
}
// getDefaultResourceMessage provides default messages for resource alerts
func (snm *ServerNotificationManager) getDefaultResourceMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "⚠️"
statusText := "Alert"
if payload.Status == "up" {
statusEmoji = "✅"
statusText = "Recovery"
} else if payload.Status == "down" {
statusEmoji = "❌"
statusText = "Critical"
}
switch strings.ToLower(resourceType) {
case "cpu":
if payload.Status == "up" {
return fmt.Sprintf("%s CPU %s: Server %s CPU usage has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.CPUUsage)
}
return fmt.Sprintf("%s CPU %s: Server %s CPU usage is %s", statusEmoji, statusText, payload.ServiceName, payload.CPUUsage)
case "ram", "memory":
if payload.Status == "up" {
return fmt.Sprintf("%s RAM %s: Server %s RAM usage has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.RAMUsage)
}
return fmt.Sprintf("%s RAM %s: Server %s RAM usage is %s", statusEmoji, statusText, payload.ServiceName, payload.RAMUsage)
case "disk":
if payload.Status == "up" {
return fmt.Sprintf("%s Disk %s: Server %s disk usage has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.DiskUsage)
}
return fmt.Sprintf("%s Disk %s: Server %s disk usage is %s", statusEmoji, statusText, payload.ServiceName, payload.DiskUsage)
case "network":
if payload.Status == "up" {
return fmt.Sprintf("%s Network %s: Server %s network usage has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.NetworkUsage)
}
return fmt.Sprintf("%s Network %s: Server %s network usage is %s", statusEmoji, statusText, payload.ServiceName, payload.NetworkUsage)
case "cpu_temp", "cpu_temperature":
if payload.Status == "up" {
return fmt.Sprintf("%s CPU Temperature %s: Server %s CPU temperature has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.CPUTemp)
}
return fmt.Sprintf("%s CPU Temperature %s: Server %s CPU temperature is %s", statusEmoji, statusText, payload.ServiceName, payload.CPUTemp)
case "disk_io":
if payload.Status == "up" {
return fmt.Sprintf("%s Disk I/O %s: Server %s disk I/O has returned to normal: %s", statusEmoji, statusText, payload.ServiceName, payload.DiskIO)
}
return fmt.Sprintf("%s Disk I/O %s: Server %s disk I/O is %s", statusEmoji, statusText, payload.ServiceName, payload.DiskIO)
default:
if payload.Status == "up" {
return fmt.Sprintf("%s Resource %s: Server %s %s has recovered", statusEmoji, statusText, payload.ServiceName, payload.Message)
}
return fmt.Sprintf("%s Resource %s: Server %s %s", statusEmoji, statusText, payload.ServiceName, payload.Message)
}
}
// generateMessage creates notification message from template or default
func (snm *ServerNotificationManager) generateMessage(payload *NotificationPayload, template *ServerNotificationTemplate) string {
var baseMessage string
// Select appropriate message based on status and template
if template != nil {
// log.Printf("🔧 Using template for status: %s", strings.ToLower(payload.Status))
switch strings.ToLower(payload.Status) {
case "up":
baseMessage = template.UpMessage
case "down":
baseMessage = template.DownMessage
case "warning":
baseMessage = template.WarningMessage
case "paused":
baseMessage = template.PausedMessage
default:
baseMessage = template.UpMessage // fallback
}
// log.Printf("📝 Template message selected: %s", baseMessage)
}
// If no template or empty message, use default
if baseMessage == "" {
// log.Printf("🔧 Using default message (no template or empty template message)")
baseMessage = snm.getDefaultMessage(payload)
}
// Replace placeholders with actual values
message := snm.replacePlaceholders(baseMessage, payload)
// log.Printf("📝 Final message after placeholder replacement: %s", message)
return message
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (snm *ServerNotificationManager) replacePlaceholders(message string, payload *NotificationPayload) string {
// Service/Server name placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${server_name}", payload.ServiceName) // For server notifications
// Status placeholder
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
// Host/IP placeholders
message = strings.ReplaceAll(message, "${host}", payload.Host)
message = strings.ReplaceAll(message, "${ip}", payload.Host) // Alternative placeholder
message = strings.ReplaceAll(message, "${ip_address}", payload.Host) // New server monitoring placeholder
// Hostname placeholder
message = strings.ReplaceAll(message, "${hostname}", payload.Hostname)
// Port placeholder
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
// Response time placeholder
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
// Timestamp placeholders
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${date}", payload.Timestamp.Format("2006-01-02"))
// Service type placeholder
message = strings.ReplaceAll(message, "${service_type}", payload.ServiceType)
// Error message placeholder
if payload.ErrorMessage != "" {
message = strings.ReplaceAll(message, "${error}", payload.ErrorMessage)
message = strings.ReplaceAll(message, "${error_message}", payload.ErrorMessage)
} else {
message = strings.ReplaceAll(message, "${error}", "")
message = strings.ReplaceAll(message, "${error_message}", "")
}
// Custom message placeholder
if payload.Message != "" {
message = strings.ReplaceAll(message, "${message}", payload.Message)
}
// Server monitoring specific placeholders
if payload.CPUUsage != "" {
message = strings.ReplaceAll(message, "${cpu_usage}", payload.CPUUsage)
} else {
message = strings.ReplaceAll(message, "${cpu_usage}", "N/A")
}
if payload.RAMUsage != "" {
message = strings.ReplaceAll(message, "${ram_usage}", payload.RAMUsage)
} else {
message = strings.ReplaceAll(message, "${ram_usage}", "N/A")
}
if payload.DiskUsage != "" {
message = strings.ReplaceAll(message, "${disk_usage}", payload.DiskUsage)
} else {
message = strings.ReplaceAll(message, "${disk_usage}", "N/A")
}
if payload.NetworkUsage != "" {
message = strings.ReplaceAll(message, "${network_usage}", payload.NetworkUsage)
} else {
message = strings.ReplaceAll(message, "${network_usage}", "N/A")
}
if payload.CPUTemp != "" {
message = strings.ReplaceAll(message, "${cpu_temp}", payload.CPUTemp)
} else {
message = strings.ReplaceAll(message, "${cpu_temp}", "N/A")
}
if payload.DiskIO != "" {
message = strings.ReplaceAll(message, "${disk_io}", payload.DiskIO)
} else {
message = strings.ReplaceAll(message, "${disk_io}", "N/A")
}
if payload.Threshold != "" {
message = strings.ReplaceAll(message, "${threshold}", payload.Threshold)
} else {
message = strings.ReplaceAll(message, "${threshold}", "N/A")
}
return message
}
// getDefaultMessage provides a default notification message
func (snm *ServerNotificationManager) getDefaultMessage(payload *NotificationPayload) string {
statusEmoji := "✅"
if payload.Status == "down" {
statusEmoji = "❌"
} else if payload.Status == "warning" {
statusEmoji = "⚠️"
}
message := fmt.Sprintf("%s Server Alert\n\n", statusEmoji)
message += fmt.Sprintf("Server: %s\n", payload.ServiceName)
message += fmt.Sprintf("Status: %s\n", strings.ToUpper(payload.Status))
message += fmt.Sprintf("IP Address: %s\n", payload.Host)
if payload.CPUUsage != "" {
message += fmt.Sprintf("CPU Usage: %s\n", payload.CPUUsage)
}
if payload.RAMUsage != "" {
message += fmt.Sprintf("RAM Usage: %s\n", payload.RAMUsage)
}
if payload.DiskUsage != "" {
message += fmt.Sprintf("Disk Usage: %s\n", payload.DiskUsage)
}
message += fmt.Sprintf("Time: %s\n", payload.Timestamp.Format("2006-01-02 15:04:05"))
if payload.ErrorMessage != "" {
message += fmt.Sprintf("Error: %s\n", payload.ErrorMessage)
}
return message
}
@@ -0,0 +1,141 @@
package notification
import (
"encoding/json"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"service-operation/pocketbase"
)
// parseNotificationIDs parses comma-separated notification IDs
func parseNotificationIDs(notificationID string) []string {
if notificationID == "" {
return []string{}
}
// Split by comma and trim whitespace
ids := strings.Split(notificationID, ",")
var cleanIDs []string
for _, id := range ids {
cleanID := strings.TrimSpace(id)
if cleanID != "" {
cleanIDs = append(cleanIDs, cleanID)
}
}
//log.Printf("📋 Parsed notification IDs: %v", cleanIDs)
return cleanIDs
}
// isNotificationEnabled checks if the notification is enabled
func isNotificationEnabled(pbClient *pocketbase.PocketBaseClient, notificationID string) bool {
config, err := getAlertConfiguration(pbClient, notificationID)
if err != nil {
//log.Printf("❌ Error getting alert configuration for enabled check: %v", err)
return false
}
enabled, err := strconv.ParseBool(config.Enabled)
if err != nil {
//log.Printf("❌ Error parsing enabled field: %v", err)
return false
}
//log.Printf("️ Notification %s enabled status: %t", notificationID, enabled)
return enabled
}
// getAlertConfiguration fetches alert configuration from PocketBase
func getAlertConfiguration(pbClient *pocketbase.PocketBaseClient, notificationID string) (*AlertConfiguration, error) {
url := fmt.Sprintf("%s/api/collections/alert_configurations/records/%s", pbClient.GetBaseURL(), notificationID)
//log.Printf("🌐 Fetching alert configuration from: %s", url)
resp, err := http.Get(url)
if err != nil {
//log.Printf("❌ HTTP error fetching alert configuration: %v", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Printf("❌ Failed to fetch alert configuration, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch alert configuration, status: %d", resp.StatusCode)
}
var config AlertConfiguration
if err := json.NewDecoder(resp.Body).Decode(&config); err != nil {
//log.Printf("❌ Error decoding alert configuration JSON: %v", err)
return nil, err
}
//log.Printf("✅ Successfully fetched alert configuration: %+v", config)
return &config, nil
}
// getNotificationTemplate fetches server notification template from PocketBase
func getNotificationTemplate(pbClient *pocketbase.PocketBaseClient, templateID string) (*ServerNotificationTemplate, error) {
url := fmt.Sprintf("%s/api/collections/server_notification_templates/records/%s", pbClient.GetBaseURL(), templateID)
//log.Printf("🌐 Fetching notification template from: %s", url)
resp, err := http.Get(url)
if err != nil {
//log.Printf("❌ HTTP error fetching notification template: %v", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Printf("❌ Failed to fetch notification template, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch notification template, status: %d", resp.StatusCode)
}
var template ServerNotificationTemplate
if err := json.NewDecoder(resp.Body).Decode(&template); err != nil {
//log.Printf("❌ Error decoding notification template JSON: %v", err)
return nil, err
}
//log.Printf("✅ Successfully fetched notification template: %+v", template)
return &template, nil
}
// getServiceNotificationTemplate fetches service notification template from PocketBase
func getServiceNotificationTemplate(pbClient *pocketbase.PocketBaseClient, templateID string) (*ServiceNotificationTemplate, error) {
url := fmt.Sprintf("%s/api/collections/service_notification_templates/records/%s", pbClient.GetBaseURL(), templateID)
//log.Printf("🌐 Fetching service notification template from: %s", url)
resp, err := http.Get(url)
if err != nil {
//log.Printf("❌ HTTP error fetching service notification template: %v", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//log.Printf("❌ Failed to fetch service notification template, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch service notification template, status: %d", resp.StatusCode)
}
var template ServiceNotificationTemplate
if err := json.NewDecoder(resp.Body).Decode(&template); err != nil {
//log.Printf("❌ Error decoding service notification template JSON: %v", err)
return nil, err
}
//log.Printf("✅ Successfully fetched service notification template: %+v", template)
return &template, nil
}
// Helper function to get map keys
func getKeys(m map[string]NotificationService) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}
@@ -0,0 +1,371 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// SignalService handles Signal notifications
type SignalService struct{}
// NewSignalService creates a new Signal notification service
func NewSignalService() *SignalService {
return &SignalService{}
}
// SignalPayload represents the payload for Signal REST API
type SignalPayload struct {
Number string `json:"number"`
Recipients []string `json:"recipients"`
Message string `json:"message"`
}
// SendNotification sends a notification via Signal REST API
func (ss *SignalService) SendNotification(config *AlertConfiguration, message string) error {
// fmt.Printf("📱 [SIGNAL] Attempting to send notification...\n")
// fmt.Printf("📱 [SIGNAL] Config - Phone Number: %s, API Endpoint: %s\n", config.SignalNumber, config.SignalAPIEndpoint)
// fmt.Printf("📱 [SIGNAL] Message: %s\n", message)
if config.SignalNumber == "" {
return fmt.Errorf("signal phone number is required")
}
if config.SignalAPIEndpoint == "" {
return fmt.Errorf("signal API endpoint is required")
}
// Use the configured endpoint directly (it should already include the full path like /v2/send)
apiURL := config.SignalAPIEndpoint
// fmt.Printf("📱 [SIGNAL] API URL: %s\n", apiURL)
// Prepare payload for Signal REST API
payload := SignalPayload{
Number: config.SignalNumber, // This should be the sender's number (registered with signal-cli)
Recipients: []string{config.SignalNumber}, // Send to the same number for now
Message: message,
}
jsonData, err := json.Marshal(payload)
if err != nil {
// fmt.Printf("❌ [SIGNAL] JSON marshal error: %v\n", err)
return fmt.Errorf("failed to marshal signal payload: %v", err)
}
// fmt.Printf("📱 [SIGNAL] Payload: %s\n", string(jsonData))
// fmt.Printf("📱 [SIGNAL] Sending POST request...\n")
// Send POST request to Signal REST API
resp, err := http.Post(apiURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
// fmt.Printf("❌ [SIGNAL] HTTP error: %v\n", err)
return fmt.Errorf("failed to send signal request: %v", err)
}
defer resp.Body.Close()
// fmt.Printf("📱 [SIGNAL] Response Status: %d\n", resp.StatusCode)
// Check if the request was successful
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
// fmt.Printf("❌ [SIGNAL] HTTP Status error: %d\n", resp.StatusCode)
// Try to read error response
var errorResponse map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&errorResponse); err == nil {
// fmt.Printf("❌ [SIGNAL] Error response: %+v\n", errorResponse)
return fmt.Errorf("signal API error (status %d): %v", resp.StatusCode, errorResponse)
}
return fmt.Errorf("signal API returned status %d", resp.StatusCode)
}
// Parse response
var result map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
// fmt.Printf("❌ [SIGNAL] JSON decode error: %v\n", err)
// Don't fail if we can't decode response but got success status
// fmt.Printf("✅ [SIGNAL] Message sent successfully (response decode failed but status was OK)!\n")
return nil
}
// fmt.Printf("📱 [SIGNAL] Response: %+v\n", result)
// fmt.Printf("✅ [SIGNAL] Message sent successfully!\n")
return nil
}
// SendServerNotification sends a server-specific notification via Signal
func (ss *SignalService) SendServerNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) error {
message := ss.generateServerMessage(payload, template, resourceType)
return ss.SendNotification(config, message)
}
// SendServiceNotification sends a service-specific notification via Signal
func (ss *SignalService) SendServiceNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServiceNotificationTemplate) error {
message := ss.generateServiceMessage(payload, template)
return ss.SendNotification(config, message)
}
// generateServerMessage creates a message for server notifications using server template
func (ss *SignalService) generateServerMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var templateMessage string
// Select appropriate template message based on status and resource type
switch strings.ToLower(payload.Status) {
case "down":
templateMessage = template.DownMessage
case "up":
templateMessage = template.UpMessage
case "warning":
templateMessage = template.WarningMessage
case "paused":
templateMessage = template.PausedMessage
default:
// Handle resource-specific messages
switch resourceType {
case "cpu":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUMessage
} else {
templateMessage = template.CPUMessage
}
case "ram", "memory":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreRAMMessage
} else {
templateMessage = template.RAMMessage
}
case "disk":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskMessage
} else {
templateMessage = template.DiskMessage
}
case "network":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreNetworkMessage
} else {
templateMessage = template.NetworkMessage
}
case "cpu_temp", "cpu_temperature":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUTempMessage
} else {
templateMessage = template.CPUTempMessage
}
case "disk_io":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskIOMessage
} else {
templateMessage = template.DiskIOMessage
}
default:
templateMessage = template.WarningMessage
}
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ss.generateDefaultServerMessage(payload, resourceType)
}
return ss.replacePlaceholders(templateMessage, payload)
}
// generateServiceMessage creates a message for service notifications using service template
func (ss *SignalService) generateServiceMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var templateMessage string
// Select appropriate template message based on status
switch strings.ToLower(payload.Status) {
case "up":
templateMessage = template.UpMessage
case "down":
templateMessage = template.DownMessage
case "maintenance":
templateMessage = template.MaintenanceMessage
case "incident":
templateMessage = template.IncidentMessage
case "resolved":
templateMessage = template.ResolvedMessage
case "warning":
templateMessage = template.WarningMessage
default:
templateMessage = template.WarningMessage
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ss.generateDefaultUptimeMessage(payload)
}
return ss.replacePlaceholders(templateMessage, payload)
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (ss *SignalService) replacePlaceholders(message string, payload *NotificationPayload) string {
// Replace basic placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", ss.safeString(payload.Host))
message = strings.ReplaceAll(message, "${hostname}", ss.safeString(payload.Hostname))
// Replace URL with fallback to host
url := ss.safeString(payload.URL)
if url == "N/A" && payload.Host != "" {
url = payload.Host
}
message = strings.ReplaceAll(message, "${url}", url)
// Replace domain
message = strings.ReplaceAll(message, "${domain}", ss.safeString(payload.Domain))
// Replace service type
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
// Replace region and agent info
message = strings.ReplaceAll(message, "${region_name}", ss.safeString(payload.RegionName))
message = strings.ReplaceAll(message, "${agent_id}", ss.safeString(payload.AgentID))
// Handle numeric fields safely
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
// Replace server monitoring fields
message = strings.ReplaceAll(message, "${cpu_usage}", ss.safeString(payload.CPUUsage))
message = strings.ReplaceAll(message, "${ram_usage}", ss.safeString(payload.RAMUsage))
message = strings.ReplaceAll(message, "${disk_usage}", ss.safeString(payload.DiskUsage))
message = strings.ReplaceAll(message, "${network_usage}", ss.safeString(payload.NetworkUsage))
message = strings.ReplaceAll(message, "${cpu_temp}", ss.safeString(payload.CPUTemp))
message = strings.ReplaceAll(message, "${disk_io}", ss.safeString(payload.DiskIO))
message = strings.ReplaceAll(message, "${threshold}", ss.safeString(payload.Threshold))
// Replace error message - important for uptime services
message = strings.ReplaceAll(message, "${error_message}", ss.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${error}", ss.safeString(payload.ErrorMessage))
// Replace time placeholders
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
return message
}
// safeString returns the string value or "N/A" if empty
func (ss *SignalService) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// generateDefaultUptimeMessage creates a default uptime message with proper formatting
func (ss *SignalService) generateDefaultUptimeMessage(payload *NotificationPayload) string {
// Status emoji mapping
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
case "maintenance", "paused":
statusEmoji = "🟠"
}
message := fmt.Sprintf("%s Service %s is %s.", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
// Build formatted details
details := []string{}
// Add URL or host
if payload.URL != "" {
details = append(details, fmt.Sprintf(" - Host URL: %s", payload.URL))
} else if payload.Host != "" {
details = append(details, fmt.Sprintf(" - Host: %s", payload.Host))
}
// Add service type
if payload.ServiceType != "" {
details = append(details, fmt.Sprintf(" - Type: %s", strings.ToUpper(payload.ServiceType)))
}
// Add port if available
if payload.Port > 0 {
details = append(details, fmt.Sprintf(" - Port: %d", payload.Port))
}
// Add domain if available
if payload.Domain != "" {
details = append(details, fmt.Sprintf(" - Domain: %s", payload.Domain))
}
// Add response time
if payload.ResponseTime > 0 {
details = append(details, fmt.Sprintf(" - Response time: %dms", payload.ResponseTime))
} else {
details = append(details, " - Response time: N/A")
}
// Add region info
if payload.RegionName != "" {
details = append(details, fmt.Sprintf(" - Region: %s", payload.RegionName))
}
// Add agent info
if payload.AgentID != "" {
details = append(details, fmt.Sprintf(" - Agent: %s", payload.AgentID))
}
// Add uptime if available
if payload.Uptime > 0 {
details = append(details, fmt.Sprintf(" - Uptime: %d%%", payload.Uptime))
}
// Add timestamp
details = append(details, fmt.Sprintf(" - Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05")))
// Combine message with details
if len(details) > 0 {
message += "\n" + strings.Join(details, "\n")
}
return message
}
// generateDefaultServerMessage creates a default server message
func (ss *SignalService) generateDefaultServerMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
}
return fmt.Sprintf("%s 🖥️ Server %s (%s) status: %s", statusEmoji, payload.ServiceName, payload.Hostname, strings.ToUpper(payload.Status))
}
@@ -0,0 +1,538 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
)
// SlackService handles Slack notifications
type SlackService struct{}
// NewSlackService creates a new Slack notification service
func NewSlackService() *SlackService {
return &SlackService{}
}
// SlackPayload represents the payload for Slack webhook
type SlackPayload struct {
Text string `json:"text"`
Username string `json:"username,omitempty"`
Channel string `json:"channel,omitempty"`
IconEmoji string `json:"icon_emoji,omitempty"`
Attachments []SlackAttachment `json:"attachments,omitempty"`
Blocks []SlackBlock `json:"blocks,omitempty"`
}
// SlackAttachment represents a Slack attachment
type SlackAttachment struct {
Color string `json:"color,omitempty"`
Title string `json:"title,omitempty"`
Text string `json:"text,omitempty"`
Timestamp int64 `json:"ts,omitempty"`
Fields []SlackField `json:"fields,omitempty"`
}
// SlackField represents a field in Slack attachment
type SlackField struct {
Title string `json:"title"`
Value string `json:"value"`
Short bool `json:"short"`
}
// SlackBlock represents a Slack block element
type SlackBlock struct {
Type string `json:"type"`
Text *SlackText `json:"text,omitempty"`
}
// SlackText represents text in Slack blocks
type SlackText struct {
Type string `json:"type"`
Text string `json:"text"`
}
// SendNotification sends a notification via Slack webhook
func (ss *SlackService) SendNotification(config *AlertConfiguration, message string) error {
if config.SlackWebhookURL == "" {
return fmt.Errorf("slack webhook URL is required")
}
// Parse and format the message for better Slack presentation
formattedPayload := ss.createSlackPayload(config, message)
jsonData, err := json.Marshal(formattedPayload)
if err != nil {
return err
}
resp, err := http.Post(config.SlackWebhookURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("slack webhook error, status: %d", resp.StatusCode)
}
return nil
}
// SendServerNotification sends a server-specific notification via Slack
func (ss *SlackService) SendServerNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) error {
message := ss.generateServerMessage(payload, template, resourceType)
return ss.SendNotification(config, message)
}
// SendServiceNotification sends a service-specific notification via Slack
func (ss *SlackService) SendServiceNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServiceNotificationTemplate) error {
message := ss.generateServiceMessage(payload, template)
return ss.SendNotification(config, message)
}
// createSlackPayload creates a properly formatted Slack payload with rich formatting
func (ss *SlackService) createSlackPayload(config *AlertConfiguration, message string) SlackPayload {
// Determine color based on message content
color := ss.getColorFromMessage(message)
// Parse message for structured presentation
title, fields := ss.parseMessageForSlack(message)
payload := SlackPayload{
Username: config.NotifyName,
IconEmoji: ss.getEmojiFromMessage(message),
Text: title,
}
// Create attachment for better formatting
if len(fields) > 0 {
attachment := SlackAttachment{
Color: color,
Title: title,
Timestamp: time.Now().Unix(),
Fields: fields,
}
payload.Attachments = []SlackAttachment{attachment}
payload.Text = "" // Clear text when using attachments
}
return payload
}
// parseMessageForSlack parses the message to extract title and create structured fields
func (ss *SlackService) parseMessageForSlack(message string) (string, []SlackField) {
lines := strings.Split(message, "\n")
if len(lines) == 0 {
return message, nil
}
title := strings.TrimSpace(lines[0])
var fields []SlackField
// Process bullet points and key-value pairs
for i, line := range lines {
if i == 0 {
continue // Skip title
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Handle bullet points (•, -, *)
if strings.HasPrefix(line, "•") || strings.HasPrefix(line, "-") || strings.HasPrefix(line, "*") {
// Remove bullet and clean up
cleaned := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(line, "•"), "-"), "*"))
// Try to split on colon for key-value pairs
if strings.Contains(cleaned, ":") {
parts := strings.SplitN(cleaned, ":", 2)
if len(parts) == 2 {
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
fields = append(fields, SlackField{
Title: key,
Value: value,
Short: len(value) < 30, // Short fields for compact display
})
}
} else {
// Add as a single field
fields = append(fields, SlackField{
Title: "Info",
Value: cleaned,
Short: false,
})
}
} else if strings.Contains(line, ":") {
// Direct key-value pairs
parts := strings.SplitN(line, ":", 2)
if len(parts) == 2 {
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
fields = append(fields, SlackField{
Title: key,
Value: value,
Short: len(value) < 30,
})
}
}
}
return title, fields
}
// getColorFromMessage determines the appropriate color based on message content
func (ss *SlackService) getColorFromMessage(message string) string {
lowerMsg := strings.ToLower(message)
// Critical/Error states - Red
if strings.Contains(lowerMsg, "down") || strings.Contains(lowerMsg, "failed") ||
strings.Contains(lowerMsg, "error") || strings.Contains(lowerMsg, "expired") ||
strings.Contains(lowerMsg, "🔴") || strings.Contains(lowerMsg, "🚨") {
return "#FF0000" // Red
}
// Warning states - Yellow/Orange
if strings.Contains(lowerMsg, "warning") || strings.Contains(lowerMsg, "expiring") ||
strings.Contains(lowerMsg, "🟡") || strings.Contains(lowerMsg, "⚠️") {
return "#FFA500" // Orange
}
// Success states - Green
if strings.Contains(lowerMsg, "up") || strings.Contains(lowerMsg, "restored") ||
strings.Contains(lowerMsg, "resolved") || strings.Contains(lowerMsg, "🟢") ||
strings.Contains(lowerMsg, "✅") {
return "#00FF00" // Green
}
// Maintenance/Info states - Blue
if strings.Contains(lowerMsg, "maintenance") || strings.Contains(lowerMsg, "paused") ||
strings.Contains(lowerMsg, "🟠") || strings.Contains(lowerMsg, "🔵") {
return "#0080FF" // Blue
}
return "#808080" // Default gray
}
// getEmojiFromMessage extracts or determines appropriate emoji for the message
func (ss *SlackService) getEmojiFromMessage(message string) string {
lowerMsg := strings.ToLower(message)
// Check for specific service types first
if strings.Contains(lowerMsg, "server") {
if strings.Contains(lowerMsg, "down") || strings.Contains(lowerMsg, "failed") {
return ":red_circle:"
} else if strings.Contains(lowerMsg, "warning") {
return ":warning:"
} else if strings.Contains(lowerMsg, "up") || strings.Contains(lowerMsg, "restored") {
return ":white_check_mark:"
}
return ":desktop_computer:"
}
// SSL Certificate notifications
if strings.Contains(lowerMsg, "certificate") || strings.Contains(lowerMsg, "ssl") {
if strings.Contains(lowerMsg, "expired") {
return ":no_entry:"
} else if strings.Contains(lowerMsg, "expiring") {
return ":warning:"
}
return ":lock:"
}
// General status-based emojis
if strings.Contains(lowerMsg, "down") || strings.Contains(lowerMsg, "failed") {
return ":red_circle:"
} else if strings.Contains(lowerMsg, "warning") {
return ":warning:"
} else if strings.Contains(lowerMsg, "up") || strings.Contains(lowerMsg, "restored") {
return ":white_check_mark:"
} else if strings.Contains(lowerMsg, "maintenance") {
return ":construction:"
}
return ":information_source:" // Default info emoji
}
// generateServerMessage creates a message for server notifications using server template
func (ss *SlackService) generateServerMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var templateMessage string
// Select appropriate template message based on status and resource type
switch strings.ToLower(payload.Status) {
case "down":
templateMessage = template.DownMessage
case "up":
templateMessage = template.UpMessage
case "warning":
templateMessage = template.WarningMessage
case "paused":
templateMessage = template.PausedMessage
default:
// Handle resource-specific messages
switch resourceType {
case "cpu":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUMessage
} else {
templateMessage = template.CPUMessage
}
case "ram", "memory":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreRAMMessage
} else {
templateMessage = template.RAMMessage
}
case "disk":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskMessage
} else {
templateMessage = template.DiskMessage
}
case "network":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreNetworkMessage
} else {
templateMessage = template.NetworkMessage
}
case "cpu_temp", "cpu_temperature":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUTempMessage
} else {
templateMessage = template.CPUTempMessage
}
case "disk_io":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskIOMessage
} else {
templateMessage = template.DiskIOMessage
}
default:
templateMessage = template.WarningMessage
}
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ss.generateDefaultServerMessage(payload, resourceType)
}
return ss.replacePlaceholders(templateMessage, payload)
}
// generateServiceMessage creates a message for service notifications using service template
func (ss *SlackService) generateServiceMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var templateMessage string
// Select appropriate template message based on status
switch strings.ToLower(payload.Status) {
case "up":
templateMessage = template.UpMessage
case "down":
templateMessage = template.DownMessage
case "maintenance":
templateMessage = template.MaintenanceMessage
case "incident":
templateMessage = template.IncidentMessage
case "resolved":
templateMessage = template.ResolvedMessage
case "warning":
templateMessage = template.WarningMessage
default:
templateMessage = template.WarningMessage
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ss.generateDefaultUptimeMessage(payload)
}
return ss.replacePlaceholders(templateMessage, payload)
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (ss *SlackService) replacePlaceholders(message string, payload *NotificationPayload) string {
// Replace basic placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", ss.safeString(payload.Host))
message = strings.ReplaceAll(message, "${hostname}", ss.safeString(payload.Hostname))
// Replace URL with fallback to host
url := ss.safeString(payload.URL)
if url == "N/A" && payload.Host != "" {
url = payload.Host
}
message = strings.ReplaceAll(message, "${url}", url)
// Replace domain
message = strings.ReplaceAll(message, "${domain}", ss.safeString(payload.Domain))
// Replace service type
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
// Replace region and agent info
message = strings.ReplaceAll(message, "${region_name}", ss.safeString(payload.RegionName))
message = strings.ReplaceAll(message, "${agent_id}", ss.safeString(payload.AgentID))
// Handle numeric fields safely
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
// Replace server monitoring fields
message = strings.ReplaceAll(message, "${cpu_usage}", ss.safeString(payload.CPUUsage))
message = strings.ReplaceAll(message, "${ram_usage}", ss.safeString(payload.RAMUsage))
message = strings.ReplaceAll(message, "${disk_usage}", ss.safeString(payload.DiskUsage))
message = strings.ReplaceAll(message, "${network_usage}", ss.safeString(payload.NetworkUsage))
message = strings.ReplaceAll(message, "${cpu_temp}", ss.safeString(payload.CPUTemp))
message = strings.ReplaceAll(message, "${disk_io}", ss.safeString(payload.DiskIO))
message = strings.ReplaceAll(message, "${threshold}", ss.safeString(payload.Threshold))
// Replace SSL certificate fields
message = strings.ReplaceAll(message, "${certificate_name}", ss.safeString(payload.CertificateName))
message = strings.ReplaceAll(message, "${expiry_date}", ss.safeString(payload.ExpiryDate))
message = strings.ReplaceAll(message, "${days_left}", ss.safeString(payload.DaysLeft))
message = strings.ReplaceAll(message, "${issuer_cn}", ss.safeString(payload.IssuerCN))
message = strings.ReplaceAll(message, "${serial_number}", ss.safeString(payload.SerialNumber))
// Replace error message - important for uptime services
message = strings.ReplaceAll(message, "${error_message}", ss.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${error}", ss.safeString(payload.ErrorMessage))
// Replace time placeholders
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${date}", payload.Timestamp.Format("2006-01-02"))
// Replace message placeholder
if payload.Message != "" {
message = strings.ReplaceAll(message, "${message}", payload.Message)
} else {
message = strings.ReplaceAll(message, "${message}", "N/A")
}
return message
}
// safeString returns the string value or "N/A" if empty
func (ss *SlackService) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// generateDefaultUptimeMessage creates a default uptime message with proper formatting
func (ss *SlackService) generateDefaultUptimeMessage(payload *NotificationPayload) string {
// Status emoji mapping
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
case "maintenance", "paused":
statusEmoji = "🟠"
}
message := fmt.Sprintf("%s Service %s is %s", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
// Build formatted details
details := []string{}
// Add URL or host
if payload.URL != "" {
details = append(details, fmt.Sprintf("• Host URL: %s", payload.URL))
} else if payload.Host != "" {
details = append(details, fmt.Sprintf("• Host: %s", payload.Host))
}
// Add service type
if payload.ServiceType != "" {
details = append(details, fmt.Sprintf("• Type: %s", strings.ToUpper(payload.ServiceType)))
}
// Add port if available
if payload.Port > 0 {
details = append(details, fmt.Sprintf("• Port: %d", payload.Port))
}
// Add domain if available
if payload.Domain != "" {
details = append(details, fmt.Sprintf("• Domain: %s", payload.Domain))
}
// Add response time
if payload.ResponseTime > 0 {
details = append(details, fmt.Sprintf("• Response time: %dms", payload.ResponseTime))
} else {
details = append(details, "• Response time: N/A")
}
// Add region info
if payload.RegionName != "" {
details = append(details, fmt.Sprintf("• Region: %s", payload.RegionName))
}
// Add agent info
if payload.AgentID != "" {
details = append(details, fmt.Sprintf("• Agent: %s", payload.AgentID))
}
// Add uptime if available
if payload.Uptime > 0 {
details = append(details, fmt.Sprintf("• Uptime: %d%%", payload.Uptime))
}
// Add timestamp
details = append(details, fmt.Sprintf("• Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05")))
// Combine message with details
if len(details) > 0 {
message += "\n" + strings.Join(details, "\n")
}
return message
}
// generateDefaultServerMessage creates a default server message
func (ss *SlackService) generateDefaultServerMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
}
return fmt.Sprintf("%s 🖥️ Server %s (%s) status: %s", statusEmoji, payload.ServiceName, payload.Hostname, strings.ToUpper(payload.Status))
}
@@ -0,0 +1,285 @@
package notification
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"service-operation/pocketbase"
)
// SSLNotificationTemplate represents an SSL notification template
type SSLNotificationTemplate struct {
ID string `json:"id"`
Name string `json:"name"`
Expired string `json:"expired"`
ExpiringSoon string `json:"exiring_soon"`
Warning string `json:"warning"`
Placeholder string `json:"placeholder"`
}
// SSLNotificationManager handles SSL certificate notifications
type SSLNotificationManager struct {
pbClient *pocketbase.PocketBaseClient
services map[string]NotificationService
}
// NewSSLNotificationManager creates a new SSL notification manager
func NewSSLNotificationManager(pbClient *pocketbase.PocketBaseClient, services map[string]NotificationService) *SSLNotificationManager {
return &SSLNotificationManager{
pbClient: pbClient,
services: services,
}
}
// SendSSLNotification sends notification for SSL certificate status
func (snm *SSLNotificationManager) SendSSLNotification(payload *NotificationPayload, notificationID, templateID string) error {
// log.Printf("📨 [SSL-MANAGER] IMMEDIATE send for certificate: %s, status: %s", payload.Domain, payload.Status)
// log.Printf("📨 [SSL-MANAGER] Notification ID: %s, Template ID: %s", notificationID, templateID)
if notificationID == "" {
return fmt.Errorf("notification ID required for SSL certificate: %s", payload.Domain)
}
// Parse notification IDs
notificationIDs := parseNotificationIDs(notificationID)
if len(notificationIDs) == 0 {
return fmt.Errorf("no valid notification IDs for SSL certificate: %s", payload.Domain)
}
var errors []string
successCount := 0
// Send to each notification channel IMMEDIATELY
for _, id := range notificationIDs {
// log.Printf("📤 [SSL-SEND] Processing notification ID: %s for certificate %s", id, payload.Domain)
// Check if enabled
if !isNotificationEnabled(snm.pbClient, id) {
// log.Printf("⚠️ Notification %s disabled for SSL certificate %s, skipping", id, payload.Domain)
_ = id
continue
}
// Get alert configuration
alertConfig, err := getAlertConfiguration(snm.pbClient, id)
if err != nil {
// log.Printf("❌ Failed to get alert config for %s (certificate: %s): %v", id, payload.Domain, err)
errors = append(errors, fmt.Sprintf("config error %s: %v", id, err))
continue
}
// log.Printf("📋 [SSL-CONFIG] Alert config for %s: Type=%s, ChatID=%s, Token present=%v",
// id, alertConfig.NotificationType, alertConfig.TelegramChatID, alertConfig.BotToken != "")
// Get SSL template
var sslTemplate *SSLNotificationTemplate
if templateID != "" {
sslTemplate, err = snm.getSSLNotificationTemplate(templateID)
if err != nil {
// log.Printf("⚠️ Template error for %s (certificate: %s): %v", templateID, payload.Domain, err)
_ = err
} else {
// log.Printf("📄 [SSL-TEMPLATE] Retrieved template: Expired=%s, ExpiringSoon=%s, Warning=%s",
// sslTemplate.Expired, sslTemplate.ExpiringSoon, sslTemplate.Warning)
_ = sslTemplate
}
}
// Generate message
message := snm.generateSSLMessage(payload, sslTemplate)
// log.Printf("📝 [SSL-MESSAGE] Generated for %s (%s): %s", payload.Domain, id, message)
// Get notification service
service, exists := snm.services[alertConfig.NotificationType]
if !exists {
// log.Printf("❌ Unsupported notification type for SSL: %s", alertConfig.NotificationType)
errors = append(errors, fmt.Sprintf("unsupported type %s", alertConfig.NotificationType))
continue
}
// SEND IMMEDIATELY - NO DELAYS
// log.Printf("⚡ [SSL-TELEGRAM] Sending via %s for SSL certificate %s", alertConfig.NotificationType, payload.Domain)
err = service.SendNotification(alertConfig, message)
if err != nil {
// log.Printf("❌ [SSL-FAILED] Failed to send via %s for %s: %v", alertConfig.NotificationType, payload.Domain, err)
errors = append(errors, fmt.Sprintf("send failed %s: %v", alertConfig.NotificationType, err))
} else {
// log.Printf("✅ [SSL-SUCCESS] Successfully sent via %s for %s", alertConfig.NotificationType, payload.Domain)
successCount++
}
_ = alertConfig
_ = message
}
// Report results
if successCount > 0 {
// log.Printf("✅ [SSL-FINAL] Sent %d/%d SSL notifications for %s", successCount, len(notificationIDs), payload.Domain)
_ = successCount
_ = notificationIDs
}
if len(errors) > 0 && successCount == 0 {
return fmt.Errorf("all SSL notifications failed for %s: %v", payload.Domain, errors)
}
return nil
}
// getSSLNotificationTemplate fetches SSL notification template from PocketBase
func (snm *SSLNotificationManager) getSSLNotificationTemplate(templateID string) (*SSLNotificationTemplate, error) {
url := fmt.Sprintf("%s/api/collections/ssl_notification_templates/records/%s", snm.pbClient.GetBaseURL(), templateID)
// log.Printf("🌐 Fetching SSL notification template from: %s", url)
resp, err := http.Get(url)
if err != nil {
// log.Printf("❌ HTTP error fetching SSL notification template: %v", err)
_ = err
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// log.Printf("❌ Failed to fetch SSL notification template, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch SSL notification template, status: %d", resp.StatusCode)
}
var template SSLNotificationTemplate
if err := json.NewDecoder(resp.Body).Decode(&template); err != nil {
// log.Printf("❌ Error decoding SSL notification template JSON: %v", err)
_ = err
return nil, err
}
// log.Printf("✅ Successfully fetched SSL notification template: %+v", template)
_ = url
_ = template
return &template, nil
}
// generateSSLMessage creates notification message for SSL certificates
func (snm *SSLNotificationManager) generateSSLMessage(payload *NotificationPayload, template *SSLNotificationTemplate) string {
var baseMessage string
// Use template if available
if template != nil {
// log.Printf("🔧 Using SSL template for status: %s", strings.ToLower(payload.Status))
// log.Printf("🔧 [SSL-TEMPLATE-DEBUG] Available templates - Expired: '%s', ExpiringSoon: '%s', Warning: '%s'",
// template.Expired, template.ExpiringSoon, template.Warning)
switch strings.ToLower(payload.Status) {
case "expired":
baseMessage = template.Expired
// log.Printf("🔧 [SSL-EXPIRED] Selected expired template: '%s'", baseMessage)
case "expiring_soon":
baseMessage = template.ExpiringSoon
// log.Printf("🔧 [SSL-EXPIRING] Selected expiring soon template: '%s'", baseMessage)
case "warning":
baseMessage = template.Warning
// log.Printf("🔧 [SSL-WARNING] Selected warning template: '%s'", baseMessage)
default:
baseMessage = template.Warning
// log.Printf("🔧 [SSL-DEFAULT] Using warning template for status '%s': '%s'", payload.Status, baseMessage)
}
_ = baseMessage
_ = template
}
// Use default if no template or template message is empty
if baseMessage == "" {
// log.Printf("🔧 Using default SSL message (no template or empty template)")
baseMessage = snm.getDefaultSSLMessage(payload)
}
// Replace placeholders
message := snm.replaceSSLPlaceholders(baseMessage, payload)
// log.Printf("📝 Final SSL message: %s", message)
_ = message
return message
}
// replaceSSLPlaceholders replaces all placeholders in the SSL message
func (snm *SSLNotificationManager) replaceSSLPlaceholders(message string, payload *NotificationPayload) string {
// log.Printf("🔄 [SSL-REPLACE] Before replacement: %s", message)
// log.Printf("🔄 [SSL-DATA] Payload data - Domain: %s, ExpiryDate: %s, DaysLeft: %s, IssuerCN: %s",
// payload.Domain, payload.ExpiryDate, payload.DaysLeft, payload.IssuerCN)
// SSL Certificate specific placeholders - ensure all are replaced
message = strings.ReplaceAll(message, "${domain}", snm.safeString(payload.Domain))
message = strings.ReplaceAll(message, "${certificate_name}", snm.safeString(payload.CertificateName))
message = strings.ReplaceAll(message, "${expiry_date}", snm.safeString(payload.ExpiryDate))
message = strings.ReplaceAll(message, "${days_left}", snm.safeString(payload.DaysLeft))
message = strings.ReplaceAll(message, "${issuer_cn}", snm.safeString(payload.IssuerCN))
message = strings.ReplaceAll(message, "${serial_number}", snm.safeString(payload.SerialNumber))
// Basic placeholders
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${service_name}", snm.safeString(payload.ServiceName))
message = strings.ReplaceAll(message, "${host}", snm.safeString(payload.Host))
// Time placeholders
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("15:04:05"))
message = strings.ReplaceAll(message, "${date}", payload.Timestamp.Format("2006-01-02"))
// Message placeholder
if payload.Message != "" {
message = strings.ReplaceAll(message, "${message}", payload.Message)
} else {
message = strings.ReplaceAll(message, "${message}", "N/A")
}
// log.Printf("🔄 [SSL-REPLACE] After replacement: %s", message)
_ = payload
return message
}
// safeString returns the string value or "N/A" if empty
func (snm *SSLNotificationManager) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// getDefaultSSLMessage provides a default notification message for SSL certificates
func (snm *SSLNotificationManager) getDefaultSSLMessage(payload *NotificationPayload) string {
statusEmoji := "🔒"
if payload.Status == "expired" {
statusEmoji = "🚨"
} else if payload.Status == "expiring_soon" {
statusEmoji = "⚠️"
} else if payload.Status == "warning" {
statusEmoji = "🔔"
}
// Create the default message with all SSL details
message := fmt.Sprintf("%s SSL certificate for %s has %s", statusEmoji, payload.Domain, strings.ToUpper(payload.Status))
// Add certificate details if available
if payload.CertificateName != "" && payload.CertificateName != payload.Domain {
message += fmt.Sprintf("\n • Certs Name: %s", payload.CertificateName)
}
if payload.ExpiryDate != "" {
message += fmt.Sprintf("\n • Expiry Date: %s", payload.ExpiryDate)
}
if payload.DaysLeft != "" {
message += fmt.Sprintf("\n • Days Left: %s", payload.DaysLeft)
}
if payload.IssuerCN != "" {
message += fmt.Sprintf("\n • Issuer: %s", payload.IssuerCN)
}
// Add timestamp
message += fmt.Sprintf("\n • Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05"))
return message
}
@@ -0,0 +1,347 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// TelegramService handles Telegram notifications
type TelegramService struct{}
// NewTelegramService creates a new Telegram notification service
func NewTelegramService() *TelegramService {
return &TelegramService{}
}
// TelegramPayload represents the payload for Telegram API
type TelegramPayload struct {
ChatID string `json:"chat_id"`
Text string `json:"text"`
ParseMode string `json:"parse_mode,omitempty"`
}
// SendNotification sends a notification via Telegram
func (ts *TelegramService) SendNotification(config *AlertConfiguration, message string) error {
// fmt.Printf("📱 [TELEGRAM] Attempting to send notification...\n")
// fmt.Printf("📱 [TELEGRAM] Config - Chat ID: %s, Bot Token present: %v\n", config.TelegramChatID, config.BotToken != "")
// fmt.Printf("📱 [TELEGRAM] Message: %s\n", message)
if config.BotToken == "" || config.TelegramChatID == "" {
return fmt.Errorf("telegram bot token and chat ID are required")
}
url := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", config.BotToken)
// fmt.Printf("📱 [TELEGRAM] API URL: %s\n", strings.Replace(url, config.BotToken, "[REDACTED]", 1))
payload := TelegramPayload{
ChatID: config.TelegramChatID,
Text: message,
}
jsonData, err := json.Marshal(payload)
if err != nil {
return err
}
// fmt.Printf("📱 [TELEGRAM] Sending POST request...\n")
resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
// fmt.Printf("❌ [TELEGRAM] HTTP error: %v\n", err)
return err
}
defer resp.Body.Close()
var result NotificationResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
// fmt.Printf("❌ [TELEGRAM] JSON decode error: %v\n", err)
return err
}
// fmt.Printf("📱 [TELEGRAM] Response - OK: %v, Description: %s\n", result.OK, result.Description)
if !result.OK {
// fmt.Printf("❌ [TELEGRAM] API error: %s\n", result.Description)
return fmt.Errorf("telegram API error: %s", result.Description)
}
// fmt.Printf("✅ [TELEGRAM] Message sent successfully!\n")
return nil
}
// SendServerNotification sends a server-specific notification via Telegram
func (ts *TelegramService) SendServerNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) error {
message := ts.generateServerMessage(payload, template, resourceType)
return ts.SendNotification(config, message)
}
// SendServiceNotification sends a service-specific notification via Telegram
func (ts *TelegramService) SendServiceNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServiceNotificationTemplate) error {
message := ts.generateServiceMessage(payload, template)
return ts.SendNotification(config, message)
}
// generateServerMessage creates a message for server notifications using server template
func (ts *TelegramService) generateServerMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var templateMessage string
// Select appropriate template message based on status and resource type
switch strings.ToLower(payload.Status) {
case "down":
templateMessage = template.DownMessage
case "up":
templateMessage = template.UpMessage
case "warning":
templateMessage = template.WarningMessage
case "paused":
templateMessage = template.PausedMessage
default:
// Handle resource-specific messages
switch resourceType {
case "cpu":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUMessage
} else {
templateMessage = template.CPUMessage
}
case "ram", "memory":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreRAMMessage
} else {
templateMessage = template.RAMMessage
}
case "disk":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskMessage
} else {
templateMessage = template.DiskMessage
}
case "network":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreNetworkMessage
} else {
templateMessage = template.NetworkMessage
}
case "cpu_temp", "cpu_temperature":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUTempMessage
} else {
templateMessage = template.CPUTempMessage
}
case "disk_io":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskIOMessage
} else {
templateMessage = template.DiskIOMessage
}
default:
templateMessage = template.WarningMessage
}
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ts.generateDefaultServerMessage(payload, resourceType)
}
return ts.replacePlaceholders(templateMessage, payload)
}
// generateServiceMessage creates a message for service notifications using service template
func (ts *TelegramService) generateServiceMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var templateMessage string
// Select appropriate template message based on status
switch strings.ToLower(payload.Status) {
case "up":
templateMessage = template.UpMessage
case "down":
templateMessage = template.DownMessage
case "maintenance":
templateMessage = template.MaintenanceMessage
case "incident":
templateMessage = template.IncidentMessage
case "resolved":
templateMessage = template.ResolvedMessage
case "warning":
templateMessage = template.WarningMessage
default:
templateMessage = template.WarningMessage
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ts.generateDefaultUptimeMessage(payload)
}
return ts.replacePlaceholders(templateMessage, payload)
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (ts *TelegramService) replacePlaceholders(message string, payload *NotificationPayload) string {
// Replace basic placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", ts.safeString(payload.Host))
message = strings.ReplaceAll(message, "${hostname}", ts.safeString(payload.Hostname))
// Replace URL with fallback to host
url := ts.safeString(payload.URL)
if url == "N/A" && payload.Host != "" {
url = payload.Host
}
message = strings.ReplaceAll(message, "${url}", url)
// Replace domain
message = strings.ReplaceAll(message, "${domain}", ts.safeString(payload.Domain))
// Replace service type
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
// Replace region and agent info
message = strings.ReplaceAll(message, "${region_name}", ts.safeString(payload.RegionName))
message = strings.ReplaceAll(message, "${agent_id}", ts.safeString(payload.AgentID))
// Handle numeric fields safely
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
// Replace server monitoring fields
message = strings.ReplaceAll(message, "${cpu_usage}", ts.safeString(payload.CPUUsage))
message = strings.ReplaceAll(message, "${ram_usage}", ts.safeString(payload.RAMUsage))
message = strings.ReplaceAll(message, "${disk_usage}", ts.safeString(payload.DiskUsage))
message = strings.ReplaceAll(message, "${network_usage}", ts.safeString(payload.NetworkUsage))
message = strings.ReplaceAll(message, "${cpu_temp}", ts.safeString(payload.CPUTemp))
message = strings.ReplaceAll(message, "${disk_io}", ts.safeString(payload.DiskIO))
message = strings.ReplaceAll(message, "${threshold}", ts.safeString(payload.Threshold))
// Replace error message - important for uptime services
message = strings.ReplaceAll(message, "${error_message}", ts.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${error}", ts.safeString(payload.ErrorMessage))
// Replace time placeholders
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
return message
}
// safeString returns the string value or "N/A" if empty
func (ts *TelegramService) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// generateDefaultUptimeMessage creates a default uptime message with proper formatting
func (ts *TelegramService) generateDefaultUptimeMessage(payload *NotificationPayload) string {
// Status emoji mapping
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
case "maintenance", "paused":
statusEmoji = "🟠"
}
message := fmt.Sprintf("%sService %s is %s.", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
// Build formatted details
details := []string{}
// Add URL or host
if payload.URL != "" {
details = append(details, fmt.Sprintf(" - Host URL: %s", payload.URL))
} else if payload.Host != "" {
details = append(details, fmt.Sprintf(" - Host: %s", payload.Host))
}
// Add service type
if payload.ServiceType != "" {
details = append(details, fmt.Sprintf(" - Type: %s", strings.ToUpper(payload.ServiceType)))
}
// Add port if available
if payload.Port > 0 {
details = append(details, fmt.Sprintf(" - Port: %d", payload.Port))
}
// Add domain if available
if payload.Domain != "" {
details = append(details, fmt.Sprintf(" - Domain: %s", payload.Domain))
}
// Add response time
if payload.ResponseTime > 0 {
details = append(details, fmt.Sprintf(" - Response time: %dms", payload.ResponseTime))
} else {
details = append(details, " - Response time: N/A")
}
// Add region info
if payload.RegionName != "" {
details = append(details, fmt.Sprintf(" - Region: %s", payload.RegionName))
}
// Add agent info
if payload.AgentID != "" {
details = append(details, fmt.Sprintf(" - Agent: %s", payload.AgentID))
}
// Add uptime if available
if payload.Uptime > 0 {
details = append(details, fmt.Sprintf(" - Uptime: %d%%", payload.Uptime))
}
// Add timestamp
details = append(details, fmt.Sprintf(" - Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05")))
// Combine message with details
if len(details) > 0 {
message += "\n" + strings.Join(details, "\n")
}
return message
}
// generateDefaultServerMessage creates a default server message
func (ts *TelegramService) generateDefaultServerMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
}
return fmt.Sprintf("%s🖥️ Server %s (%s) status: %s", statusEmoji, payload.ServiceName, payload.Hostname, strings.ToUpper(payload.Status))
}
@@ -0,0 +1,117 @@
package notification
import "time"
// NotificationPayload represents the data sent in notifications
type NotificationPayload struct {
ServiceName string `json:"service_name"`
Status string `json:"status"`
Host string `json:"host"`
Hostname string `json:"hostname"`
Port int `json:"port"`
ServiceType string `json:"service_type"`
ResponseTime int64 `json:"response_time"`
Timestamp time.Time `json:"timestamp"`
Message string `json:"message"`
ErrorMessage string `json:"error_message,omitempty"`
// Service-specific fields
URL string `json:"url,omitempty"`
Domain string `json:"domain,omitempty"`
RegionName string `json:"region_name,omitempty"`
AgentID string `json:"agent_id,omitempty"`
Uptime int `json:"uptime,omitempty"`
// Server monitoring specific fields
CPUUsage string `json:"cpu_usage,omitempty"`
RAMUsage string `json:"ram_usage,omitempty"`
DiskUsage string `json:"disk_usage,omitempty"`
NetworkUsage string `json:"network_usage,omitempty"`
CPUTemp string `json:"cpu_temp,omitempty"`
DiskIO string `json:"disk_io,omitempty"`
Threshold string `json:"threshold,omitempty"`
// SSL Certificate specific fields
CertificateName string `json:"certificate_name,omitempty"`
ExpiryDate string `json:"expiry_date,omitempty"`
DaysLeft string `json:"days_left,omitempty"`
IssuerCN string `json:"issuer_cn,omitempty"`
SerialNumber string `json:"serial_number,omitempty"`
}
// AlertConfiguration represents an alert configuration from PocketBase
type AlertConfiguration struct {
ID string `json:"id"`
NotificationType string `json:"notification_type"`
TelegramChatID string `json:"telegram_chat_id"`
DiscordWebhookURL string `json:"discord_webhook_url"`
SignalNumber string `json:"signal_number"`
SignalAPIEndpoint string `json:"signal_api_endpoint"`
NotifyName string `json:"notify_name"`
BotToken string `json:"bot_token"`
TemplateID string `json:"template_id"`
SlackWebhookURL string `json:"slack_webhook_url"`
GoogleChatWebhookURL string `json:"google_chat_webhook_url"`
Enabled string `json:"enabled"` // String because PocketBase returns it as string
EmailAddress string `json:"email_address"`
EmailSenderName string `json:"email_sender_name"`
SMTPServer string `json:"smtp_server"`
SMTPPassword string `json:"smtp_password,omitempty"`
SMTPPort string `json:"smtp_port"`
WebhookID string `json:"webhook_id"`
ChannelID string `json:"channel_id"`
// New webhook fields
WebhookURL string `json:"webhook_url"`
WebhookPayloadTemplate string `json:"webhook_payload_template"`
}
// ServerNotificationTemplate represents a server notification template
type ServerNotificationTemplate struct {
ID string `json:"id"`
Name string `json:"name"`
UpMessage string `json:"up_message"`
DownMessage string `json:"down_message"`
WarningMessage string `json:"warning_message"`
PausedMessage string `json:"paused_message"`
// Resource-specific messages
RAMMessage string `json:"ram_message"`
CPUMessage string `json:"cpu_message"`
DiskMessage string `json:"disk_message"`
NetworkMessage string `json:"network_message"`
CPUTempMessage string `json:"cpu_temp_message"`
DiskIOMessage string `json:"disk_io_message"`
// Resource restore messages
RestoreRAMMessage string `json:"restore_ram_message"`
RestoreCPUMessage string `json:"restore_cpu_message"`
RestoreDiskMessage string `json:"restore_disk_message"`
RestoreNetworkMessage string `json:"restore_network_message"`
RestoreCPUTempMessage string `json:"restore_cpu_temp_message"`
RestoreDiskIOMessage string `json:"restore_disk_io_message"`
Placeholder string `json:"placeholder"`
}
// ServiceNotificationTemplate represents a service notification template
type ServiceNotificationTemplate struct {
ID string `json:"id"`
Name string `json:"name"`
UpMessage string `json:"up_message"`
DownMessage string `json:"down_message"`
MaintenanceMessage string `json:"maintenance_message"`
IncidentMessage string `json:"incident_message"`
ResolvedMessage string `json:"resolved_message"`
WarningMessage string `json:"warning_message"`
Placeholder string `json:"placeholder"`
}
// NotificationResponse represents the response from notification APIs (like Telegram)
type NotificationResponse struct {
OK bool `json:"ok"`
Description string `json:"description,omitempty"`
ErrorCode int `json:"error_code,omitempty"`
Result interface{} `json:"result,omitempty"`
}
// NotificationService interface for different notification services
type NotificationService interface {
SendNotification(config *AlertConfiguration, message string) error
}
@@ -0,0 +1,244 @@
package notification
import (
"fmt"
// "log"
"strings"
"service-operation/pocketbase"
)
// UptimeNotificationManager handles uptime service notifications with IMMEDIATE sending
type UptimeNotificationManager struct {
pbClient *pocketbase.PocketBaseClient
services map[string]NotificationService
}
// NewUptimeNotificationManager creates a new uptime notification manager
func NewUptimeNotificationManager(pbClient *pocketbase.PocketBaseClient, services map[string]NotificationService) *UptimeNotificationManager {
return &UptimeNotificationManager{
pbClient: pbClient,
services: services,
}
}
// SendUptimeServiceNotification sends notification IMMEDIATELY - NO QUEUING
func (unm *UptimeNotificationManager) SendUptimeServiceNotification(payload *NotificationPayload, notificationID, templateID string) error {
// log.Printf("📨 [UPTIME-MANAGER] IMMEDIATE send for service: %s, status: %s", payload.ServiceName, payload.Status)
// log.Printf("📨 [UPTIME-MANAGER] Notification ID: %s, Template ID: %s", notificationID, templateID)
if notificationID == "" {
return fmt.Errorf("notification ID required for uptime service: %s", payload.ServiceName)
}
// Parse notification IDs
notificationIDs := parseNotificationIDs(notificationID)
if len(notificationIDs) == 0 {
return fmt.Errorf("no valid notification IDs for uptime service: %s", payload.ServiceName)
}
var errors []string
successCount := 0
// Send to each notification channel IMMEDIATELY
for _, id := range notificationIDs {
// log.Printf("📤 [UPTIME-SEND] Processing notification ID: %s for service %s", id, payload.ServiceName)
// Check if enabled
if !isNotificationEnabled(unm.pbClient, id) {
// log.Printf("⚠️ Notification %s disabled for uptime service %s, skipping", id, payload.ServiceName)
continue
}
// Get alert configuration
alertConfig, err := getAlertConfiguration(unm.pbClient, id)
if err != nil {
// log.Printf("❌ Failed to get alert config for %s (service: %s): %v", id, payload.ServiceName, err)
errors = append(errors, fmt.Sprintf("config error %s: %v", id, err))
continue
}
// log.Printf("📋 [UPTIME-CONFIG] Alert config for %s: Type=%s, ChatID=%s, Token present=%v",
// id, alertConfig.NotificationType, alertConfig.TelegramChatID, alertConfig.BotToken != "")
// Get service template
var serviceTemplate *ServiceNotificationTemplate
if templateID != "" {
serviceTemplate, err = getServiceNotificationTemplate(unm.pbClient, templateID)
if err != nil {
// log.Printf("⚠️ Template error for %s (service: %s): %v", templateID, payload.ServiceName, err)
}
}
// Generate message
message := unm.generateUptimeMessage(payload, serviceTemplate)
// log.Printf("📝 [UPTIME-MESSAGE] Generated for %s (%s): %s", payload.ServiceName, id, message)
// Get notification service
service, exists := unm.services[alertConfig.NotificationType]
if !exists {
// log.Printf("❌ Unsupported notification type for uptime: %s", alertConfig.NotificationType)
errors = append(errors, fmt.Sprintf("unsupported type %s", alertConfig.NotificationType))
continue
}
// SEND IMMEDIATELY - NO DELAYS
// log.Printf("⚡ [UPTIME-TELEGRAM] Sending via %s for uptime service %s", alertConfig.NotificationType, payload.ServiceName)
err = service.SendNotification(alertConfig, message)
if err != nil {
// log.Printf("❌ [UPTIME-FAILED] Failed to send via %s for %s: %v", alertConfig.NotificationType, payload.ServiceName, err)
errors = append(errors, fmt.Sprintf("send failed %s: %v", alertConfig.NotificationType, err))
} else {
// log.Printf("✅ [UPTIME-SUCCESS] Successfully sent via %s for %s", alertConfig.NotificationType, payload.ServiceName)
successCount++
}
}
// Report results
if successCount > 0 {
// log.Printf("✅ [UPTIME-FINAL] Sent %d/%d uptime notifications for %s", successCount, len(notificationIDs), payload.ServiceName)
}
if len(errors) > 0 && successCount == 0 {
return fmt.Errorf("all uptime notifications failed for %s: %v", payload.ServiceName, errors)
}
return nil
}
// generateUptimeMessage creates notification message for uptime services
func (unm *UptimeNotificationManager) generateUptimeMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var baseMessage string
// Use template if available
if template != nil {
// log.Printf("🔧 Using uptime template for status: %s", strings.ToLower(payload.Status))
switch strings.ToLower(payload.Status) {
case "up":
baseMessage = template.UpMessage
case "down":
baseMessage = template.DownMessage
case "maintenance":
baseMessage = template.MaintenanceMessage
case "incident":
baseMessage = template.IncidentMessage
case "resolved":
baseMessage = template.ResolvedMessage
case "warning":
baseMessage = template.WarningMessage
default:
baseMessage = template.WarningMessage
}
}
// Use default if no template
if baseMessage == "" {
// log.Printf("🔧 Using default uptime message (no template)")
baseMessage = unm.getDefaultUptimeMessage(payload)
}
// Replace placeholders
message := unm.replacePlaceholders(baseMessage, payload)
// log.Printf("📝 Final uptime message: %s", message)
return message
}
// replacePlaceholders replaces all placeholders in the message
func (unm *UptimeNotificationManager) replacePlaceholders(message string, payload *NotificationPayload) string {
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", payload.Host)
message = strings.ReplaceAll(message, "${ip}", payload.Host)
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.URL != "" {
message = strings.ReplaceAll(message, "${url}", payload.URL)
} else {
message = strings.ReplaceAll(message, "${url}", "N/A")
}
if payload.Domain != "" {
message = strings.ReplaceAll(message, "${domain}", payload.Domain)
} else {
message = strings.ReplaceAll(message, "${domain}", "N/A")
}
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
if payload.RegionName != "" {
message = strings.ReplaceAll(message, "${region_name}", payload.RegionName)
} else {
message = strings.ReplaceAll(message, "${region_name}", "N/A")
}
if payload.AgentID != "" {
message = strings.ReplaceAll(message, "${agent_id}", payload.AgentID)
} else {
message = strings.ReplaceAll(message, "${agent_id}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${date}", payload.Timestamp.Format("2006-01-02"))
// Handle error message placeholder - critical for uptime services
if payload.ErrorMessage != "" {
message = strings.ReplaceAll(message, "${error}", payload.ErrorMessage)
message = strings.ReplaceAll(message, "${error_message}", payload.ErrorMessage)
} else {
message = strings.ReplaceAll(message, "${error}", "")
message = strings.ReplaceAll(message, "${error_message}", "")
}
if payload.Message != "" {
message = strings.ReplaceAll(message, "${message}", payload.Message)
}
return message
}
// getDefaultUptimeMessage provides a default notification message for uptime services
func (unm *UptimeNotificationManager) getDefaultUptimeMessage(payload *NotificationPayload) string {
statusEmoji := "✅"
if payload.Status == "down" {
statusEmoji = "❌"
} else if payload.Status == "warning" {
statusEmoji = "⚠️"
}
message := fmt.Sprintf("%s [UPTIME] %s is %s", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
if payload.Host != "" {
message += fmt.Sprintf(" | Host: %s", payload.Host)
}
if payload.ResponseTime > 0 {
message += fmt.Sprintf(" | %dms", payload.ResponseTime)
}
message += fmt.Sprintf(" | %s", payload.Timestamp.Format("15:04:05"))
return message
}
@@ -0,0 +1,440 @@
package notification
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"text/template"
"time"
)
// WebhookService handles webhook notifications
type WebhookService struct{}
// NewWebhookService creates a new webhook notification service
func NewWebhookService() *WebhookService {
return &WebhookService{}
}
// WebhookPayload represents the payload for webhook
type WebhookPayload struct {
Message string `json:"message"`
NotifyName string `json:"notify_name"`
Timestamp string `json:"timestamp"`
ServiceName string `json:"service_name,omitempty"`
Status string `json:"status,omitempty"`
Host string `json:"host,omitempty"`
URL string `json:"url,omitempty"`
Port int `json:"port,omitempty"`
ServiceType string `json:"service_type,omitempty"`
ResponseTime int64 `json:"response_time,omitempty"`
// Server monitoring fields
CPUUsage string `json:"cpu_usage,omitempty"`
RAMUsage string `json:"ram_usage,omitempty"`
DiskUsage string `json:"disk_usage,omitempty"`
NetworkUsage string `json:"network_usage,omitempty"`
CPUTemp string `json:"cpu_temp,omitempty"`
DiskIO string `json:"disk_io,omitempty"`
Threshold string `json:"threshold,omitempty"`
// SSL fields
CertificateName string `json:"certificate_name,omitempty"`
ExpiryDate string `json:"expiry_date,omitempty"`
DaysLeft string `json:"days_left,omitempty"`
IssuerCN string `json:"issuer_cn,omitempty"`
}
// DiscordWebhookPayload represents Discord-specific webhook payload
type DiscordWebhookPayload struct {
Content string `json:"content"`
}
// SendNotification sends a notification via webhook
func (ws *WebhookService) SendNotification(config *AlertConfiguration, message string) error {
// fmt.Printf("📡 [WEBHOOK] Attempting to send notification...\n")
// fmt.Printf("📡 [WEBHOOK] Config - URL: %s, Notify Name: %s\n", config.WebhookURL, config.NotifyName)
// fmt.Printf("📡 [WEBHOOK] Message: %s\n", message)
// Use the webhook_url field from alert_configurations
if config.WebhookURL == "" {
return fmt.Errorf("webhook URL is required")
}
// Create payload using webhook_payload_template if available
var jsonData []byte
var err error
if config.WebhookPayloadTemplate != "" && strings.TrimSpace(config.WebhookPayloadTemplate) != "" {
// Use custom payload template
jsonData, err = ws.generateCustomPayload(config, message)
if err != nil {
// fmt.Printf("❌ [WEBHOOK] Custom payload generation error: %v\n", err)
return err
}
} else {
// Detect Discord webhook and use appropriate payload
if strings.Contains(strings.ToLower(config.WebhookURL), "discord") {
jsonData, err = ws.generateDiscordPayload(message)
} else {
// Use default payload structure for other webhooks
jsonData, err = ws.generateDefaultPayload(config, message)
}
if err != nil {
// fmt.Printf("❌ [WEBHOOK] JSON marshal error: %v\n", err)
return err
}
}
// fmt.Printf("📡 [WEBHOOK] Payload: %s\n", string(jsonData))
// fmt.Printf("📡 [WEBHOOK] Sending POST request...\n")
resp, err := http.Post(config.WebhookURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
// fmt.Printf("❌ [WEBHOOK] HTTP error: %v\n", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
// fmt.Printf("❌ [WEBHOOK] API error, status: %d\n", resp.StatusCode)
return fmt.Errorf("webhook error, status: %d", resp.StatusCode)
}
// fmt.Printf("✅ [WEBHOOK] Message sent successfully!\n")
return nil
}
// generateDiscordPayload generates a Discord-compatible webhook payload
func (ws *WebhookService) generateDiscordPayload(message string) ([]byte, error) {
payload := DiscordWebhookPayload{
Content: message,
}
return json.Marshal(payload)
}
// generateDefaultPayload generates the default webhook payload
func (ws *WebhookService) generateDefaultPayload(config *AlertConfiguration, message string) ([]byte, error) {
payload := WebhookPayload{
Message: message,
NotifyName: config.NotifyName,
Timestamp: time.Now().Format(time.RFC3339),
}
return json.Marshal(payload)
}
// generateCustomPayload generates payload using the custom template
func (ws *WebhookService) generateCustomPayload(config *AlertConfiguration, message string) ([]byte, error) {
// Parse the template
tmpl, err := template.New("webhook").Parse(config.WebhookPayloadTemplate)
if err != nil {
return nil, fmt.Errorf("failed to parse webhook payload template: %v", err)
}
// Create template data
templateData := map[string]interface{}{
"message": message,
"notify_name": config.NotifyName,
"timestamp": time.Now().Format(time.RFC3339),
}
// Execute template
var buf bytes.Buffer
err = tmpl.Execute(&buf, templateData)
if err != nil {
return nil, fmt.Errorf("failed to execute webhook payload template: %v", err)
}
// Validate that the result is valid JSON
var jsonCheck interface{}
if err := json.Unmarshal(buf.Bytes(), &jsonCheck); err != nil {
return nil, fmt.Errorf("webhook payload template did not generate valid JSON: %v", err)
}
return buf.Bytes(), nil
}
// SendServerNotification sends a server-specific notification via webhook
func (ws *WebhookService) SendServerNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) error {
message := ws.generateServerMessage(payload, template, resourceType)
return ws.SendNotification(config, message)
}
// SendServiceNotification sends a service-specific notification via webhook
func (ws *WebhookService) SendServiceNotification(config *AlertConfiguration, payload *NotificationPayload, template *ServiceNotificationTemplate) error {
message := ws.generateServiceMessage(payload, template)
return ws.SendNotification(config, message)
}
// generateServerMessage creates a message for server notifications using server template
func (ws *WebhookService) generateServerMessage(payload *NotificationPayload, template *ServerNotificationTemplate, resourceType string) string {
var templateMessage string
// Select appropriate template message based on status and resource type
switch strings.ToLower(payload.Status) {
case "down":
templateMessage = template.DownMessage
case "up":
templateMessage = template.UpMessage
case "warning":
templateMessage = template.WarningMessage
case "paused":
templateMessage = template.PausedMessage
default:
// Handle resource-specific messages
switch resourceType {
case "cpu":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUMessage
} else {
templateMessage = template.CPUMessage
}
case "ram", "memory":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreRAMMessage
} else {
templateMessage = template.RAMMessage
}
case "disk":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskMessage
} else {
templateMessage = template.DiskMessage
}
case "network":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreNetworkMessage
} else {
templateMessage = template.NetworkMessage
}
case "cpu_temp", "cpu_temperature":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreCPUTempMessage
} else {
templateMessage = template.CPUTempMessage
}
case "disk_io":
if strings.Contains(strings.ToLower(payload.Status), "restore") {
templateMessage = template.RestoreDiskIOMessage
} else {
templateMessage = template.DiskIOMessage
}
default:
templateMessage = template.WarningMessage
}
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ws.generateDefaultServerMessage(payload, resourceType)
}
return ws.replacePlaceholders(templateMessage, payload)
}
// generateServiceMessage creates a message for service notifications using service template
func (ws *WebhookService) generateServiceMessage(payload *NotificationPayload, template *ServiceNotificationTemplate) string {
var templateMessage string
// Select appropriate template message based on status
switch strings.ToLower(payload.Status) {
case "up":
templateMessage = template.UpMessage
case "down":
templateMessage = template.DownMessage
case "maintenance":
templateMessage = template.MaintenanceMessage
case "incident":
templateMessage = template.IncidentMessage
case "resolved":
templateMessage = template.ResolvedMessage
case "warning":
templateMessage = template.WarningMessage
default:
templateMessage = template.WarningMessage
}
// If no template message found, use a default
if templateMessage == "" {
templateMessage = ws.generateDefaultUptimeMessage(payload)
}
return ws.replacePlaceholders(templateMessage, payload)
}
// replacePlaceholders replaces all placeholders in the message with actual values
func (ws *WebhookService) replacePlaceholders(message string, payload *NotificationPayload) string {
// Replace basic placeholders
message = strings.ReplaceAll(message, "${service_name}", payload.ServiceName)
message = strings.ReplaceAll(message, "${server_name}", payload.ServiceName) // server_name maps to service_name
message = strings.ReplaceAll(message, "${status}", strings.ToUpper(payload.Status))
message = strings.ReplaceAll(message, "${host}", ws.safeString(payload.Host))
message = strings.ReplaceAll(message, "${hostname}", ws.safeString(payload.Hostname))
// Replace URL with fallback to host
url := ws.safeString(payload.URL)
if url == "N/A" && payload.Host != "" {
url = payload.Host
}
message = strings.ReplaceAll(message, "${url}", url)
// Replace domain
message = strings.ReplaceAll(message, "${domain}", ws.safeString(payload.Domain))
// Replace service type
if payload.ServiceType != "" {
message = strings.ReplaceAll(message, "${service_type}", strings.ToUpper(payload.ServiceType))
} else {
message = strings.ReplaceAll(message, "${service_type}", "N/A")
}
// Replace region and agent info
message = strings.ReplaceAll(message, "${region_name}", ws.safeString(payload.RegionName))
message = strings.ReplaceAll(message, "${agent_id}", ws.safeString(payload.AgentID))
// Handle numeric fields safely
if payload.Port > 0 {
message = strings.ReplaceAll(message, "${port}", fmt.Sprintf("%d", payload.Port))
} else {
message = strings.ReplaceAll(message, "${port}", "N/A")
}
if payload.ResponseTime > 0 {
message = strings.ReplaceAll(message, "${response_time}", fmt.Sprintf("%dms", payload.ResponseTime))
} else {
message = strings.ReplaceAll(message, "${response_time}", "N/A")
}
if payload.Uptime > 0 {
message = strings.ReplaceAll(message, "${uptime}", fmt.Sprintf("%d%%", payload.Uptime))
} else {
message = strings.ReplaceAll(message, "${uptime}", "N/A")
}
// Replace server monitoring fields
message = strings.ReplaceAll(message, "${cpu_usage}", ws.safeString(payload.CPUUsage))
message = strings.ReplaceAll(message, "${ram_usage}", ws.safeString(payload.RAMUsage))
message = strings.ReplaceAll(message, "${disk_usage}", ws.safeString(payload.DiskUsage))
message = strings.ReplaceAll(message, "${network_usage}", ws.safeString(payload.NetworkUsage))
message = strings.ReplaceAll(message, "${cpu_temp}", ws.safeString(payload.CPUTemp))
message = strings.ReplaceAll(message, "${disk_io}", ws.safeString(payload.DiskIO))
message = strings.ReplaceAll(message, "${threshold}", ws.safeString(payload.Threshold))
// Replace SSL certificate fields
message = strings.ReplaceAll(message, "${certificate_name}", ws.safeString(payload.CertificateName))
message = strings.ReplaceAll(message, "${expiry_date}", ws.safeString(payload.ExpiryDate))
message = strings.ReplaceAll(message, "${days_left}", ws.safeString(payload.DaysLeft))
message = strings.ReplaceAll(message, "${issuer_cn}", ws.safeString(payload.IssuerCN))
message = strings.ReplaceAll(message, "${issuer}", ws.safeString(payload.IssuerCN)) // alias
// Replace error message - important for uptime services
message = strings.ReplaceAll(message, "${error_message}", ws.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${error}", ws.safeString(payload.ErrorMessage))
message = strings.ReplaceAll(message, "${message}", ws.safeString(payload.Message))
// Replace time placeholders
message = strings.ReplaceAll(message, "${time}", payload.Timestamp.Format("2006-01-02 15:04:05"))
message = strings.ReplaceAll(message, "${timestamp}", payload.Timestamp.Format("2006-01-02 15:04:05"))
return message
}
// safeString returns the string value or "N/A" if empty
func (ws *WebhookService) safeString(value string) string {
if value == "" {
return "N/A"
}
return value
}
// generateDefaultUptimeMessage creates a default uptime message with proper formatting
func (ws *WebhookService) generateDefaultUptimeMessage(payload *NotificationPayload) string {
// Status emoji mapping
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
case "maintenance", "paused":
statusEmoji = "🟠"
}
message := fmt.Sprintf("%s Service %s is %s.", statusEmoji, payload.ServiceName, strings.ToUpper(payload.Status))
// Build formatted details
details := []string{}
// Add URL or host
if payload.URL != "" {
details = append(details, fmt.Sprintf(" - Host URL: %s", payload.URL))
} else if payload.Host != "" {
details = append(details, fmt.Sprintf(" - Host: %s", payload.Host))
}
// Add service type
if payload.ServiceType != "" {
details = append(details, fmt.Sprintf(" - Type: %s", strings.ToUpper(payload.ServiceType)))
}
// Add port if available
if payload.Port > 0 {
details = append(details, fmt.Sprintf(" - Port: %d", payload.Port))
}
// Add domain if available
if payload.Domain != "" {
details = append(details, fmt.Sprintf(" - Domain: %s", payload.Domain))
}
// Add response time
if payload.ResponseTime > 0 {
details = append(details, fmt.Sprintf(" - Response time: %dms", payload.ResponseTime))
} else {
details = append(details, " - Response time: N/A")
}
// Add region info
if payload.RegionName != "" {
details = append(details, fmt.Sprintf(" - Region: %s", payload.RegionName))
}
// Add agent info
if payload.AgentID != "" {
details = append(details, fmt.Sprintf(" - Agent: %s", payload.AgentID))
}
// Add uptime if available
if payload.Uptime > 0 {
details = append(details, fmt.Sprintf(" - Uptime: %d%%", payload.Uptime))
}
// Add timestamp
details = append(details, fmt.Sprintf(" - Time: %s", payload.Timestamp.Format("2006-01-02 15:04:05")))
// Combine message with details
if len(details) > 0 {
message += "\n" + strings.Join(details, "\n")
}
return message
}
// generateDefaultServerMessage creates a default server message
func (ws *WebhookService) generateDefaultServerMessage(payload *NotificationPayload, resourceType string) string {
statusEmoji := "🔵"
switch strings.ToLower(payload.Status) {
case "up":
statusEmoji = "🟢"
case "down":
statusEmoji = "🔴"
case "warning":
statusEmoji = "🟡"
}
return fmt.Sprintf("%s 🖥️ Server %s (%s) status: %s", statusEmoji, payload.ServiceName, payload.Hostname, strings.ToUpper(payload.Status))
}
+68 -35
View File
@@ -18,41 +18,6 @@ func (c *PocketBaseClient) parseResponse(resp *http.Response, target interface{}
return json.Unmarshal(body, target)
}
func (c *PocketBaseClient) updateRecord(collection string, recordID string, data interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
req, err := http.NewRequest("PATCH",
fmt.Sprintf("%s/api/collections/%s/records/%s", c.baseURL, collection, recordID),
bytes.NewBuffer(jsonData))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
bodyString := string(bodyBytes)
fmt.Printf("Failed to update record in %s collection. Status: %d, Response: %s\n",
collection, resp.StatusCode, bodyString)
return fmt.Errorf("failed to update record in %s, status: %d, response: %s",
collection, resp.StatusCode, bodyString)
}
return nil
}
func (c *PocketBaseClient) createRecord(collection string, data interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
@@ -88,4 +53,72 @@ func (c *PocketBaseClient) createRecord(collection string, data interface{}) err
}
return nil
}
func (c *PocketBaseClient) updateRecord(collection string, recordID string, data interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
req, err := http.NewRequest("PATCH",
fmt.Sprintf("%s/api/collections/%s/records/%s", c.baseURL, collection, recordID),
bytes.NewBuffer(jsonData))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
bodyString := string(bodyBytes)
fmt.Printf("Failed to update record in %s collection. Status: %d, Response: %s\n",
collection, resp.StatusCode, bodyString)
return fmt.Errorf("failed to update record in %s, status: %d, response: %s",
collection, resp.StatusCode, bodyString)
}
return nil
}
// UpdateRecord is the public method for server monitoring that preserves other fields
func (c *PocketBaseClient) UpdateRecord(collection string, id string, data map[string]interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
req, err := http.NewRequest("PATCH",
fmt.Sprintf("%s/api/collections/%s/records/%s", c.baseURL, collection, id),
bytes.NewBuffer(jsonData))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to update record, status: %d", resp.StatusCode)
}
return nil
}
func (c *PocketBaseClient) GetHTTPClient() *http.Client {
return c.httpClient
}
@@ -0,0 +1,16 @@
package servermonitoring
import (
"time"
)
// LogServerUpdateAttempt logs when we attempt to update a server record
func LogServerUpdateAttempt(serverID, operation, details string) {
// log.Printf("🔄 [UPDATE ATTEMPT] Server: %s | Operation: %s | Details: %s | Time: %s",
// serverID, operation, details, time.Now().Format("2006-01-02 15:04:05"))
_ = serverID
_ = operation
_ = details
_ = time.Now()
}
@@ -0,0 +1,46 @@
package servermonitoring
// DebugServerFields helps debug server field preservation issues
func (c *ServerPocketBaseClient) DebugServerFields(serverID string, operation string) {
// log.Printf("🔍 === DEBUG: %s for server %s ===", operation, serverID)
servers, err := c.GetAllServers()
if err != nil {
// log.Printf("❌ Debug failed to fetch servers: %v", err)
_ = err
return
}
for _, server := range servers {
if server.ID == serverID || server.ServerID == serverID {
// log.Printf("📊 Server Debug Info:")
// log.Printf(" - Server ID: %s", server.ID)
// log.Printf(" - Server Name: %s", server.Name)
// log.Printf(" - Status: %s", server.Status)
// log.Printf(" - Notification ID: '%s'", server.NotificationID)
// log.Printf(" - Template ID: '%s'", server.TemplateID)
// log.Printf(" - Threshold ID: '%s'", server.ThresholdID)
// log.Printf(" - Agent Status: %s", server.AgentStatus)
// log.Printf(" - Check Interval: %d", server.CheckInterval)
// Check for empty strings vs nil
if server.NotificationID == "" {
// log.Printf("⚠️ WARNING: Notification ID is empty string")
}
if server.TemplateID == "" {
// log.Printf("⚠️ WARNING: Template ID is empty string")
}
if server.ThresholdID == "" {
// log.Printf("⚠️ WARNING: Threshold ID is empty string")
}
_ = server
break
}
}
// log.Printf("🔍 === END DEBUG: %s ===", operation)
_ = operation
_ = serverID
}
@@ -0,0 +1,40 @@
package servermonitoring
import (
"service-operation/pocketbase"
)
// ServerMonitoringService provides an easy interface for server monitoring
type ServerMonitoringService struct {
monitor *ServerMonitor
}
// NewServerMonitoringService creates a new server monitoring service
func NewServerMonitoringService(pbClient *pocketbase.PocketBaseClient) *ServerMonitoringService {
return &ServerMonitoringService{
monitor: NewServerMonitor(pbClient),
}
}
// Start starts the server monitoring service
func (sms *ServerMonitoringService) Start() {
sms.monitor.Start()
}
// Stop stops the server monitoring service
func (sms *ServerMonitoringService) Stop() {
sms.monitor.Stop()
}
// GetMonitor returns the underlying monitor for advanced usage
func (sms *ServerMonitoringService) GetMonitor() *ServerMonitor {
return sms.monitor
}
// IsRunning returns whether the monitoring service is currently running
func (sms *ServerMonitoringService) IsRunning() bool {
sms.monitor.mu.RLock()
defer sms.monitor.mu.RUnlock()
return sms.monitor.isRunning
}
@@ -0,0 +1,462 @@
package servermonitoring
import (
"fmt"
"strconv"
"sync"
"time"
"service-operation/pocketbase"
)
// StatusAlert tracks alert state for server status changes
type StatusAlert struct {
IsActive bool
Status string
LastAlerted time.Time
RetryCount int
}
// ServerMonitor tracks server metrics and sends notifications
type ServerMonitor struct {
pbClient *ServerPocketBaseClient
notificationService *ServerNotificationService
thresholdMonitor *ThresholdMonitor
lastStatuses map[string]string // Track last known status for each server
activeStatusAlerts map[string]*StatusAlert // Track status alerts with retry counts
mu sync.RWMutex
stopChan chan bool
isRunning bool
}
// NewServerMonitor creates a new server monitor
func NewServerMonitor(pbClient *pocketbase.PocketBaseClient) *ServerMonitor {
// log.Println("🔧 Creating new ServerMonitor instance")
serverPBClient := NewServerPocketBaseClient(pbClient)
notificationService := NewServerNotificationService(pbClient)
thresholdMonitor := NewThresholdMonitor(serverPBClient, notificationService)
return &ServerMonitor{
pbClient: serverPBClient,
notificationService: notificationService,
thresholdMonitor: thresholdMonitor,
lastStatuses: make(map[string]string),
activeStatusAlerts: make(map[string]*StatusAlert),
stopChan: make(chan bool),
isRunning: false,
}
}
// Start begins monitoring servers
func (sm *ServerMonitor) Start() {
sm.mu.Lock()
defer sm.mu.Unlock()
if sm.isRunning {
// log.Println("⚠️ Server monitor is already running")
return
}
sm.isRunning = true
// log.Println("🚀 Starting server monitoring service...")
go sm.monitoringLoop()
}
// Stop stops monitoring servers
func (sm *ServerMonitor) Stop() {
sm.mu.Lock()
defer sm.mu.Unlock()
if !sm.isRunning {
// log.Println("⚠️ Server monitor is not running")
return
}
// log.Println("🛑 Stopping server monitoring service...")
sm.isRunning = false
sm.stopChan <- true
}
// monitoringLoop runs the main monitoring loop
func (sm *ServerMonitor) monitoringLoop() {
// log.Println("🔄 Starting monitoring loop with 30-second intervals")
ticker := time.NewTicker(30 * time.Second) // Check every 30 seconds for more responsive monitoring
defer ticker.Stop()
// Initial check
// log.Println("🔍 Performing initial server check...")
sm.checkAllServers()
for {
select {
case <-ticker.C:
// log.Println("⏰ Timer triggered - checking all servers")
sm.checkAllServers()
case <-sm.stopChan:
// log.Println("🛑 Stop signal received - exiting monitoring loop")
return
}
}
}
// checkAllServers checks all servers for missing metrics
func (sm *ServerMonitor) checkAllServers() {
// log.Println("📋 Fetching all servers from database...")
servers, err := sm.pbClient.GetAllServers()
if err != nil {
// log.Printf("❌ CRITICAL ERROR: Failed to fetch servers: %v", err)
_ = err
return
}
// log.Printf("✅ Successfully fetched %d servers", len(servers))
if len(servers) == 0 {
// log.Println("⚠️ No servers found in database")
return
}
for i, server := range servers {
// log.Printf("🔍 [%d/%d] Processing server: %s (ID: %s)", i+1, len(servers), server.Name, server.ServerID)
sm.checkServerMetrics(server)
// log.Printf("✅ [%d/%d] Completed processing server: %s", i+1, len(servers), server.Name)
_ = i
}
// log.Println("📋 Completed checking all servers")
}
// checkServerMetrics checks if a server has recent metrics and agent status
func (sm *ServerMonitor) checkServerMetrics(server Server) {
// log.Printf("🔍 === DETAILED SERVER CHECK FOR: %s ===", server.Name)
// Debug server fields BEFORE any operations
sm.pbClient.DebugServerFields(server.ID, "BEFORE_CHECK")
// log.Printf("📊 Server Details:")
// log.Printf(" - ID: %s", server.ID)
// log.Printf(" - Server ID: %s", server.ServerID)
// log.Printf(" - Name: %s", server.Name)
// log.Printf(" - Current Status: %s", server.Status)
// log.Printf(" - Agent Status: %s", server.AgentStatus)
// log.Printf(" - Check Interval: %d minutes", server.CheckInterval)
// log.Printf(" - Notification ID: '%s'", server.NotificationID)
// log.Printf(" - Template ID: '%s'", server.TemplateID)
// log.Printf(" - Threshold ID: '%s'", server.ThresholdID)
// log.Printf(" - IP Address: %s", server.IPAddress)
// log.Printf(" - OS Type: %s", server.OSType)
// log.Printf(" - Max Retries: %s", server.MaxRetries)
// Track the notification configuration BEFORE any updates
beforeNotificationID := server.NotificationID
beforeTemplateID := server.TemplateID
_ = beforeNotificationID
_ = beforeTemplateID
// Skip paused servers
if server.Status == "paused" {
// log.Printf("⏸️ Server %s is paused - skipping monitoring", server.Name)
return
}
sm.mu.Lock()
lastStatus := sm.lastStatuses[server.ServerID]
sm.mu.Unlock()
// log.Printf("📈 Status Tracking:")
// log.Printf(" - Last Known Status: '%s'", lastStatus)
// log.Printf(" - Current Database Status: '%s'", server.Status)
currentStatus := "up"
var message string
var statusReason string
// Check agent status first - if agent is stopped, server is definitely down
if server.AgentStatus == "stopped" {
currentStatus = "down"
statusReason = "agent is stopped"
message = fmt.Sprintf("🔴 Server %s is DOWN - Agent has stopped running", server.Name)
// log.Printf("❌ SERVER DOWN DETECTED: %s - Agent is stopped", server.Name)
} else if server.AgentStatus == "running" {
// log.Printf("✅ Agent is running for server %s - checking metrics...", server.Name)
// If agent is running, check metrics
checkIntervalMinutes := time.Duration(server.CheckInterval) * time.Minute
gracePeriod := 30 * time.Second
metricsTimeout := checkIntervalMinutes + gracePeriod
// Minimum timeout of 2 minutes to avoid false positives
if metricsTimeout < 2*time.Minute {
metricsTimeout = 2 * time.Minute
}
// log.Printf("🕐 Metrics Check Configuration:")
// log.Printf(" - Check Interval: %v", checkIntervalMinutes)
// log.Printf(" - Grace Period: %v", gracePeriod)
// log.Printf(" - Total Timeout: %v", metricsTimeout)
// Get recent metrics for this server
// log.Printf("🔍 Fetching recent metrics for server %s...", server.Name)
metrics, err := sm.pbClient.GetLatestServerMetrics(server.ServerID, metricsTimeout)
if err != nil {
// log.Printf("❌ ERROR: Failed to fetch metrics for server %s: %v", server.Name, err)
_ = err
return
}
// log.Printf("📊 Metrics Query Results:")
// log.Printf(" - Found %d metrics within timeout period", len(metrics))
if len(metrics) == 0 {
// No recent metrics found - server is down
currentStatus = "down"
statusReason = fmt.Sprintf("no metrics received in last %v", metricsTimeout)
message = fmt.Sprintf("🔴 Server %s is DOWN - No metrics received in the last %v (check interval: %d minutes)",
server.Name, metricsTimeout, server.CheckInterval)
// log.Printf("❌ SERVER DOWN DETECTED: %s - No recent metrics", server.Name)
} else {
// Recent metrics found - server is up
currentStatus = "up"
latestMetric := metrics[0]
age := time.Since(latestMetric.CreatedTime)
statusReason = fmt.Sprintf("metrics received %v ago", age.Round(time.Second))
message = fmt.Sprintf("✅ Server %s is operational - Latest metrics received %v ago (check interval: %d minutes)",
server.Name, age.Round(time.Second), server.CheckInterval)
// log.Printf("✅ SERVER UP: %s - Latest metric %v ago", server.Name, age.Round(time.Second))
// log.Printf("📊 Latest Metric Details:")
// log.Printf(" - Metric ID: %s", latestMetric.ID)
// log.Printf(" - Created: %v", latestMetric.CreatedTime)
// log.Printf(" - Age: %v", age.Round(time.Second))
// log.Printf(" - Status: %s", latestMetric.Status)
// Check thresholds if server is up and has metrics
// log.Printf("🎯 Checking thresholds for server %s...", server.Name)
sm.thresholdMonitor.CheckServerThresholds(server, metrics)
}
} else {
// log.Printf("⚠️ Unknown agent status '%s' for server %s", server.AgentStatus, server.Name)
currentStatus = "up" // Default to up for unknown status
statusReason = fmt.Sprintf("unknown agent status: %s", server.AgentStatus)
message = fmt.Sprintf("⚠️ Server %s has unknown agent status: %s", server.Name, server.AgentStatus)
}
// log.Printf("📊 Status Determination:")
// log.Printf(" - Determined Status: %s", currentStatus)
// log.Printf(" - Reason: %s", statusReason)
// log.Printf(" - Message: %s", message)
// Check if status has changed
statusChanged := lastStatus != currentStatus && lastStatus != ""
isFirstCheck := lastStatus == ""
_ = statusChanged
_ = isFirstCheck
// log.Printf("🔄 Status Change Analysis:")
// log.Printf(" - Status Changed: %t", statusChanged)
// log.Printf(" - First Check: %t", isFirstCheck)
// log.Printf(" - Previous Status: '%s'", lastStatus)
// log.Printf(" - New Status: '%s'", currentStatus)
// Update last known status
sm.mu.Lock()
sm.lastStatuses[server.ServerID] = currentStatus
sm.mu.Unlock()
// Update server status in database if it changed
if server.Status != currentStatus {
// log.Printf("💾 === DATABASE UPDATE FOR: %s ===", server.Name)
// log.Printf(" - BEFORE UPDATE:")
// log.Printf(" * Status: %s → %s", server.Status, currentStatus)
// log.Printf(" * Notification ID: '%s'", beforeNotificationID)
// log.Printf(" * Template ID: '%s'", beforeTemplateID)
// Log the update attempt
LogServerUpdateAttempt(server.ID, "UpdateServerStatus", fmt.Sprintf("Status change from %s to %s", server.Status, currentStatus))
// Debug fields before update
sm.pbClient.DebugServerFields(server.ID, "BEFORE_UPDATE")
if err := sm.pbClient.UpdateServerStatus(server.ID, currentStatus); err != nil {
// log.Printf("❌ CRITICAL ERROR: Failed to update server status for %s: %v", server.Name, err)
_ = err
} else {
// log.Printf("✅ Successfully updated database status for server %s", server.Name)
// Debug fields after update
sm.pbClient.DebugServerFields(server.ID, "AFTER_UPDATE")
}
} else {
// log.Printf("️ Database status already matches current status (%s) for server %s", currentStatus, server.Name)
}
// Handle status notifications with retry logic
sm.handleStatusNotifications(server, currentStatus, message, statusChanged, isFirstCheck)
// Final status summary
// log.Printf("📊 === FINAL STATUS SUMMARY FOR: %s ===", server.Name)
// log.Printf(" - Final Status: %s (%s)", currentStatus, statusReason)
// log.Printf(" - Database Updated: %t", server.Status != currentStatus)
// log.Printf(" - Notification Config Preserved: ID='%s', Template='%s'", server.NotificationID, server.TemplateID)
// Final debug after all operations
sm.pbClient.DebugServerFields(server.ID, "FINAL_CHECK")
// log.Printf("=== END SERVER CHECK FOR: %s ===\n", server.Name)
_ = statusReason
}
// handleStatusNotifications handles sending notifications based on status changes and retry logic
func (sm *ServerMonitor) handleStatusNotifications(server Server, currentStatus, message string, statusChanged, isFirstCheck bool) {
// log.Printf("📱 === NOTIFICATION ANALYSIS FOR: %s ===", server.Name)
// log.Printf("🔧 Notification Configuration:")
// log.Printf(" - Notification ID: '%s'", server.NotificationID)
// log.Printf(" - Template ID: '%s'", server.TemplateID)
// log.Printf(" - Has Notification Config: %t", server.NotificationID != "")
// log.Printf(" - Max Retries: %s", server.MaxRetries)
if server.NotificationID == "" {
// log.Printf("🔕 NOTIFICATION SKIPPED: No notification ID configured for server %s", server.Name)
return
}
// Parse max retries
maxRetries, err := strconv.Atoi(server.MaxRetries)
if err != nil {
// log.Printf("Invalid max_retries value '%s' for server %s, using default 3", server.MaxRetries, server.Name)
maxRetries = 3
}
// Determine if notification should be sent
shouldSendNotification := false
notificationReason := ""
if currentStatus == "down" {
if sm.shouldSendStatusAlert(server.ServerID, "down", maxRetries) {
shouldSendNotification = true
retryCount := sm.getStatusRetryCount(server.ServerID)
notificationReason = fmt.Sprintf("server down - retry %d/%d", retryCount+1, maxRetries)
} else {
notificationReason = "server down but max retries reached"
}
} else if currentStatus == "up" && sm.shouldSendRecoveryStatusAlert(server.ServerID) {
shouldSendNotification = true
notificationReason = "server recovery notification"
}
// log.Printf("🎯 Notification Decision:")
// log.Printf(" - Should Send: %t", shouldSendNotification)
// log.Printf(" - Reason: %s", notificationReason)
if shouldSendNotification {
// log.Printf("📤 SENDING NOTIFICATION for server %s", server.Name)
// log.Printf("📧 Notification Details:")
// log.Printf(" - Server Name: %s", server.Name)
// log.Printf(" - Status: %s", currentStatus)
// log.Printf(" - Message: %s", message)
// log.Printf(" - Notification ID: %s", server.NotificationID)
// log.Printf(" - Template ID: %s", server.TemplateID)
// log.Printf(" - Reason: %s", notificationReason)
err := sm.notificationService.SendServerNotification(server, currentStatus, message)
if err != nil {
// log.Printf("❌ NOTIFICATION FAILED for server %s: %v", server.Name, err)
_ = err
} else {
// log.Printf("✅ NOTIFICATION SENT SUCCESSFULLY for server %s", server.Name)
// Update alert tracking based on status
if currentStatus == "down" {
sm.setStatusAlert(server.ServerID, "down")
} else if currentStatus == "up" {
sm.clearStatusAlert(server.ServerID)
}
}
} else {
// log.Printf("📱 NOTIFICATION SKIPPED for server %s: %s", server.Name, notificationReason)
}
_ = statusChanged
_ = isFirstCheck
_ = notificationReason
}
// shouldSendStatusAlert determines if a status alert should be sent based on retry count
func (sm *ServerMonitor) shouldSendStatusAlert(serverID, status string, maxRetries int) bool {
sm.mu.RLock()
alert, exists := sm.activeStatusAlerts[serverID]
sm.mu.RUnlock()
if !exists {
return true // No existing alert, send new one
}
// Check if we've reached max retries
if alert.RetryCount >= maxRetries {
// log.Printf("🔇 Status alert for server %s has reached max retries (%d), not sending notification", serverID, maxRetries)
return false
}
// Check if enough time has passed since last alert (resend every 5 minutes)
timeSinceLastAlert := time.Since(alert.LastAlerted)
return timeSinceLastAlert > 5*time.Minute
}
// getStatusRetryCount returns the current retry count for a status alert
func (sm *ServerMonitor) getStatusRetryCount(serverID string) int {
sm.mu.RLock()
defer sm.mu.RUnlock()
if alert, exists := sm.activeStatusAlerts[serverID]; exists {
return alert.RetryCount
}
return 0
}
// shouldSendRecoveryStatusAlert determines if a recovery alert should be sent
func (sm *ServerMonitor) shouldSendRecoveryStatusAlert(serverID string) bool {
sm.mu.RLock()
defer sm.mu.RUnlock()
_, exists := sm.activeStatusAlerts[serverID]
return exists // Send recovery only if there was an active alert
}
// setStatusAlert creates or updates a status alert with incremented retry count
func (sm *ServerMonitor) setStatusAlert(serverID, status string) {
sm.mu.Lock()
defer sm.mu.Unlock()
alert, exists := sm.activeStatusAlerts[serverID]
if exists {
// Update existing alert and increment retry count
alert.Status = status
alert.LastAlerted = time.Now()
alert.RetryCount++
// log.Printf("📊 Updated status alert for server %s - Retry count: %d", serverID, alert.RetryCount)
} else {
// Create new alert
sm.activeStatusAlerts[serverID] = &StatusAlert{
IsActive: true,
Status: status,
LastAlerted: time.Now(),
RetryCount: 1, // Start with 1 since we're sending the first notification
}
// log.Printf("🆕 Created new status alert for server %s - Retry count: 1", serverID)
}
}
// clearStatusAlert removes a status alert
func (sm *ServerMonitor) clearStatusAlert(serverID string) {
sm.mu.Lock()
defer sm.mu.Unlock()
if _, exists := sm.activeStatusAlerts[serverID]; exists {
// log.Printf("🗑️ Cleared status alert for server %s", serverID)
delete(sm.activeStatusAlerts, serverID)
}
}
@@ -0,0 +1,236 @@
package servermonitoring
import (
"time"
"service-operation/notification"
"service-operation/pocketbase"
)
// ServerNotificationService handles server-specific notifications
type ServerNotificationService struct {
notificationManager *notification.NotificationManager
}
// NewServerNotificationService creates a new server notification service
func NewServerNotificationService(pbClient *pocketbase.PocketBaseClient) *ServerNotificationService {
// log.Println("🔧 Creating new ServerNotificationService instance")
return &ServerNotificationService{
notificationManager: notification.NewNotificationManager(pbClient),
}
}
// SendResourceNotification sends a resource-specific notification (CPU, RAM, Disk, etc.)
func (sns *ServerNotificationService) SendResourceNotification(server Server, status string, message string, resourceType string) error {
return sns.SendResourceNotificationWithValues(server, status, message, resourceType, "N/A", "N/A")
}
// SendResourceNotificationWithValues sends a resource-specific notification with actual usage and threshold values
func (sns *ServerNotificationService) SendResourceNotificationWithValues(server Server, status string, message string, resourceType string, usageValue string, thresholdValue string) error {
// log.Printf("📨 === SENDING RESOURCE NOTIFICATION WITH VALUES ===")
// log.Printf("🔔 SendResourceNotificationWithValues called for server: %s", server.Name)
// log.Printf("📊 Resource Notification Parameters:")
// log.Printf(" - Server Name: %s", server.Name)
// log.Printf(" - Resource Type: %s", resourceType)
// log.Printf(" - Status: %s", status)
// log.Printf(" - Message: %s", message)
// log.Printf(" - Usage Value: %s", usageValue)
// log.Printf(" - Threshold Value: %s", thresholdValue)
// log.Printf(" - Notification ID: '%s'", server.NotificationID)
// log.Printf(" - Template ID: '%s'", server.TemplateID)
// Validate notification configuration
if server.NotificationID == "" {
// log.Printf("❌ NOTIFICATION FAILED: No notification ID configured for server %s", server.Name)
return nil
}
// Get server metrics for notification
cpuUsage, ramUsage, diskUsage, networkUsage, cpuTemp, diskIO := sns.getServerMetrics(server.ServerID)
// Override specific resource metric with actual value
switch resourceType {
case "cpu":
cpuUsage = usageValue
case "ram", "memory":
ramUsage = usageValue
case "disk":
diskUsage = usageValue
case "network":
networkUsage = usageValue
case "cpu_temp", "cpu_temperature":
cpuTemp = usageValue
case "disk_io":
diskIO = usageValue
}
// Convert to generic notification payload
notificationPayload := &notification.NotificationPayload{
ServiceName: server.Name,
Status: status,
Host: server.IPAddress,
Hostname: server.Hostname,
Port: 0,
ServiceType: "server",
ResponseTime: 0,
Timestamp: time.Now(),
Message: message,
// Server monitoring metrics
CPUUsage: cpuUsage,
RAMUsage: ramUsage,
DiskUsage: diskUsage,
NetworkUsage: networkUsage,
CPUTemp: cpuTemp,
DiskIO: diskIO,
Threshold: thresholdValue,
}
// log.Printf("📤 About to call SendResourceNotification...")
// log.Printf(" - Resource Type: %s", resourceType)
// log.Printf(" - Notification ID: %s", server.NotificationID)
// log.Printf(" - Template ID: %s", server.TemplateID)
// log.Printf(" - Usage Value: %s", usageValue)
// log.Printf(" - Threshold Value: %s", thresholdValue)
// Send resource-specific notification using the notification manager
err := sns.notificationManager.SendResourceNotification(notificationPayload, server.NotificationID, server.TemplateID, resourceType)
if err != nil {
// log.Printf("❌ RESOURCE NOTIFICATION ERROR for server %s: %v", server.Name, err)
return err
}
// log.Printf("✅ RESOURCE NOTIFICATION SENT SUCCESSFULLY for server %s (%s)", server.Name, resourceType)
// log.Printf("=== RESOURCE NOTIFICATION COMPLETE ===")
return nil
}
// SendServerNotification sends a notification for server status change
func (sns *ServerNotificationService) SendServerNotification(server Server, status string, message string) error {
// log.Printf("📨 === SENDING SERVER NOTIFICATION ===")
// log.Printf("🔔 SendServerNotification called for server: %s", server.Name)
// log.Printf("📊 Notification Parameters:")
// log.Printf(" - Server Name: %s", server.Name)
// log.Printf(" - Server ID: %s", server.ServerID)
// log.Printf(" - Status: %s", status)
// log.Printf(" - Message: %s", message)
// log.Printf(" - Notification ID: '%s'", server.NotificationID)
// log.Printf(" - Template ID: '%s'", server.TemplateID)
// log.Printf(" - IP Address: %s", server.IPAddress)
// log.Printf(" - Hostname: %s", server.Hostname)
// Validate notification configuration
if server.NotificationID == "" {
// log.Printf("❌ NOTIFICATION FAILED: No notification ID configured for server %s", server.Name)
return nil
}
// log.Printf("✅ Notification ID validation passed: %s", server.NotificationID)
// Get server metrics for notification
cpuUsage, ramUsage, diskUsage, networkUsage, cpuTemp, diskIO := sns.getServerMetrics(server.ServerID)
// Convert to generic notification payload
// log.Printf("🔄 Converting to generic notification payload...")
notificationPayload := &notification.NotificationPayload{
ServiceName: server.Name,
Status: status,
Host: server.IPAddress,
Hostname: server.Hostname,
Port: 0, // Servers don't have ports
ServiceType: "server",
ResponseTime: 0,
Timestamp: time.Now(),
Message: message,
// Server monitoring metrics
CPUUsage: cpuUsage,
RAMUsage: ramUsage,
DiskUsage: diskUsage,
NetworkUsage: networkUsage,
CPUTemp: cpuTemp,
DiskIO: diskIO,
Threshold: "N/A", // Can be set based on server configuration
}
// log.Printf("📦 Generic notification payload created:")
// log.Printf(" - ServiceName: %s", notificationPayload.ServiceName)
// log.Printf(" - Status: %s", notificationPayload.Status)
// log.Printf(" - Host: %s", notificationPayload.Host)
// log.Printf(" - ServiceType: %s", notificationPayload.ServiceType)
// log.Printf(" - Message: %s", notificationPayload.Message)
// log.Printf(" - CPUUsage: %s", notificationPayload.CPUUsage)
// log.Printf(" - RAMUsage: %s", notificationPayload.RAMUsage)
// log.Printf(" - DiskUsage: %s", notificationPayload.DiskUsage)
// log.Printf(" - NetworkUsage: %s", notificationPayload.NetworkUsage)
// log.Printf(" - CPUTemp: %s", notificationPayload.CPUTemp)
// log.Printf(" - DiskIO: %s", notificationPayload.DiskIO)
// log.Printf(" - Timestamp: %v", notificationPayload.Timestamp)
// log.Printf("📤 About to call SendServiceNotification...")
// log.Printf(" - Notification ID: %s", server.NotificationID)
// log.Printf(" - Template ID: %s", server.TemplateID)
// Send notification using the notification manager
err := sns.notificationManager.SendServiceNotification(notificationPayload, server.NotificationID, server.TemplateID)
if err != nil {
// log.Printf("❌ NOTIFICATION MANAGER ERROR for server %s: %v", server.Name, err)
// log.Printf("❌ Error Details:")
// log.Printf(" - Error Type: %T", err)
// log.Printf(" - Error Message: %s", err.Error())
// log.Printf(" - Notification ID used: %s", server.NotificationID)
// log.Printf(" - Template ID used: %s", server.TemplateID)
return err
}
// log.Printf("✅ NOTIFICATION SENT SUCCESSFULLY for server %s", server.Name)
// log.Printf("✅ Notification Details:")
// log.Printf(" - Status: %s", status)
// log.Printf(" - Notification ID: %s", server.NotificationID)
// log.Printf(" - Template ID: %s", server.TemplateID)
// log.Printf("=== NOTIFICATION COMPLETE ===")
return nil
}
// getServerMetrics retrieves the latest server metrics (placeholder implementation)
// This should be replaced with actual metric retrieval from your monitoring system
func (sns *ServerNotificationService) getServerMetrics(serverID string) (cpuUsage, ramUsage, diskUsage, networkUsage, cpuTemp, diskIO string) {
// TODO: Implement actual metric retrieval from your server monitoring system
// For now, returning placeholder values
// You should replace this with actual calls to your metrics collection system
// log.Printf("🔍 Retrieving server metrics for server ID: %s", serverID)
// These should be replaced with actual metric values from your monitoring system
cpuUsage = "N/A"
ramUsage = "N/A"
diskUsage = "N/A"
networkUsage = "N/A"
cpuTemp = "N/A"
diskIO = "N/A"
// Example of how you might retrieve metrics:
// You could query your metrics database, call an API, or read from system files
// For example:
// metrics, err := sns.getLatestMetrics(serverID)
// if err == nil {
// cpuUsage = fmt.Sprintf("%.1f%%", metrics.CPUUsage)
// ramUsage = fmt.Sprintf("%.1f%%", metrics.RAMUsage)
// diskUsage = fmt.Sprintf("%.1f%%", metrics.DiskUsage)
// networkUsage = fmt.Sprintf("%.2f MB/s", metrics.NetworkUsage)
// cpuTemp = fmt.Sprintf("%.1f°C", metrics.CPUTemp)
// diskIO = fmt.Sprintf("%.2f MB/s", metrics.DiskIO)
// }
// log.Printf("📊 Retrieved server metrics:")
// log.Printf(" - CPU Usage: %s", cpuUsage)
// log.Printf(" - RAM Usage: %s", ramUsage)
// log.Printf(" - Disk Usage: %s", diskUsage)
// log.Printf(" - Network Usage: %s", networkUsage)
// log.Printf(" - CPU Temperature: %s", cpuTemp)
// log.Printf(" - Disk I/O: %s", diskIO)
_ = serverID
return cpuUsage, ramUsage, diskUsage, networkUsage, cpuTemp, diskIO
}
@@ -0,0 +1,287 @@
package servermonitoring
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"time"
"service-operation/pocketbase"
)
// ServerMetrics represents server metrics record in PocketBase
type ServerMetrics struct {
ID string `json:"id"`
ServerID string `json:"server_id"`
Timestamp string `json:"timestamp"` // Changed to string for custom parsing
RAMTotal string `json:"ram_total"`
RAMUsed string `json:"ram_used"`
RAMFree string `json:"ram_free"`
CPUCores string `json:"cpu_cores"`
CPUUsage string `json:"cpu_usage"`
CPUFree string `json:"cpu_free"`
DiskTotal string `json:"disk_total"`
DiskUsed string `json:"disk_used"`
DiskFree string `json:"disk_free"`
Status string `json:"status"`
NetworkRxBytes int64 `json:"network_rx_bytes"`
NetworkTxBytes int64 `json:"network_tx_bytes"`
NetworkRxSpeed int64 `json:"network_rx_speed"`
NetworkTxSpeed int64 `json:"network_tx_speed"`
Created string `json:"created"` // Changed to string for custom parsing
Updated string `json:"updated"` // Changed to string for custom parsing
}
// ParsedServerMetrics represents ServerMetrics with parsed time fields
type ParsedServerMetrics struct {
ServerMetrics
CreatedTime time.Time
UpdatedTime time.Time
TimestampTime time.Time
}
// ParseServerMetrics converts ServerMetrics to ParsedServerMetrics with proper time parsing
func ParseServerMetrics(sm ServerMetrics) (ParsedServerMetrics, error) {
psm := ParsedServerMetrics{ServerMetrics: sm}
var err error
// Parse Created time
if sm.Created != "" {
psm.CreatedTime, err = parsePocketBaseTime(sm.Created)
if err != nil {
log.Printf("Warning: Failed to parse Created time '%s': %v", sm.Created, err)
}
}
// Parse Updated time
if sm.Updated != "" {
psm.UpdatedTime, err = parsePocketBaseTime(sm.Updated)
if err != nil {
log.Printf("Warning: Failed to parse Updated time '%s': %v", sm.Updated, err)
}
}
// Parse Timestamp
if sm.Timestamp != "" {
psm.TimestampTime, err = parsePocketBaseTime(sm.Timestamp)
if err != nil {
log.Printf("Warning: Failed to parse Timestamp '%s': %v", sm.Timestamp, err)
}
}
return psm, nil
}
// parsePocketBaseTime parses PocketBase time format "2025-08-11 13:09:13.243Z"
func parsePocketBaseTime(timeStr string) (time.Time, error) {
// First try the PocketBase format with space separator
if t, err := time.Parse("2006-01-02 15:04:05.000Z", timeStr); err == nil {
return t, nil
}
// Fallback to RFC3339 format with T separator
if t, err := time.Parse(time.RFC3339, timeStr); err == nil {
return t, nil
}
// Fallback to RFC3339Nano format
if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil {
return t, nil
}
return time.Time{}, fmt.Errorf("unable to parse time string: %s", timeStr)
}
// ServerPocketBaseClient is a wrapper around the PocketBase client for server monitoring
type ServerPocketBaseClient struct {
client *pocketbase.PocketBaseClient
}
// NewServerPocketBaseClient creates a new server PocketBase client
func NewServerPocketBaseClient(client *pocketbase.PocketBaseClient) *ServerPocketBaseClient {
return &ServerPocketBaseClient{
client: client,
}
}
// GetServerThreshold fetches threshold configuration for a server
func (spc *ServerPocketBaseClient) GetServerThreshold(thresholdID string) (*ServerThreshold, error) {
if thresholdID == "" {
log.Printf("No threshold ID provided")
_ = thresholdID
return nil, nil
}
url := fmt.Sprintf("%s/api/collections/server_threshold_templates/records/%s", spc.client.GetBaseURL(), thresholdID)
//log.Printf("🔍 Fetching server threshold from: %s", url)
resp, err := http.Get(url)
if err != nil {
//log.Printf("❌ HTTP error fetching server threshold: %v", err)
_ = err
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//log.Printf("❌ Failed to fetch server threshold, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch server threshold, status: %d", resp.StatusCode)
}
var threshold ServerThreshold
if err := json.NewDecoder(resp.Body).Decode(&threshold); err != nil {
//log.Printf("❌ Error decoding server threshold JSON: %v", err)
_ = err
return nil, err
}
//log.Printf("✅ Successfully fetched server threshold: %+v", threshold)
_ = url
_ = threshold
return &threshold, nil
}
// GetAllServers retrieves all servers from PocketBase
func (spc *ServerPocketBaseClient) GetAllServers() ([]Server, error) {
url := fmt.Sprintf("%s/api/collections/servers/records?perPage=500", spc.client.GetBaseURL())
//log.Printf("🌐 Fetching all servers from: %s", url)
resp, err := http.Get(url)
if err != nil {
//log.Printf("❌ HTTP error fetching servers: %v", err)
_ = err
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//log.Printf("❌ Failed to fetch servers, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch servers, status: %d", resp.StatusCode)
}
var response struct {
Items []Server `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
//log.Printf("❌ Error decoding servers JSON: %v", err)
_ = err
return nil, err
}
//log.Printf("✅ Successfully fetched %d servers", len(response.Items))
_ = url
return response.Items, nil
}
// GetLatestServerMetrics retrieves the latest server metrics from PocketBase
func (spc *ServerPocketBaseClient) GetLatestServerMetrics(serverID string, timeout time.Duration) ([]ParsedServerMetrics, error) {
// Calculate the time before which metrics are considered too old
cutoff := time.Now().Add(-timeout).UTC().Format("2006-01-02 15:04:05.000Z")
// Construct the filter string with proper formatting (no spaces around operators)
filter := fmt.Sprintf("server_id='%s'&&created>'%s'", serverID, cutoff)
// URL encode the filter parameter
encodedFilter := url.QueryEscape(filter)
// Construct the URL with properly encoded parameters
requestURL := fmt.Sprintf("%s/api/collections/server_metrics/records?filter=%s&sort=-created&perPage=1",
spc.client.GetBaseURL(), encodedFilter)
//log.Printf("🌐 Fetching latest server metrics from: %s", requestURL)
//log.Printf("🔍 Filter used: %s", filter)
//log.Printf("🔍 Cutoff time: %s", cutoff)
resp, err := http.Get(requestURL)
if err != nil {
//log.Printf("❌ HTTP error fetching server metrics: %v", err)
_ = err
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//log.Printf("❌ Failed to fetch server metrics, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch server metrics, status: %d", resp.StatusCode)
}
var response struct {
Items []ServerMetrics `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
//log.Printf("❌ Error decoding server metrics JSON: %v", err)
_ = err
return nil, err
}
// Convert to ParsedServerMetrics
var parsedMetrics []ParsedServerMetrics
for _, metric := range response.Items {
parsed, err := ParseServerMetrics(metric)
if err != nil {
//log.Printf("❌ Error parsing server metric: %v", err)
_ = err
continue
}
parsedMetrics = append(parsedMetrics, parsed)
}
//log.Printf("✅ Successfully fetched and parsed %d server metrics", len(parsedMetrics))
_ = requestURL
_ = filter
_ = cutoff
return parsedMetrics, nil
}
// UpdateServerStatus updates the server status in PocketBase
func (spc *ServerPocketBaseClient) UpdateServerStatus(serverID string, status string) error {
url := fmt.Sprintf("%s/api/collections/servers/records/%s", spc.client.GetBaseURL(), serverID)
//log.Printf("🌐 Updating server status at: %s", url)
payload := map[string]interface{}{
"status": status,
"last_checked": time.Now().UTC().Format("2006-01-02 15:04:05.000Z"),
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
//log.Printf("❌ Error marshaling payload: %v", err)
_ = err
return err
}
//log.Printf("📝 Update payload: %s", string(payloadBytes))
req, err := http.NewRequest(http.MethodPatch, url, bytes.NewBuffer(payloadBytes))
if err != nil {
//log.Printf("❌ Error creating request: %v", err)
_ = err
return err
}
req.Header.Set("Content-Type", "application/json")
httpClient := &http.Client{Timeout: 10 * time.Second}
resp, err := httpClient.Do(req)
if err != nil {
//log.Printf("❌ HTTP error updating server status: %v", err)
_ = err
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//log.Printf("❌ Failed to update server status, status: %d", resp.StatusCode)
return fmt.Errorf("failed to update server status, status: %d", resp.StatusCode)
}
//log.Printf("✅ Successfully updated server status to %s", status)
_ = url
_ = payloadBytes
_ = status
return nil
}
@@ -0,0 +1,333 @@
package servermonitoring
import (
"fmt"
"strconv"
"strings"
"time"
)
// ThresholdAlert tracks alert state and retry count
type ThresholdAlert struct {
IsActive bool
Threshold int
CurrentValue float64
LastAlerted time.Time
RetryCount int // Track how many times we've sent this alert
}
// ThresholdMonitor handles server threshold monitoring and alerting
type ThresholdMonitor struct {
pbClient *ServerPocketBaseClient
notificationService *ServerNotificationService
activeAlerts map[string]*ThresholdAlert // serverID-metricType -> alert
}
// NewThresholdMonitor creates a new threshold monitor
func NewThresholdMonitor(pbClient *ServerPocketBaseClient, notificationService *ServerNotificationService) *ThresholdMonitor {
return &ThresholdMonitor{
pbClient: pbClient,
notificationService: notificationService,
activeAlerts: make(map[string]*ThresholdAlert),
}
}
// CheckServerThresholds checks all thresholds for a server
func (tm *ThresholdMonitor) CheckServerThresholds(server Server, metrics []ParsedServerMetrics) {
if server.ThresholdID == "" {
// log.Printf("No threshold configuration for server %s", server.Name)
return
}
// Get threshold configuration
threshold, err := tm.pbClient.GetServerThreshold(server.ThresholdID)
if err != nil {
// log.Printf("❌ Failed to get threshold config for server %s: %v", server.Name, err)
_ = err
return
}
if threshold == nil {
// log.Printf("No threshold configuration found for server %s", server.Name)
return
}
if len(metrics) == 0 {
// log.Printf("No metrics available for threshold checking on server %s", server.Name)
return
}
latestMetric := metrics[0]
// log.Printf("🎯 Checking thresholds for server %s with threshold config: %+v", server.Name, threshold)
// Parse MaxRetries from string to int
maxRetries, err := strconv.Atoi(server.MaxRetries)
if err != nil {
// log.Printf("Invalid max_retries value '%s' for server %s, using default 3", server.MaxRetries, server.Name)
maxRetries = 3
}
// Check CPU threshold
tm.checkCPUThreshold(server, latestMetric, threshold, maxRetries)
// Check RAM threshold
tm.checkRAMThreshold(server, latestMetric, threshold, maxRetries)
// Check Disk threshold
tm.checkDiskThreshold(server, latestMetric, threshold, maxRetries)
}
// checkCPUThreshold checks CPU usage against threshold
func (tm *ThresholdMonitor) checkCPUThreshold(server Server, metric ParsedServerMetrics, threshold *ServerThreshold, maxRetries int) {
cpuThresholdValue, err := strconv.Atoi(threshold.CPUThreshold)
if err != nil {
// log.Printf("Invalid CPU threshold value '%s' for server %s", threshold.CPUThreshold, server.Name)
return
}
// Parse CPU usage from metric (format: "XX.XX%")
cpuUsageStr := strings.TrimSuffix(metric.CPUUsage, "%")
cpuUsage, err := strconv.ParseFloat(cpuUsageStr, 64)
if err != nil {
// log.Printf("Failed to parse CPU usage '%s' for server %s", metric.CPUUsage, server.Name)
return
}
alertKey := fmt.Sprintf("%s-cpu", server.ServerID)
if cpuUsage > float64(cpuThresholdValue) {
// Threshold exceeded
if tm.shouldCreateAlert(alertKey, cpuUsage, float64(cpuThresholdValue), maxRetries) {
// log.Printf("🚨 CPU threshold exceeded for server %s: %.2f%% > %d%% (retry %d/%d)",
// server.Name, cpuUsage, cpuThresholdValue, tm.getRetryCount(alertKey)+1, maxRetries)
message := fmt.Sprintf("🚨 CPU Alert: Server %s CPU usage is %.2f%% (threshold: %d%%)",
server.Name, cpuUsage, cpuThresholdValue)
tm.sendThresholdAlert(server, "cpu", message, cpuUsage, float64(cpuThresholdValue), fmt.Sprintf("%.2f%%", cpuUsage), fmt.Sprintf("%d%%", cpuThresholdValue))
tm.setAlert(alertKey, cpuUsage, float64(cpuThresholdValue))
} else {
// log.Printf("🔇 CPU threshold exceeded for server %s but max retries (%d) reached, skipping notification", server.Name, maxRetries)
}
} else {
// Check if we need to send recovery notification
if tm.shouldSendRecoveryAlert(alertKey) {
// log.Printf("✅ CPU recovered for server %s: %.2f%% <= %d%%", server.Name, cpuUsage, cpuThresholdValue)
message := fmt.Sprintf("✅ CPU Recovery: Server %s CPU usage is back to normal: %.2f%% (threshold: %d%%)",
server.Name, cpuUsage, cpuThresholdValue)
tm.sendThresholdRecovery(server, "cpu", message, fmt.Sprintf("%.2f%%", cpuUsage), fmt.Sprintf("%d%%", cpuThresholdValue))
tm.clearAlert(alertKey)
}
}
}
// checkRAMThreshold checks RAM usage against threshold
func (tm *ThresholdMonitor) checkRAMThreshold(server Server, metric ParsedServerMetrics, threshold *ServerThreshold, maxRetries int) {
ramThresholdValue, err := strconv.Atoi(threshold.RAMThreshold)
if err != nil {
// log.Printf("Invalid RAM threshold value '%s' for server %s", threshold.RAMThreshold, server.Name)
return
}
// Parse RAM usage from metric (format: "X.XX GB (XX.X%)")
ramUsageStr := metric.RAMUsed
if strings.Contains(ramUsageStr, "(") && strings.Contains(ramUsageStr, "%)") {
// Extract percentage from parentheses
start := strings.Index(ramUsageStr, "(") + 1
end := strings.Index(ramUsageStr, "%)")
if start < end {
ramPercentageStr := ramUsageStr[start:end]
ramUsage, err := strconv.ParseFloat(ramPercentageStr, 64)
if err != nil {
// log.Printf("Failed to parse RAM usage percentage '%s' for server %s", ramPercentageStr, server.Name)
return
}
alertKey := fmt.Sprintf("%s-ram", server.ServerID)
if ramUsage > float64(ramThresholdValue) {
// Threshold exceeded
if tm.shouldCreateAlert(alertKey, ramUsage, float64(ramThresholdValue), maxRetries) {
// log.Printf("🚨 RAM threshold exceeded for server %s: %.2f%% > %d%% (retry %d/%d)",
// server.Name, ramUsage, ramThresholdValue, tm.getRetryCount(alertKey)+1, maxRetries)
message := fmt.Sprintf("🚨 RAM Alert: Server %s RAM usage is %.2f%% (threshold: %d%%)",
server.Name, ramUsage, ramThresholdValue)
tm.sendThresholdAlert(server, "ram", message, ramUsage, float64(ramThresholdValue), fmt.Sprintf("%.2f%%", ramUsage), fmt.Sprintf("%d%%", ramThresholdValue))
tm.setAlert(alertKey, ramUsage, float64(ramThresholdValue))
} else {
// log.Printf("🔇 RAM threshold exceeded for server %s but max retries (%d) reached, skipping notification", server.Name, maxRetries)
}
} else {
// Check if we need to send recovery notification
if tm.shouldSendRecoveryAlert(alertKey) {
// log.Printf("✅ RAM recovered for server %s: %.2f%% <= %d%%", server.Name, ramUsage, ramThresholdValue)
message := fmt.Sprintf("✅ RAM Recovery: Server %s RAM usage is back to normal: %.2f%% (threshold: %d%%)",
server.Name, ramUsage, ramThresholdValue)
tm.sendThresholdRecovery(server, "ram", message, fmt.Sprintf("%.2f%%", ramUsage), fmt.Sprintf("%d%%", ramThresholdValue))
tm.clearAlert(alertKey)
}
}
}
}
}
// checkDiskThreshold checks disk usage against threshold
func (tm *ThresholdMonitor) checkDiskThreshold(server Server, metric ParsedServerMetrics, threshold *ServerThreshold, maxRetries int) {
diskThresholdValue, err := strconv.Atoi(threshold.DiskThreshold)
if err != nil {
// log.Printf("Invalid disk threshold value '%s' for server %s", threshold.DiskThreshold, server.Name)
return
}
// Parse disk usage from metric (format: "X.XX GB (XX.X%)")
diskUsageStr := metric.DiskUsed
if strings.Contains(diskUsageStr, "(") && strings.Contains(diskUsageStr, "%)") {
// Extract percentage from parentheses
start := strings.Index(diskUsageStr, "(") + 1
end := strings.Index(diskUsageStr, "%)")
if start < end {
diskPercentageStr := diskUsageStr[start:end]
diskUsage, err := strconv.ParseFloat(diskPercentageStr, 64)
if err != nil {
// log.Printf("Failed to parse disk usage percentage '%s' for server %s", diskPercentageStr, server.Name)
return
}
alertKey := fmt.Sprintf("%s-disk", server.ServerID)
if diskUsage > float64(diskThresholdValue) {
// Threshold exceeded
if tm.shouldCreateAlert(alertKey, diskUsage, float64(diskThresholdValue), maxRetries) {
// log.Printf("🚨 Disk threshold exceeded for server %s: %.2f%% > %d%% (retry %d/%d)",
// server.Name, diskUsage, diskThresholdValue, tm.getRetryCount(alertKey)+1, maxRetries)
message := fmt.Sprintf("🚨 Disk Alert: Server %s disk usage is %.2f%% (threshold: %d%%)",
server.Name, diskUsage, diskThresholdValue)
tm.sendThresholdAlert(server, "disk", message, diskUsage, float64(diskThresholdValue), fmt.Sprintf("%.2f%%", diskUsage), fmt.Sprintf("%d%%", diskThresholdValue))
tm.setAlert(alertKey, diskUsage, float64(diskThresholdValue))
} else {
// log.Printf("🔇 Disk threshold exceeded for server %s but max retries (%d) reached, skipping notification", server.Name, maxRetries)
}
} else {
// Check if we need to send recovery notification
if tm.shouldSendRecoveryAlert(alertKey) {
// log.Printf("✅ Disk recovered for server %s: %.2f%% <= %d%%", server.Name, diskUsage, diskThresholdValue)
message := fmt.Sprintf("✅ Disk Recovery: Server %s disk usage is back to normal: %.2f%% (threshold: %d%%)",
server.Name, diskUsage, diskThresholdValue)
tm.sendThresholdRecovery(server, "disk", message, fmt.Sprintf("%.2f%%", diskUsage), fmt.Sprintf("%d%%", diskThresholdValue))
tm.clearAlert(alertKey)
}
}
}
}
}
// shouldCreateAlert determines if an alert should be created based on retry count and max retries
func (tm *ThresholdMonitor) shouldCreateAlert(alertKey string, currentValue, threshold float64, maxRetries int) bool {
alert, exists := tm.activeAlerts[alertKey]
if !exists {
return true // No existing alert, create new one
}
// Check if we've reached max retries
if alert.RetryCount >= maxRetries {
// log.Printf("🔇 Alert %s has reached max retries (%d), not sending notification", alertKey, maxRetries)
return false
}
// Check if enough time has passed since last alert (resend every 5 minutes)
timeSinceLastAlert := time.Since(alert.LastAlerted)
return timeSinceLastAlert > 5*time.Minute
}
// getRetryCount returns the current retry count for an alert
func (tm *ThresholdMonitor) getRetryCount(alertKey string) int {
if alert, exists := tm.activeAlerts[alertKey]; exists {
return alert.RetryCount
}
return 0
}
// shouldSendRecoveryAlert determines if a recovery alert should be sent
func (tm *ThresholdMonitor) shouldSendRecoveryAlert(alertKey string) bool {
_, exists := tm.activeAlerts[alertKey]
return exists // Send recovery only if there was an active alert
}
// setAlert creates or updates an alert with incremented retry count
func (tm *ThresholdMonitor) setAlert(alertKey string, currentValue, threshold float64) {
alert, exists := tm.activeAlerts[alertKey]
if exists {
// Update existing alert and increment retry count
alert.CurrentValue = currentValue
alert.LastAlerted = time.Now()
alert.RetryCount++
// log.Printf("📊 Updated alert %s - Retry count: %d", alertKey, alert.RetryCount)
} else {
// Create new alert
tm.activeAlerts[alertKey] = &ThresholdAlert{
IsActive: true,
Threshold: int(threshold),
CurrentValue: currentValue,
LastAlerted: time.Now(),
RetryCount: 1, // Start with 1 since we're sending the first notification
}
// log.Printf("🆕 Created new alert %s - Retry count: 1", alertKey)
}
}
// clearAlert removes an alert
func (tm *ThresholdMonitor) clearAlert(alertKey string) {
if _, exists := tm.activeAlerts[alertKey]; exists {
// log.Printf("🗑️ Cleared alert %s", alertKey)
delete(tm.activeAlerts, alertKey)
}
}
// sendThresholdAlert sends a threshold exceeded notification using resource-specific templates
func (tm *ThresholdMonitor) sendThresholdAlert(server Server, metricType, message string, currentValue, threshold float64, usageStr, thresholdStr string) {
if server.NotificationID == "" {
// log.Printf("No notification ID configured for server %s", server.Name)
return
}
// log.Printf("📤 Sending resource-specific threshold alert for server %s (%s): %s", server.Name, metricType, message)
// Use the new resource-specific notification method with actual values
err := tm.notificationService.SendResourceNotificationWithValues(server, "warning", message, metricType, usageStr, thresholdStr)
if err != nil {
// log.Printf("❌ Failed to send threshold alert for server %s: %v", server.Name, err)
_ = err
} else {
// log.Printf("✅ Threshold alert sent successfully for server %s", server.Name)
}
}
// sendThresholdRecovery sends a threshold recovery notification using resource-specific templates
func (tm *ThresholdMonitor) sendThresholdRecovery(server Server, metricType, message, usageStr, thresholdStr string) {
if server.NotificationID == "" {
// log.Printf("No notification ID configured for server %s", server.Name)
return
}
// log.Printf("📤 Sending resource-specific threshold recovery for server %s (%s): %s", server.Name, metricType, message)
// Use the new resource-specific notification method with actual values
err := tm.notificationService.SendResourceNotificationWithValues(server, "up", message, metricType, usageStr, thresholdStr)
if err != nil {
// log.Printf("❌ Failed to send threshold recovery for server %s: %v", server.Name, err)
_ = err
} else {
// log.Printf("✅ Threshold recovery sent successfully for server %s", server.Name)
}
}
@@ -0,0 +1,77 @@
package servermonitoring
import "time"
// Server represents a server record in PocketBase
type Server struct {
CollectionId string `json:"collectionId"`
CollectionName string `json:"collectionName"`
ID string `json:"id"`
ServerID string `json:"server_id"`
Name string `json:"name"`
Hostname string `json:"hostname"`
IPAddress string `json:"ip_address"`
OSType string `json:"os_type"`
Status string `json:"status"` // 'up' | 'down' | 'warning' | 'paused'
Uptime string `json:"uptime"`
RAMTotal float64 `json:"ram_total"`
RAMUsed float64 `json:"ram_used"`
CPUCores int `json:"cpu_cores"`
CPUUsage float64 `json:"cpu_usage"`
DiskTotal float64 `json:"disk_total"`
DiskUsed float64 `json:"disk_used"`
LastChecked string `json:"last_checked"`
ServerToken string `json:"server_token"`
TemplateID string `json:"template_id"`
ThresholdID string `json:"threshold_id"`
NotificationID string `json:"notification_id"`
NotificationStatus bool `json:"notification_status"`
MaxRetries string `json:"max_retries"`
Timestamp string `json:"timestamp"`
Connection string `json:"connection"`
AgentStatus string `json:"agent_status"`
SystemInfo string `json:"system_info"`
NetworkRxBytes string `json:"network_rx_bytes"`
NetworkTxBytes string `json:"network_tx_bytes"`
NetworkRxSpeed string `json:"network_rx_speed"`
NetworkTxSpeed string `json:"network_tx_speed"`
CheckInterval int `json:"check_interval"`
Docker string `json:"docker"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// ServerStats represents server statistics
type ServerStats struct {
Total int `json:"total"`
Online int `json:"online"`
Offline int `json:"offline"`
Warning int `json:"warning"`
}
// ServerThreshold represents threshold configuration for server monitoring
type ServerThreshold struct {
ID string `json:"id"`
Name string `json:"name"`
CPUThreshold string `json:"cpu_threshold"`
RAMThreshold string `json:"ram_threshold"`
DiskThreshold string `json:"disk_threshold"`
NetworkThreshold string `json:"network_threshold"`
DiskIOThreshold string `json:"disk_io_threshold"`
CPUTempThreshold string `json:"cpu_temp_threshold"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// ServerNotificationPayload represents server-specific notification data
type ServerNotificationPayload struct {
ServerName string `json:"server_name"`
ServerID string `json:"server_id"`
Hostname string `json:"hostname"`
IPAddress string `json:"ip_address"`
Status string `json:"status"`
OSType string `json:"os_type"`
Timestamp time.Time `json:"timestamp"`
Message string `json:"message"`
}
@@ -0,0 +1,98 @@
package sslmonitoring
import (
"time"
"service-operation/pocketbase"
)
// SSLMonitor monitors SSL certificates and sends notifications
type SSLMonitor struct {
client *SSLClient
notificationService *SSLNotificationService
checkInterval time.Duration
stopChan chan bool
}
// NewSSLMonitor creates a new SSL monitor
func NewSSLMonitor(pbClient *pocketbase.PocketBaseClient) *SSLMonitor {
// Initialize SSL client
sslClient := NewSSLClient(pbClient)
// Initialize notification service
notificationService := NewSSLNotificationService(sslClient, pbClient)
// Set check interval to 30 seconds to match other services
checkInterval := 30 * time.Second
return &SSLMonitor{
client: sslClient,
notificationService: notificationService,
checkInterval: checkInterval,
stopChan: make(chan bool, 1),
}
}
// Start begins monitoring SSL certificates
func (sm *SSLMonitor) Start() {
// log.Printf("🚀 [SSL-MONITOR] Starting with check interval: %v", sm.checkInterval)
// log.Printf("⏰ [SSL-MONITOR] Startup grace period: 2 minutes (no notifications for valid certificates)")
// Run initial check
sm.checkCertificates()
// Set up periodic checking every 30 seconds
ticker := time.NewTicker(sm.checkInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
sm.checkCertificates()
case <-sm.stopChan:
// log.Println("🛑 [SSL-MONITOR] Monitoring stopped")
return
}
}
}
// Stop gracefully stops the SSL monitoring
func (sm *SSLMonitor) Stop() {
// log.Printf("🛑 [SSL-MONITOR] Stopping SSL monitoring...")
select {
case sm.stopChan <- true:
default:
}
}
// checkCertificates fetches all SSL certificates and checks their notification requirements
func (sm *SSLMonitor) checkCertificates() {
// log.Printf("🔍 [SSL-CHECK] Checking SSL certificates for expiration alerts...")
certificates, err := sm.client.GetSSLCertificates()
if err != nil {
// log.Printf("❌ [SSL-ERROR] Failed to fetch SSL certificates: %v", err)
return
}
// log.Printf("📊 [SSL-CERTIFICATES] Found %d certificates to check", len(certificates))
successCount := 0
errorCount := 0
for _, cert := range certificates {
// Check each certificate for notification requirements
if err := sm.notificationService.CheckAndNotifySSLCertificate(cert); err != nil {
// log.Printf("❌ [SSL-FAILED] Failed to process certificate %s: %v", cert.Domain, err)
errorCount++
} else {
successCount++
}
}
// log.Printf("✅ [SSL-SUMMARY] Processed %d certificates (success: %d, errors: %d)",
// len(certificates), successCount, errorCount)
_ = successCount // Prevent unused variable warning
_ = errorCount // Prevent unused variable warning
}
@@ -0,0 +1,192 @@
package sslmonitoring
import (
"fmt"
"strconv"
"time"
"service-operation/notification"
"service-operation/pocketbase"
)
// SSLNotificationService handles SSL certificate notifications
type SSLNotificationService struct {
client *SSLClient
pbClient *pocketbase.PocketBaseClient
notificationManager *notification.NotificationManager
statusTracker *SSLStatusTracker
}
// NewSSLNotificationService creates a new SSL notification service
func NewSSLNotificationService(client *SSLClient, pbClient *pocketbase.PocketBaseClient) *SSLNotificationService {
return &SSLNotificationService{
client: client,
pbClient: pbClient,
notificationManager: notification.NewNotificationManager(pbClient),
statusTracker: NewSSLStatusTracker(pbClient), // Pass pbClient to status tracker
}
}
// CheckAndNotifySSLCertificate checks SSL certificate and sends notification if needed
func (sns *SSLNotificationService) CheckAndNotifySSLCertificate(cert SSLCertificate) error {
// Always recalculate days left from the actual expiry date to ensure accuracy
actualDaysLeft := sns.calculateDaysLeft(cert.ValidTill)
// Use the calculated value as it's more accurate
cert.DaysLeft = actualDaysLeft
// Determine current status based on thresholds with calculated days left
currentStatus := sns.determineSSLStatus(cert.DaysLeft, cert.WarningThreshold, cert.ExpiryThreshold)
// Check if notification should be sent
if !sns.shouldSendNotification(cert.ID, currentStatus) {
return nil
}
// Skip if no notification configuration
if cert.NotificationID == "" {
return nil
}
// Send notification
err := sns.sendSSLNotification(cert, currentStatus)
if err != nil {
return err
}
// Update tracking in database
sns.statusTracker.UpdateStatus(cert.ID, currentStatus)
sns.statusTracker.SetLastNotificationTime(cert.ID, time.Now())
return nil
}
// calculateDaysLeft calculates days remaining until expiration with better error handling
func (sns *SSLNotificationService) calculateDaysLeft(validTill string) int {
// Try multiple date formats to parse the expiry date
var expiryTime time.Time
var err error
// Common date formats - updated to handle PocketBase format correctly
formats := []string{
"2006-01-02 15:04:05.000Z", // PocketBase format with space (most common)
"2006-01-02T15:04:05.000Z", // ISO 8601 with milliseconds
time.RFC3339, // Standard RFC3339
time.RFC3339Nano, // RFC3339 with nanoseconds
"2006-01-02 15:04:05Z", // Without milliseconds
"2006-01-02T15:04:05Z", // ISO without milliseconds
"2006-01-02 15:04:05.999Z", // Alternative milliseconds format
"2006-01-02T15:04:05.999Z", // Alternative ISO milliseconds
"2006-01-02 15:04:05.000000Z", // Microseconds format
"2006-01-02T15:04:05.000000Z", // ISO microseconds format
"2006-01-02 15:04:05", // Simple format without timezone
"2006-01-02", // Date only
}
for _, format := range formats {
expiryTime, err = time.Parse(format, validTill)
if err == nil {
break
}
}
if err != nil {
return 0
}
now := time.Now()
duration := expiryTime.Sub(now)
daysLeft := int(duration.Hours() / 24)
// If the result is negative, the certificate is expired
if daysLeft < 0 {
daysLeft = 0
}
return daysLeft
}
// sendSSLNotification sends the actual notification
func (sns *SSLNotificationService) sendSSLNotification(cert SSLCertificate, status string) error {
// Determine the correct issuer value - prioritize IssuerCN over IssuerO
issuerValue := ""
if cert.IssuerCN != "" {
issuerValue = cert.IssuerCN
} else if cert.IssuerO != "" {
issuerValue = cert.IssuerO
} else {
issuerValue = "Unknown"
}
// Create notification payload with SSL-specific data using calculated values
payload := &notification.NotificationPayload{
ServiceName: fmt.Sprintf("SSL Certificate - %s", cert.Domain),
Status: status,
Host: cert.Domain,
Domain: cert.Domain,
ServiceType: "ssl",
Timestamp: time.Now(),
Message: sns.generateStatusMessage(cert, status),
// SSL-specific fields with calculated/corrected values
CertificateName: cert.Domain,
ExpiryDate: cert.ValidTill,
DaysLeft: strconv.Itoa(cert.DaysLeft), // Use the calculated value
IssuerCN: issuerValue,
SerialNumber: cert.SerialNumber,
}
// Send SSL notification using the manager
return sns.notificationManager.SendSSLNotification(payload, cert.NotificationID, cert.TemplateID)
}
// generateStatusMessage creates appropriate message based on certificate status using calculated days
func (sns *SSLNotificationService) generateStatusMessage(cert SSLCertificate, status string) string {
switch status {
case "expired":
return fmt.Sprintf("SSL certificate for %s expired on %s", cert.Domain, cert.ValidTill)
case "expiring_soon":
return fmt.Sprintf("SSL certificate for %s expires in %d days on %s", cert.Domain, cert.DaysLeft, cert.ValidTill)
case "warning":
return fmt.Sprintf("SSL certificate for %s expires in %d days", cert.Domain, cert.DaysLeft)
default:
return fmt.Sprintf("SSL certificate for %s is valid (%d days remaining)", cert.Domain, cert.DaysLeft)
}
}
// shouldSendNotification determines if notification should be sent with enhanced logic
func (sns *SSLNotificationService) shouldSendNotification(certID, currentStatus string) bool {
lastStatus := sns.statusTracker.GetLastStatus(certID)
lastNotified := sns.statusTracker.GetLastNotificationTime(certID)
// Send if status changed or first check
if lastStatus == "" || lastStatus != currentStatus {
return true
}
// For critical statuses (expired/expiring_soon), resend after 24 hours
if (currentStatus == "expired" || currentStatus == "expiring_soon") &&
!lastNotified.IsZero() && time.Since(lastNotified) > 24*time.Hour {
return true
}
// For warning status, resend after 7 days to avoid spam
if currentStatus == "warning" &&
!lastNotified.IsZero() && time.Since(lastNotified) > 7*24*time.Hour {
return true
}
return false
}
// determineSSLStatus determines SSL certificate status based on days left and thresholds
func (sns *SSLNotificationService) determineSSLStatus(daysLeft, warningThreshold, expiryThreshold int) string {
if daysLeft <= 0 {
return "expired"
} else if daysLeft <= expiryThreshold {
return "expiring_soon"
} else if daysLeft <= warningThreshold {
return "warning"
}
return "valid"
}
@@ -0,0 +1,78 @@
package sslmonitoring
import (
"encoding/json"
"fmt"
"net/http"
"service-operation/pocketbase"
)
// SSLClient handles SSL certificate data operations
type SSLClient struct {
pbClient *pocketbase.PocketBaseClient
}
// NewSSLClient creates a new SSL client
func NewSSLClient(pbClient *pocketbase.PocketBaseClient) *SSLClient {
return &SSLClient{
pbClient: pbClient,
}
}
// GetSSLCertificates fetches all SSL certificates from PocketBase
func (sc *SSLClient) GetSSLCertificates() ([]SSLCertificate, error) {
url := fmt.Sprintf("%s/api/collections/ssl_certificates/records", sc.pbClient.GetBaseURL())
// log.Printf("🔍 [SSL-CLIENT] Fetching SSL certificates from: %s", url)
resp, err := http.Get(url)
if err != nil {
// log.Printf("❌ [SSL-CLIENT] HTTP error: %v", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// log.Printf("❌ [SSL-CLIENT] Failed to fetch SSL certificates, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch SSL certificates, status: %d", resp.StatusCode)
}
var response struct {
Items []SSLCertificate `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
// log.Printf("❌ [SSL-CLIENT] Error decoding response: %v", err)
return nil, err
}
// log.Printf("✅ [SSL-CLIENT] Successfully fetched %d SSL certificates", len(response.Items))
return response.Items, nil
}
// UpdateSSLCertificate updates SSL certificate status and notification time
func (sc *SSLClient) UpdateSSLCertificate(certID string, data map[string]interface{}) error {
url := fmt.Sprintf("%s/api/collections/ssl_certificates/records/%s", sc.pbClient.GetBaseURL(), certID)
req, err := http.NewRequest(http.MethodPatch, url, nil)
if err != nil {
return fmt.Errorf("failed to create SSL certificate update request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to update SSL certificate: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to update SSL certificate, status: %d", resp.StatusCode)
}
// log.Printf("✅ [SSL-CLIENT] Successfully updated SSL certificate: %s", certID)
return nil
}
@@ -0,0 +1,111 @@
package sslmonitoring
import (
"sync"
"time"
"service-operation/pocketbase"
)
// SSLStatusTracker tracks SSL certificate status changes and notification timing using database persistence
type SSLStatusTracker struct {
pbClient *pocketbase.PocketBaseClient
mu sync.RWMutex
}
// NewSSLStatusTracker creates a new SSL status tracker with database persistence
func NewSSLStatusTracker(pbClient *pocketbase.PocketBaseClient) *SSLStatusTracker {
return &SSLStatusTracker{
pbClient: pbClient,
}
}
// GetLastStatus returns the last known status for an SSL certificate from database
func (sst *SSLStatusTracker) GetLastStatus(certID string) string {
sst.mu.RLock()
defer sst.mu.RUnlock()
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("🔍 [SSL-TRACKER] Error getting last status for %s: %v", certID, err)
return ""
}
status := cert.Status
// log.Printf("🔍 [SSL-TRACKER] GetLastStatus for %s: %s", certID, status)
return status
}
// UpdateStatus updates the last known status for an SSL certificate in database
func (sst *SSLStatusTracker) UpdateStatus(certID, status string) {
sst.mu.Lock()
defer sst.mu.Unlock()
// Get current certificate to preserve other fields
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("📝 [SSL-TRACKER] Error getting certificate for status update %s: %v", certID, err)
return
}
oldStatus := cert.Status
// Update only the status field
updateData := map[string]interface{}{
"status": status,
}
err = sst.pbClient.UpdateSSLCertificate(certID, updateData)
if err != nil {
// log.Printf("📝 [SSL-TRACKER] Error updating status for %s: %v", certID, err)
return
}
// log.Printf("📝 [SSL-TRACKER] UpdateStatus for %s: %s -> %s", certID, oldStatus, status)
_ = oldStatus // Prevent unused variable warning
}
// GetLastNotificationTime returns the last time a notification was sent for an SSL certificate from database
func (sst *SSLStatusTracker) GetLastNotificationTime(certID string) time.Time {
sst.mu.RLock()
defer sst.mu.RUnlock()
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error getting last notification time for %s: %v", certID, err)
return time.Time{}
}
if cert.LastNotified == "" {
// log.Printf("⏰ [SSL-TRACKER] GetLastNotificationTime for %s: never notified", certID)
return time.Time{}
}
lastTime, err := time.Parse(time.RFC3339, cert.LastNotified)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error parsing last notification time for %s: %v", certID, err)
return time.Time{}
}
// log.Printf("⏰ [SSL-TRACKER] GetLastNotificationTime for %s: %v", certID, lastTime)
return lastTime
}
// SetLastNotificationTime sets the last notification time for an SSL certificate in database
func (sst *SSLStatusTracker) SetLastNotificationTime(certID string, t time.Time) {
sst.mu.Lock()
defer sst.mu.Unlock()
updateData := map[string]interface{}{
"last_notified": t.Format(time.RFC3339),
}
err := sst.pbClient.UpdateSSLCertificate(certID, updateData)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error setting last notification time for %s: %v", certID, err)
return
}
// log.Printf("⏰ [SSL-TRACKER] SetLastNotificationTime for %s: %v", certID, t)
}
@@ -0,0 +1,110 @@
package sslmonitoring
import (
"encoding/json"
"strconv"
"time"
)
type SSLCertificate struct {
ID string `json:"id"`
CollectionID string `json:"collectionId"`
CollectionName string `json:"collectionName"`
Domain string `json:"domain"`
IssuerO string `json:"issuer_o"`
Status string `json:"status"`
LastNotified string `json:"last_notified"`
WarningThreshold int `json:"warning_threshold"`
ExpiryThreshold int `json:"expiry_threshold"`
NotificationChannel string `json:"notification_channel"`
NotificationID string `json:"notification_id"`
TemplateID string `json:"template_id"`
ValidFrom string `json:"valid_from"`
SerialNumber string `json:"serial_number"`
IssuedTo string `json:"issued_to"`
ValidTill string `json:"valid_till"`
ValidityDays int `json:"validity_days"`
DaysLeft int `json:"days_left"`
ValidDaysToExpire int `json:"valid_days_to_expire"`
ResolvedIP string `json:"resolved_ip"`
IssuerCN string `json:"issuer_cn"`
CertAlg string `json:"cert_alg"`
CertSans string `json:"cert_sans"`
CheckInterval int `json:"check_interval"`
CheckAt string `json:"check_at"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// Custom unmarshaler to handle check_interval as both string and int, and serial_number as string
func (s *SSLCertificate) UnmarshalJSON(data []byte) error {
// Create a temporary struct with flexible types
type Alias SSLCertificate
aux := &struct {
CheckInterval interface{} `json:"check_interval"`
SerialNumber interface{} `json:"serial_number"`
*Alias
}{
Alias: (*Alias)(s),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
// Handle check_interval conversion
switch v := aux.CheckInterval.(type) {
case string:
if v == "" {
s.CheckInterval = 1440 // Default 24 hours in minutes
} else {
interval, err := strconv.Atoi(v)
if err != nil {
s.CheckInterval = 1440 // Default on error
} else {
s.CheckInterval = interval
}
}
case float64:
s.CheckInterval = int(v)
case int:
s.CheckInterval = v
default:
s.CheckInterval = 1440 // Default 24 hours in minutes
}
// Handle serial_number conversion to string
switch v := aux.SerialNumber.(type) {
case string:
s.SerialNumber = v
case float64:
// Handle scientific notation by converting to string
s.SerialNumber = strconv.FormatFloat(v, 'f', 0, 64)
case int64:
s.SerialNumber = strconv.FormatInt(v, 10)
case int:
s.SerialNumber = strconv.Itoa(v)
default:
s.SerialNumber = ""
}
return nil
}
type SSLCheckResult struct {
Domain string `json:"domain"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
ValidFrom time.Time `json:"valid_from"`
ValidTill time.Time `json:"valid_till"`
DaysLeft int `json:"days_left"`
Issuer string `json:"issuer"`
Subject string `json:"subject"`
SerialNumber string `json:"serial_number"`
Algorithm string `json:"algorithm"`
SANs []string `json:"sans"`
ResponseTime time.Duration `json:"response_time"`
ResolvedIP string `json:"resolved_ip"`
CheckedAt time.Time `json:"checked_at"`
}
@@ -0,0 +1,113 @@
package uptimemonitoring
import (
"encoding/json"
"fmt"
"net/http"
"time"
"service-operation/pocketbase"
)
// UptimeClient handles uptime service operations
type UptimeClient struct {
pbClient *pocketbase.PocketBaseClient
}
// MetricRecord represents a metric record from any collection
type MetricRecord struct {
ID string `json:"id"`
ServiceID string `json:"service_id"`
Status string `json:"status"`
ResponseTime int `json:"response_time"`
ErrorMessage string `json:"error_message"` // Added missing ErrorMessage field
Timestamp time.Time `json:"timestamp"`
Created time.Time `json:"created"`
}
// NewUptimeClient creates a new uptime client
func NewUptimeClient(pbClient *pocketbase.PocketBaseClient) *UptimeClient {
return &UptimeClient{
pbClient: pbClient,
}
}
// GetServices retrieves all services for monitoring
func (uc *UptimeClient) GetServices() ([]UptimeService, error) {
url := fmt.Sprintf("%s/api/collections/services/records", uc.pbClient.GetBaseURL())
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch services: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to fetch services, status: %d", resp.StatusCode)
}
var response struct {
Items []UptimeService `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to decode services: %v", err)
}
// log.Printf("✅ Fetched %d services for monitoring", len(response.Items))
return response.Items, nil
}
// GetLatestMetricRecord gets the latest metric record for a service from specified collection
func (uc *UptimeClient) GetLatestMetricRecord(serviceID, collection string) ([]MetricRecord, error) {
url := fmt.Sprintf("%s/api/collections/%s/records?filter=service_id='%s'&sort=-timestamp&perPage=1",
uc.pbClient.GetBaseURL(), collection, serviceID)
// log.Printf("🔍 [METRICS-QUERY] Getting latest record for service %s from %s", serviceID, collection)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch metric record: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to fetch metric record, status: %d", resp.StatusCode)
}
var response struct {
Items []struct {
ID string `json:"id"`
ServiceID string `json:"service_id"`
Status string `json:"status"`
ResponseTime int `json:"response_time"`
ErrorMessage string `json:"error_message"` // Added missing ErrorMessage field
Timestamp string `json:"timestamp"`
Created string `json:"created"`
} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to decode metric record: %v", err)
}
var records []MetricRecord
for _, item := range response.Items {
timestamp, _ := time.Parse(time.RFC3339, item.Timestamp)
created, _ := time.Parse(time.RFC3339, item.Created)
records = append(records, MetricRecord{
ID: item.ID,
ServiceID: item.ServiceID,
Status: item.Status,
ResponseTime: item.ResponseTime,
ErrorMessage: item.ErrorMessage, // Include ErrorMessage in the record
Timestamp: timestamp,
Created: created,
})
}
// log.Printf("📊 [METRICS-RESULT] Found %d records for service %s in %s", len(records), serviceID, collection)
return records, nil
}
@@ -0,0 +1,106 @@
package uptimemonitoring
import (
"time"
"service-operation/pocketbase"
)
// UptimeMonitor monitors uptime services and sends notifications
type UptimeMonitor struct {
client *UptimeClient
notificationService *UptimeNotificationService
checkInterval time.Duration
stopChan chan bool
}
// NewUptimeMonitor creates a new uptime monitor
func NewUptimeMonitor(pbClient *pocketbase.PocketBaseClient) *UptimeMonitor {
// Initialize uptime client
uptimeClient := NewUptimeClient(pbClient)
// Initialize notification service
notificationService := NewUptimeNotificationService(uptimeClient, pbClient)
// Set check interval to 30 seconds to match server monitoring
checkInterval := 30 * time.Second
return &UptimeMonitor{
client: uptimeClient,
notificationService: notificationService,
checkInterval: checkInterval,
stopChan: make(chan bool, 1),
}
}
// Start begins monitoring uptime services
func (um *UptimeMonitor) Start() {
// log.Printf("🚀 [UPTIME-MONITOR] Starting with check interval: %v", um.checkInterval)
// log.Printf("⏰ [UPTIME-MONITOR] Startup grace period: 2 minutes (no notifications for UP services)")
// Run initial check
um.checkServices()
// Set up periodic checking every 30 seconds
ticker := time.NewTicker(um.checkInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
um.checkServices()
case <-um.stopChan:
// log.Println("🛑 [UPTIME-MONITOR] Monitoring stopped")
return
}
}
}
// Stop gracefully stops the uptime monitoring
func (um *UptimeMonitor) Stop() {
// log.Printf("🛑 [UPTIME-MONITOR] Stopping uptime monitoring...")
select {
case um.stopChan <- true:
default:
}
}
// checkServices fetches all services and checks their notification requirements
func (um *UptimeMonitor) checkServices() {
// log.Printf("🔍 [UPTIME-CHECK] Checking uptime services for status changes...")
services, err := um.client.GetServices()
if err != nil {
// log.Printf("❌ [UPTIME-ERROR] Failed to fetch services: %v", err)
_ = err
return
}
// log.Printf("📊 [UPTIME-SERVICES] Found %d services to check", len(services))
successCount := 0
errorCount := 0
for _, service := range services {
// Skip paused services
if service.Status == "paused" {
// log.Printf("⏸️ [UPTIME-SKIP] Service %s is paused - skipping", service.Name)
continue
}
// Check each service for notification requirements
if err := um.notificationService.CheckAndNotifyService(service); err != nil {
// log.Printf("❌ [UPTIME-FAILED] Failed to process service %s: %v", service.Name, err)
_ = err
errorCount++
} else {
successCount++
}
}
// log.Printf("✅ [UPTIME-SUMMARY] Processed %d services (success: %d, errors: %d)",
// len(services), successCount, errorCount)
_ = successCount
_ = errorCount
}
@@ -0,0 +1,272 @@
package uptimemonitoring
import (
"fmt"
"strings"
"time"
"service-operation/notification"
"service-operation/pocketbase"
)
// UptimeNotificationService handles notifications for uptime services
type UptimeNotificationService struct {
client *UptimeClient
notificationMgr *notification.NotificationManager
statusTracker *ServiceStatusTracker
startupTime time.Time
gracePeriod time.Duration
}
// NewUptimeNotificationService creates a new uptime notification service
func NewUptimeNotificationService(client *UptimeClient, pbClient *pocketbase.PocketBaseClient) *UptimeNotificationService {
return &UptimeNotificationService{
client: client,
notificationMgr: notification.NewNotificationManager(pbClient),
statusTracker: NewServiceStatusTracker(),
startupTime: time.Now(),
gracePeriod: 2 * time.Minute, // 2 minute grace period after startup
}
}
// isInGracePeriod checks if we're still in the startup grace period
func (uns *UptimeNotificationService) isInGracePeriod() bool {
return time.Since(uns.startupTime) < uns.gracePeriod
}
// CheckAndNotifyService checks a service and sends notifications for status changes
func (uns *UptimeNotificationService) CheckAndNotifyService(service UptimeService) error {
// log.Printf("🔍 [UPTIME-CHECK] Checking service: %s (ID: %s, Type: %s)", service.Name, service.ID, service.ServiceType)
// Check if notifications are enabled for this service
if !uns.isNotificationEnabled(service) {
// log.Printf("⚠️ [SKIP] Notifications disabled for service %s", service.Name)
return nil
}
// Get ACTUAL current status from PocketBase metrics (including error message)
actualCurrentStatus, responseTime, errorMessage, err := uns.getActualServiceStatusWithError(service)
if err != nil {
// log.Printf("❌ [ERROR] Failed to get actual status for %s: %v", service.Name, err)
// If we can't get status, assume it's down
actualCurrentStatus = "down"
responseTime = 0
errorMessage = fmt.Sprintf("Failed to retrieve service metrics: %v", err)
}
// Get previously tracked status
previousStatus := uns.statusTracker.GetLastStatus(service.ID)
// log.Printf("📊 [STATUS-COMPARE] Service %s: PREVIOUS='%s' -> ACTUAL_CURRENT='%s', Error: '%s'", service.Name, previousStatus, actualCurrentStatus, errorMessage)
// Initialize tracking for new services
if previousStatus == "" {
// log.Printf("🆕 [INIT] First time tracking service %s - setting initial status: %s", service.Name, actualCurrentStatus)
// Check if we're in startup grace period
if uns.isInGracePeriod() {
// log.Printf("⏰ [GRACE-PERIOD] Service %s - within startup grace period, skipping initial notification", service.Name)
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
return nil
}
// Only send initial notification if service is DOWN (critical status)
if actualCurrentStatus == "down" {
// log.Printf("🚨 [CRITICAL-INIT] Service %s is DOWN on first check - sending notification", service.Name)
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, "INITIAL_DOWN_DETECTED")
if err != nil {
// log.Printf("❌ [FAILED] Initial DOWN notification failed for %s: %v", service.Name, err)
_ = err
} else {
// log.Printf("✅ [SUCCESS] Initial DOWN notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
}
} else {
// log.Printf("️ [INIT-SKIP] Service %s is %s on first check - skipping notification (not critical)", service.Name, actualCurrentStatus)
}
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
return nil
}
// Check if there's a REAL status change
if previousStatus != actualCurrentStatus {
// log.Printf("🚨 [REAL-CHANGE] STATUS CHANGE for %s: %s -> %s", service.Name, previousStatus, actualCurrentStatus)
// Send notification immediately for real status change
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, fmt.Sprintf("STATUS_CHANGE_%s_TO_%s", previousStatus, actualCurrentStatus))
if err != nil {
// log.Printf("❌ [FAILED] Status change notification failed for %s: %v", service.Name, err)
return err
}
// log.Printf("✅ [SUCCESS] Status change notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
uns.statusTracker.UpdateStatus(service.ID, actualCurrentStatus)
} else if actualCurrentStatus == "down" {
// For services that remain down, send periodic notifications (every 5 minutes)
lastNotified := uns.statusTracker.GetLastNotificationTime(service.ID)
if time.Since(lastNotified) >= 5*time.Minute {
// log.Printf("⏰ [PERIODIC] Service %s still down for 5+ minutes - sending reminder", service.Name)
err := uns.sendImmediateNotification(service, actualCurrentStatus, responseTime, errorMessage, "PERIODIC_DOWN_REMINDER")
if err != nil {
// log.Printf("❌ [FAILED] Periodic down notification failed for %s: %v", service.Name, err)
return err
}
// log.Printf("✅ [SUCCESS] Periodic down notification sent for %s", service.Name)
uns.statusTracker.SetLastNotificationTime(service.ID, time.Now())
}
} else {
// log.Printf("️ [NO-CHANGE] No status change for %s (both %s) - no notification needed", service.Name, actualCurrentStatus)
}
return nil
}
// getActualServiceStatusWithError gets the real current status from PocketBase metrics including error message
func (uns *UptimeNotificationService) getActualServiceStatusWithError(service UptimeService) (string, int64, string, error) {
// Get the appropriate collection based on service type
collection := uns.getCollectionForServiceType(service.ServiceType)
// log.Printf("🔍 [METRICS-CHECK] Getting actual status for %s from collection: %s", service.Name, collection)
// Get the latest metric record for this service
records, err := uns.client.GetLatestMetricRecord(service.ID, collection)
if err != nil {
// log.Printf("❌ [METRICS-ERROR] Failed to get metrics for %s: %v", service.Name, err)
return "unknown", 0, fmt.Sprintf("Failed to retrieve metrics: %v", err), err
}
if len(records) == 0 {
// log.Printf("⚠️ [NO-METRICS] No metrics found for service %s", service.Name)
return "down", 0, "No metrics data available", fmt.Errorf("no metrics found for service %s", service.Name)
}
// Get the latest record
latestRecord := records[0]
actualStatus := latestRecord.Status
responseTime := int64(latestRecord.ResponseTime)
errorMessage := latestRecord.ErrorMessage // Get the error message from the record
// log.Printf("📈 [METRICS-RESULT] Latest metric for %s: Status=%s, ResponseTime=%dms, Error='%s', Timestamp=%s",
// service.Name, actualStatus, responseTime, errorMessage, latestRecord.Timestamp)
return actualStatus, responseTime, errorMessage, nil
}
// getCollectionForServiceType maps service types to their corresponding collections
func (uns *UptimeNotificationService) getCollectionForServiceType(serviceType string) string {
switch strings.ToLower(serviceType) {
case "ping", "icmp":
return "ping_data"
case "dns":
return "dns_data"
case "tcp":
return "tcp_data"
case "http", "https":
return "uptime_data"
default:
return "uptime_data"
}
}
// sendImmediateNotification sends notification with zero delay for status changes
func (uns *UptimeNotificationService) sendImmediateNotification(service UptimeService, status string, responseTime int64, errorMessage, reason string) error {
// log.Printf("⚡ [IMMEDIATE-SEND] Sending notification for %s (status: %s, error: '%s', reason: %s)", service.Name, status, errorMessage, reason)
// Build notification payload with actual current status and error message
payload := &notification.NotificationPayload{
ServiceName: service.Name,
Status: status,
Host: service.Host,
Port: service.Port,
ServiceType: service.ServiceType,
ResponseTime: responseTime,
ErrorMessage: errorMessage, // Include the error message from metrics
Timestamp: time.Now(),
Message: uns.generateMessage(service, status, reason),
}
// Add optional fields
if service.Domain != "" {
payload.Domain = service.Domain
}
if service.URL != "" {
payload.URL = service.URL
}
if service.RegionName != "" {
payload.RegionName = service.RegionName
}
if service.AgentID != "" {
payload.AgentID = service.AgentID
}
payload.Uptime = service.Uptime
// log.Printf("⚡ [TELEGRAM-SEND] Sending uptime notification for %s via NotificationManager with error: '%s'", service.Name, errorMessage)
// Send notification using the uptime manager
return uns.notificationMgr.SendUptimeServiceNotification(payload, service.NotificationID, service.TemplateID)
}
// generateMessage creates a status message for notifications
func (uns *UptimeNotificationService) generateMessage(service UptimeService, status, reason string) string {
statusEmoji := "✅"
action := "is operational"
switch strings.ToLower(status) {
case "down":
statusEmoji = "❌"
action = "is DOWN"
case "warning":
statusEmoji = "⚠️"
action = "has issues"
case "up":
statusEmoji = "✅"
previousStatus := uns.statusTracker.GetLastStatus(service.ID)
if previousStatus == "down" {
action = "has RECOVERED"
statusEmoji = "🔄"
} else {
action = "is operational"
}
}
message := fmt.Sprintf("%s [UPTIME] %s %s", statusEmoji, service.Name, action)
if service.Host != "" {
message += fmt.Sprintf(" | Host: %s", service.Host)
}
// Add reason for debugging (only in logs, not in user message)
// log.Printf("📝 [MSG-DEBUG] Generated uptime message for %s: %s [%s]", service.Name, message, reason)
return message
}
// isNotificationEnabled checks if notifications should be sent
func (uns *UptimeNotificationService) isNotificationEnabled(service UptimeService) bool {
// Check notification_status boolean field
if !service.NotificationStatus {
// log.Printf("📵 Service %s notification_status: false", service.Name)
return false
}
// Check alerts field for muted status
if service.Alerts == "muted" {
// log.Printf("🔇 Service %s alerts muted", service.Name)
return false
}
// Check if notification_id is configured
if service.NotificationID == "" {
// log.Printf("📵 Service %s has no notification_id", service.Name)
return false
}
// log.Printf("✅ Service %s notifications enabled (ID: %s)", service.Name, service.NotificationID)
return true
}
@@ -0,0 +1,73 @@
package uptimemonitoring
import (
"sync"
"time"
)
// ServiceStatusTracker tracks service status changes and notification timing
type ServiceStatusTracker struct {
lastStatus map[string]string
lastNotificationTime map[string]time.Time
mu sync.RWMutex
}
// NewServiceStatusTracker creates a new service status tracker
func NewServiceStatusTracker() *ServiceStatusTracker {
return &ServiceStatusTracker{
lastStatus: make(map[string]string),
lastNotificationTime: make(map[string]time.Time),
}
}
// GetLastStatus returns the last known status for a service
func (sst *ServiceStatusTracker) GetLastStatus(serviceID string) string {
sst.mu.RLock()
defer sst.mu.RUnlock()
status := sst.lastStatus[serviceID]
// log.Printf("🔍 GetLastStatus for %s: %s", serviceID, status)
return status
}
// UpdateStatus updates the last known status for a service
func (sst *ServiceStatusTracker) UpdateStatus(serviceID, status string) {
sst.mu.Lock()
defer sst.mu.Unlock()
oldStatus := sst.lastStatus[serviceID]
sst.lastStatus[serviceID] = status
// log.Printf("📝 UpdateStatus for %s: %s -> %s", serviceID, oldStatus, status)
_ = oldStatus
}
// GetLastNotificationTime returns the last time a notification was sent for a service
func (sst *ServiceStatusTracker) GetLastNotificationTime(serviceID string) time.Time {
sst.mu.RLock()
defer sst.mu.RUnlock()
lastTime := sst.lastNotificationTime[serviceID]
// log.Printf("⏰ GetLastNotificationTime for %s: %v", serviceID, lastTime)
return lastTime
}
// SetLastNotificationTime sets the last notification time for a service
func (sst *ServiceStatusTracker) SetLastNotificationTime(serviceID string, t time.Time) {
sst.mu.Lock()
defer sst.mu.Unlock()
sst.lastNotificationTime[serviceID] = t
// log.Printf("⏰ SetLastNotificationTime for %s: %v", serviceID, t)
}
// ShouldNotify determines if a notification should be sent (legacy method for compatibility)
func (sst *ServiceStatusTracker) ShouldNotify(serviceID, currentStatus string) bool {
sst.mu.RLock()
lastStatus := sst.lastStatus[serviceID]
sst.mu.RUnlock()
shouldNotify := lastStatus == "" || lastStatus != currentStatus
// log.Printf("🤔 ShouldNotify for %s: lastStatus='%s', currentStatus='%s', result=%t", serviceID, lastStatus, currentStatus, shouldNotify)
// Send notification if status changed or if it's the first check
return shouldNotify
}
@@ -0,0 +1,41 @@
package uptimemonitoring
// UptimeService represents a service from PocketBase
type UptimeService struct {
ID string `json:"id"`
Name string `json:"name"`
Host string `json:"host"`
Uptime int `json:"uptime"`
ResponseTime int64 `json:"response_time"`
LastChecked string `json:"last_checked"`
Port int `json:"port"`
Domain string `json:"domain"`
HeartbeatInterval int `json:"heartbeat_interval"`
MaxRetries int `json:"max_retries"`
NotificationID string `json:"notification_id"`
TemplateID string `json:"template_id"`
ServiceType string `json:"service_type"`
Status string `json:"status"`
URL string `json:"url"`
Alerts string `json:"alerts"`
StatusCodes string `json:"status_codes"`
Keyword string `json:"keyword"`
RegionName string `json:"region_name"`
AgentID string `json:"agent_id"`
RegionalStatus string `json:"regional_status"`
NotificationStatus bool `json:"notification_status"` // Changed from string to bool
GroupID string `json:"group_id"`
WebhookID string `json:"webhook_id"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// UptimeServicesResponse represents the response from PocketBase services API
type UptimeServicesResponse struct {
Page int `json:"page"`
PerPage int `json:"perPage"`
TotalItems int `json:"totalItems"`
TotalPages int `json:"totalPages"`
Items []UptimeService `json:"items"`
}