feat: Implement comprehensive multi-channel notification system with templating and SSL monitoring

- Add robust notification system supporting 7+ communication channels with intelligent message templating, resource monitoring, and SSL certificate alerts. Notification Channels: (Telegram, Discord, Slack, Signal, Email, Google Chat, Webhooks).

- This notification system provides enterprise-grade alerting capabilities with extensive customization options and multi-channel redundancy for critical service monitoring.
This commit is contained in:
Tola Leng
2025-08-14 20:50:31 +07:00
parent 58242c33f8
commit 933e70a3f6
40 changed files with 7159 additions and 97 deletions
@@ -0,0 +1,98 @@
package sslmonitoring
import (
"time"
"service-operation/pocketbase"
)
// SSLMonitor monitors SSL certificates and sends notifications
type SSLMonitor struct {
client *SSLClient
notificationService *SSLNotificationService
checkInterval time.Duration
stopChan chan bool
}
// NewSSLMonitor creates a new SSL monitor
func NewSSLMonitor(pbClient *pocketbase.PocketBaseClient) *SSLMonitor {
// Initialize SSL client
sslClient := NewSSLClient(pbClient)
// Initialize notification service
notificationService := NewSSLNotificationService(sslClient, pbClient)
// Set check interval to 30 seconds to match other services
checkInterval := 30 * time.Second
return &SSLMonitor{
client: sslClient,
notificationService: notificationService,
checkInterval: checkInterval,
stopChan: make(chan bool, 1),
}
}
// Start begins monitoring SSL certificates
func (sm *SSLMonitor) Start() {
// log.Printf("🚀 [SSL-MONITOR] Starting with check interval: %v", sm.checkInterval)
// log.Printf("⏰ [SSL-MONITOR] Startup grace period: 2 minutes (no notifications for valid certificates)")
// Run initial check
sm.checkCertificates()
// Set up periodic checking every 30 seconds
ticker := time.NewTicker(sm.checkInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
sm.checkCertificates()
case <-sm.stopChan:
// log.Println("🛑 [SSL-MONITOR] Monitoring stopped")
return
}
}
}
// Stop gracefully stops the SSL monitoring
func (sm *SSLMonitor) Stop() {
// log.Printf("🛑 [SSL-MONITOR] Stopping SSL monitoring...")
select {
case sm.stopChan <- true:
default:
}
}
// checkCertificates fetches all SSL certificates and checks their notification requirements
func (sm *SSLMonitor) checkCertificates() {
// log.Printf("🔍 [SSL-CHECK] Checking SSL certificates for expiration alerts...")
certificates, err := sm.client.GetSSLCertificates()
if err != nil {
// log.Printf("❌ [SSL-ERROR] Failed to fetch SSL certificates: %v", err)
return
}
// log.Printf("📊 [SSL-CERTIFICATES] Found %d certificates to check", len(certificates))
successCount := 0
errorCount := 0
for _, cert := range certificates {
// Check each certificate for notification requirements
if err := sm.notificationService.CheckAndNotifySSLCertificate(cert); err != nil {
// log.Printf("❌ [SSL-FAILED] Failed to process certificate %s: %v", cert.Domain, err)
errorCount++
} else {
successCount++
}
}
// log.Printf("✅ [SSL-SUMMARY] Processed %d certificates (success: %d, errors: %d)",
// len(certificates), successCount, errorCount)
_ = successCount // Prevent unused variable warning
_ = errorCount // Prevent unused variable warning
}
@@ -0,0 +1,192 @@
package sslmonitoring
import (
"fmt"
"strconv"
"time"
"service-operation/notification"
"service-operation/pocketbase"
)
// SSLNotificationService handles SSL certificate notifications
type SSLNotificationService struct {
client *SSLClient
pbClient *pocketbase.PocketBaseClient
notificationManager *notification.NotificationManager
statusTracker *SSLStatusTracker
}
// NewSSLNotificationService creates a new SSL notification service
func NewSSLNotificationService(client *SSLClient, pbClient *pocketbase.PocketBaseClient) *SSLNotificationService {
return &SSLNotificationService{
client: client,
pbClient: pbClient,
notificationManager: notification.NewNotificationManager(pbClient),
statusTracker: NewSSLStatusTracker(pbClient), // Pass pbClient to status tracker
}
}
// CheckAndNotifySSLCertificate checks SSL certificate and sends notification if needed
func (sns *SSLNotificationService) CheckAndNotifySSLCertificate(cert SSLCertificate) error {
// Always recalculate days left from the actual expiry date to ensure accuracy
actualDaysLeft := sns.calculateDaysLeft(cert.ValidTill)
// Use the calculated value as it's more accurate
cert.DaysLeft = actualDaysLeft
// Determine current status based on thresholds with calculated days left
currentStatus := sns.determineSSLStatus(cert.DaysLeft, cert.WarningThreshold, cert.ExpiryThreshold)
// Check if notification should be sent
if !sns.shouldSendNotification(cert.ID, currentStatus) {
return nil
}
// Skip if no notification configuration
if cert.NotificationID == "" {
return nil
}
// Send notification
err := sns.sendSSLNotification(cert, currentStatus)
if err != nil {
return err
}
// Update tracking in database
sns.statusTracker.UpdateStatus(cert.ID, currentStatus)
sns.statusTracker.SetLastNotificationTime(cert.ID, time.Now())
return nil
}
// calculateDaysLeft calculates days remaining until expiration with better error handling
func (sns *SSLNotificationService) calculateDaysLeft(validTill string) int {
// Try multiple date formats to parse the expiry date
var expiryTime time.Time
var err error
// Common date formats - updated to handle PocketBase format correctly
formats := []string{
"2006-01-02 15:04:05.000Z", // PocketBase format with space (most common)
"2006-01-02T15:04:05.000Z", // ISO 8601 with milliseconds
time.RFC3339, // Standard RFC3339
time.RFC3339Nano, // RFC3339 with nanoseconds
"2006-01-02 15:04:05Z", // Without milliseconds
"2006-01-02T15:04:05Z", // ISO without milliseconds
"2006-01-02 15:04:05.999Z", // Alternative milliseconds format
"2006-01-02T15:04:05.999Z", // Alternative ISO milliseconds
"2006-01-02 15:04:05.000000Z", // Microseconds format
"2006-01-02T15:04:05.000000Z", // ISO microseconds format
"2006-01-02 15:04:05", // Simple format without timezone
"2006-01-02", // Date only
}
for _, format := range formats {
expiryTime, err = time.Parse(format, validTill)
if err == nil {
break
}
}
if err != nil {
return 0
}
now := time.Now()
duration := expiryTime.Sub(now)
daysLeft := int(duration.Hours() / 24)
// If the result is negative, the certificate is expired
if daysLeft < 0 {
daysLeft = 0
}
return daysLeft
}
// sendSSLNotification sends the actual notification
func (sns *SSLNotificationService) sendSSLNotification(cert SSLCertificate, status string) error {
// Determine the correct issuer value - prioritize IssuerCN over IssuerO
issuerValue := ""
if cert.IssuerCN != "" {
issuerValue = cert.IssuerCN
} else if cert.IssuerO != "" {
issuerValue = cert.IssuerO
} else {
issuerValue = "Unknown"
}
// Create notification payload with SSL-specific data using calculated values
payload := &notification.NotificationPayload{
ServiceName: fmt.Sprintf("SSL Certificate - %s", cert.Domain),
Status: status,
Host: cert.Domain,
Domain: cert.Domain,
ServiceType: "ssl",
Timestamp: time.Now(),
Message: sns.generateStatusMessage(cert, status),
// SSL-specific fields with calculated/corrected values
CertificateName: cert.Domain,
ExpiryDate: cert.ValidTill,
DaysLeft: strconv.Itoa(cert.DaysLeft), // Use the calculated value
IssuerCN: issuerValue,
SerialNumber: cert.SerialNumber,
}
// Send SSL notification using the manager
return sns.notificationManager.SendSSLNotification(payload, cert.NotificationID, cert.TemplateID)
}
// generateStatusMessage creates appropriate message based on certificate status using calculated days
func (sns *SSLNotificationService) generateStatusMessage(cert SSLCertificate, status string) string {
switch status {
case "expired":
return fmt.Sprintf("SSL certificate for %s expired on %s", cert.Domain, cert.ValidTill)
case "expiring_soon":
return fmt.Sprintf("SSL certificate for %s expires in %d days on %s", cert.Domain, cert.DaysLeft, cert.ValidTill)
case "warning":
return fmt.Sprintf("SSL certificate for %s expires in %d days", cert.Domain, cert.DaysLeft)
default:
return fmt.Sprintf("SSL certificate for %s is valid (%d days remaining)", cert.Domain, cert.DaysLeft)
}
}
// shouldSendNotification determines if notification should be sent with enhanced logic
func (sns *SSLNotificationService) shouldSendNotification(certID, currentStatus string) bool {
lastStatus := sns.statusTracker.GetLastStatus(certID)
lastNotified := sns.statusTracker.GetLastNotificationTime(certID)
// Send if status changed or first check
if lastStatus == "" || lastStatus != currentStatus {
return true
}
// For critical statuses (expired/expiring_soon), resend after 24 hours
if (currentStatus == "expired" || currentStatus == "expiring_soon") &&
!lastNotified.IsZero() && time.Since(lastNotified) > 24*time.Hour {
return true
}
// For warning status, resend after 7 days to avoid spam
if currentStatus == "warning" &&
!lastNotified.IsZero() && time.Since(lastNotified) > 7*24*time.Hour {
return true
}
return false
}
// determineSSLStatus determines SSL certificate status based on days left and thresholds
func (sns *SSLNotificationService) determineSSLStatus(daysLeft, warningThreshold, expiryThreshold int) string {
if daysLeft <= 0 {
return "expired"
} else if daysLeft <= expiryThreshold {
return "expiring_soon"
} else if daysLeft <= warningThreshold {
return "warning"
}
return "valid"
}
@@ -0,0 +1,78 @@
package sslmonitoring
import (
"encoding/json"
"fmt"
"net/http"
"service-operation/pocketbase"
)
// SSLClient handles SSL certificate data operations
type SSLClient struct {
pbClient *pocketbase.PocketBaseClient
}
// NewSSLClient creates a new SSL client
func NewSSLClient(pbClient *pocketbase.PocketBaseClient) *SSLClient {
return &SSLClient{
pbClient: pbClient,
}
}
// GetSSLCertificates fetches all SSL certificates from PocketBase
func (sc *SSLClient) GetSSLCertificates() ([]SSLCertificate, error) {
url := fmt.Sprintf("%s/api/collections/ssl_certificates/records", sc.pbClient.GetBaseURL())
// log.Printf("🔍 [SSL-CLIENT] Fetching SSL certificates from: %s", url)
resp, err := http.Get(url)
if err != nil {
// log.Printf("❌ [SSL-CLIENT] HTTP error: %v", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// log.Printf("❌ [SSL-CLIENT] Failed to fetch SSL certificates, status: %d", resp.StatusCode)
return nil, fmt.Errorf("failed to fetch SSL certificates, status: %d", resp.StatusCode)
}
var response struct {
Items []SSLCertificate `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
// log.Printf("❌ [SSL-CLIENT] Error decoding response: %v", err)
return nil, err
}
// log.Printf("✅ [SSL-CLIENT] Successfully fetched %d SSL certificates", len(response.Items))
return response.Items, nil
}
// UpdateSSLCertificate updates SSL certificate status and notification time
func (sc *SSLClient) UpdateSSLCertificate(certID string, data map[string]interface{}) error {
url := fmt.Sprintf("%s/api/collections/ssl_certificates/records/%s", sc.pbClient.GetBaseURL(), certID)
req, err := http.NewRequest(http.MethodPatch, url, nil)
if err != nil {
return fmt.Errorf("failed to create SSL certificate update request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to update SSL certificate: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to update SSL certificate, status: %d", resp.StatusCode)
}
// log.Printf("✅ [SSL-CLIENT] Successfully updated SSL certificate: %s", certID)
return nil
}
@@ -0,0 +1,111 @@
package sslmonitoring
import (
"sync"
"time"
"service-operation/pocketbase"
)
// SSLStatusTracker tracks SSL certificate status changes and notification timing using database persistence
type SSLStatusTracker struct {
pbClient *pocketbase.PocketBaseClient
mu sync.RWMutex
}
// NewSSLStatusTracker creates a new SSL status tracker with database persistence
func NewSSLStatusTracker(pbClient *pocketbase.PocketBaseClient) *SSLStatusTracker {
return &SSLStatusTracker{
pbClient: pbClient,
}
}
// GetLastStatus returns the last known status for an SSL certificate from database
func (sst *SSLStatusTracker) GetLastStatus(certID string) string {
sst.mu.RLock()
defer sst.mu.RUnlock()
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("🔍 [SSL-TRACKER] Error getting last status for %s: %v", certID, err)
return ""
}
status := cert.Status
// log.Printf("🔍 [SSL-TRACKER] GetLastStatus for %s: %s", certID, status)
return status
}
// UpdateStatus updates the last known status for an SSL certificate in database
func (sst *SSLStatusTracker) UpdateStatus(certID, status string) {
sst.mu.Lock()
defer sst.mu.Unlock()
// Get current certificate to preserve other fields
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("📝 [SSL-TRACKER] Error getting certificate for status update %s: %v", certID, err)
return
}
oldStatus := cert.Status
// Update only the status field
updateData := map[string]interface{}{
"status": status,
}
err = sst.pbClient.UpdateSSLCertificate(certID, updateData)
if err != nil {
// log.Printf("📝 [SSL-TRACKER] Error updating status for %s: %v", certID, err)
return
}
// log.Printf("📝 [SSL-TRACKER] UpdateStatus for %s: %s -> %s", certID, oldStatus, status)
_ = oldStatus // Prevent unused variable warning
}
// GetLastNotificationTime returns the last time a notification was sent for an SSL certificate from database
func (sst *SSLStatusTracker) GetLastNotificationTime(certID string) time.Time {
sst.mu.RLock()
defer sst.mu.RUnlock()
cert, err := sst.pbClient.GetSSLCertificateByID(certID)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error getting last notification time for %s: %v", certID, err)
return time.Time{}
}
if cert.LastNotified == "" {
// log.Printf("⏰ [SSL-TRACKER] GetLastNotificationTime for %s: never notified", certID)
return time.Time{}
}
lastTime, err := time.Parse(time.RFC3339, cert.LastNotified)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error parsing last notification time for %s: %v", certID, err)
return time.Time{}
}
// log.Printf("⏰ [SSL-TRACKER] GetLastNotificationTime for %s: %v", certID, lastTime)
return lastTime
}
// SetLastNotificationTime sets the last notification time for an SSL certificate in database
func (sst *SSLStatusTracker) SetLastNotificationTime(certID string, t time.Time) {
sst.mu.Lock()
defer sst.mu.Unlock()
updateData := map[string]interface{}{
"last_notified": t.Format(time.RFC3339),
}
err := sst.pbClient.UpdateSSLCertificate(certID, updateData)
if err != nil {
// log.Printf("⏰ [SSL-TRACKER] Error setting last notification time for %s: %v", certID, err)
return
}
// log.Printf("⏰ [SSL-TRACKER] SetLastNotificationTime for %s: %v", certID, t)
}
@@ -0,0 +1,110 @@
package sslmonitoring
import (
"encoding/json"
"strconv"
"time"
)
type SSLCertificate struct {
ID string `json:"id"`
CollectionID string `json:"collectionId"`
CollectionName string `json:"collectionName"`
Domain string `json:"domain"`
IssuerO string `json:"issuer_o"`
Status string `json:"status"`
LastNotified string `json:"last_notified"`
WarningThreshold int `json:"warning_threshold"`
ExpiryThreshold int `json:"expiry_threshold"`
NotificationChannel string `json:"notification_channel"`
NotificationID string `json:"notification_id"`
TemplateID string `json:"template_id"`
ValidFrom string `json:"valid_from"`
SerialNumber string `json:"serial_number"`
IssuedTo string `json:"issued_to"`
ValidTill string `json:"valid_till"`
ValidityDays int `json:"validity_days"`
DaysLeft int `json:"days_left"`
ValidDaysToExpire int `json:"valid_days_to_expire"`
ResolvedIP string `json:"resolved_ip"`
IssuerCN string `json:"issuer_cn"`
CertAlg string `json:"cert_alg"`
CertSans string `json:"cert_sans"`
CheckInterval int `json:"check_interval"`
CheckAt string `json:"check_at"`
Created string `json:"created"`
Updated string `json:"updated"`
}
// Custom unmarshaler to handle check_interval as both string and int, and serial_number as string
func (s *SSLCertificate) UnmarshalJSON(data []byte) error {
// Create a temporary struct with flexible types
type Alias SSLCertificate
aux := &struct {
CheckInterval interface{} `json:"check_interval"`
SerialNumber interface{} `json:"serial_number"`
*Alias
}{
Alias: (*Alias)(s),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
// Handle check_interval conversion
switch v := aux.CheckInterval.(type) {
case string:
if v == "" {
s.CheckInterval = 1440 // Default 24 hours in minutes
} else {
interval, err := strconv.Atoi(v)
if err != nil {
s.CheckInterval = 1440 // Default on error
} else {
s.CheckInterval = interval
}
}
case float64:
s.CheckInterval = int(v)
case int:
s.CheckInterval = v
default:
s.CheckInterval = 1440 // Default 24 hours in minutes
}
// Handle serial_number conversion to string
switch v := aux.SerialNumber.(type) {
case string:
s.SerialNumber = v
case float64:
// Handle scientific notation by converting to string
s.SerialNumber = strconv.FormatFloat(v, 'f', 0, 64)
case int64:
s.SerialNumber = strconv.FormatInt(v, 10)
case int:
s.SerialNumber = strconv.Itoa(v)
default:
s.SerialNumber = ""
}
return nil
}
type SSLCheckResult struct {
Domain string `json:"domain"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
ValidFrom time.Time `json:"valid_from"`
ValidTill time.Time `json:"valid_till"`
DaysLeft int `json:"days_left"`
Issuer string `json:"issuer"`
Subject string `json:"subject"`
SerialNumber string `json:"serial_number"`
Algorithm string `json:"algorithm"`
SANs []string `json:"sans"`
ResponseTime time.Duration `json:"response_time"`
ResolvedIP string `json:"resolved_ip"`
CheckedAt time.Time `json:"checked_at"`
}