Refactor: Integrate regional agents in service-operation

Integrate distributed monitoring agents for HTTP, PING, TCP, and DNS checks
This commit is contained in:
Tola Leng
2025-06-21 17:48:55 +07:00
parent ea95490f13
commit 57a65106cb
16 changed files with 422 additions and 40 deletions
@@ -114,9 +114,10 @@ func (ms *MonitoringService) performCheck(service pocketbase.Service) {
log.Printf("Failed to update service status for %s: %v", latestService.Name, err)
}
// Save metrics data in ONE place to prevent duplicates
// Save metrics data with regional information
if result != nil {
metricsSaver := savers.NewMetricsSaver(ms.pbClient)
regionName, agentID := ms.GetRegionalInfo()
metricsSaver := savers.NewMetricsSaverWithRegion(ms.pbClient, regionName, agentID)
metricsSaver.SaveMetricsForService(*latestService, result)
}
}
@@ -0,0 +1,114 @@
package monitoring
import (
"log"
"time"
"service-operation/pocketbase"
)
type RegionalMonitor struct {
pbClient *pocketbase.PocketBaseClient
regionalService *pocketbase.RegionalService
isOnline bool
ticker *time.Ticker
stopChan chan bool
}
func NewRegionalMonitor(pbClient *pocketbase.PocketBaseClient) *RegionalMonitor {
return &RegionalMonitor{
pbClient: pbClient,
isOnline: false,
stopChan: make(chan bool),
}
}
func (rm *RegionalMonitor) Start() {
// Get or create the "Default" regional service
service, err := rm.pbClient.GetDefaultRegionalService()
if err != nil {
log.Printf("Warning: Could not get default regional service: %v", err)
log.Printf("Regional monitoring will continue with fallback values")
// Continue with fallback service
service = &pocketbase.RegionalService{
ID: "default",
RegionName: "Default",
Status: "active",
AgentID: "1",
AgentIPAddress: "127.0.0.1",
Connection: "offline",
Token: "default-token",
}
}
rm.regionalService = service
rm.ticker = time.NewTicker(30 * time.Second) // Check connection every 30 seconds
// Initial connection status update
rm.updateConnectionStatus("online")
rm.isOnline = true
log.Printf("Regional monitor started for region: %s (Agent ID: %s)",
service.RegionName, service.AgentID)
go func() {
for {
select {
case <-rm.ticker.C:
rm.checkConnection()
case <-rm.stopChan:
rm.ticker.Stop()
return
}
}
}()
}
func (rm *RegionalMonitor) Stop() {
if rm.regionalService != nil {
rm.updateConnectionStatus("offline")
log.Printf("Regional monitor stopped for region: %s", rm.regionalService.RegionName)
}
if rm.ticker != nil {
rm.ticker.Stop()
}
rm.stopChan <- true
}
func (rm *RegionalMonitor) checkConnection() {
// Test PocketBase connection to determine if we're online
err := rm.pbClient.TestConnection()
if err != nil && rm.isOnline {
// We were online but now we're offline
rm.updateConnectionStatus("offline")
rm.isOnline = false
log.Printf("Regional agent went offline: %v", err)
} else if err == nil && !rm.isOnline {
// We were offline but now we're online
rm.updateConnectionStatus("online")
rm.isOnline = true
log.Printf("Regional agent back online")
}
}
func (rm *RegionalMonitor) updateConnectionStatus(status string) {
if rm.regionalService == nil {
return
}
if err := rm.pbClient.UpdateRegionalServiceConnection(rm.regionalService.ID, status); err != nil {
// Don't log errors for update failures - might be due to collection access issues
// log.Printf("Failed to update regional service connection status: %v", err)
}
}
func (rm *RegionalMonitor) GetRegionalInfo() (string, string) {
if rm.regionalService == nil {
return "Default", "1" // Fallback values
}
return rm.regionalService.RegionName, rm.regionalService.AgentID
}
+21 -10
View File
@@ -1,4 +1,3 @@
package monitoring
import (
@@ -10,19 +9,21 @@ import (
)
type MonitoringService struct {
pbClient *pocketbase.PocketBaseClient
activeServices map[string]*ServiceMonitor
mu sync.RWMutex
stopChan chan bool
isRunning bool
pbClient *pocketbase.PocketBaseClient
activeServices map[string]*ServiceMonitor
regionalMonitor *RegionalMonitor
mu sync.RWMutex
stopChan chan bool
isRunning bool
}
func NewMonitoringService(pbClient *pocketbase.PocketBaseClient) *MonitoringService {
return &MonitoringService{
pbClient: pbClient,
activeServices: make(map[string]*ServiceMonitor),
stopChan: make(chan bool),
isRunning: false,
pbClient: pbClient,
activeServices: make(map[string]*ServiceMonitor),
regionalMonitor: NewRegionalMonitor(pbClient),
stopChan: make(chan bool),
isRunning: false,
}
}
@@ -38,6 +39,9 @@ func (ms *MonitoringService) Start() {
ms.isRunning = true
log.Println("Starting monitoring service...")
// Start regional monitoring
ms.regionalMonitor.Start()
// Start monitoring all services from PocketBase
go ms.monitoringLoop()
}
@@ -53,6 +57,9 @@ func (ms *MonitoringService) Stop() {
log.Println("Stopping monitoring service...")
ms.isRunning = false
// Stop regional monitoring
ms.regionalMonitor.Stop()
// Stop all active monitors
for serviceID, monitor := range ms.activeServices {
ms.stopMonitor(serviceID, monitor)
@@ -61,6 +68,10 @@ func (ms *MonitoringService) Stop() {
ms.stopChan <- true
}
func (ms *MonitoringService) GetRegionalInfo() (string, string) {
return ms.regionalMonitor.GetRegionalInfo()
}
func (ms *MonitoringService) monitoringLoop() {
ticker := time.NewTicker(30 * time.Second) // Check for new services every 30 seconds
defer ticker.Stop()
@@ -77,7 +77,7 @@ func (s *SSLMonitoringService) shouldCheckCertificate(cert types.SSLCertificate)
return true
} else {
//log.Printf("Certificate %s scheduled for later check (check_at: %s)",
//cert.Domain, checkAt.Format("2006-01-02 15:04:05"))
// cert.Domain, checkAt.Format("2006-01-02 15:04:05"))
//return false
}
} else {