Implement CheckCle Microservice Operation
A Go-based microservice for service operations including ICMP ping, DNS resolution, and TCP connectivity
This commit is contained in:
Tola Leng
@@ -0,0 +1,122 @@
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"service-operation/operations"
|
||||
"service-operation/pocketbase"
|
||||
"service-operation/shared/savers"
|
||||
"service-operation/types"
|
||||
)
|
||||
|
||||
func (ms *MonitoringService) performCheck(service pocketbase.Service) {
|
||||
// First, fetch the latest service status from PocketBase to ensure we have current data
|
||||
latestService, err := ms.pbClient.GetService(service.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed to fetch latest service status for %s: %v", service.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Respect the service status - don't check paused services
|
||||
if latestService.Status == "paused" {
|
||||
return // Silently skip paused services
|
||||
}
|
||||
|
||||
timeout := 10 * time.Second // Default timeout
|
||||
var result *types.OperationResult
|
||||
|
||||
serviceType := strings.ToLower(latestService.ServiceType)
|
||||
|
||||
// Single log message for check start
|
||||
//log.Printf("Checking %s (%s)", latestService.Name, serviceType)
|
||||
|
||||
switch serviceType {
|
||||
case "ping", "icmp":
|
||||
pingOp := operations.NewPingOperation(timeout)
|
||||
host := latestService.Host
|
||||
if host == "" {
|
||||
host = latestService.URL
|
||||
}
|
||||
result, err = pingOp.Execute(host, 1) // Single ping for monitoring
|
||||
|
||||
case "dns":
|
||||
dnsOp := operations.NewDNSOperation(timeout)
|
||||
host := latestService.Host
|
||||
if host == "" {
|
||||
host = latestService.Domain
|
||||
}
|
||||
// Default to A record, but could be made configurable
|
||||
queryType := "A"
|
||||
result, err = dnsOp.Execute(host, queryType)
|
||||
|
||||
case "tcp":
|
||||
tcpOp := operations.NewTCPOperation(timeout)
|
||||
host := latestService.Host
|
||||
if host == "" {
|
||||
host = latestService.URL
|
||||
}
|
||||
port := latestService.Port
|
||||
if port <= 0 {
|
||||
port = 80 // Default port
|
||||
}
|
||||
result, err = tcpOp.Execute(host, port)
|
||||
|
||||
case "http", "https":
|
||||
httpOp := operations.NewHTTPOperation(timeout)
|
||||
url := latestService.URL
|
||||
if url == "" {
|
||||
url = latestService.Host
|
||||
}
|
||||
result, err = httpOp.Execute(url, "GET")
|
||||
|
||||
default:
|
||||
log.Printf("Unknown service type: %s for service %s", latestService.ServiceType, latestService.Name)
|
||||
return
|
||||
}
|
||||
|
||||
// Determine status based on result
|
||||
status := "down"
|
||||
errorMessage := ""
|
||||
responseTime := int64(0)
|
||||
|
||||
if err != nil {
|
||||
errorMessage = err.Error()
|
||||
log.Printf("❌ %s failed: %v", latestService.Name, err)
|
||||
} else if result != nil {
|
||||
responseTime = result.ResponseTime.Milliseconds()
|
||||
if result.Success {
|
||||
status = "up"
|
||||
//log.Printf("✅ %s: %.0fms", latestService.Name, float64(responseTime))
|
||||
} else {
|
||||
status = "down"
|
||||
errorMessage = result.Error
|
||||
log.Printf("❌ %s failed: %s", latestService.Name, errorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
// Only update service status if the service is not paused
|
||||
// Check one more time before updating to prevent race conditions
|
||||
currentService, err := ms.pbClient.GetService(latestService.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed to verify service status before update for %s: %v", latestService.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
if currentService.Status == "paused" {
|
||||
return // Silently skip status update for paused services
|
||||
}
|
||||
|
||||
// Update service status in PocketBase only if not paused
|
||||
if err := ms.pbClient.UpdateServiceStatus(latestService.ID, status, responseTime, errorMessage); err != nil {
|
||||
log.Printf("Failed to update service status for %s: %v", latestService.Name, err)
|
||||
}
|
||||
|
||||
// Save metrics data in ONE place to prevent duplicates
|
||||
if result != nil {
|
||||
metricsSaver := savers.NewMetricsSaver(ms.pbClient)
|
||||
metricsSaver.SaveMetricsForService(*latestService, result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"service-operation/pocketbase"
|
||||
)
|
||||
|
||||
// Simplified metrics saver for basic service status updates only
|
||||
func (ms *MonitoringService) saveMetrics(service pocketbase.Service, status string, responseTime int64, errorMessage string) {
|
||||
// Save general metrics using the new structure
|
||||
metrics := pocketbase.MetricsRecord{
|
||||
ServiceName: service.Name,
|
||||
Host: service.Host,
|
||||
Uptime: 0, // This would need to be calculated based on your requirements
|
||||
ResponseTime: responseTime,
|
||||
LastChecked: time.Now().Format(time.RFC3339),
|
||||
Port: service.Port,
|
||||
ServiceType: service.ServiceType,
|
||||
Status: status,
|
||||
ErrorMessage: errorMessage,
|
||||
CheckedAt: time.Now().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
if err := ms.pbClient.SaveMetrics(metrics); err != nil {
|
||||
log.Printf("Failed to save metrics for %s: %v", service.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// This method is now removed to prevent duplicate saves
|
||||
// The shared savers handle everything in SaveMetricsForService
|
||||
@@ -0,0 +1,52 @@
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"service-operation/pocketbase"
|
||||
)
|
||||
|
||||
type ServiceMonitor struct {
|
||||
service pocketbase.Service
|
||||
ticker *time.Ticker
|
||||
stopChan chan bool
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) startMonitor(service pocketbase.Service) {
|
||||
if service.HeartbeatInterval <= 0 {
|
||||
service.HeartbeatInterval = 60 // Default to 60 seconds
|
||||
}
|
||||
|
||||
monitor := &ServiceMonitor{
|
||||
service: service,
|
||||
ticker: time.NewTicker(time.Duration(service.HeartbeatInterval) * time.Second),
|
||||
stopChan: make(chan bool),
|
||||
}
|
||||
|
||||
ms.activeServices[service.ID] = monitor
|
||||
|
||||
log.Printf("Starting monitor for service: %s (%s)", service.Name, service.ServiceType)
|
||||
|
||||
go func() {
|
||||
// Perform initial check
|
||||
ms.performCheck(service)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-monitor.ticker.C:
|
||||
ms.performCheck(service)
|
||||
case <-monitor.stopChan:
|
||||
monitor.ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) stopMonitor(serviceID string, monitor *ServiceMonitor) {
|
||||
log.Printf("Stopping monitor for service: %s", serviceID)
|
||||
monitor.stopChan <- true
|
||||
delete(ms.activeServices, serviceID)
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"service-operation/pocketbase"
|
||||
)
|
||||
|
||||
type MonitoringService struct {
|
||||
pbClient *pocketbase.PocketBaseClient
|
||||
activeServices map[string]*ServiceMonitor
|
||||
mu sync.RWMutex
|
||||
stopChan chan bool
|
||||
isRunning bool
|
||||
}
|
||||
|
||||
func NewMonitoringService(pbClient *pocketbase.PocketBaseClient) *MonitoringService {
|
||||
return &MonitoringService{
|
||||
pbClient: pbClient,
|
||||
activeServices: make(map[string]*ServiceMonitor),
|
||||
stopChan: make(chan bool),
|
||||
isRunning: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) Start() {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
|
||||
if ms.isRunning {
|
||||
log.Println("Monitoring service is already running")
|
||||
return
|
||||
}
|
||||
|
||||
ms.isRunning = true
|
||||
log.Println("Starting monitoring service...")
|
||||
|
||||
// Start monitoring all services from PocketBase
|
||||
go ms.monitoringLoop()
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) Stop() {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
|
||||
if !ms.isRunning {
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Stopping monitoring service...")
|
||||
ms.isRunning = false
|
||||
|
||||
// Stop all active monitors
|
||||
for serviceID, monitor := range ms.activeServices {
|
||||
ms.stopMonitor(serviceID, monitor)
|
||||
}
|
||||
|
||||
ms.stopChan <- true
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) monitoringLoop() {
|
||||
ticker := time.NewTicker(30 * time.Second) // Check for new services every 30 seconds
|
||||
defer ticker.Stop()
|
||||
|
||||
// Initial load of services
|
||||
ms.loadAndStartServices()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
ms.loadAndStartServices()
|
||||
case <-ms.stopChan:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *MonitoringService) loadAndStartServices() {
|
||||
// Only get services that are NOT paused
|
||||
services, err := ms.pbClient.GetActiveServices()
|
||||
if err != nil {
|
||||
log.Printf("Failed to load services: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
|
||||
// Filter out paused services and start monitoring for active ones
|
||||
activeServiceIDs := make(map[string]bool)
|
||||
for _, service := range services {
|
||||
if service.Status != "paused" {
|
||||
activeServiceIDs[service.ID] = true
|
||||
|
||||
// Start monitoring if not already active
|
||||
if _, exists := ms.activeServices[service.ID]; !exists {
|
||||
ms.startMonitor(service)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop monitoring for paused or removed services
|
||||
for serviceID, monitor := range ms.activeServices {
|
||||
if !activeServiceIDs[serviceID] {
|
||||
log.Printf("Stopping monitoring for service %s (paused or removed)", serviceID)
|
||||
ms.stopMonitor(serviceID, monitor)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
|
||||
package monitoring
|
||||
|
||||
// This file previously contained saveDetailedDataByType which was causing duplicate saves
|
||||
// All detailed data saving is now handled by the shared savers in a single place
|
||||
// to prevent duplicate records in PocketBase
|
||||
|
||||
// The monitoring service now uses only:
|
||||
// 1. SaveMetricsForService from shared/savers for complete data saving
|
||||
// 2. Direct service status updates for the services table
|
||||
Reference in New Issue
Block a user