feat: Phase 3 & 4 — monitoring, discovery, polish, deployment
Phase 3 — Discovery & Monitoring: - Network scanner: nmap wrapper + mock fallback, fingerprint service (35 signatures) - Status checker: ping/http/https/tcp/ssh/prometheus/health per-node checks - APScheduler: status checks every 60s, WebSocket broadcast - WebSocket /ws/status: live node status updates to frontend - Sidebar panels: Pending Devices, Hidden Devices, Scan History - Auth token persisted to localStorage (survive page refresh) - 24 new backend tests (scan flow + status_checker) Phase 4 — Polish & Deployment: - Auto-layout: Dagre hierarchical TB via Toolbar button - Export PNG: html-to-image download via Toolbar button - Scan config modal: CIDR ranges + check interval, GET/POST /api/v1/scan/config - Dockerfile.backend (Python 3.13 slim + nmap), Dockerfile.frontend (nginx) - docker-compose.yml with data volume and NET_RAW cap for ping - scripts/lxc-install.sh: Proxmox VE systemd bootstrap - README.md: quick-start, config reference, stack overview
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
"""Match nmap scan results against service_signatures.json."""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_SIGNATURES: list[dict] | None = None
|
||||
|
||||
|
||||
def _load() -> list[dict]:
|
||||
global _SIGNATURES
|
||||
if _SIGNATURES is None:
|
||||
path = Path(__file__).parent.parent.parent / "data" / "service_signatures.json"
|
||||
with open(path) as f:
|
||||
_SIGNATURES = json.load(f)
|
||||
return _SIGNATURES
|
||||
|
||||
|
||||
def match_port(port: int, protocol: str, banner: str | None = None) -> dict | None:
|
||||
"""Return the first signature matching port+protocol, optionally banner."""
|
||||
for sig in _load():
|
||||
if sig["port"] != port or sig["protocol"] != protocol:
|
||||
continue
|
||||
if sig.get("banner_regex") and banner and not re.search(sig["banner_regex"], banner, re.IGNORECASE):
|
||||
continue
|
||||
return sig
|
||||
return None
|
||||
|
||||
|
||||
def fingerprint_ports(open_ports: list[dict]) -> list[dict]:
|
||||
"""
|
||||
Given a list of {port, protocol, banner?} dicts, return matched services.
|
||||
Unknown ports are included as unknown_service.
|
||||
"""
|
||||
results = []
|
||||
for p in open_ports:
|
||||
sig = match_port(p["port"], p.get("protocol", "tcp"), p.get("banner"))
|
||||
if sig:
|
||||
results.append({
|
||||
"port": p["port"],
|
||||
"protocol": p.get("protocol", "tcp"),
|
||||
"service_name": sig["service_name"],
|
||||
"icon": sig.get("icon"),
|
||||
"category": sig.get("category"),
|
||||
})
|
||||
else:
|
||||
results.append({
|
||||
"port": p["port"],
|
||||
"protocol": p.get("protocol", "tcp"),
|
||||
"service_name": "unknown_service",
|
||||
"icon": None,
|
||||
"category": None,
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def suggest_node_type(open_ports: list[dict]) -> str:
|
||||
"""Suggest a node type based on the most specific matched signature."""
|
||||
priority = ["proxmox", "nas", "router", "lxc", "vm", "server", "ap", "iot", "switch"]
|
||||
found: set[str] = set()
|
||||
for p in open_ports:
|
||||
sig = match_port(p["port"], p.get("protocol", "tcp"))
|
||||
if sig and sig.get("suggested_node_type"):
|
||||
found.add(sig["suggested_node_type"])
|
||||
for t in priority:
|
||||
if t in found:
|
||||
return t
|
||||
return "generic"
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Network scanner: ARP sweep + nmap service detection."""
|
||||
import logging
|
||||
import socket
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import PendingDevice, ScanRun
|
||||
from app.services.fingerprint import fingerprint_ports, suggest_node_type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import nmap # type: ignore[import-untyped]
|
||||
_NMAP_AVAILABLE = True
|
||||
except ImportError:
|
||||
_NMAP_AVAILABLE = False
|
||||
logger.warning("python-nmap not available — scanner will run in mock mode")
|
||||
|
||||
|
||||
def _nmap_scan(target: str) -> list[dict]:
|
||||
"""Run nmap -sV --open on target, return list of host dicts."""
|
||||
if not _NMAP_AVAILABLE:
|
||||
return _mock_scan(target)
|
||||
|
||||
nm = nmap.PortScanner()
|
||||
try:
|
||||
nm.scan(hosts=target, arguments="-sV --open -T4 --host-timeout 30s")
|
||||
except Exception as exc:
|
||||
logger.error("nmap scan failed: %s", exc)
|
||||
return []
|
||||
|
||||
hosts = []
|
||||
for host in nm.all_hosts():
|
||||
if nm[host].state() != "up":
|
||||
continue
|
||||
open_ports = []
|
||||
for proto in nm[host].all_protocols():
|
||||
for port, info in nm[host][proto].items():
|
||||
if info["state"] == "open":
|
||||
open_ports.append({
|
||||
"port": port,
|
||||
"protocol": proto,
|
||||
"banner": info.get("product", "") + " " + info.get("version", ""),
|
||||
})
|
||||
hosts.append({
|
||||
"ip": host,
|
||||
"hostname": _resolve_hostname(host),
|
||||
"mac": nm[host].get("addresses", {}).get("mac"),
|
||||
"os": _extract_os(nm, host),
|
||||
"open_ports": open_ports,
|
||||
})
|
||||
return hosts
|
||||
|
||||
|
||||
def _resolve_hostname(ip: str) -> str | None:
|
||||
try:
|
||||
return socket.gethostbyaddr(ip)[0]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_os(nm: object, host: str) -> str | None:
|
||||
try:
|
||||
osmatch = nm[host].get("osmatch", []) # type: ignore[index]
|
||||
if osmatch:
|
||||
return osmatch[0]["name"]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _mock_scan(target: str) -> list[dict]:
|
||||
"""Return fake results for dev/test environments without nmap."""
|
||||
return [
|
||||
{
|
||||
"ip": "192.168.1.99",
|
||||
"hostname": "unknown-device.lan",
|
||||
"mac": "AA:BB:CC:DD:EE:FF",
|
||||
"os": None,
|
||||
"open_ports": [
|
||||
{"port": 80, "protocol": "tcp", "banner": "nginx"},
|
||||
{"port": 22, "protocol": "tcp", "banner": "OpenSSH 9.0"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:
|
||||
"""Execute scan for given CIDR ranges and populate pending_devices."""
|
||||
devices_found = 0
|
||||
try:
|
||||
for cidr in ranges:
|
||||
hosts = _nmap_scan(cidr)
|
||||
for host in hosts:
|
||||
services = fingerprint_ports(host["open_ports"])
|
||||
suggested_type = suggest_node_type(host["open_ports"])
|
||||
|
||||
# Skip if already pending or already a node (by IP)
|
||||
existing = await db.execute(
|
||||
__import__("sqlalchemy", fromlist=["select"]).select(PendingDevice).where(
|
||||
PendingDevice.ip == host["ip"],
|
||||
PendingDevice.status == "pending",
|
||||
)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
continue
|
||||
|
||||
device = PendingDevice(
|
||||
ip=host["ip"],
|
||||
mac=host.get("mac"),
|
||||
hostname=host.get("hostname"),
|
||||
os=host.get("os"),
|
||||
services=services,
|
||||
suggested_type=suggested_type,
|
||||
status="pending",
|
||||
)
|
||||
db.add(device)
|
||||
devices_found += 1
|
||||
|
||||
await db.commit()
|
||||
|
||||
# Update scan run
|
||||
run = await db.get(ScanRun, run_id)
|
||||
if run:
|
||||
run.status = "done"
|
||||
run.devices_found = devices_found
|
||||
run.finished_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Scan failed: %s", exc)
|
||||
run = await db.get(ScanRun, run_id)
|
||||
if run:
|
||||
run.status = "error"
|
||||
run.error = str(exc)
|
||||
run.finished_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Per-node status checks: ping, http, https, tcp, ssh, prometheus, health."""
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def check_node(check_method: str, target: str | None, ip: str | None) -> dict:
|
||||
"""
|
||||
Run the appropriate check and return {status, response_time_ms}.
|
||||
status is one of: online, offline, unknown.
|
||||
"""
|
||||
host = target or ip
|
||||
if not host:
|
||||
return {"status": "unknown", "response_time_ms": None}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
match check_method:
|
||||
case "ping":
|
||||
ok = await _ping(host)
|
||||
case "http":
|
||||
url = host if host.startswith("http") else f"http://{host}"
|
||||
ok = await _http_get(url)
|
||||
case "https":
|
||||
url = host if host.startswith("https") else f"https://{host}"
|
||||
ok = await _http_get(url, verify=True)
|
||||
case "tcp":
|
||||
host_part, _, port_str = host.rpartition(":")
|
||||
port = int(port_str) if port_str.isdigit() else 80
|
||||
ok = await _tcp_connect(host_part or host, port)
|
||||
case "ssh":
|
||||
ok = await _tcp_connect(host, 22)
|
||||
case "prometheus":
|
||||
url = host if host.startswith("http") else f"http://{host}/metrics"
|
||||
ok = await _http_get(url)
|
||||
case "health":
|
||||
url = host if host.startswith("http") else f"http://{host}/health"
|
||||
ok = await _http_get(url)
|
||||
case _:
|
||||
ok = await _ping(host)
|
||||
|
||||
elapsed_ms = int((time.monotonic() - start) * 1000)
|
||||
return {"status": "online" if ok else "offline", "response_time_ms": elapsed_ms}
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug("Check failed for %s (%s): %s", host, check_method, exc)
|
||||
return {"status": "offline", "response_time_ms": None}
|
||||
|
||||
|
||||
async def _ping(host: str) -> bool:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ping", "-c", "1", "-W", "1", host,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
await proc.wait()
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
async def _http_get(url: str, verify: bool = False) -> bool:
|
||||
async with httpx.AsyncClient(verify=verify, timeout=5) as client:
|
||||
resp = await client.get(url, follow_redirects=True)
|
||||
return resp.status_code < 500
|
||||
|
||||
|
||||
async def _tcp_connect(host: str, port: int) -> bool:
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(host, port), timeout=3
|
||||
)
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return True
|
||||
except (TimeoutError, OSError, socket.gaierror):
|
||||
return False
|
||||
Reference in New Issue
Block a user