From b20c47a0a5f3a4e0f0b13c564eba401abe39a68c Mon Sep 17 00:00:00 2001 From: Pouzor Date: Tue, 23 Jun 2026 12:25:10 +0200 Subject: [PATCH 01/12] feat: add tiered service matching with http_regex and port-agnostic signatures Adds match_service() priority walk for fingerprinting: 1. port + http_regex confirmed 2. port + banner_regex confirmed 3. port:null + http_regex confirmed (custom-port services) 4. port-only fallback http_regex is strict only once a probe has run; with no probe signals it degrades to port-only matching, preserving pre-probe behaviour. match_port kept as a probe-less alias. ha-relevant: yes --- backend/app/services/fingerprint.py | 102 ++++++++++++++++++++++++---- backend/tests/test_fingerprint.py | 84 +++++++++++++++++++++++ 2 files changed, 173 insertions(+), 13 deletions(-) diff --git a/backend/app/services/fingerprint.py b/backend/app/services/fingerprint.py index 460bdf3..4532644 100644 --- a/backend/app/services/fingerprint.py +++ b/backend/app/services/fingerprint.py @@ -50,25 +50,101 @@ def _load_oui() -> dict[str, str]: return _OUI_MAP -def match_port(port: int, protocol: str, banner: str | None = None) -> dict[str, Any] | None: - """Return the first signature matching port+protocol, optionally banner.""" - for sig in _load(): - if sig["port"] != port or sig["protocol"] != protocol: - continue - if sig.get("banner_regex") and (not banner or not re.search(sig["banner_regex"], banner, re.IGNORECASE)): - continue - return sig - return None +def _http_regex_hit(sig: dict[str, Any], http_signals: dict[str, Any] | None) -> bool: + """True when the signature's http_regex matches the probe's title/headers.""" + rx = sig.get("http_regex") + if not rx or not http_signals: + return False + headers = http_signals.get("headers") or {} + haystack = " ".join( + s for s in ( + http_signals.get("title"), + headers.get("Server"), + headers.get("X-Powered-By"), + ) if s + ) + return bool(haystack and re.search(rx, haystack, re.IGNORECASE)) -def fingerprint_ports(open_ports: list[dict[str, Any]]) -> list[dict[str, Any]]: +def _service_tier( + sig: dict[str, Any], + port: int, + protocol: str, + banner: str | None, + http_signals: dict[str, Any] | None, +) -> int | None: """ - Given a list of {port, protocol, banner?} dicts, return matched services. - Unknown ports are included as unknown_service. + Rank how well a signature matches (lower = stronger). None = not a match. + + Tier 1: port match + http_regex confirmed + Tier 2: port match + banner_regex confirmed + Tier 3: port-agnostic (port: null) + http_regex confirmed + Tier 4: port match only (no regex, or http_regex with probe disabled) + + When http_signals is None (probe not run) an http_regex entry degrades to + a port-only match — identical to pre-probe behaviour, no regression. + When http_signals is provided, http_regex is strict: a miss disqualifies. + """ + probe_ran = http_signals is not None + has_http = bool(sig.get("http_regex")) + + # Port-agnostic entries (port: null) match purely on HTTP signals. + if sig.get("port") is None: + if has_http and _http_regex_hit(sig, http_signals): + return 3 + return None + + if sig["port"] != port or sig["protocol"] != protocol: + return None + + # http_regex is authoritative once a probe has run. + if has_http and probe_ran: + return 1 if _http_regex_hit(sig, http_signals) else None + + if sig.get("banner_regex"): + if banner and re.search(sig["banner_regex"], banner, re.IGNORECASE): + return 2 + return None + + # No regex constraint (or http_regex but probe disabled) → port-only guess. + return 4 + + +def match_service( + port: int, + protocol: str, + banner: str | None = None, + http_signals: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """Return the best signature for a port, walking tiers most-specific first.""" + best: dict[str, Any] | None = None + best_tier = 99 + for sig in _load(): + tier = _service_tier(sig, port, protocol, banner, http_signals) + if tier is not None and tier < best_tier: + best, best_tier = sig, tier + if best_tier == 1: + break # strongest possible — stop early + return best + + +def match_port(port: int, protocol: str, banner: str | None = None) -> dict[str, Any] | None: + """Back-compat alias: match without HTTP-probe signals.""" + return match_service(port, protocol, banner) + + +def fingerprint_ports( + open_ports: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """ + Given a list of {port, protocol, banner?, http_signals?} dicts, return + matched services. Unknown ports are included as unknown_service. """ results = [] for p in open_ports: - sig = match_port(p["port"], p.get("protocol", "tcp"), p.get("banner")) + sig = match_service( + p["port"], p.get("protocol", "tcp"), p.get("banner"), p.get("http_signals") + ) if sig: results.append({ "port": p["port"], diff --git a/backend/tests/test_fingerprint.py b/backend/tests/test_fingerprint.py index 168b437..bdf487e 100644 --- a/backend/tests/test_fingerprint.py +++ b/backend/tests/test_fingerprint.py @@ -5,6 +5,7 @@ import pytest from app.services.fingerprint import ( fingerprint_ports, match_port, + match_service, suggest_node_type, suggest_type_from_mac, ) @@ -249,3 +250,86 @@ def test_suggest_node_type_ubiquiti_mac_with_bgp_upgrades_to_router(): mac="24:a4:3c:11:22:33", ) assert result == "router" + + +# ── match_service: HTTP probe + port-agnostic ────────────────────────────────── + +HTTP_SIGNATURES = [ + # Generic web fallback on 8096 (port-only guess) + {"port": 8096, "protocol": "tcp", "banner_regex": None, "http_regex": None, + "service_name": "HTTP", "icon": "🌐", "category": "web", "suggested_node_type": "server"}, + # Same port, but confirmed by HTML title → should win when probe confirms + {"port": 8096, "protocol": "tcp", "banner_regex": None, "http_regex": "Jellyfin", + "service_name": "Jellyfin", "icon": "🎬", "category": "media", "suggested_node_type": "server"}, + # Port-agnostic: matches on HTTP content regardless of port + {"port": None, "protocol": "tcp", "banner_regex": None, "http_regex": "Portainer", + "service_name": "Portainer", "icon": "🐳", "category": "container", "suggested_node_type": "server"}, + # Banner-based entry, no http + {"port": 9090, "protocol": "tcp", "banner_regex": "prometheus", "http_regex": None, + "service_name": "Prometheus", "icon": "🔥", "category": "monitoring", "suggested_node_type": "server"}, +] + + +@pytest.fixture +def http_signatures(): + with patch("app.services.fingerprint._load", return_value=HTTP_SIGNATURES): + yield + + +def test_http_regex_confirmed_beats_port_only(http_signatures): + # Probe ran and title matches → Jellyfin (tier 1) beats generic HTTP (tier 4) + sig = match_service(8096, "tcp", banner=None, + http_signals={"title": "Jellyfin", "headers": {}}) + assert sig["service_name"] == "Jellyfin" + + +def test_http_regex_matches_on_header(http_signatures): + sig = match_service(8096, "tcp", banner=None, + http_signals={"title": None, "headers": {"Server": "Jellyfin"}}) + assert sig["service_name"] == "Jellyfin" + + +def test_http_regex_miss_falls_back_to_port_only(http_signatures): + # Probe ran but nothing matched the http_regex → generic port-only entry wins + sig = match_service(8096, "tcp", banner=None, + http_signals={"title": "Some Other App", "headers": {}}) + assert sig["service_name"] == "HTTP" + + +def test_probe_disabled_ignores_http_regex(http_signatures): + # http_signals=None (deep scan off) → http_regex entry degrades to port-only, + # generic entry (listed first) wins — identical to pre-probe behaviour. + sig = match_service(8096, "tcp", banner=None, http_signals=None) + assert sig["service_name"] == "HTTP" + + +def test_port_agnostic_match_on_custom_port(http_signatures): + # Portainer found on a non-standard port, recognised purely by HTTP content + sig = match_service(54321, "tcp", banner=None, + http_signals={"title": "Portainer", "headers": {}}) + assert sig["service_name"] == "Portainer" + + +def test_port_agnostic_requires_probe(http_signatures): + # Same custom port, probe off → no signal → no match + assert match_service(54321, "tcp", banner=None, http_signals=None) is None + + +def test_banner_match_still_works_with_probe(http_signatures): + sig = match_service(9090, "tcp", banner="prometheus 2.x", + http_signals={"title": "x", "headers": {}}) + assert sig["service_name"] == "Prometheus" + + +def test_match_port_alias_has_no_http(http_signatures): + # match_port() is the probe-less alias → http_regex entry degrades to port-only + sig = match_port(8096, "tcp") + assert sig["service_name"] == "HTTP" + + +def test_fingerprint_ports_uses_http_signals(http_signatures): + results = fingerprint_ports([ + {"port": 8096, "protocol": "tcp", "banner": None, + "http_signals": {"title": "Jellyfin", "headers": {}}}, + ]) + assert results[0]["service_name"] == "Jellyfin" From 44e0029f2b9d9eee7d9b123aead56b2e267aa076 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Tue, 23 Jun 2026 13:41:32 +0200 Subject: [PATCH 02/12] feat: add async HTTP prober for deep-scan service identification probe_port() GETs https:// then http:// for an open port and returns the page plus Server/X-Powered-By headers as identifying signals. Non-web ports (SSH, DB, etc.) are skipped; body read is capped at 64KB; 3s timeout. probe_open_ports() fans out over all open ports with a bounded semaphore. ha-relevant: yes --- backend/app/services/http_probe.py | 85 +++++++++++++++++++++ backend/tests/test_http_probe.py | 118 +++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 backend/app/services/http_probe.py create mode 100644 backend/tests/test_http_probe.py diff --git a/backend/app/services/http_probe.py b/backend/app/services/http_probe.py new file mode 100644 index 0000000..8ed9319 --- /dev/null +++ b/backend/app/services/http_probe.py @@ -0,0 +1,85 @@ +"""HTTP probe: GET a discovered port and extract identifying signals. + +Used by the optional deep-scan mode to confirm what service sits behind an +open port, regardless of port number. Returns the page <title> plus a small +set of identifying response headers, which fingerprint.match_service() then +matches against signature http_regex fields. +""" +import asyncio +import logging +import re +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + +# Headers that commonly carry the application name. +_SIGNAL_HEADERS = ("Server", "X-Powered-By") +# Cap how much body we read when hunting for <title> — avoids large downloads. +_MAX_BODY_BYTES = 64 * 1024 +_TITLE_RE = re.compile(r"<title[^>]*>(.*?)", re.IGNORECASE | re.DOTALL) +_PROBE_TIMEOUT = 3.0 +# Ports we never bother probing over HTTP (not web services). +_NON_HTTP_PORTS = frozenset({22, 21, 23, 25, 53, 110, 143, 161, 162, 179, 445, 3306, 5432, 6379}) + + +def _extract_title(body: str) -> str | None: + m = _TITLE_RE.search(body) + if not m: + return None + title = re.sub(r"\s+", " ", m.group(1)).strip() + return title or None + + +async def _probe_scheme(client: httpx.AsyncClient, url: str) -> dict[str, Any] | None: + try: + resp = await client.get(url, follow_redirects=True) + except (httpx.HTTPError, OSError): + return None + headers = {h: resp.headers[h] for h in _SIGNAL_HEADERS if h in resp.headers} + body = resp.text[:_MAX_BODY_BYTES] if resp.text else "" + title = _extract_title(body) + if not title and not headers: + return None + return {"title": title, "headers": headers} + + +async def probe_port( + ip: str, port: int, verify_tls: bool = False +) -> dict[str, Any] | None: + """ + GET https:// then http:// for a port and return {title, headers} or None. + + None means the port did not answer HTTP or yielded no usable signal. + """ + if port in _NON_HTTP_PORTS: + return None + async with httpx.AsyncClient(verify=verify_tls, timeout=_PROBE_TIMEOUT) as client: + for scheme in ("https", "http"): + result = await _probe_scheme(client, f"{scheme}://{ip}:{port}/") + if result is not None: + return result + return None + + +async def probe_open_ports( + ip: str, + open_ports: list[dict[str, Any]], + verify_tls: bool = False, + concurrency: int = 50, +) -> list[dict[str, Any]]: + """ + Probe every open port for HTTP signals (option 2: probe all, match after). + + Returns the same port dicts, each enriched with an http_signals key + (None when the port gave no HTTP signal). + """ + sem = asyncio.Semaphore(concurrency) + + async def _one(p: dict[str, Any]) -> dict[str, Any]: + async with sem: + signals = await probe_port(ip, p["port"], verify_tls) + return {**p, "http_signals": signals} + + return await asyncio.gather(*(_one(p) for p in open_ports)) diff --git a/backend/tests/test_http_probe.py b/backend/tests/test_http_probe.py new file mode 100644 index 0000000..4175d63 --- /dev/null +++ b/backend/tests/test_http_probe.py @@ -0,0 +1,118 @@ +"""Tests for the HTTP probe used by deep-scan service identification.""" +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +from app.services.http_probe import ( + _extract_title, + probe_open_ports, + probe_port, +) + + +def _response(text: str = "", headers: dict | None = None, status: int = 200) -> httpx.Response: + return httpx.Response(status_code=status, text=text, headers=headers or {}) + + +# ── _extract_title ────────────────────────────────────────────────────────── + +def test_extract_title_basic(): + assert _extract_title("Jellyfin") == "Jellyfin" + + +def test_extract_title_collapses_whitespace(): + assert _extract_title("\n My App\n") == "My App" + + +def test_extract_title_missing(): + assert _extract_title("no title") is None + + +def test_extract_title_case_insensitive(): + assert _extract_title("Portainer") == "Portainer" + + +# ── probe_port ────────────────────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_probe_port_reads_title(): + with patch("httpx.AsyncClient.get", new=AsyncMock(return_value=_response("Jellyfin"))): + result = await probe_port("10.0.0.5", 8096) + assert result == {"title": "Jellyfin", "headers": {}} + + +@pytest.mark.asyncio +async def test_probe_port_reads_headers(): + resp = _response("", headers={"Server": "nginx", "X-Powered-By": "Express"}) + with patch("httpx.AsyncClient.get", new=AsyncMock(return_value=resp)): + result = await probe_port("10.0.0.5", 3000) + assert result["headers"] == {"Server": "nginx", "X-Powered-By": "Express"} + + +@pytest.mark.asyncio +async def test_probe_port_falls_back_to_http(): + # https raises, http succeeds + calls = {"n": 0} + + async def fake_get(self, url, **kw): + calls["n"] += 1 + if url.startswith("https"): + raise httpx.ConnectError("tls fail") + return _response("HTTP App") + + with patch("httpx.AsyncClient.get", new=fake_get): + result = await probe_port("10.0.0.5", 8080) + assert result["title"] == "HTTP App" + assert calls["n"] == 2 # tried https then http + + +@pytest.mark.asyncio +async def test_probe_port_no_signal_returns_none(): + with patch("httpx.AsyncClient.get", new=AsyncMock(return_value=_response(""))): + result = await probe_port("10.0.0.5", 8080) + assert result is None + + +@pytest.mark.asyncio +async def test_probe_port_timeout_returns_none(): + with patch("httpx.AsyncClient.get", new=AsyncMock(side_effect=httpx.TimeoutException("slow"))): + result = await probe_port("10.0.0.5", 8080) + assert result is None + + +@pytest.mark.asyncio +async def test_probe_port_skips_non_http_ports(): + # SSH should never trigger an HTTP request + get = AsyncMock() + with patch("httpx.AsyncClient.get", new=get): + result = await probe_port("10.0.0.5", 22) + assert result is None + get.assert_not_called() + + +@pytest.mark.asyncio +async def test_probe_port_verify_tls_flag_passed(): + with patch("app.services.http_probe.httpx.AsyncClient") as client_cls: + instance = client_cls.return_value.__aenter__.return_value + instance.get = AsyncMock(return_value=_response("X")) + await probe_port("10.0.0.5", 8443, verify_tls=True) + assert client_cls.call_args.kwargs["verify"] is True + + +# ── probe_open_ports ───────────────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_probe_open_ports_enriches_each_port(): + async def fake_get(self, url, **kw): + if ":8096" in url: + return _response("Jellyfin") + return _response("") + + ports = [{"port": 8096, "protocol": "tcp"}, {"port": 9999, "protocol": "tcp"}] + with patch("httpx.AsyncClient.get", new=fake_get): + result = await probe_open_ports("10.0.0.5", ports) + + by_port = {p["port"]: p for p in result} + assert by_port[8096]["http_signals"]["title"] == "Jellyfin" + assert by_port[9999]["http_signals"] is None From 9d1ebd6c6eb24b643a9887de27d955ea45bb3cb8 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Tue, 23 Jun 2026 14:04:15 +0200 Subject: [PATCH 03/12] feat: wire deep-scan port ranges and HTTP probe into scanner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_scan() accepts DeepScanOptions: user port ranges are validated and appended to the nmap -p list; when http_probe_enabled, open ports are probed for HTTP signals before fingerprinting. Defaults (no options) reproduce the standard scan exactly — probe is never called, port list unchanged. ha-relevant: yes --- backend/app/services/scanner.py | 74 +++++++++++++++--- backend/tests/test_scan.py | 2 +- backend/tests/test_scanner.py | 134 +++++++++++++++++++++++++++++++- 3 files changed, 197 insertions(+), 13 deletions(-) diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.py index 4f0ac43..fccb315 100644 --- a/backend/app/services/scanner.py +++ b/backend/app/services/scanner.py @@ -7,6 +7,7 @@ import re import socket import subprocess import threading +from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any @@ -15,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import Node, PendingDevice, ScanRun from app.services.fingerprint import fingerprint_ports, suggest_node_type +from app.services.http_probe import probe_open_ports logger = logging.getLogger(__name__) @@ -34,6 +36,37 @@ _EXTRA_PORTS = ( "16686,34567,37777,51413,64738" ) +# nmap -p accepts "N" or "N-M"; user ranges are validated against this. +_PORT_RANGE_RE = re.compile(r"^\d{1,5}(-\d{1,5})?$") + + +@dataclass +class DeepScanOptions: + """Per-scan deep-scan settings (None/empty → standard scan, today's behaviour).""" + + http_ranges: list[str] = field(default_factory=list) + http_probe_enabled: bool = False + verify_tls: bool = False + + +def _valid_port_range(spec: str) -> bool: + if not _PORT_RANGE_RE.match(spec): + return False + parts = [int(p) for p in spec.split("-")] + if any(p < 1 or p > 65535 for p in parts): + return False + return len(parts) == 1 or parts[0] <= parts[1] + + +def _build_port_spec(http_ranges: list[str] | None) -> str: + """Combine the default port list with validated user ranges for nmap -p.""" + if not http_ranges: + return _EXTRA_PORTS + extra = [r.strip() for r in http_ranges if _valid_port_range(r.strip())] + if not extra: + return _EXTRA_PORTS + return _EXTRA_PORTS + "," + ",".join(extra) + _MDNS_SERVICE_TYPES = [ "_http._tcp.local.", "_shelly._tcp.local.", @@ -195,7 +228,7 @@ async def _ping_sweep(target: str) -> dict[str, dict[str, Any]]: return alive -def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]: +def _nmap_scan_single(host_dict: dict[str, Any], port_spec: str = _EXTRA_PORTS) -> dict[str, Any]: """ Phase 2 — single-IP port scan with service detection. Runs in a thread (blocking). Returns the host dict enriched with open_ports. @@ -210,11 +243,11 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]: is_root = os.geteuid() == 0 if is_root: # SYN scan + version detection (fastest, most accurate) - scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}" + scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}" else: # TCP connect scan (-sT) — no raw sockets needed, works without root. # nmap auto-selects -sT without root but being explicit avoids edge cases. - scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}" + scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}" logger.debug("[Phase 2] %s args: %s", ip, scan_args) nm = nmap.PortScanner() @@ -252,7 +285,9 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]: return host_dict -async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: +async def _nmap_port_scan( + alive: dict[str, dict[str, Any]], port_spec: str = _EXTRA_PORTS +) -> list[dict[str, Any]]: """ Phase 2: Per-IP service detection with bounded concurrency. Each host is scanned independently in a thread — no inter-host timeout interference. @@ -266,7 +301,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, An async def _scan_with_sem(host_dict: dict[str, Any]) -> dict[str, Any]: async with semaphore: - return await asyncio.to_thread(_nmap_scan_single, host_dict) + return await asyncio.to_thread(_nmap_scan_single, host_dict, port_spec) raw = await asyncio.gather(*[_scan_with_sem(h) for h in alive.values()], return_exceptions=True) results = [] @@ -279,7 +314,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, An return results -async def _nmap_scan(target: str) -> list[dict[str, Any]]: +async def _nmap_scan(target: str, port_spec: str = _EXTRA_PORTS) -> list[dict[str, Any]]: """ Two-phase scan for a CIDR range. Phase 1: Concurrent ping sweep to find alive hosts (fast, no false positives). @@ -296,7 +331,7 @@ async def _nmap_scan(target: str) -> list[dict[str, Any]]: except Exception as exc: logger.error("Phase 1 ping sweep failed: %s", exc) raise RuntimeError(str(exc)) from exc - return await _nmap_port_scan(alive) + return await _nmap_port_scan(alive, port_spec) async def _mdns_discover(timeout: float = 4.0) -> list[dict[str, Any]]: @@ -375,10 +410,18 @@ def _mock_scan(target: str) -> list[dict[str, Any]]: ] -async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None: +async def run_scan( + ranges: list[str], + db: AsyncSession, + run_id: str, + deep_scan: DeepScanOptions | None = None, +) -> None: """Execute scan for given CIDR ranges and populate pending_devices.""" from app.api.routes.status import broadcast_scan_update + deep_scan = deep_scan or DeepScanOptions() + port_spec = _build_port_spec(deep_scan.http_ranges) + devices_found = 0 mdns_task: asyncio.Task[list[dict[str, Any]]] | None = None try: @@ -427,8 +470,17 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None: logger.debug("Skipping %s — hidden by user", ip) return - services = fingerprint_ports(host["open_ports"]) - suggested_type = suggest_node_type(host["open_ports"], host.get("mac")) + open_ports = host["open_ports"] + # Deep-scan HTTP probe: enrich open ports with title/header signals so + # fingerprint can confirm services on custom ports. No-op when disabled + # or when the host has no open ports (e.g. mDNS-only discovery). + if deep_scan.http_probe_enabled and open_ports: + open_ports = await probe_open_ports( + ip, open_ports, verify_tls=deep_scan.verify_tls + ) + + services = fingerprint_ports(open_ports) + suggested_type = suggest_node_type(open_ports, host.get("mac")) existing_result = await db.execute( select(PendingDevice).where( @@ -463,7 +515,7 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None: for cidr in ranges: if _is_cancelled(run_id): break - hosts = await _nmap_scan(cidr) + hosts = await _nmap_scan(cidr, port_spec) for host in hosts: if _is_cancelled(run_id): break diff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.py index 6feb0b9..56ea3ed 100644 --- a/backend/tests/test_scan.py +++ b/backend/tests/test_scan.py @@ -518,7 +518,7 @@ async def test_run_scan_cancelled_mid_scan_skips_remaining_cidrs(db_session: Asy call_count = 0 - def nmap_side_effect(target: str): + def nmap_side_effect(target: str, port_spec: str | None = None): nonlocal call_count call_count += 1 # Signal cancellation after the first CIDR scan completes diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index e29e2fc..b7d66fa 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -359,7 +359,7 @@ async def test_nmap_port_scan_tolerates_single_host_exception(): call_count = 0 - def _flaky_scan(host_dict): + def _flaky_scan(host_dict, port_spec=None): nonlocal call_count call_count += 1 if host_dict["ip"] == "192.168.1.1": @@ -533,3 +533,135 @@ async def test_run_scan_cancelled_marks_status_cancelled(mem_db): run = await session.get(ScanRun, run_id) assert run is not None assert run.status == "cancelled" + + +# --------------------------------------------------------------------------- +# Deep scan: port-range plumbing + HTTP probe +# --------------------------------------------------------------------------- + +def test_valid_port_range(): + from app.services.scanner import _valid_port_range + + assert _valid_port_range("8080") + assert _valid_port_range("8000-8100") + assert not _valid_port_range("8100-8000") # reversed + assert not _valid_port_range("0") # below 1 + assert not _valid_port_range("70000") # above 65535 + assert not _valid_port_range("abc") + assert not _valid_port_range("80,443") # not a single range + + +def test_build_port_spec_default_when_empty(): + from app.services.scanner import _EXTRA_PORTS, _build_port_spec + + assert _build_port_spec([]) == _EXTRA_PORTS + assert _build_port_spec(None) == _EXTRA_PORTS + + +def test_build_port_spec_appends_valid_ranges(): + from app.services.scanner import _EXTRA_PORTS, _build_port_spec + + spec = _build_port_spec(["8000-8100", "9000"]) + assert spec == _EXTRA_PORTS + ",8000-8100,9000" + + +def test_build_port_spec_drops_invalid_ranges(): + from app.services.scanner import _EXTRA_PORTS, _build_port_spec + + # invalid entries silently dropped; only valid kept + assert _build_port_spec(["bad", "70000"]) == _EXTRA_PORTS + assert _build_port_spec(["bad", "9000"]) == _EXTRA_PORTS + ",9000" + + +@pytest.mark.asyncio +async def test_run_scan_deep_scan_passes_port_spec_to_nmap(mem_db): + from app.services.scanner import DeepScanOptions, run_scan + + run_id = _make_run_id() + async with mem_db() as session: + session.add(_make_scan_run(run_id)) + await session.commit() + + captured = {} + + async def fake_nmap(target, port_spec): + captured["port_spec"] = port_spec + return [] + + async with mem_db() as session: + with patch("app.services.scanner._nmap_scan", new=fake_nmap), \ + patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \ + patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock): + await run_scan( + ["192.168.1.0/24"], session, run_id, + deep_scan=DeepScanOptions(http_ranges=["8000-8100"]), + ) + + assert "8000-8100" in captured["port_spec"] + + +@pytest.mark.asyncio +async def test_run_scan_probe_enriches_services(mem_db): + """With probe enabled, a custom-port service is identified via HTTP signals.""" + from app.services.scanner import DeepScanOptions, run_scan + + run_id = _make_run_id() + async with mem_db() as session: + session.add(_make_scan_run(run_id)) + await session.commit() + + nmap_hosts = [{ + "ip": "192.168.1.50", "hostname": None, "mac": None, "os": None, + "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}], + }] + jellyfin_sig = [{ + "port": 8096, "protocol": "tcp", "banner_regex": None, "http_regex": "Jellyfin", + "service_name": "Jellyfin", "icon": "🎬", "category": "media", "suggested_node_type": "server", + }] + + async def fake_probe(ip, ports, verify_tls=False, concurrency=50): + return [{**p, "http_signals": {"title": "Jellyfin", "headers": {}}} for p in ports] + + async with mem_db() as session: + with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \ + patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \ + patch("app.services.scanner.probe_open_ports", new=fake_probe), \ + patch("app.services.fingerprint._load", return_value=jellyfin_sig), \ + patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock): + await run_scan( + ["192.168.1.0/24"], session, run_id, + deep_scan=DeepScanOptions(http_probe_enabled=True), + ) + + async with mem_db() as session: + result = await session.execute(sa_select(PendingDevice).where(PendingDevice.ip == "192.168.1.50")) + device = result.scalar_one_or_none() + + assert device is not None + assert any(s["service_name"] == "Jellyfin" for s in device.services) + + +@pytest.mark.asyncio +async def test_run_scan_no_probe_when_disabled(mem_db): + """Probe must not be called on a standard (non-deep) scan.""" + from app.services.scanner import run_scan + + run_id = _make_run_id() + async with mem_db() as session: + session.add(_make_scan_run(run_id)) + await session.commit() + + nmap_hosts = [{ + "ip": "192.168.1.51", "hostname": None, "mac": None, "os": None, + "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}], + }] + probe = AsyncMock() + + async with mem_db() as session: + with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \ + patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \ + patch("app.services.scanner.probe_open_ports", new=probe), \ + patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock): + await run_scan(["192.168.1.0/24"], session, run_id) + + probe.assert_not_called() From b6423c0115f9501bc0d6f6bbf52e8f101c781511 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Wed, 24 Jun 2026 10:04:31 +0200 Subject: [PATCH 04/12] feat: expose deep-scan settings via API and persisted config Adds scanner_http_ranges / scanner_http_probe_enabled / scanner_http_verify_tls to Settings (persisted in scan_config.json, Options page defaults). /scan/trigger accepts an optional body to override these per-scan; /scan/config GET/POST read and persist the defaults. Port ranges validated at the API boundary. ha-relevant: yes --- .env.example | 6 ++ backend/app/api/routes/scan.py | 84 ++++++++++++++++++++++-- backend/app/core/config.py | 15 +++++ backend/tests/test_scan.py | 114 ++++++++++++++++++++++++++++++++- 4 files changed, 211 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 52fa748..1f2b9b9 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,12 @@ AUTH_PASSWORD_HASH='$2b$12$RtMbyw17l4N5UGzeXMNAWuzCaVV.XFBY7ZetWheQhxcBDcxahapkG # Scanner — JSON array of CIDR ranges to scan SCANNER_RANGES=["192.168.1.0/24"] +# Deep scan (optional) — extra nmap port ranges + HTTP probe for service ID on +# custom ports. Defaults below are overridable per-scan from the scan dialog. +SCANNER_HTTP_RANGES=[] +SCANNER_HTTP_PROBE_ENABLED=false +SCANNER_HTTP_VERIFY_TLS=false + # Status checker interval in seconds STATUS_CHECKER_INTERVAL=60 diff --git a/backend/app/api/routes/scan.py b/backend/app/api/routes/scan.py index 390c86a..4e1214a 100644 --- a/backend/app/api/routes/scan.py +++ b/backend/app/api/routes/scan.py @@ -14,7 +14,7 @@ from app.db.database import AsyncSessionLocal, get_db from app.db.models import Design, Edge, Node, PendingDevice, PendingDeviceLink, ScanRun from app.schemas.nodes import NodeCreate from app.schemas.scan import PendingDeviceResponse, ScanRunResponse -from app.services.scanner import request_cancel, run_scan +from app.services.scanner import DeepScanOptions, _valid_port_range, request_cancel, run_scan from app.services.zigbee_service import build_zigbee_properties _ZIGBEE_TYPES = {"zigbee_coordinator", "zigbee_router", "zigbee_enddevice"} @@ -52,8 +52,20 @@ class BulkActionRequest(BaseModel): device_ids: list[str] +def _check_port_ranges(v: list[str]) -> list[str]: + for r in v: + if not _valid_port_range(r.strip()): + raise ValueError(f"Invalid port range: {r!r}") + return v + + class ScanConfig(BaseModel): + """Persisted scan defaults (Options page). Deep-scan fields are optional.""" + ranges: list[str] + http_ranges: list[str] = [] + http_probe_enabled: bool = False + verify_tls: bool = False @field_validator("ranges") @classmethod @@ -65,15 +77,35 @@ class ScanConfig(BaseModel): raise ValueError(f"Invalid CIDR range: {r!r}") from exc return v + @field_validator("http_ranges") + @classmethod + def validate_http_ranges(cls, v: list[str]) -> list[str]: + return _check_port_ranges(v) + + +class TriggerScanRequest(BaseModel): + """Per-scan deep-scan overrides (scan dialog). None → use persisted default.""" + + http_ranges: list[str] | None = None + http_probe_enabled: bool | None = None + verify_tls: bool | None = None + + @field_validator("http_ranges") + @classmethod + def validate_http_ranges(cls, v: list[str] | None) -> list[str] | None: + return None if v is None else _check_port_ranges(v) + logger = logging.getLogger(__name__) router = APIRouter() -async def _background_scan(run_id: str, ranges: list[str]) -> None: +async def _background_scan( + run_id: str, ranges: list[str], deep_scan: DeepScanOptions | None = None +) -> None: async with AsyncSessionLocal() as db: try: - await run_scan(ranges, db, run_id) + await run_scan(ranges, db, run_id, deep_scan=deep_scan or DeepScanOptions()) except Exception: logger.exception("Scan run %s failed unexpectedly", run_id) await db.rollback() @@ -83,18 +115,38 @@ async def _background_scan(run_id: str, ranges: list[str]) -> None: await db.commit() +def _resolve_deep_scan(payload: TriggerScanRequest | None) -> DeepScanOptions: + """Merge per-scan overrides over persisted settings defaults.""" + p = payload or TriggerScanRequest() + return DeepScanOptions( + http_ranges=( + p.http_ranges if p.http_ranges is not None else settings.scanner_http_ranges + ), + http_probe_enabled=( + p.http_probe_enabled + if p.http_probe_enabled is not None + else settings.scanner_http_probe_enabled + ), + verify_tls=( + p.verify_tls if p.verify_tls is not None else settings.scanner_http_verify_tls + ), + ) + + @router.post("/trigger", response_model=ScanRunResponse) async def trigger_scan( background_tasks: BackgroundTasks, + payload: TriggerScanRequest | None = None, db: AsyncSession = Depends(get_db), _: str = Depends(get_current_user), ) -> ScanRun: ranges = settings.scanner_ranges + deep_scan = _resolve_deep_scan(payload) run = ScanRun(status="running", ranges=ranges) db.add(run) await db.commit() await db.refresh(run) - background_tasks.add_task(_background_scan, run.id, ranges) + background_tasks.add_task(_background_scan, run.id, ranges, deep_scan) return run @@ -427,17 +479,35 @@ async def list_runs(db: AsyncSession = Depends(get_db), _: str = Depends(get_cur @router.get("/config", response_model=ScanConfig) async def get_scan_config(_: str = Depends(get_current_user)) -> ScanConfig: - return ScanConfig(ranges=settings.scanner_ranges) + return ScanConfig( + ranges=settings.scanner_ranges, + http_ranges=settings.scanner_http_ranges, + http_probe_enabled=settings.scanner_http_probe_enabled, + verify_tls=settings.scanner_http_verify_tls, + ) @router.post("/config", response_model=ScanConfig) async def update_scan_config(payload: ScanConfig, _: str = Depends(get_current_user)) -> ScanConfig: - previous = settings.scanner_ranges + previous = ( + settings.scanner_ranges, + settings.scanner_http_ranges, + settings.scanner_http_probe_enabled, + settings.scanner_http_verify_tls, + ) settings.scanner_ranges = payload.ranges + settings.scanner_http_ranges = payload.http_ranges + settings.scanner_http_probe_enabled = payload.http_probe_enabled + settings.scanner_http_verify_tls = payload.verify_tls try: settings.save_overrides() return payload except Exception as exc: - settings.scanner_ranges = previous + ( + settings.scanner_ranges, + settings.scanner_http_ranges, + settings.scanner_http_probe_enabled, + settings.scanner_http_verify_tls, + ) = previous logger.error("Failed to save scan config: %s", exc) raise HTTPException(status_code=500, detail="Failed to save scan config") from exc diff --git a/backend/app/core/config.py b/backend/app/core/config.py index f599f74..b9f4016 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -48,6 +48,12 @@ class Settings(BaseSettings): # Scanner scanner_ranges: list[str] = ["192.168.1.0/24"] + # Deep scan — persisted defaults (overridable per-scan from the scan dialog). + # http_ranges: extra nmap port ranges, opt-in, no default. Probe + TLS off by default. + scanner_http_ranges: list[str] = [] + scanner_http_probe_enabled: bool = False + scanner_http_verify_tls: bool = False + # Status checker status_checker_interval: int = 60 @@ -85,6 +91,12 @@ class Settings(BaseSettings): self.service_check_enabled = bool(data["service_check_enabled"]) if "service_check_interval" in data: self.service_check_interval = int(data["service_check_interval"]) + if "scanner_http_ranges" in data: + self.scanner_http_ranges = list(data["scanner_http_ranges"]) + if "scanner_http_probe_enabled" in data: + self.scanner_http_probe_enabled = bool(data["scanner_http_probe_enabled"]) + if "scanner_http_verify_tls" in data: + self.scanner_http_verify_tls = bool(data["scanner_http_verify_tls"]) except Exception: pass @@ -96,6 +108,9 @@ class Settings(BaseSettings): "status_checker_interval": self.status_checker_interval, "service_check_enabled": self.service_check_enabled, "service_check_interval": self.service_check_interval, + "scanner_http_ranges": self.scanner_http_ranges, + "scanner_http_probe_enabled": self.scanner_http_probe_enabled, + "scanner_http_verify_tls": self.scanner_http_verify_tls, })) diff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.py index 56ea3ed..7032772 100644 --- a/backend/tests/test_scan.py +++ b/backend/tests/test_scan.py @@ -122,7 +122,8 @@ async def test_background_scan_success_path_invokes_run_scan(mem_db): patch("app.api.routes.scan.AsyncSessionLocal", mem_db), patch("app.api.routes.scan.run_scan", new_callable=AsyncMock) as mock_run_scan, ): - await _background_scan(run_id, ["10.0.0.0/24"]) + from app.services.scanner import DeepScanOptions + await _background_scan(run_id, ["10.0.0.0/24"], DeepScanOptions()) mock_run_scan.assert_awaited_once() @@ -1122,3 +1123,114 @@ async def test_approve_zigbee_resolves_link_after_second_approval( assert len(edges) == 1 links = (await db_session.execute(select(PendingDeviceLink))).scalars().all() assert links == [] # consumed + + +# --- Deep scan: trigger overrides + config persistence --- + +@pytest.mark.asyncio +async def test_resolve_deep_scan_falls_back_to_settings(): + from app.api.routes.scan import TriggerScanRequest, _resolve_deep_scan + + with patch("app.api.routes.scan.settings") as mock_settings: + mock_settings.scanner_http_ranges = ["7000-7100"] + mock_settings.scanner_http_probe_enabled = True + mock_settings.scanner_http_verify_tls = False + # Empty payload → all values come from settings defaults + ds = _resolve_deep_scan(TriggerScanRequest()) + assert ds.http_ranges == ["7000-7100"] + assert ds.http_probe_enabled is True + assert ds.verify_tls is False + + +@pytest.mark.asyncio +async def test_resolve_deep_scan_override_wins(): + from app.api.routes.scan import TriggerScanRequest, _resolve_deep_scan + + with patch("app.api.routes.scan.settings") as mock_settings: + mock_settings.scanner_http_ranges = [] + mock_settings.scanner_http_probe_enabled = False + mock_settings.scanner_http_verify_tls = False + ds = _resolve_deep_scan( + TriggerScanRequest(http_ranges=["9000"], http_probe_enabled=True, verify_tls=True) + ) + assert ds.http_ranges == ["9000"] + assert ds.http_probe_enabled is True + assert ds.verify_tls is True + + +@pytest.mark.asyncio +async def test_trigger_scan_passes_deep_scan_options(client: AsyncClient, headers): + captured = {} + + async def fake_bg(run_id, ranges, deep_scan): + captured["deep_scan"] = deep_scan + + with ( + patch("app.api.routes.scan._background_scan", new=fake_bg), + patch("app.api.routes.scan.settings") as mock_settings, + ): + mock_settings.scanner_ranges = ["192.168.1.0/24"] + mock_settings.scanner_http_ranges = [] + mock_settings.scanner_http_probe_enabled = False + mock_settings.scanner_http_verify_tls = False + res = await client.post( + "/api/v1/scan/trigger", + json={"http_probe_enabled": True, "http_ranges": ["8000-8100"]}, + headers=headers, + ) + assert res.status_code == 200 + assert captured["deep_scan"].http_probe_enabled is True + assert captured["deep_scan"].http_ranges == ["8000-8100"] + + +@pytest.mark.asyncio +async def test_trigger_scan_rejects_invalid_port_range(client: AsyncClient, headers): + with patch("app.api.routes.scan.settings") as mock_settings: + mock_settings.scanner_ranges = ["192.168.1.0/24"] + res = await client.post( + "/api/v1/scan/trigger", + json={"http_ranges": ["70000-80000"]}, + headers=headers, + ) + assert res.status_code == 422 + + +@pytest.mark.asyncio +async def test_get_scan_config_includes_deep_scan(client: AsyncClient, headers): + with patch("app.api.routes.scan.settings") as mock_settings: + mock_settings.scanner_ranges = ["192.168.1.0/24"] + mock_settings.scanner_http_ranges = ["8000-8100"] + mock_settings.scanner_http_probe_enabled = True + mock_settings.scanner_http_verify_tls = False + res = await client.get("/api/v1/scan/config", headers=headers) + assert res.status_code == 200 + data = res.json() + assert data["http_ranges"] == ["8000-8100"] + assert data["http_probe_enabled"] is True + + +@pytest.mark.asyncio +async def test_update_scan_config_persists_deep_scan(client: AsyncClient, headers): + saved = {} + + with patch("app.api.routes.scan.settings") as mock_settings: + mock_settings.scanner_ranges = ["192.168.1.0/24"] + mock_settings.scanner_http_ranges = [] + mock_settings.scanner_http_probe_enabled = False + mock_settings.scanner_http_verify_tls = False + mock_settings.save_overrides = lambda: saved.update( + http_ranges=mock_settings.scanner_http_ranges, + probe=mock_settings.scanner_http_probe_enabled, + ) + res = await client.post( + "/api/v1/scan/config", + json={ + "ranges": ["192.168.1.0/24"], + "http_ranges": ["9000-9100"], + "http_probe_enabled": True, + "verify_tls": True, + }, + headers=headers, + ) + assert res.status_code == 200 + assert saved == {"http_ranges": ["9000-9100"], "probe": True} From d01630bf37c06d98daa26096b13238f6f773175a Mon Sep 17 00:00:00 2001 From: Pouzor Date: Wed, 24 Jun 2026 10:31:32 +0200 Subject: [PATCH 05/12] feat: add Deep Scan toggle to scan dialog Collapsible Deep Scan section in ScanConfigModal exposes extra port ranges, HTTP probe and TLS-verify switches. Pre-filled from saved defaults; edits are passed to trigger() as a per-scan override and do not change the persisted defaults (those live in the Options/scan config). scanApi.trigger now accepts an optional deep-scan body. ha-relevant: yes --- frontend/src/api/client.ts | 14 ++- .../src/components/modals/ScanConfigModal.tsx | 93 ++++++++++++++++++- .../modals/__tests__/ScanConfigModal.test.tsx | 62 ++++++++++++- 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 96a8ebe..22b8e8c 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -58,8 +58,16 @@ export const liveviewApi = { getConfig: () => api.get<{ enabled: boolean; key: string | null }>('/liveview/config'), } +export interface DeepScanConfig { + http_ranges: string[] + http_probe_enabled: boolean + verify_tls: boolean +} + +export type ScanConfigData = { ranges: string[] } & DeepScanConfig + export const scanApi = { - trigger: () => api.post('/scan/trigger'), + trigger: (deepScan?: Partial) => api.post('/scan/trigger', deepScan ?? {}), pending: () => api.get('/scan/pending'), hidden: () => api.get('/scan/hidden'), runs: () => api.get('/scan/runs'), @@ -86,8 +94,8 @@ export const scanApi = { restore: (id: string) => api.post<{ restored: boolean; device_id: string }>(`/scan/pending/${id}/restore`), bulkRestore: (ids: string[]) => api.post<{ restored: number; skipped: number }>('/scan/pending/bulk-restore', { device_ids: ids }), stop: (runId: string) => api.post(`/scan/${runId}/stop`), - getConfig: () => api.get<{ ranges: string[] }>('/scan/config'), - saveConfig: (data: { ranges: string[] }) => api.post('/scan/config', data), + getConfig: () => api.get('/scan/config'), + saveConfig: (data: ScanConfigData) => api.post('/scan/config', data), } export interface AppSettings { diff --git a/frontend/src/components/modals/ScanConfigModal.tsx b/frontend/src/components/modals/ScanConfigModal.tsx index f1d3742..c9b779a 100644 --- a/frontend/src/components/modals/ScanConfigModal.tsx +++ b/frontend/src/components/modals/ScanConfigModal.tsx @@ -1,10 +1,10 @@ import { useState, useEffect } from 'react' -import { Plus, Trash2, Settings } from 'lucide-react' +import { Plus, Trash2, Settings, ChevronRight, ChevronDown } from 'lucide-react' import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' -import { scanApi } from '@/api/client' +import { scanApi, type DeepScanConfig } from '@/api/client' import { toast } from 'sonner' interface ScanConfigModalProps { @@ -13,24 +13,56 @@ interface ScanConfigModalProps { onScanNow: () => void } +const DEEP_DEFAULTS: DeepScanConfig = { http_ranges: [], http_probe_enabled: false, verify_tls: false } + export function ScanConfigModal({ open, onClose, onScanNow }: ScanConfigModalProps) { const [ranges, setRanges] = useState(['']) const [saving, setSaving] = useState(false) + // Deep-scan section. Pre-filled from persisted defaults; edits here are a + // per-scan override passed to trigger() — they do NOT change the saved defaults. + const [deepOpen, setDeepOpen] = useState(false) + const [deepDefaults, setDeepDefaults] = useState(DEEP_DEFAULTS) + const [httpProbe, setHttpProbe] = useState(false) + const [verifyTls, setVerifyTls] = useState(false) + const [httpRangesText, setHttpRangesText] = useState('') + useEffect(() => { if (!open) return scanApi.getConfig() - .then((res) => setRanges(res.data.ranges.length > 0 ? res.data.ranges : [''])) + .then((res) => { + const d = res.data + setRanges(d.ranges.length > 0 ? d.ranges : ['']) + const deep: DeepScanConfig = { + http_ranges: d.http_ranges ?? [], + http_probe_enabled: d.http_probe_enabled ?? false, + verify_tls: d.verify_tls ?? false, + } + setDeepDefaults(deep) + setHttpProbe(deep.http_probe_enabled) + setVerifyTls(deep.verify_tls) + setHttpRangesText(deep.http_ranges.join(', ')) + setDeepOpen(deep.http_probe_enabled || deep.http_ranges.length > 0) + }) .catch(() => {/* use defaults */}) }, [open]) + const parseHttpRanges = () => + httpRangesText.split(',').map((r) => r.trim()).filter(Boolean) + const handleScanNow = async () => { const cleaned = ranges.map((r) => r.trim()).filter(Boolean) if (cleaned.length === 0) { toast.error('Add at least one IP range'); return } setSaving(true) try { - await scanApi.saveConfig({ ranges: cleaned }) - await scanApi.trigger() + // Persist IP ranges; leave deep-scan defaults as configured in Options. + await scanApi.saveConfig({ ranges: cleaned, ...deepDefaults }) + // Per-scan deep-scan override from this dialog. + await scanApi.trigger({ + http_ranges: parseHttpRanges(), + http_probe_enabled: httpProbe, + verify_tls: verifyTls, + }) onScanNow() onClose() } catch { @@ -84,6 +116,57 @@ export function ScanConfigModal({ open, onClose, onScanNow }: ScanConfigModalPro + {/* Deep Scan (opt-in) */} +
+ + + {deepOpen && ( +
+

+ Scan extra ports and probe HTTP services to identify apps on custom ports. + Overrides the saved defaults for this scan only. +

+ +
+ + setHttpRangesText(e.target.value)} + placeholder="8000-8100, 9000-9100" + className="font-mono text-sm bg-[#0d1117] border-border" + /> +
+ + + + +
+ )} +
+

Status check interval can be configured in the sidebar Settings. diff --git a/frontend/src/components/modals/__tests__/ScanConfigModal.test.tsx b/frontend/src/components/modals/__tests__/ScanConfigModal.test.tsx index 988be69..8bdb4e8 100644 --- a/frontend/src/components/modals/__tests__/ScanConfigModal.test.tsx +++ b/frontend/src/components/modals/__tests__/ScanConfigModal.test.tsx @@ -82,7 +82,12 @@ describe('ScanConfigModal', () => { await screen.findByDisplayValue('192.168.1.0/24') fireEvent.click(screen.getByRole('button', { name: 'Scan Now' })) await waitFor(() => { - expect(scanApi.saveConfig).toHaveBeenCalledWith({ ranges: ['192.168.1.0/24'] }) + expect(scanApi.saveConfig).toHaveBeenCalledWith({ + ranges: ['192.168.1.0/24'], + http_ranges: [], + http_probe_enabled: false, + verify_tls: false, + }) expect(scanApi.trigger).toHaveBeenCalledOnce() expect(onScanNow).toHaveBeenCalledOnce() expect(onClose).toHaveBeenCalledOnce() @@ -108,4 +113,59 @@ describe('ScanConfigModal', () => { ) }) }) + + // --- Deep scan --- + + it('reveals deep-scan fields when the section is toggled', async () => { + render() + await screen.findByDisplayValue('192.168.1.0/24') + expect(screen.queryByText('Enable HTTP probe')).toBeNull() + fireEvent.click(screen.getByText('Deep Scan')) + expect(screen.getByText('Enable HTTP probe')).toBeDefined() + }) + + it('passes deep-scan overrides to trigger() as a per-scan override', async () => { + render() + await screen.findByDisplayValue('192.168.1.0/24') + fireEvent.click(screen.getByText('Deep Scan')) + fireEvent.change(screen.getByPlaceholderText('8000-8100, 9000-9100'), { + target: { value: '8000-8100, 9000' }, + }) + fireEvent.click(screen.getByLabelText('Enable HTTP probe')) + fireEvent.click(screen.getByRole('button', { name: 'Scan Now' })) + await waitFor(() => { + expect(scanApi.trigger).toHaveBeenCalledWith({ + http_ranges: ['8000-8100', '9000'], + http_probe_enabled: true, + verify_tls: false, + }) + }) + }) + + it('auto-opens deep-scan section when a default probe is enabled', async () => { + vi.mocked(scanApi.getConfig).mockResolvedValue({ + data: { ranges: ['192.168.1.0/24'], http_ranges: ['7000-7100'], http_probe_enabled: true, verify_tls: false }, + } as never) + render() + await screen.findByDisplayValue('192.168.1.0/24') + expect(screen.getByText('Enable HTTP probe')).toBeDefined() + expect(screen.getByDisplayValue('7000-7100')).toBeDefined() + }) + + it('saving keeps deep-scan defaults untouched (modal only overrides per-scan)', async () => { + vi.mocked(scanApi.getConfig).mockResolvedValue({ + data: { ranges: ['192.168.1.0/24'], http_ranges: ['7000-7100'], http_probe_enabled: true, verify_tls: true }, + } as never) + render() + await screen.findByDisplayValue('192.168.1.0/24') + fireEvent.click(screen.getByRole('button', { name: 'Scan Now' })) + await waitFor(() => { + expect(scanApi.saveConfig).toHaveBeenCalledWith({ + ranges: ['192.168.1.0/24'], + http_ranges: ['7000-7100'], + http_probe_enabled: true, + verify_tls: true, + }) + }) + }) }) From c68084b751b415f584974593ddad41fe11479e2e Mon Sep 17 00:00:00 2001 From: Pouzor Date: Wed, 24 Jun 2026 10:40:03 +0200 Subject: [PATCH 06/12] test: update client trigger assertion for deep-scan body ha-relevant: yes --- frontend/src/api/__tests__/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/api/__tests__/client.test.ts b/frontend/src/api/__tests__/client.test.ts index 46de58d..ea434d4 100644 --- a/frontend/src/api/__tests__/client.test.ts +++ b/frontend/src/api/__tests__/client.test.ts @@ -170,7 +170,7 @@ describe('api/client', () => { it('scanApi endpoints route correctly', () => { mod.scanApi.trigger() - expect(api.post).toHaveBeenCalledWith('/scan/trigger') + expect(api.post).toHaveBeenCalledWith('/scan/trigger', {}) mod.scanApi.pending() expect(api.get).toHaveBeenCalledWith('/scan/pending') mod.scanApi.hidden() From 96107cc65789031339c3ee84a4ff6e364922ea9b Mon Sep 17 00:00:00 2001 From: Pouzor Date: Thu, 25 Jun 2026 14:47:58 +0200 Subject: [PATCH 07/12] feat: seed port-agnostic http_regex signatures for ~50 popular homelab apps Adds port:null signatures matched by HTTP page title for the most-used self-hosted and home-automation apps (Jellyfin, Plex, *Arr, Home Assistant, Node-RED, Zigbee2MQTT, ESPHome, Domoticz, Jeedom, Portainer, Grafana, Uptime Kuma, Pi-hole, AdGuard Home, Nextcloud, Vaultwarden, Authelia, OpenMediaVault, Unraid, etc.). These identify a service on any port once the deep-scan HTTP probe is enabled. Existing port-keyed entries are unchanged. ha-relevant: yes --- backend/app/data/service_signatures.json | 66 +++++++++++++++++++++++- backend/tests/test_signatures_data.py | 63 ++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_signatures_data.py diff --git a/backend/app/data/service_signatures.json b/backend/app/data/service_signatures.json index 412b14c..54821fb 100644 --- a/backend/app/data/service_signatures.json +++ b/backend/app/data/service_signatures.json @@ -142,5 +142,69 @@ {"port": 1194, "protocol": "udp", "banner_regex": null, "service_name": "OpenVPN", "icon": "shield", "category": "vpn", "suggested_node_type": "router"}, {"port": 500, "protocol": "udp", "banner_regex": null, "service_name": "IPsec IKE", "icon": "shield", "category": "vpn", "suggested_node_type": "router"}, {"port": 53, "protocol": "udp", "banner_regex": null, "service_name": "DNS", "icon": "search", "category": "network", "suggested_node_type": "router"}, - {"port": 67, "protocol": "udp", "banner_regex": null, "service_name": "DHCP", "icon": "wifi", "category": "network", "suggested_node_type": "router"} + {"port": 67, "protocol": "udp", "banner_regex": null, "service_name": "DHCP", "icon": "wifi", "category": "network", "suggested_node_type": "router"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Jellyfin", "service_name": "Jellyfin", "icon": "film", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Plex", "service_name": "Plex", "icon": "play-circle", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Emby", "service_name": "Emby", "icon": "film", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Overseerr", "service_name": "Overseerr", "icon": "tv", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Jellyseerr", "service_name": "Jellyseerr", "icon": "tv", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Tautulli", "service_name": "Tautulli", "icon": "bar-chart", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Navidrome", "service_name": "Navidrome", "icon": "music", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "[Aa]udiobookshelf", "service_name": "Audiobookshelf", "icon": "book-open", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Immich", "service_name": "Immich", "icon": "camera", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "PhotoPrism", "service_name": "PhotoPrism", "icon": "camera", "category": "media", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Calibre[- ]Web", "service_name": "Calibre-Web", "icon": "book", "category": "media", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Sonarr", "service_name": "Sonarr", "icon": "tv", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Radarr", "service_name": "Radarr", "icon": "film", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Lidarr", "service_name": "Lidarr", "icon": "music", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Readarr", "service_name": "Readarr", "icon": "book", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Prowlarr", "service_name": "Prowlarr", "icon": "search", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Bazarr", "service_name": "Bazarr", "icon": "subtitles", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "qBittorrent", "service_name": "qBittorrent", "icon": "download", "category": "download", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "SABnzbd", "service_name": "SABnzbd", "icon": "download", "category": "download", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Homarr", "service_name": "Homarr", "icon": "home", "category": "web", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Heimdall", "service_name": "Heimdall", "icon": "home", "category": "web", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Dashy", "service_name": "Dashy", "icon": "home", "category": "web", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Organizr", "service_name": "Organizr", "icon": "home", "category": "web", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Portainer", "service_name": "Portainer", "icon": "box", "category": "containers", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Dockge", "service_name": "Dockge", "icon": "box", "category": "containers", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Yacht", "service_name": "Yacht", "icon": "box", "category": "containers", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Home Assistant", "service_name": "Home Assistant", "icon": "home", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Node-RED", "service_name": "Node-RED", "icon": "share-2", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Zigbee2MQTT", "service_name": "Zigbee2MQTT", "icon": "radio", "category": "automation", "suggested_node_type": "iot"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "ESPHome", "service_name": "ESPHome", "icon": "cpu", "category": "automation", "suggested_node_type": "iot"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "openHAB", "service_name": "openHAB", "icon": "home", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Domoticz", "service_name": "Domoticz", "icon": "home", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Homebridge", "service_name": "Homebridge", "icon": "home", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Jeedom", "service_name": "Jeedom", "icon": "home", "category": "automation", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Scrypted", "service_name": "Scrypted", "icon": "video", "category": "automation", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Grafana", "service_name": "Grafana", "icon": "bar-chart-2", "category": "monitoring", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Uptime Kuma", "service_name": "Uptime Kuma", "icon": "activity", "category": "monitoring", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Netdata", "service_name": "Netdata", "icon": "activity", "category": "monitoring", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Glances", "service_name": "Glances", "icon": "activity", "category": "monitoring", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Dozzle", "service_name": "Dozzle", "icon": "terminal", "category": "monitoring", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "AdGuard Home", "service_name": "AdGuard Home", "icon": "shield", "category": "network", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Pi-hole", "service_name": "Pi-hole", "icon": "shield", "category": "network", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Nginx Proxy Manager", "service_name": "Nginx Proxy Manager", "icon": "share-2", "category": "network", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Traefik", "service_name": "Traefik", "icon": "share-2", "category": "network", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Vaultwarden|Bitwarden", "service_name": "Vaultwarden", "icon": "lock", "category": "auth", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Authelia", "service_name": "Authelia", "icon": "lock", "category": "auth", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "[Aa]uthentik", "service_name": "Authentik", "icon": "lock", "category": "auth", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Nextcloud", "service_name": "Nextcloud", "icon": "hard-drive", "category": "storage", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Paperless", "service_name": "Paperless-ngx", "icon": "book", "category": "storage", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Syncthing", "service_name": "Syncthing", "icon": "refresh-cw", "category": "storage", "suggested_node_type": "server"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Gitea", "service_name": "Gitea", "icon": "git-branch", "category": "dev", "suggested_node_type": "server"}, + + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "openmediavault", "service_name": "OpenMediaVault", "icon": "hard-drive", "category": "nas", "suggested_node_type": "nas"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Unraid", "service_name": "Unraid", "icon": "hard-drive", "category": "nas", "suggested_node_type": "nas"}, + {"port": null, "protocol": "tcp", "banner_regex": null, "http_regex": "Cockpit", "service_name": "Cockpit", "icon": "monitor", "category": "nas", "suggested_node_type": "server"} ] diff --git a/backend/tests/test_signatures_data.py b/backend/tests/test_signatures_data.py new file mode 100644 index 0000000..4d34221 --- /dev/null +++ b/backend/tests/test_signatures_data.py @@ -0,0 +1,63 @@ +"""Integrity + matching tests against the real service_signatures.json.""" +import re + +import pytest + +from app.services.fingerprint import _load, match_service + +_NODE_TYPES = { + "isp", "router", "switch", "server", "proxmox", "vm", "lxc", + "nas", "iot", "ap", "camera", "generic", +} + + +@pytest.fixture +def signatures(): + return _load() + + +def test_all_entries_well_formed(signatures): + for sig in signatures: + # port is an int or explicitly null (port-agnostic) + assert sig.get("port") is None or isinstance(sig["port"], int) + assert isinstance(sig["service_name"], str) and sig["service_name"] + assert sig["suggested_node_type"] in _NODE_TYPES + if sig.get("banner_regex"): + re.compile(sig["banner_regex"]) + if sig.get("http_regex"): + re.compile(sig["http_regex"]) + + +def test_port_agnostic_entries_require_http_regex(signatures): + for sig in signatures: + if sig.get("port") is None: + assert sig.get("http_regex"), f"port:null entry needs http_regex: {sig}" + + +def test_popular_apps_have_port_agnostic_signatures(signatures): + names = {s["service_name"] for s in signatures if s.get("port") is None} + for expected in { + "Jellyfin", "Plex", "Home Assistant", "Portainer", "Pi-hole", + "AdGuard Home", "Grafana", "Nextcloud", "Vaultwarden", "Sonarr", + }: + assert expected in names, f"missing port-agnostic signature for {expected}" + + +@pytest.mark.parametrize(("title", "expected"), [ + ("Jellyfin", "Jellyfin"), + ("Home Assistant", "Home Assistant"), + ("Portainer", "Portainer"), + ("Vaultwarden Web Vault", "Vaultwarden"), + ("Pi-hole - Dashboard", "Pi-hole"), + ("Audiobookshelf", "Audiobookshelf"), +]) +def test_custom_port_identified_via_http_title(title, expected): + # A service on a non-standard port, recognised purely by its HTML title. + sig = match_service(58000, "tcp", banner=None, http_signals={"title": title, "headers": {}}) + assert sig is not None + assert sig["service_name"] == expected + + +def test_custom_port_without_probe_is_unknown(): + # Same custom port, deep scan off → no signal → no port-agnostic match. + assert match_service(58000, "tcp", banner=None, http_signals=None) is None From d7ab4ba49aeba28a5be52e836cea082a2890e9f9 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Thu, 25 Jun 2026 18:05:13 +0200 Subject: [PATCH 08/12] =?UTF-8?q?feat:=20Device=20Inventory=20=E2=80=94=20?= =?UTF-8?q?show=20all=20scanned=20devices=20with=20canvas-presence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks the "Pending Devices" panel into a "Device Inventory": scanned devices already placed on a canvas are no longer suppressed — they stay listed and badged with how many canvases they appear on. - scanner: stop deleting/skipping on-canvas IPs (hidden still suppressed) - scan API: /pending returns all non-hidden devices; compute canvas_count by correlating ip/ieee_address against nodes grouped by design - frontend: rename to "Device Inventory", top-right canvas-count corner, toggle to show/hide on-canvas devices (default show) ha-relevant: maybe --- backend/app/api/routes/scan.py | 56 ++++++++++- backend/app/schemas/scan.py | 3 + backend/app/services/scanner.py | 26 ++---- backend/tests/test_scan.py | 92 ++++++++++++++++--- backend/tests/test_scanner.py | 9 +- .../components/modals/PendingDeviceModal.tsx | 2 + .../components/modals/PendingDevicesModal.tsx | 39 +++++++- .../__tests__/PendingDevicesModal.test.tsx | 51 ++++++++++ 8 files changed, 236 insertions(+), 42 deletions(-) diff --git a/backend/app/api/routes/scan.py b/backend/app/api/routes/scan.py index 4e1214a..7f0da05 100644 --- a/backend/app/api/routes/scan.py +++ b/backend/app/api/routes/scan.py @@ -169,10 +169,60 @@ async def stop_scan( return {"stopping": True} +async def _canvas_counts( + db: AsyncSession, devices: list[PendingDevice] +) -> dict[str, int]: + """Map each device id → number of distinct canvases (designs) it appears on. + + Correlates a scanned device to existing nodes by ``ieee_address`` (exact) or + ``ip`` (exact). Runs a single node query and groups in Python — node counts are + small for a homelab, so this avoids an N+1 per device. + """ + if not devices: + return {} + rows = ( + await db.execute( + select(Node.ip, Node.ieee_address, Node.design_id).where( + Node.design_id.isnot(None) + ) + ) + ).all() + # ip → set(design_id), ieee → set(design_id) + by_ip: dict[str, set[str]] = {} + by_ieee: dict[str, set[str]] = {} + for ip, ieee, design_id in rows: + if ip: + by_ip.setdefault(ip, set()).add(design_id) + if ieee: + by_ieee.setdefault(ieee, set()).add(design_id) + + counts: dict[str, int] = {} + for d in devices: + designs: set[str] = set() + if d.ieee_address: + designs |= by_ieee.get(d.ieee_address, set()) + if d.ip: + designs |= by_ip.get(d.ip, set()) + counts[d.id] = len(designs) + return counts + + +async def _with_canvas_counts( + db: AsyncSession, devices: list[PendingDevice] +) -> list[PendingDevice]: + """Attach a transient ``canvas_count`` to each device for the response schema.""" + counts = await _canvas_counts(db, devices) + for d in devices: + d.canvas_count = counts.get(d.id, 0) + return devices + + @router.get("/pending", response_model=list[PendingDeviceResponse]) async def list_pending(db: AsyncSession = Depends(get_db), _: str = Depends(get_current_user)) -> list[PendingDevice]: - result = await db.execute(select(PendingDevice).where(PendingDevice.status == "pending")) - return list(result.scalars().all()) + # Inventory: every scanned device except the user-hidden ones. Approved devices + # stay listed so they keep showing with a canvas-presence badge. + result = await db.execute(select(PendingDevice).where(PendingDevice.status != "hidden")) + return await _with_canvas_counts(db, list(result.scalars().all())) @router.delete("/pending", response_model=dict) @@ -189,7 +239,7 @@ async def clear_pending( @router.get("/hidden", response_model=list[PendingDeviceResponse]) async def list_hidden(db: AsyncSession = Depends(get_db), _: str = Depends(get_current_user)) -> list[PendingDevice]: result = await db.execute(select(PendingDevice).where(PendingDevice.status == "hidden")) - return list(result.scalars().all()) + return await _with_canvas_counts(db, list(result.scalars().all())) @router.post("/pending/bulk-approve", response_model=dict) diff --git a/backend/app/schemas/scan.py b/backend/app/schemas/scan.py index 92151bf..b0f5b78 100644 --- a/backend/app/schemas/scan.py +++ b/backend/app/schemas/scan.py @@ -21,6 +21,9 @@ class PendingDeviceResponse(BaseModel): vendor: str | None = None lqi: int | None = None discovered_at: datetime + # Number of distinct canvases (designs) this device already appears on, + # correlated by ip / ieee_address against existing nodes. Computed per-request. + canvas_count: int = 0 model_config = {"from_attributes": True} diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.py index fccb315..909f268 100644 --- a/backend/app/services/scanner.py +++ b/backend/app/services/scanner.py @@ -14,7 +14,7 @@ from typing import Any from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db.models import Node, PendingDevice, ScanRun +from app.db.models import PendingDevice, ScanRun from app.services.fingerprint import fingerprint_ports, suggest_node_type from app.services.http_probe import probe_open_ports @@ -432,26 +432,14 @@ async def run_scan( except ValueError: raise ValueError(f"Invalid CIDR range: {r!r}") from None - # Pre-fetch canvas IPs and hidden IPs once — avoids N+1 queries per host - canvas_ips_result = await db.execute(select(Node.ip).where(Node.ip.isnot(None))) - canvas_ips: set[str] = {row[0] for row in canvas_ips_result.fetchall()} - + # Pre-fetch hidden IPs once — avoids N+1 queries per host. + # Devices already on a canvas are intentionally NOT suppressed: they stay + # in the inventory and are badged "In N canvas" via per-request correlation. hidden_ips_result = await db.execute( select(PendingDevice.ip).where(PendingDevice.status == "hidden") ) hidden_ips: set[str] = {row[0] for row in hidden_ips_result.fetchall()} - # Clean up stale pending devices whose IPs are already in the canvas - if canvas_ips: - from sqlalchemy import delete as sa_delete - await db.execute( - sa_delete(PendingDevice).where( - PendingDevice.status == "pending", - PendingDevice.ip.in_(canvas_ips), - ) - ) - await db.commit() - # Start mDNS discovery in the background while nmap scans run mdns_task = asyncio.create_task(_mdns_discover()) @@ -462,10 +450,8 @@ async def run_scan( nonlocal devices_found ip = host["ip"] - # Skip canvas nodes and user-hidden devices (sets pre-fetched before loop) - if ip in canvas_ips: - logger.debug("Skipping %s — already in canvas", ip) - return + # Skip only user-hidden devices. On-canvas devices are kept so they + # surface in the inventory with a canvas-presence badge. if ip in hidden_ips: logger.debug("Skipping %s — hidden by user", ip) return diff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.py index 7032772..5879390 100644 --- a/backend/tests/test_scan.py +++ b/backend/tests/test_scan.py @@ -7,7 +7,7 @@ from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db.models import Node, PendingDevice, ScanRun +from app.db.models import Design, Node, PendingDevice, ScanRun from app.services.scanner import _cancelled_runs, request_cancel, run_scan @@ -167,6 +167,66 @@ async def test_list_pending_returns_device(client: AsyncClient, headers, pending assert len(data) == 1 assert data[0]["ip"] == "192.168.1.100" assert data[0]["hostname"] == "my-server" + # No matching node → not on any canvas. + assert data[0]["canvas_count"] == 0 + + +# --- Canvas-presence correlation (canvas_count) --- + +async def _add_design(db_session, name: str) -> str: + design = Design(id=str(uuid.uuid4()), name=name) + db_session.add(design) + await db_session.commit() + return design.id + + +def _node(design_id: str, *, ip=None, ieee=None) -> Node: + return Node( + id=str(uuid.uuid4()), label="n", type="server", status="online", + ip=ip, ieee_address=ieee, services=[], pos_x=0.0, pos_y=0.0, + design_id=design_id, + ) + + +@pytest.mark.asyncio +async def test_canvas_count_counts_distinct_designs_by_ip(client, headers, db_session, pending_device): + # Same IP placed on two different canvases → canvas_count == 2. + d1 = await _add_design(db_session, "Home") + d2 = await _add_design(db_session, "Lab") + db_session.add(_node(d1, ip="192.168.1.100")) + db_session.add(_node(d2, ip="192.168.1.100")) + await db_session.commit() + + res = await client.get("/api/v1/scan/pending", headers=headers) + data = res.json() + assert len(data) == 1 + assert data[0]["canvas_count"] == 2 + + +@pytest.mark.asyncio +async def test_canvas_count_correlates_by_ieee(client, headers, db_session): + device = PendingDevice( + id=str(uuid.uuid4()), ieee_address="0x00124b001", discovery_source="zigbee", + suggested_type="zigbee_enddevice", services=[], status="pending", + ) + db_session.add(device) + d1 = await _add_design(db_session, "Zigbee") + db_session.add(_node(d1, ieee="0x00124b001")) + await db_session.commit() + + res = await client.get("/api/v1/scan/pending", headers=headers) + by_id = {d["id"]: d for d in res.json()} + assert by_id[device.id]["canvas_count"] == 1 + + +@pytest.mark.asyncio +async def test_canvas_count_ignores_nodes_without_design(client, headers, db_session, pending_device): + # A node with no design_id is not "on a canvas". + db_session.add(_node(None, ip="192.168.1.100")) + await db_session.commit() + + res = await client.get("/api/v1/scan/pending", headers=headers) + assert res.json()[0]["canvas_count"] == 0 # --- Approve device --- @@ -191,9 +251,13 @@ async def test_approve_device(client: AsyncClient, headers, pending_device): assert data["approved"] is True assert "node_id" in data - # Device should no longer appear in pending list + # Approved devices stay in the inventory (status != "hidden") so they keep + # showing with an "In N canvas" badge — they are no longer dropped. pending_res = await client.get("/api/v1/scan/pending", headers=headers) - assert pending_res.json() == [] + inventory = pending_res.json() + assert len(inventory) == 1 + assert inventory[0]["id"] == pending_device.id + assert inventory[0]["status"] == "approved" @pytest.mark.asyncio @@ -332,8 +396,9 @@ async def test_run_scan_creates_new_pending_device(db_session: AsyncSession): @pytest.mark.asyncio -async def test_run_scan_purges_stale_pending_for_canvas_nodes(db_session: AsyncSession): - """Pending devices that were already in canvas before scan starts must be removed.""" +async def test_run_scan_keeps_stale_pending_for_canvas_nodes(db_session: AsyncSession): + """Pending devices whose IP is already on a canvas are NOT purged — they stay + in the inventory and are surfaced with an "In N canvas" badge.""" node = Node( id=str(uuid.uuid4()), label="Existing Server", @@ -372,12 +437,13 @@ async def test_run_scan_purges_stale_pending_for_canvas_nodes(db_session: AsyncS result = await db_session.execute( select(PendingDevice).where(PendingDevice.ip == "192.168.1.50") ) - assert result.scalar_one_or_none() is None + assert result.scalar_one_or_none() is not None @pytest.mark.asyncio -async def test_run_scan_skips_ip_already_in_canvas(db_session: AsyncSession): - """Devices whose IP already exists as a canvas Node must not appear in pending.""" +async def test_run_scan_records_ip_already_in_canvas(db_session: AsyncSession): + """A scanned IP that already exists as a canvas Node still produces a pending + device (no longer suppressed).""" node = Node( id=str(uuid.uuid4()), label="Existing Server", @@ -405,7 +471,9 @@ async def test_run_scan_skips_ip_already_in_canvas(db_session: AsyncSession): result = await db_session.execute( select(PendingDevice).where(PendingDevice.ip == "192.168.1.50") ) - assert result.scalar_one_or_none() is None + device = result.scalar_one_or_none() + assert device is not None + assert device.status == "pending" @pytest.mark.asyncio @@ -613,9 +681,11 @@ async def test_bulk_approve_approves_devices(client: AsyncClient, headers, two_p assert all(nid is not None for nid in data["node_ids"]), "node_ids must be non-null UUIDs" assert len(data["device_ids"]) == 2 assert data["skipped"] == 0 - # Pending list should now be empty + # Approved devices stay in the inventory, now marked "approved". pending_res = await client.get("/api/v1/scan/pending", headers=headers) - assert pending_res.json() == [] + inventory = pending_res.json() + assert len(inventory) == 2 + assert all(d["status"] == "approved" for d in inventory) @pytest.fixture diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index b7d66fa..2b02326 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -457,8 +457,9 @@ async def test_run_scan_mdns_skipped_if_already_in_nmap(mem_db): @pytest.mark.asyncio -async def test_run_scan_skips_canvas_nodes(mem_db): - """Hosts already approved onto the canvas must be skipped.""" +async def test_run_scan_keeps_canvas_nodes(mem_db): + """Hosts already on a canvas are NOT suppressed — they stay in the inventory + (badged "In N canvas" via correlation), so a re-scan still records them.""" from app.services.scanner import run_scan run_id = _make_run_id() @@ -481,7 +482,9 @@ async def test_run_scan_skips_canvas_nodes(mem_db): async with mem_db() as session: result = await session.execute(sa_select(PendingDevice).where(PendingDevice.ip == "192.168.1.100")) - assert result.scalar_one_or_none() is None + device = result.scalar_one_or_none() + assert device is not None + assert device.status == "pending" @pytest.mark.asyncio diff --git a/frontend/src/components/modals/PendingDeviceModal.tsx b/frontend/src/components/modals/PendingDeviceModal.tsx index aa76069..542363b 100644 --- a/frontend/src/components/modals/PendingDeviceModal.tsx +++ b/frontend/src/components/modals/PendingDeviceModal.tsx @@ -27,6 +27,8 @@ export interface PendingDevice { vendor?: string | null lqi?: number | null discovered_at: string + // How many canvases (designs) this device already appears on. Computed server-side. + canvas_count?: number } interface PendingDeviceModalProps { diff --git a/frontend/src/components/modals/PendingDevicesModal.tsx b/frontend/src/components/modals/PendingDevicesModal.tsx index de35096..02657cb 100644 --- a/frontend/src/components/modals/PendingDevicesModal.tsx +++ b/frontend/src/components/modals/PendingDevicesModal.tsx @@ -119,6 +119,8 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus const [sourceFilter, setSourceFilter] = useState('all') const [typeFilter, setTypeFilter] = useState('all') const [statusFilter, setStatusFilter] = useState(initialStatus) + // Inventory shows on-canvas devices by default; toggle off to hide them. + const [showOnCanvas, setShowOnCanvas] = useState(true) const { addNode, scanEventTs } = useCanvasStore() const highlightRef = useRef(null) @@ -159,6 +161,8 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus return devices.filter((d) => { if (sourceFilter !== 'all' && inferSource(d) !== sourceFilter) return false if (typeFilter !== 'all' && d.suggested_type !== typeFilter) return false + // Inventory-only: optionally hide devices already placed on a canvas. + if (statusFilter === 'pending' && !showOnCanvas && (d.canvas_count ?? 0) > 0) return false if (q) { const hay = [ d.friendly_name, d.hostname, d.ip, d.mac, d.ieee_address, d.vendor, d.model, @@ -168,7 +172,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus } return true }) - }, [devices, search, sourceFilter, typeFilter]) + }, [devices, search, sourceFilter, typeFilter, statusFilter, showOnCanvas]) useEffect(() => { if (!highlightId || loading || !open) return @@ -241,9 +245,11 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus if (failed > 0) toast.error(`Removed ${removedIds.size}, ${failed} failed`) else toast.success(`Removed ${removedIds.size} device${removedIds.size !== 1 ? 's' : ''}`) } else { + // Clears only pending rows server-side; approved/on-canvas devices stay, + // so reload rather than blanking the whole inventory. await scanApi.clearPending() - setDevices([]) setSelectedIds(new Set()) + await load() toast.success('Pending devices cleared') } } catch { @@ -398,7 +404,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus

- {statusFilter === 'pending' ? 'Pending Devices' : 'Hidden Devices'} + {statusFilter === 'pending' ? 'Device Inventory' : 'Hidden Devices'} ({filtered.length}{filtered.length !== devices.length && ` of ${devices.length}`}) @@ -479,7 +485,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus onClick={() => setStatusFilter('pending')} className={`px-2.5 py-1.5 transition-colors ${statusFilter === 'pending' ? 'bg-[#00d4ff]/20 text-[#00d4ff]' : 'bg-[#0d1117] text-muted-foreground hover:text-foreground'}`} > - Pending + Inventory
+ {statusFilter === 'pending' && ( + + )} )} +