url/homelable
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: merge IP-scanned and Proxmox-imported devices by MAC

Browse Source 
Reconcile the same physical device discovered by both the nmap IP scan and
the Proxmox importer into a single inventory row, keyed on MAC. Previously
each path only deduped by IP, and the importer captured no MAC (and no IP for
stopped guests), so most guests double-listed.

Backend:
- mac_utils.normalize_mac: canonical MAC (lowercase, ':'-separated), the
  cross-source join key. Normalized on write and on compare.
- proxmox_service: capture the guest NIC MAC agent-free from the net0 config
  (qemu virtio=<MAC>, lxc hwaddr=<MAC>); works for stopped guests. Resolver
  now returns (ip, mac).
- proxmox persist: match existing Node/PendingDevice by ieee OR ip OR MAC;
  fill mac, keep the vm/lxc type, union sources.
- scanner persist: match PendingDevice by ip OR MAC; fill the IP a Proxmox
  import lacked, keep a pve row's type, union the scan source. Stamp query
  matches raw + normalized MAC (legacy-safe).
- Multi-source tags: new PendingDevice.discovery_sources JSON column so a
  merged device shows under both the IP and Proxmox filters. Idempotent
  migration backfills from discovery_source (legacy NULL-scalar rows with an
  IP become ["arp"]). _sources_after_merge preserves a scanned row's IP origin
  through the merge without tagging a pure Proxmox guest.
- Import now broadcasts a scan update on completion so an open inventory
  reloads without a manual refresh.

Frontend:
- pendingSources: sourceBuckets/orderedSources map discovery_sources to filter
  buckets; a device with ["arp","proxmox"] matches both filters and renders
  both badges. PendingDevicesModal filter + badges use them.

Tests: MAC normalization, config MAC capture, cross-source merge both
directions, legacy-row IP-tag preservation, no-false-IP-tag guard, refresh
broadcast, and the frontend bucket mapping.

ha-relevant: maybe
This commit is contained in:
Pouzor
2026-07-06 20:11:03 +02:00
parent 05ef746f22
commit adf82f8f01
16 changed files with 572 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/discovery_sources.py
+19
View File
@@ -0,0 +1,19 @@
"""Helpers for the multi-valued ``PendingDevice.discovery_sources`` set.
A device discovered by more than one path (e.g. an IP scan *and* a Proxmox
import) accumulates every source that has seen it, so it surfaces under each
matching inventory filter. Order is preserved (origin first) and duplicates are
dropped.
"""
from __future__ import annotations
from collections.abc import Iterable
def add_source(sources: Iterable[str] | None, source: str | None) -> list[str]:
"""Return ``sources`` with ``source`` appended if not already present."""
out = [s for s in (sources or []) if s]
if source and source not in out:
out.append(source)
return out
backend/app/services/mac_utils.py
+18
View File
@@ -0,0 +1,18 @@
"""MAC-address normalization, shared by the scan + Proxmox persist paths.
Different discovery sources emit MACs in different casing/separators (ARP is
lowercase ``bc:24:11:..``, Proxmox config is often uppercase ``BC:24:11:..``).
Canonicalizing on write *and* on compare lets cross-source dedup match a device
by MAC with a plain ``==`` — the join key for merging an IP-scanned row with a
Proxmox-imported one.
"""
from __future__ import annotations
def normalize_mac(mac: str | None) -> str | None:
"""Canonical MAC: lowercase, ``-`` → ``:``, stripped. Blank/None → None."""
if not mac:
return None
normalized = mac.strip().lower().replace("-", ":")
return normalized or None
backend/app/services/proxmox_service.py
+53 -17
View File
@@ -18,6 +18,7 @@ from typing import Any
import httpx
from app.services.mac_utils import normalize_mac
from app.services.zigbee_service import merge_zigbee_properties
logger = logging.getLogger(__name__)
@@ -32,6 +33,9 @@ _BYTES_PER_GB = 1024 ** 3
# net0 config line: "name=eth0,bridge=vmbr0,ip=192.168.1.5/24,gw=..."
_LXC_IP_RE = re.compile(r"(?:^|,)ip=([0-9]{1,3}(?:\.[0-9]{1,3}){3})(?:/\d+)?")
# NIC MAC inside a net0 string — qemu "virtio=BC:24:11:..,bridge=.." or lxc
# "..,hwaddr=BC:24:11:..,..". A bare 6-octet MAC match works for both forms.
_NET_MAC_RE = re.compile(r"([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})")
def _sanitize_proxmox_error(exc: BaseException) -> str:
@@ -121,6 +125,21 @@ def _extract_lxc_ip(config_payload: dict[str, Any] | None) -> str | None:
return match.group(1) if match else None
def _extract_net_mac(config_payload: dict[str, Any] | None) -> str | None:
"""Parse the NIC MAC from a qemu/lxc ``net0`` config string (normalized).
Works agent-free for both guest kinds and for stopped guests — the sole
identity we can reliably cross-match against an ARP-scanned device.
"""
if not config_payload:
return None
net0 = config_payload.get("net0")
if not isinstance(net0, str):
return None
match = _NET_MAC_RE.search(net0)
return normalize_mac(match.group(1)) if match else None
def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None:
"""Build a homelable ``proxmox`` host node from a ``/nodes`` entry."""
name = raw.get("node")
@@ -144,7 +163,9 @@ def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None:
}
def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None) -> dict[str, Any] | None:
def _guest_node(
raw: dict[str, Any], host_name: str, kind: str, ip: str | None, mac: str | None = None
) -> dict[str, Any] | None:
"""Build a homelable vm/lxc node from a ``/qemu`` or ``/lxc`` list entry."""
vmid = raw.get("vmid")
if vmid is None:
@@ -159,6 +180,7 @@ def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None)
"ieee_address": ieee,
"hostname": name,
"ip": ip,
"mac": mac,
"status": "online" if raw.get("status") == "running" else "offline",
"cpu_count": _int_or_none(raw.get("maxcpu") or raw.get("cpus")),
"ram_gb": _gb(raw.get("maxmem")),
@@ -320,34 +342,48 @@ async def _fetch_host_guests(
if not isinstance(entries, list):
continue
for raw in entries:
ip = await _resolve_guest_ip(client, host_name, kind, raw)
node = _guest_node(raw, host_name, kind, ip)
ip, mac = await _resolve_guest_net(client, host_name, kind, raw)
node = _guest_node(raw, host_name, kind, ip, mac)
if node:
guests.append(node)
return guests
async def _resolve_guest_ip(
async def _resolve_guest_net(
client: httpx.AsyncClient, host_name: str, kind: str, raw: dict[str, Any]
) -> str | None:
"""Best-effort guest IP. qemu → guest agent, lxc → net0 config. Never raises."""
) -> tuple[str | None, str | None]:
"""Best-effort (ip, mac) for a guest. Never raises.
- MAC comes from the guest ``/config`` net0 line for both kinds (agent-free,
works for stopped guests) — the cross-source dedup key.
- IP: qemu → guest agent (running only); lxc → static net0 config.
Partial results are returned even if a later call fails (e.g. config MAC is
kept when the qemu agent call errors).
"""
vmid = raw.get("vmid")
if vmid is None:
return None
return None, None
ip: str | None = None
mac: str | None = None
try:
if kind == "qemu":
if raw.get("status") != "running":
return None
data = await _get_json(
client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces"
)
return _extract_qemu_ip(data)
data = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config")
return _extract_lxc_ip(data)
config = await _get_json(client, f"/nodes/{host_name}/qemu/{vmid}/config")
mac = _extract_net_mac(config)
if raw.get("status") == "running":
data = await _get_json(
client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces"
)
ip = _extract_qemu_ip(data)
else:
config = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config")
ip = _extract_lxc_ip(config)
mac = _extract_net_mac(config)
except httpx.HTTPError:
# Guest agent not installed / container stopped / no perms → no IP. Fine.
return None
# Guest agent not installed / container stopped / no perms. Keep whatever
# was resolved before the failure.
pass
return ip, mac
async def test_proxmox_connection(
backend/app/services/scanner.py
+29 -13
View File
@@ -15,8 +15,10 @@ from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Node, PendingDevice, ScanRun
from app.services.discovery_sources import add_source
from app.services.fingerprint import fingerprint_ports, suggest_node_type
from app.services.http_probe import probe_open_ports
from app.services.mac_utils import normalize_mac
logger = logging.getLogger(__name__)
@@ -522,16 +524,21 @@ async def run_scan(
ip, open_ports, verify_tls=deep_scan.verify_tls
)
norm_mac = normalize_mac(host.get("mac"))
services = fingerprint_ports(open_ports)
suggested_type = suggest_node_type(open_ports, host.get("mac"))
suggested_type = suggest_node_type(open_ports, norm_mac)
# One inventory row per device (by IP). Match across pending AND
# approved so a re-scan of an already-approved device refreshes its
# row instead of spawning a fresh "pending" duplicate. Hidden rows
# are already skipped above.
# One inventory row per device. Match by IP OR MAC across pending AND
# approved so a re-scan refreshes the existing row instead of spawning
# a duplicate — and so a device previously imported from Proxmox (which
# may have no IP but a known NIC MAC) reconciles with this scan instead
# of doubling up. Hidden rows are already skipped above.
match_cond = [PendingDevice.ip == ip]
if norm_mac:
match_cond.append(PendingDevice.mac == norm_mac)
existing_rows = (await db.execute(
select(PendingDevice)
.where(PendingDevice.ip == ip, PendingDevice.status != "hidden")
.where(or_(*match_cond), PendingDevice.status != "hidden")
.order_by(PendingDevice.discovered_at)
)).scalars().all()
@@ -543,32 +550,41 @@ async def run_scan(
for dup in existing_rows:
if dup is not keep:
await db.delete(dup)
keep.mac = host.get("mac") or keep.mac
keep.ip = keep.ip or ip # fill an IP a Proxmox import lacked
keep.mac = norm_mac or keep.mac
keep.hostname = host.get("hostname") or keep.hostname
keep.os = host.get("os") or keep.os
keep.services = services
keep.suggested_type = suggested_type
# Don't downgrade a Proxmox-typed guest (vm/lxc) to the generic
# scan guess; the importer knows the true type.
if not (keep.ieee_address or "").startswith("pve-"):
keep.suggested_type = suggested_type
# Merged row carries both sources (e.g. ["proxmox", "arp"]).
keep.discovery_sources = add_source(keep.discovery_sources, discovery_source)
# status preserved — an approved device stays approved.
else:
db.add(PendingDevice(
ip=ip,
mac=host.get("mac"),
mac=norm_mac,
hostname=host.get("hostname"),
os=host.get("os"),
services=services,
suggested_type=suggested_type,
status="pending",
discovery_source=discovery_source,
discovery_sources=[discovery_source],
))
devices_found += 1
# Stamp last_scan on any canvas node that matches this device by IP
# (or MAC, when known) so the inventory shows when the scanner last
# observed it. Matches across designs.
host_mac = host.get("mac")
# observed it. Match both the normalized and raw MAC so a legacy
# canvas node whose mac predates normalization still matches. Across
# designs.
node_match = [Node.ip == ip]
if host_mac:
node_match.append(Node.mac == host_mac)
for m in {norm_mac, host.get("mac")}:
if m:
node_match.append(Node.mac == m)
matching_nodes = (await db.execute(
select(Node).where(or_(*node_match))
)).scalars().all()