From ab36ba6f81d906fa35801afddfdd5e5d318d21ef Mon Sep 17 00:00:00 2001 From: Pouzor Date: Sun, 5 Jul 2026 18:58:12 +0200 Subject: [PATCH 1/6] feat: import hosts/VMs/LXC from Proxmox VE with optional auto-sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Proxmox VE importer that reads the /api2/json REST API with a read-only API token and drops hosts (proxmox), VMs (vm) and LXC containers (lxc) onto the canvas as typed nodes with run state and hardware specs (vCPU/RAM/disk). - Backend: proxmox_service (httpx) + proxmox routes (test-connection, import, import-pending, config). Two-tier dedupe β€” merge onto an existing scanned node by IP, else synthetic pve-{host}-{vmid} identity. Update-in-place, never deletes. Host->guest rendered as a 'virtual' edge via the pending-link flow. - Security: token is env-only (PROXMOX_TOKEN_*), never written to disk by the app, never returned by any endpoint; errors are credential-sanitized. - Auto-sync: optional scheduled re-import into pending (APScheduler job). - PendingDevice.properties carries specs through approve (+ migration). - Frontend: ProxmoxImportModal, sidebar entry, pending inventory source filter, Settings auto-sync section, proxmoxApi client. - Docs: docs/proxmox-import.md, README + FEATURES sections, .env.example keys. - Tests: backend service/router/scheduler, frontend modal/client/pending. ha-relevant: maybe --- .env.example | 12 + FEATURES.md | 43 +- README.md | 31 ++ backend/app/api/routes/proxmox.py | 379 +++++++++++++++++ backend/app/api/routes/scan.py | 12 +- backend/app/core/config.py | 31 ++ backend/app/core/scheduler.py | 67 +++ backend/app/db/database.py | 3 + backend/app/db/models.py | 4 + backend/app/main.py | 17 +- backend/app/schemas/proxmox.py | 75 ++++ backend/app/schemas/scan.py | 3 + backend/app/services/proxmox_service.py | 348 ++++++++++++++++ backend/tests/test_proxmox_router.py | 188 +++++++++ backend/tests/test_proxmox_service.py | 123 ++++++ backend/tests/test_scheduler.py | 52 ++- docs/proxmox-import.md | 188 +++++++++ frontend/src/App.tsx | 61 +++ frontend/src/api/__tests__/client.test.ts | 18 + frontend/src/api/client.ts | 45 ++ .../components/modals/PendingDeviceModal.tsx | 3 + .../components/modals/PendingDevicesModal.tsx | 25 +- .../src/components/modals/SettingsModal.tsx | 65 ++- .../__tests__/PendingDevicesModal.test.tsx | 35 ++ .../modals/__tests__/SettingsModal.test.tsx | 8 +- frontend/src/components/panels/Sidebar.tsx | 6 +- .../components/proxmox/ProxmoxImportModal.tsx | 385 ++++++++++++++++++ .../__tests__/ProxmoxImportModal.test.tsx | 121 ++++++ frontend/src/components/proxmox/types.ts | 30 ++ 29 files changed, 2349 insertions(+), 29 deletions(-) create mode 100644 backend/app/api/routes/proxmox.py create mode 100644 backend/app/schemas/proxmox.py create mode 100644 backend/app/services/proxmox_service.py create mode 100644 backend/tests/test_proxmox_router.py create mode 100644 backend/tests/test_proxmox_service.py create mode 100644 docs/proxmox-import.md create mode 100644 frontend/src/components/proxmox/ProxmoxImportModal.tsx create mode 100644 frontend/src/components/proxmox/__tests__/ProxmoxImportModal.test.tsx create mode 100644 frontend/src/components/proxmox/types.ts diff --git a/.env.example b/.env.example index 9dbfdcf..a53f833 100644 --- a/.env.example +++ b/.env.example @@ -46,3 +46,15 @@ MCP_SERVICE_KEY=svc_changeme # the same value in the `X-API-Key` header. # Generate: python3 -c "import secrets; print(secrets.token_urlsafe(32))" # HOMEPAGE_API_KEY= + +# Proxmox VE import β€” pull hosts/VMs/LXC from the Proxmox REST API. +# The token is a credential: kept in memory only, never written to disk by the +# app, never returned by any API. Create it under Datacenter β†’ Permissions β†’ +# API Tokens and grant the read-only PVEAuditor role at path "/". +# Token id format: user@realm!tokenname (e.g. root@pam!homelable). +# Required only for auto-sync; one-off imports can pass the token in the dialog. +# PROXMOX_TOKEN_ID=root@pam!homelable +# PROXMOX_TOKEN_SECRET=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# PROXMOX_HOST=192.168.1.10 +# PROXMOX_PORT=8006 +# PROXMOX_VERIFY_TLS=true # set false only for self-signed certs diff --git a/FEATURES.md b/FEATURES.md index 0b15749..eb68124 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -21,13 +21,14 @@ Here's what Homelable can do. One line on what each feature is, then how to swit 7. [Network Scanner (IP import)](#7-network-scanner-ip-import-) 8. [Zigbee Import](#8-zigbee-import-) 9. [Z-Wave Import](#9-z-wave-import-) -10. [Device Inventory](#10-device-inventory-) -11. [Live Status Monitoring](#11-live-status-monitoring-) -12. [Export (PNG / SVG / YAML / Markdown)](#12-export) -13. [Live View (read-only public canvas)](#13-live-view-) -14. [Gethomepage Widget](#14-gethomepage-widget-) -15. [MCP Server (AI integration)](#15-mcp-server-) -16. [Settings & Shortcuts](#16-settings--shortcuts) +10. [Proxmox VE Import](#10-proxmox-ve-import-) +11. [Device Inventory](#11-device-inventory-) +12. [Live Status Monitoring](#12-live-status-monitoring-) +13. [Export (PNG / SVG / YAML / Markdown)](#13-export) +14. [Live View (read-only public canvas)](#14-live-view-) +15. [Gethomepage Widget](#15-gethomepage-widget-) +16. [MCP Server (AI integration)](#16-mcp-server-) +17. [Settings & Shortcuts](#17-settings--shortcuts) --- @@ -140,7 +141,21 @@ Nodes: `zwave_coordinator` / `zwave_router` / `zwave_enddevice`. The hierarchy c --- -## 10. Device Inventory πŸ”’ +## 10. Proxmox VE Import πŸ”’ + +**What:** Pull your **Proxmox VE** inventory (hosts, VMs, LXC) in over the Proxmox REST API β€” typed, named nodes with run state and hardware specs. Optional scheduled **auto-sync**; guest IPs already found by a scan are merged, not duplicated. + +**Use:** +1. Create a read-only API token in Proxmox (Datacenter β†’ Permissions β†’ API Tokens, role `PVEAuditor`). +2. Sidebar β†’ **Proxmox Import**. +3. Enter host, port (default `8006`), and the token (`user@realm!tokenid` + secret) β€” or leave blank to use the server token. +4. **Test Connection** β†’ send to **Pending** or the **Canvas** β†’ import β†’ pick devices β†’ **Add N to Canvas**. + +Nodes: `proxmox` (host) / `vm` / `lxc`, linked hostβ†’guest by a `virtual` edge. The token is env-only, never stored on disk, never returned by the API. Enable auto-sync from **Settings** once `PROXMOX_TOKEN_ID` / `PROXMOX_TOKEN_SECRET` are set. More: [docs/proxmox-import.md](./docs/proxmox-import.md). + +--- + +## 11. Device Inventory πŸ”’ **What:** The holding pen for everything found by a scan or import that isn't on the canvas yet, plus a separate **Hidden Devices** list. @@ -151,7 +166,7 @@ Nodes: `zwave_coordinator` / `zwave_router` / `zwave_enddevice`. The hierarchy c --- -## 11. Live Status Monitoring πŸ”’ +## 12. Live Status Monitoring πŸ”’ **What:** Keeps checking each node and shows its status (🟒 online / πŸ”΄ offline / ⚫ unknown) right on the canvas. @@ -172,7 +187,7 @@ Nodes: `zwave_coordinator` / `zwave_router` / `zwave_enddevice`. The hierarchy c --- -## 12. Export +## 13. Export **What:** Get your canvas out as a picture or as structured data. @@ -184,7 +199,7 @@ Nodes: `zwave_coordinator` / `zwave_router` / `zwave_enddevice`. The hierarchy c --- -## 13. Live View πŸ”’ +## 14. Live View πŸ”’ **What:** A read-only, no-login snapshot of a canvas you can share on your LAN. Off by default. @@ -196,7 +211,7 @@ Pan and zoom only, no editing. Click a node with an IP and it opens in a new tab --- -## 14. Gethomepage Widget πŸ”’ +## 15. Gethomepage Widget πŸ”’ **What:** A tiny JSON stats endpoint for [gethomepage](https://gethomepage.dev)'s `customapi` widget. Off by default. @@ -208,7 +223,7 @@ Widget snippet lives in the [README](./README.md#gethomepage-widget-read-only-st --- -## 15. MCP Server πŸ”’ +## 16. MCP Server πŸ”’ **What:** A [Model Context Protocol](https://modelcontextprotocol.io) server so an MCP client (Claude Code, Claude Desktop, Open WebUI…) can read and change your topology. Optional, runs as its own service. @@ -226,7 +241,7 @@ The AI can list nodes/edges/canvas/pending/scans, add/update/delete nodes and ed --- -## 16. Settings & Shortcuts +## 17. Settings & Shortcuts **What:** App config and keyboard shortcuts. diff --git a/README.md b/README.md index 9b147a9..8118960 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,37 @@ Hierarchy is set automatically: controller β†’ routers β†’ end devices (`parent_ --- +## Proxmox VE Import + +Homelable can import your **Proxmox VE** inventory over the Proxmox REST API β€” hosts, VMs and LXC containers arrive as typed, named nodes with run state and hardware specs, and can auto-sync on a schedule. Guest IPs that were already found by a network scan are merged in place (no duplicates). + +### Prerequisites + +- A reachable **Proxmox VE** host (default API port `8006`) +- A **Proxmox API token** with the read-only **`PVEAuditor`** role (Datacenter β†’ Permissions β†’ API Tokens) + +### Usage + +1. Click **Proxmox Import** in the left sidebar (below "Z-Wave Import") +2. Enter the host, port (default `8006`), and API token (`user@realm!tokenid` + secret) β€” or leave the token blank to use the server-configured one +3. Click **Test Connection** to verify reachability + token +4. Choose a target β€” **Pending section** or **Canvas directly** β€” then **Import to Pending** / **Fetch Inventory** +5. Select the devices from the grouped list (Hosts / Virtual Machines / LXC Containers) and click **Add N to Canvas** + +### Node Types + +| Type | Proxmox object | Icon | +|------|----------------|------| +| `proxmox` | Host / cluster member | Layers | +| `vm` | QEMU virtual machine | Box | +| `lxc` | LXC container | Container | + +Each host is linked to its guests with a `virtual` edge. vCPU / RAM / disk are imported as node properties (hidden by default). Enable **auto-sync** from Settings once a server token is configured (`PROXMOX_TOKEN_ID` / `PROXMOX_TOKEN_SECRET`). + +> **Full documentation:** [docs/proxmox-import.md](./docs/proxmox-import.md) + +--- + ## Live View (read-only public canvas) Live View lets you share a read-only snapshot of your canvas with anyone on your network β€” no login required. It is disabled by default. diff --git a/backend/app/api/routes/proxmox.py b/backend/app/api/routes/proxmox.py new file mode 100644 index 0000000..6abf8c5 --- /dev/null +++ b/backend/app/api/routes/proxmox.py @@ -0,0 +1,379 @@ +"""FastAPI router for Proxmox VE import + auto-sync config. + +Fetches hosts/VMs/LXC from the Proxmox REST API and upserts them into the +pending inventory (same reviewβ†’approve flow as scans and mesh imports). + +Credentials: the API token comes from the request body when provided, else +falls back to the server-configured env token (``settings.proxmox_token_*``). +The token is never persisted by the app and never returned by any endpoint. +""" + +import logging +from datetime import datetime, timezone +from typing import Any + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from sqlalchemy import delete as sa_delete +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_current_user +from app.core.config import settings +from app.core.scheduler import reschedule_proxmox_sync, set_proxmox_sync_enabled +from app.db.database import AsyncSessionLocal, get_db +from app.db.models import Node, PendingDevice, PendingDeviceLink, ScanRun +from app.schemas.proxmox import ( + ProxmoxConfig, + ProxmoxConnectionRequest, + ProxmoxEdgeOut, + ProxmoxImportPendingResponse, + ProxmoxImportResponse, + ProxmoxNodeOut, + ProxmoxTestConnectionResponse, +) +from app.schemas.scan import ScanRunResponse +from app.services.node_dedupe import dedupe_nodes_by_ieee +from app.services.proxmox_service import ( + build_proxmox_properties, + fetch_proxmox_inventory, + merge_proxmox_properties, + test_proxmox_connection, +) + +logger = logging.getLogger(__name__) +router = APIRouter() + + +def _resolve_credentials(payload: ProxmoxConnectionRequest) -> tuple[str, str]: + """Pick the API token: request body first, else server env config. + + Raises HTTP 400 when neither carries a token. + """ + token_id = payload.token_id or settings.proxmox_token_id + token_secret = payload.token_secret or settings.proxmox_token_secret + if not token_id or not token_secret: + raise HTTPException( + status_code=400, + detail="No Proxmox API token provided and none configured on the server.", + ) + return token_id, token_secret + + +@router.post("/test-connection", response_model=ProxmoxTestConnectionResponse) +async def test_connection_endpoint( + payload: ProxmoxConnectionRequest, + _: str = Depends(get_current_user), +) -> ProxmoxTestConnectionResponse: + """Validate host reachability + token before importing.""" + token_id, token_secret = _resolve_credentials(payload) + connected, message = await test_proxmox_connection( + host=payload.host, + port=payload.port, + token_id=token_id, + token_secret=token_secret, + verify_tls=payload.verify_tls, + ) + return ProxmoxTestConnectionResponse(connected=connected, message=message) + + +@router.post("/import", response_model=ProxmoxImportResponse) +async def import_proxmox( + payload: ProxmoxConnectionRequest, + _: str = Depends(get_current_user), +) -> ProxmoxImportResponse: + """Fetch the inventory and return nodes + edges ready for canvas drop.""" + token_id, token_secret = _resolve_credentials(payload) + try: + nodes_raw, edges_raw = await fetch_proxmox_inventory( + host=payload.host, + port=payload.port, + token_id=token_id, + token_secret=token_secret, + verify_tls=payload.verify_tls, + ) + except ConnectionError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Unexpected error during Proxmox import") + raise HTTPException(status_code=500, detail="Unexpected error during Proxmox import") from exc + + nodes = [ProxmoxNodeOut(**n) for n in nodes_raw] + edges = [ProxmoxEdgeOut(**e) for e in edges_raw] + return ProxmoxImportResponse(nodes=nodes, edges=edges, device_count=len(nodes)) + + +@router.post("/import-pending", response_model=ScanRunResponse) +async def import_proxmox_to_pending( + payload: ProxmoxConnectionRequest, + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + _: str = Depends(get_current_user), +) -> ScanRun: + """Queue a Proxmox pending import as a background scan run (kind=proxmox).""" + token_id, token_secret = _resolve_credentials(payload) + run = ScanRun( + status="running", + kind="proxmox", + ranges=[f"{payload.host}:{payload.port}"], + ) + db.add(run) + await db.commit() + await db.refresh(run) + background_tasks.add_task( + _background_proxmox_import, + run.id, + payload.host, + payload.port, + token_id, + token_secret, + payload.verify_tls, + ) + return run + + +async def _background_proxmox_import( + run_id: str, + host: str, + port: int, + token_id: str, + token_secret: str, + verify_tls: bool, +) -> None: + async with AsyncSessionLocal() as db: + try: + nodes_raw, edges_raw = await fetch_proxmox_inventory( + host=host, + port=port, + token_id=token_id, + token_secret=token_secret, + verify_tls=verify_tls, + ) + result = await _persist_pending_import(db, nodes_raw, edges_raw) + run = await db.get(ScanRun, run_id) + if run: + run.status = "done" + run.devices_found = result.device_count + run.finished_at = datetime.now(timezone.utc) + await db.commit() + except Exception as exc: + logger.exception("Proxmox import %s failed", run_id) + await db.rollback() + run = await db.get(ScanRun, run_id) + if run: + run.status = "error" + run.error = str(exc)[:500] + run.finished_at = datetime.now(timezone.utc) + await db.commit() + + +async def _persist_pending_import( + db: AsyncSession, + nodes_raw: list[dict[str, Any]], + edges_raw: list[dict[str, Any]], +) -> ProxmoxImportPendingResponse: + """Upsert Proxmox nodes/edges into pending_devices + pending_device_links. + + Two-tier identity (order matters): + 1. Match an existing canvas Node or pending row by **IP** (merge into a + device previously found by a scan) β€” never duplicate. + 2. Else match by synthetic ``ieee_address`` (``pve-...``). + + Update-in-place only. Nothing is ever deleted; hidden rows stay hidden. + """ + await dedupe_nodes_by_ieee(db) + + pending_created = 0 + pending_updated = 0 + + for n in nodes_raw: + ieee = n.get("ieee_address") + if not ieee: + continue + ip = n.get("ip") + props = build_proxmox_properties(n) + + # 1) Already on a canvas? Match by ieee OR (ip when known). Refresh in + # place: merge properties, adopt the pve identity onto a scanned node, + # backfill blank specs/hostname. Do NOT stomp user-set type/status. + node_filter = [Node.ieee_address == ieee] + if ip: + node_filter.append(Node.ip == ip) + existing_nodes = ( + await db.execute(select(Node).where(or_(*node_filter)).order_by(Node.id)) + ).scalars().all() + + if existing_nodes: + for en in existing_nodes: + en.properties = merge_proxmox_properties(en.properties, props) + if not en.ieee_address: + en.ieee_address = ieee + if ip and not en.ip: + en.ip = ip + en.hostname = en.hostname or n.get("hostname") + en.cpu_count = en.cpu_count or n.get("cpu_count") + en.ram_gb = en.ram_gb or n.get("ram_gb") + en.disk_gb = en.disk_gb or n.get("disk_gb") + await _ensure_inventory_row(db, ieee, ip, n, props, approved=True) + pending_updated += 1 + continue + + # 2) Not on canvas β€” upsert the pending inventory row. + pending = await _find_pending(db, ieee, ip) + if pending is None: + db.add(_new_pending(ieee, ip, n, props, status="pending")) + pending_created += 1 + else: + _refresh_pending(pending, ieee, ip, n, props) + pending_updated += 1 + + links_recorded = await _replace_links(db, edges_raw) + await db.commit() + + return ProxmoxImportPendingResponse( + pending_created=pending_created, + pending_updated=pending_updated, + links_recorded=links_recorded, + device_count=len(nodes_raw), + ) + + +async def _find_pending( + db: AsyncSession, ieee: str, ip: str | None +) -> PendingDevice | None: + filters = [PendingDevice.ieee_address == ieee] + if ip: + filters.append(PendingDevice.ip == ip) + return ( + await db.execute(select(PendingDevice).where(or_(*filters))) + ).scalars().first() + + +def _new_pending( + ieee: str, ip: str | None, n: dict[str, Any], props: list[dict[str, Any]], status: str +) -> PendingDevice: + return PendingDevice( + ieee_address=ieee, + ip=ip, + hostname=n.get("hostname"), + friendly_name=n.get("label"), + suggested_type=n.get("type"), + vendor=n.get("vendor"), + model=n.get("model"), + properties=props, + status=status, + discovery_source="proxmox", + ) + + +def _refresh_pending( + pending: PendingDevice, + ieee: str, + ip: str | None, + n: dict[str, Any], + props: list[dict[str, Any]], +) -> None: + pending.ieee_address = pending.ieee_address or ieee + pending.ip = ip or pending.ip + pending.hostname = n.get("hostname") or pending.hostname + pending.friendly_name = n.get("label") or pending.friendly_name + pending.suggested_type = n.get("type") or pending.suggested_type + pending.vendor = n.get("vendor") or pending.vendor + pending.model = n.get("model") or pending.model + pending.properties = merge_proxmox_properties(list(pending.properties or []), props) + if pending.status == "approved": + # Approved earlier but the canvas Node is gone β€” revive so it reappears. + pending.status = "pending" + # hidden stays hidden. + + +async def _ensure_inventory_row( + db: AsyncSession, + ieee: str, + ip: str | None, + n: dict[str, Any], + props: list[dict[str, Any]], + approved: bool, +) -> None: + """Ensure an inventory row exists for a device already on a canvas, so it + shows in the inventory with an 'In N canvas' badge. Never changes status.""" + inv = await _find_pending(db, ieee, ip) + if inv is None: + db.add(_new_pending(ieee, ip, n, props, status="approved" if approved else "pending")) + else: + inv.ieee_address = inv.ieee_address or ieee + inv.ip = ip or inv.ip + inv.hostname = n.get("hostname") or inv.hostname + inv.suggested_type = n.get("type") or inv.suggested_type + inv.properties = merge_proxmox_properties(list(inv.properties or []), props) + + +async def _replace_links(db: AsyncSession, edges_raw: list[dict[str, Any]]) -> int: + """Wipe all proxmox-source links and re-insert the freshly discovered set.""" + await db.execute( + sa_delete(PendingDeviceLink).where(PendingDeviceLink.discovery_source == "proxmox") + ) + recorded = 0 + seen: set[tuple[str, str]] = set() + for e in edges_raw: + src = e.get("source") + tgt = e.get("target") + if not src or not tgt or (src, tgt) in seen: + continue + seen.add((src, tgt)) + db.add( + PendingDeviceLink( + source_ieee=src, + target_ieee=tgt, + discovery_source="proxmox", + ) + ) + recorded += 1 + return recorded + + +@router.get("/config", response_model=ProxmoxConfig) +async def get_proxmox_config(_: str = Depends(get_current_user)) -> ProxmoxConfig: + """Return non-secret Proxmox config. Never includes the token β€” only whether + one is configured on the server.""" + return ProxmoxConfig( + host=settings.proxmox_host, + port=settings.proxmox_port, + verify_tls=settings.proxmox_verify_tls, + sync_enabled=settings.proxmox_sync_enabled, + sync_interval=settings.proxmox_sync_interval, + token_configured=bool(settings.proxmox_token_id and settings.proxmox_token_secret), + ) + + +@router.post("/config", response_model=ProxmoxConfig) +async def save_proxmox_config( + payload: ProxmoxConfig, + _: str = Depends(get_current_user), +) -> ProxmoxConfig: + """Persist non-secret Proxmox config and apply the auto-sync schedule live. + + The token is NOT accepted here β€” it is env-only by design. + """ + if payload.sync_enabled and not (settings.proxmox_token_id and settings.proxmox_token_secret): + raise HTTPException( + status_code=400, + detail="Cannot enable auto-sync: no Proxmox API token configured on the server.", + ) + try: + settings.proxmox_host = payload.host + settings.proxmox_port = payload.port + settings.proxmox_verify_tls = payload.verify_tls + settings.proxmox_sync_enabled = payload.sync_enabled + settings.proxmox_sync_interval = payload.sync_interval + settings.save_overrides() + set_proxmox_sync_enabled(payload.sync_enabled) + if payload.sync_enabled: + reschedule_proxmox_sync(payload.sync_interval) + except HTTPException: + raise + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + return await get_proxmox_config() diff --git a/backend/app/api/routes/scan.py b/backend/app/api/routes/scan.py index 2b778d2..501d95e 100644 --- a/backend/app/api/routes/scan.py +++ b/backend/app/api/routes/scan.py @@ -367,7 +367,7 @@ async def bulk_approve_devices( ieee_address=device.ieee_address, properties=_wireless_properties( node_type, device.ieee_address, device.vendor, device.model, device.lqi - ) if is_wireless else build_mac_property(device.mac), + ) if is_wireless else merge_mac_property(list(device.properties or []), device.mac), # Default to ping so the status checker actually polls the new node. # Without this the scheduler skips it (check_method NULL β†’ no check). check_method="none" if is_wireless else ("ping" if device.ip else None), @@ -519,7 +519,10 @@ async def approve_device( ieee_address=device.ieee_address, properties=_wireless_properties( node_data.type, device.ieee_address, device.vendor, device.model, device.lqi - ) if wireless else merge_mac_property(node_data.properties, _mac), + ) if wireless else merge_mac_property( + merge_zigbee_properties(list(device.properties or []), node_data.properties or []), + _mac, + ), check_method="none" if wireless else (node_data.check_method or ("ping" if node_data.ip else None)), check_target=None if wireless else node_data.check_target, design_id=node_design_id, @@ -609,10 +612,13 @@ async def _resolve_pending_links_for_ieee( if edge_design_id is None: first = (await db.execute(select(Design).order_by(Design.created_at).limit(1))).scalar() edge_design_id = first.id if first else None + # Proxmox hostβ†’guest links render as 'virtual' (VM/LXC ↔ host); mesh + # links stay 'iot'. Default to iot for any legacy/other source. + edge_type = "virtual" if link.discovery_source == "proxmox" else "iot" edge = Edge( source=src_id, target=tgt_id, - type="iot", + type=edge_type, source_handle="bottom", target_handle="top-t", design_id=edge_design_id, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8dba247..e9b2a0c 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -80,6 +80,19 @@ class Settings(BaseSettings): # Leave empty to keep the feature disabled (default). homepage_api_key: str = "" + # Proxmox VE import. + # Token = a real credential β†’ env/.env ONLY, never persisted by the app to + # scan_config.json and never returned by the API. token_id is + # 'user@realm!tokenname'; use a read-only PVEAuditor role. + proxmox_token_id: str = "" + proxmox_token_secret: str = "" + # Non-secret connection + auto-sync config (persisted via save_overrides). + proxmox_host: str = "" + proxmox_port: int = 8006 + proxmox_verify_tls: bool = True + proxmox_sync_enabled: bool = False + proxmox_sync_interval: int = 3600 # seconds (floor 300 enforced on write) + def _override_path(self) -> Path: return Path(self.sqlite_path).parent / "scan_config.json" @@ -108,6 +121,17 @@ class Settings(BaseSettings): self.scanner_http_probe_enabled = bool(data["scanner_http_probe_enabled"]) if "scanner_http_verify_tls" in data: self.scanner_http_verify_tls = bool(data["scanner_http_verify_tls"]) + # Proxmox non-secret config (token stays env-only, never here). + if "proxmox_host" in data: + self.proxmox_host = str(data["proxmox_host"]) + if "proxmox_port" in data: + self.proxmox_port = int(data["proxmox_port"]) + if "proxmox_verify_tls" in data: + self.proxmox_verify_tls = bool(data["proxmox_verify_tls"]) + if "proxmox_sync_enabled" in data: + self.proxmox_sync_enabled = bool(data["proxmox_sync_enabled"]) + if "proxmox_sync_interval" in data: + self.proxmox_sync_interval = int(data["proxmox_sync_interval"]) except Exception: pass @@ -122,6 +146,13 @@ class Settings(BaseSettings): "scanner_http_ranges": self.scanner_http_ranges, "scanner_http_probe_enabled": self.scanner_http_probe_enabled, "scanner_http_verify_tls": self.scanner_http_verify_tls, + # Proxmox: only non-secret config. Token fields are intentionally + # excluded β€” they must never be written to disk by the app. + "proxmox_host": self.proxmox_host, + "proxmox_port": self.proxmox_port, + "proxmox_verify_tls": self.proxmox_verify_tls, + "proxmox_sync_enabled": self.proxmox_sync_enabled, + "proxmox_sync_interval": self.proxmox_sync_interval, })) diff --git a/backend/app/core/scheduler.py b/backend/app/core/scheduler.py index 955c8f2..ed34c15 100644 --- a/backend/app/core/scheduler.py +++ b/backend/app/core/scheduler.py @@ -106,6 +106,35 @@ async def _run_service_checks() -> None: logger.error("Service checks failed for node %s: %s", node_id, exc) +async def _run_proxmox_sync() -> None: + """Fetch the Proxmox inventory and upsert it into pending (auto-sync).""" + if not settings.proxmox_sync_enabled: + return + if not (settings.proxmox_host and settings.proxmox_token_id and settings.proxmox_token_secret): + logger.warning("Proxmox auto-sync enabled but host/token not configured β€” skipping") + return + # Lazy import to avoid a circular import at module load. + from app.api.routes.proxmox import _persist_pending_import + from app.services.proxmox_service import fetch_proxmox_inventory + + try: + nodes_raw, edges_raw = await fetch_proxmox_inventory( + host=settings.proxmox_host, + port=settings.proxmox_port, + token_id=settings.proxmox_token_id, + token_secret=settings.proxmox_token_secret, + verify_tls=settings.proxmox_verify_tls, + ) + async with AsyncSessionLocal() as db: + result = await _persist_pending_import(db, nodes_raw, edges_raw) + logger.info( + "Proxmox auto-sync: %d devices (%d new, %d updated)", + result.device_count, result.pending_created, result.pending_updated, + ) + except Exception as exc: + logger.error("Proxmox auto-sync failed: %s", exc) + + def _add_service_check_job() -> None: scheduler.add_job( _run_service_checks, @@ -117,6 +146,17 @@ def _add_service_check_job() -> None: ) +def _add_proxmox_sync_job() -> None: + scheduler.add_job( + _run_proxmox_sync, + "interval", + seconds=settings.proxmox_sync_interval, + id="proxmox_sync", + max_instances=1, + coalesce=True, + ) + + def start_scheduler() -> None: global scheduler if scheduler.running: @@ -135,6 +175,8 @@ def start_scheduler() -> None: ) if settings.service_check_enabled: _add_service_check_job() + if settings.proxmox_sync_enabled: + _add_proxmox_sync_job() scheduler.start() logger.info("Scheduler started β€” status checks every %ds", settings.status_checker_interval) @@ -175,6 +217,31 @@ def set_service_checks_enabled(enabled: bool) -> None: logger.info("Service checks disabled") +def reschedule_proxmox_sync(interval_seconds: int) -> None: + """Update the Proxmox auto-sync interval on the running scheduler (if enabled).""" + if interval_seconds < 300: + raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}") + if not scheduler.running: + logger.warning("Scheduler not running, skipping reschedule") + return + if scheduler.get_job("proxmox_sync"): + scheduler.reschedule_job("proxmox_sync", trigger="interval", seconds=interval_seconds) + logger.info("Proxmox auto-sync rescheduled to every %ds", interval_seconds) + + +def set_proxmox_sync_enabled(enabled: bool) -> None: + """Add or remove the Proxmox auto-sync job on the running scheduler.""" + if not scheduler.running: + return + job = scheduler.get_job("proxmox_sync") + if enabled and not job: + _add_proxmox_sync_job() + logger.info("Proxmox auto-sync enabled β€” every %ds", settings.proxmox_sync_interval) + elif not enabled and job: + scheduler.remove_job("proxmox_sync") + logger.info("Proxmox auto-sync disabled") + + def stop_scheduler() -> None: if scheduler.running: scheduler.shutdown(wait=False) diff --git a/backend/app/db/database.py b/backend/app/db/database.py index 2ffeaae..76470d6 100644 --- a/backend/app/db/database.py +++ b/backend/app/db/database.py @@ -115,6 +115,8 @@ async def init_db() -> None: await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN right_handles INTEGER NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_source TEXT") + with suppress(OperationalError): + await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN properties JSON") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE scan_runs ADD COLUMN kind TEXT NOT NULL DEFAULT 'ip'") # --- Zigbee schema migrations (logged variant per CLAUDE.md feedback) --- @@ -162,6 +164,7 @@ async def init_db() -> None: "model VARCHAR," "vendor VARCHAR," "lqi INTEGER," + "properties JSON," "discovered_at DATETIME" ")" ) diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 144f976..ac99f89 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -126,6 +126,10 @@ class PendingDevice(Base): model: Mapped[str | None] = mapped_column(String, nullable=True) vendor: Mapped[str | None] = mapped_column(String, nullable=True) lqi: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Display properties carried from discovery (e.g. Proxmox specs: CPU/RAM/Disk, + # VMID). Generic NodeProperty shape {key,value,icon,visible}; merged into the + # Node's properties on approve. Empty for scan/mesh sources that don't set it. + properties: Mapped[list[Any]] = mapped_column(JSON, default=list) discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now) # Transient (not persisted): populated per-request by the scan routes to report diff --git a/backend/app/main.py b/backend/app/main.py index 4ca6f6d..368a490 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -7,7 +7,21 @@ from typing import Any from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from app.api.routes import auth, canvas, designs, edges, liveview, media, nodes, scan, stats, status, zigbee, zwave +from app.api.routes import ( + auth, + canvas, + designs, + edges, + liveview, + media, + nodes, + proxmox, + scan, + stats, + status, + zigbee, + zwave, +) from app.api.routes import settings as settings_routes from app.core.config import settings from app.core.scheduler import start_scheduler, stop_scheduler @@ -58,6 +72,7 @@ app.include_router(settings_routes.router, prefix="/api/v1/settings", tags=["set app.include_router(liveview.router, prefix="/api/v1/liveview", tags=["liveview"]) app.include_router(zigbee.router, prefix="/api/v1/zigbee", tags=["zigbee"]) app.include_router(zwave.router, prefix="/api/v1/zwave", tags=["zwave"]) +app.include_router(proxmox.router, prefix="/api/v1/proxmox", tags=["proxmox"]) app.include_router(stats.router, prefix="/api/v1/stats", tags=["stats"]) app.include_router(media.router, prefix="/api/v1/media", tags=["media"]) diff --git a/backend/app/schemas/proxmox.py b/backend/app/schemas/proxmox.py new file mode 100644 index 0000000..8ebf67d --- /dev/null +++ b/backend/app/schemas/proxmox.py @@ -0,0 +1,75 @@ +"""Pydantic v2 schemas for Proxmox VE import. + +Token fields are accepted on requests only and are optional β€” when omitted the +backend falls back to the server-configured token (env). No response schema ever +carries a token; secrets are kept out of responses by structural omission. +""" + +from pydantic import BaseModel, Field + + +class ProxmoxConnectionRequest(BaseModel): + host: str = Field(..., description="Proxmox VE host or IP") + port: int = Field(8006, ge=1, le=65535, description="Proxmox API port") + token_id: str | None = Field( + None, description="API token id 'user@realm!tokenname' (falls back to server env)" + ) + token_secret: str | None = Field( + None, description="API token secret (falls back to server env)" + ) + verify_tls: bool = Field(True, description="Verify the Proxmox TLS certificate") + + +class ProxmoxTestConnectionResponse(BaseModel): + connected: bool + message: str + + +class ProxmoxNodeOut(BaseModel): + """A homelable-ready node representation of a Proxmox host / VM / LXC.""" + + id: str + label: str + type: str # proxmox | vm | lxc + ieee_address: str + hostname: str | None = None + ip: str | None = None + status: str + cpu_count: int | None = None + ram_gb: float | None = None + disk_gb: float | None = None + vendor: str | None = None + model: str | None = None + parent_ieee: str | None = None + + +class ProxmoxEdgeOut(BaseModel): + source: str + target: str + + +class ProxmoxImportResponse(BaseModel): + nodes: list[ProxmoxNodeOut] + edges: list[ProxmoxEdgeOut] + device_count: int + + +class ProxmoxImportPendingResponse(BaseModel): + """Result of importing a Proxmox inventory into the pending section.""" + + pending_created: int + pending_updated: int + links_recorded: int + device_count: int + + +class ProxmoxConfig(BaseModel): + """Non-secret Proxmox connection + auto-sync config. Never carries a token β€” + ``token_configured`` reflects whether a server-side token is present.""" + + host: str = "" + port: int = Field(8006, ge=1, le=65535) + verify_tls: bool = True + sync_enabled: bool = False + sync_interval: int = Field(3600, ge=300) + token_configured: bool = False diff --git a/backend/app/schemas/scan.py b/backend/app/schemas/scan.py index 3b7cef7..d64bd79 100644 --- a/backend/app/schemas/scan.py +++ b/backend/app/schemas/scan.py @@ -20,6 +20,9 @@ class PendingDeviceResponse(BaseModel): model: str | None = None vendor: str | None = None lqi: int | None = None + # Display properties carried from discovery (e.g. Proxmox specs). Merged into + # the node on approve; empty for scan/mesh sources that don't set them. + properties: list[Any] = [] discovered_at: datetime # Number of distinct canvases (designs) this device already appears on, # correlated by ip / ieee_address against existing nodes. Computed per-request. diff --git a/backend/app/services/proxmox_service.py b/backend/app/services/proxmox_service.py new file mode 100644 index 0000000..423be85 --- /dev/null +++ b/backend/app/services/proxmox_service.py @@ -0,0 +1,348 @@ +"""Proxmox VE inventory service: fetch hosts + VMs + LXC via the PVE REST API. + +Mirrors the Zigbee/Z-Wave import pipeline, but talks to the Proxmox VE REST API +(``/api2/json``) over HTTPS with an API token instead of MQTT. It returns plain +homelable node dicts + parentβ†’child edge hints; DB persistence lives in the +route layer (``app.api.routes.proxmox``). + +Auth uses a Proxmox **API token** (never a password): +``Authorization: PVEAPIToken==`` where ``token_id`` looks like +``user@realm!tokenname``. A read-only ``PVEAuditor`` role is all that is needed. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +import httpx + +from app.services.zigbee_service import merge_zigbee_properties + +logger = logging.getLogger(__name__) + +# Reuse the zigbee property-merge contract verbatim (same NodeProperty shape + +# visibility-preservation rules) for re-sync updates. +merge_proxmox_properties = merge_zigbee_properties + +_CONNECT_TIMEOUT = 8.0 +_READ_TIMEOUT = 20.0 +_BYTES_PER_GB = 1024 ** 3 + +# net0 config line: "name=eth0,bridge=vmbr0,ip=192.168.1.5/24,gw=..." +_LXC_IP_RE = re.compile(r"(?:^|,)ip=([0-9]{1,3}(?:\.[0-9]{1,3}){3})(?:/\d+)?") + + +def _sanitize_proxmox_error(exc: BaseException) -> str: + """Return a generic, credential-free message for a Proxmox/HTTP error. + + Raw httpx errors can echo the request URL and, worse, an + ``Authorization: PVEAPIToken=...=`` header in some stacks. Map known + patterns to coarse categories so the token never reaches an API client. The + original exception is logged at WARNING for operators. + """ + logger.warning("Proxmox error (sanitized for client): %r", exc) + if isinstance(exc, httpx.HTTPStatusError): + code = exc.response.status_code + if code in (401, 403): + return "Authentication failed β€” check the API token and its permissions" + if code == 404: + return "Proxmox API path not found β€” is this a Proxmox VE host?" + return f"Proxmox API returned HTTP {code}" + raw = str(exc).lower() + if "name or service not known" in raw or "getaddrinfo" in raw or "nodename nor servname" in raw: + return "Proxmox host could not be resolved" + if "refused" in raw: + return "Connection refused by Proxmox host" + if "certificate" in raw or "ssl" in raw or "tls" in raw: + return "TLS verification failed β€” enable 'skip TLS verify' for self-signed certs" + if "timed out" in raw or "timeout" in raw: + return "Connection to Proxmox host timed out" + return "Proxmox connection failed" + + +def _auth_header(token_id: str, token_secret: str) -> dict[str, str]: + return {"Authorization": f"PVEAPIToken={token_id}={token_secret}"} + + +def _gb(value: Any) -> float | None: + """Convert a byte count to GB (1 decimal). None/0 β†’ None.""" + try: + num = float(value) + except (TypeError, ValueError): + return None + if num <= 0: + return None + return round(num / _BYTES_PER_GB, 1) + + +def _int_or_none(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _guest_type_to_homelable(kind: str) -> str: + """qemu β†’ vm, lxc β†’ lxc (both existing homelable node types).""" + return "vm" if kind == "qemu" else "lxc" + + +def _extract_qemu_ip(agent_payload: dict[str, Any] | None) -> str | None: + """Pull the first non-loopback IPv4 from a qemu guest-agent interfaces reply.""" + if not agent_payload: + return None + result = agent_payload.get("result") + if not isinstance(result, list): + return None + for iface in result: + if not isinstance(iface, dict): + continue + for addr in iface.get("ip-addresses") or []: + if not isinstance(addr, dict): + continue + if addr.get("ip-address-type") != "ipv4": + continue + ip = addr.get("ip-address") + if isinstance(ip, str) and ip and not ip.startswith("127."): + return ip + return None + + +def _extract_lxc_ip(config_payload: dict[str, Any] | None) -> str | None: + """Parse a static IPv4 from an LXC ``net0`` config string (skip dhcp).""" + if not config_payload: + return None + net0 = config_payload.get("net0") + if not isinstance(net0, str): + return None + match = _LXC_IP_RE.search(net0) + return match.group(1) if match else None + + +def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None: + """Build a homelable ``proxmox`` host node from a ``/nodes`` entry.""" + name = raw.get("node") + if not name: + return None + ieee = f"pve-node-{name}" + return { + "id": ieee, + "label": name, + "type": "proxmox", + "ieee_address": ieee, + "hostname": name, + "ip": raw.get("ip") or None, + "status": "online" if raw.get("status") == "online" else "offline", + "cpu_count": _int_or_none(raw.get("maxcpu")), + "ram_gb": _gb(raw.get("maxmem")), + "disk_gb": _gb(raw.get("maxdisk")), + "vendor": "Proxmox VE", + "model": None, + "parent_ieee": None, + } + + +def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None) -> dict[str, Any] | None: + """Build a homelable vm/lxc node from a ``/qemu`` or ``/lxc`` list entry.""" + vmid = raw.get("vmid") + if vmid is None: + return None + ieee = f"pve-{host_name}-{vmid}" + node_type = _guest_type_to_homelable(kind) + name = raw.get("name") or f"{node_type}-{vmid}" + return { + "id": ieee, + "label": name, + "type": node_type, + "ieee_address": ieee, + "hostname": name, + "ip": ip, + "status": "online" if raw.get("status") == "running" else "offline", + "cpu_count": _int_or_none(raw.get("maxcpu") or raw.get("cpus")), + "ram_gb": _gb(raw.get("maxmem")), + "disk_gb": _gb(raw.get("maxdisk")), + "vendor": "Proxmox VE", + "model": kind.upper(), + "vmid": vmid, + "parent_ieee": f"pve-node-{host_name}", + } + + +def build_proxmox_properties(node: dict[str, Any]) -> list[dict[str, Any]]: + """Build a NodeProperty list for a Proxmox device (specs + identity). + + Icons match the existing hardware-property convention (``Cpu`` / + ``MemoryStick`` / ``HardDrive``). All rows default ``visible=False`` β€” the + user opts in from the right panel, same as the mesh importers.""" + props: list[dict[str, Any]] = [] + vmid = node.get("vmid") + if vmid is not None: + props.append({"key": "VMID", "value": str(vmid), "icon": None, "visible": False}) + if node.get("model"): + props.append({"key": "Kind", "value": node["model"], "icon": None, "visible": False}) + if node.get("cpu_count") is not None: + props.append({"key": "CPU Cores", "value": str(node["cpu_count"]), "icon": "Cpu", "visible": False}) + if node.get("ram_gb") is not None: + props.append({"key": "RAM", "value": f"{node['ram_gb']} GB", "icon": "MemoryStick", "visible": False}) + if node.get("disk_gb") is not None: + props.append({"key": "Disk", "value": f"{node['disk_gb']} GB", "icon": "HardDrive", "visible": False}) + props.append({"key": "Source", "value": "Proxmox VE", "icon": None, "visible": False}) + return props + + +def _parse_inventory( + hosts_raw: list[dict[str, Any]], + guests_by_host: dict[str, list[dict[str, Any]]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Assemble (nodes, edges) from fetched host + guest data. + + ``guests_by_host`` maps host name β†’ list of already-normalized guest node + dicts. Edges are one hostβ†’guest link per guest (materialized as canvas + edges on approval, mirroring the zigbee/zwave link mechanism). + """ + nodes: list[dict[str, Any]] = [] + edges: list[dict[str, Any]] = [] + seen: set[str] = set() + + for raw in hosts_raw: + host = _host_node(raw) + if host is None or host["id"] in seen: + continue + seen.add(host["id"]) + nodes.append(host) + + for host_name, guests in guests_by_host.items(): + host_ieee = f"pve-node-{host_name}" + for guest in guests: + if guest["id"] in seen: + continue + seen.add(guest["id"]) + nodes.append(guest) + edges.append({"source": host_ieee, "target": guest["id"]}) + + return nodes, edges + + +async def _get_json(client: httpx.AsyncClient, path: str) -> Any: + resp = await client.get(path) + resp.raise_for_status() + return resp.json().get("data") + + +async def fetch_proxmox_inventory( + host: str, + port: int, + token_id: str, + token_secret: str, + verify_tls: bool = True, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Fetch hosts + VMs + LXC from a Proxmox VE host, return (nodes, edges). + + Raises: + ConnectionError: transport/DNS/TLS failures (sanitized message). + ValueError: malformed API response. + """ + base_url = f"https://{host}:{port}/api2/json" + timeout = httpx.Timeout(_READ_TIMEOUT, connect=_CONNECT_TIMEOUT) + guests_by_host: dict[str, list[dict[str, Any]]] = {} + + try: + async with httpx.AsyncClient( + base_url=base_url, + headers=_auth_header(token_id, token_secret), + verify=verify_tls, + timeout=timeout, + ) as client: + hosts_raw = await _get_json(client, "/nodes") + if not isinstance(hosts_raw, list): + raise ValueError("Malformed /nodes response") + + for host_entry in hosts_raw: + name = host_entry.get("node") + if not name or host_entry.get("status") != "online": + # Offline nodes can't be queried for guests; still shown as host. + continue + guests_by_host[name] = await _fetch_host_guests(client, name) + except httpx.HTTPStatusError as exc: + raise ConnectionError(_sanitize_proxmox_error(exc)) from exc + except httpx.HTTPError as exc: + raise ConnectionError(_sanitize_proxmox_error(exc)) from exc + + return _parse_inventory(hosts_raw, guests_by_host) + + +async def _fetch_host_guests( + client: httpx.AsyncClient, host_name: str +) -> list[dict[str, Any]]: + """Fetch qemu + lxc guests for one host, resolving guest IPs best-effort.""" + guests: list[dict[str, Any]] = [] + + for kind in ("qemu", "lxc"): + try: + entries = await _get_json(client, f"/nodes/{host_name}/{kind}") + except httpx.HTTPError as exc: + logger.warning("Proxmox %s list failed for %s: %s", kind, host_name, exc) + continue + if not isinstance(entries, list): + continue + for raw in entries: + ip = await _resolve_guest_ip(client, host_name, kind, raw) + node = _guest_node(raw, host_name, kind, ip) + if node: + guests.append(node) + + return guests + + +async def _resolve_guest_ip( + client: httpx.AsyncClient, host_name: str, kind: str, raw: dict[str, Any] +) -> str | None: + """Best-effort guest IP. qemu β†’ guest agent, lxc β†’ net0 config. Never raises.""" + vmid = raw.get("vmid") + if vmid is None: + return None + try: + if kind == "qemu": + if raw.get("status") != "running": + return None + data = await _get_json( + client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces" + ) + return _extract_qemu_ip(data) + data = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config") + return _extract_lxc_ip(data) + except httpx.HTTPError: + # Guest agent not installed / container stopped / no perms β†’ no IP. Fine. + return None + + +async def test_proxmox_connection( + host: str, + port: int, + token_id: str, + token_secret: str, + verify_tls: bool = True, +) -> tuple[bool, str]: + """Quick reachability + auth check via ``GET /version``. + + Returns (connected, message). Never raises credentials outward. + """ + base_url = f"https://{host}:{port}/api2/json" + timeout = httpx.Timeout(_READ_TIMEOUT, connect=_CONNECT_TIMEOUT) + try: + async with httpx.AsyncClient( + base_url=base_url, + headers=_auth_header(token_id, token_secret), + verify=verify_tls, + timeout=timeout, + ) as client: + data = await _get_json(client, "/version") + version = (data or {}).get("version", "?") if isinstance(data, dict) else "?" + return True, f"Connected to Proxmox VE {version}" + except httpx.HTTPError as exc: + return False, _sanitize_proxmox_error(exc) + except Exception as exc: # noqa: BLE001 β€” surface a safe message, log the rest + logger.exception("Unexpected error during Proxmox connection test") + return False, _sanitize_proxmox_error(exc) diff --git a/backend/tests/test_proxmox_router.py b/backend/tests/test_proxmox_router.py new file mode 100644 index 0000000..2303430 --- /dev/null +++ b/backend/tests/test_proxmox_router.py @@ -0,0 +1,188 @@ +"""API + persistence tests for /api/v1/proxmox/*.""" + +from __future__ import annotations + +import uuid +from unittest.mock import AsyncMock, patch + +import pytest +from httpx import AsyncClient +from sqlalchemy import select + +from app.api.routes.proxmox import _persist_pending_import +from app.core.config import settings +from app.db.models import Node, PendingDevice + + +@pytest.fixture +async def headers(client: AsyncClient): + res = await client.post("/api/v1/auth/login", json={"username": "admin", "password": "admin"}) + token = res.json()["access_token"] + return {"Authorization": f"Bearer {token}"} + + +@pytest.fixture(autouse=True) +def _clear_env_token(): + """Ensure a clean token state per test; restore afterwards.""" + tid, sec = settings.proxmox_token_id, settings.proxmox_token_secret + settings.proxmox_token_id = "" + settings.proxmox_token_secret = "" + yield + settings.proxmox_token_id, settings.proxmox_token_secret = tid, sec + + +def _host_node() -> dict: + return { + "id": "pve-node-pve1", "label": "pve1", "type": "proxmox", + "ieee_address": "pve-node-pve1", "hostname": "pve1", "ip": None, + "status": "online", "cpu_count": 8, "ram_gb": 16.0, "disk_gb": 500.0, + "vendor": "Proxmox VE", "model": None, "parent_ieee": None, + } + + +def _guest_node(vmid: int, ip: str | None, status: str = "online") -> dict: + return { + "id": f"pve-pve1-{vmid}", "label": f"vm{vmid}", "type": "vm", + "ieee_address": f"pve-pve1-{vmid}", "hostname": f"vm{vmid}", "ip": ip, + "status": status, "cpu_count": 2, "ram_gb": 4.0, "disk_gb": 32.0, + "vendor": "Proxmox VE", "model": "QEMU", "vmid": vmid, + "parent_ieee": "pve-node-pve1", + } + + +# --- endpoints ------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_test_connection_uses_body_token(client: AsyncClient, headers: dict) -> None: + with patch("app.api.routes.proxmox.test_proxmox_connection", new=AsyncMock(return_value=(True, "ok"))): + res = await client.post( + "/api/v1/proxmox/test-connection", + json={"host": "pve", "port": 8006, "token_id": "u@pam!t", "token_secret": "s"}, + headers=headers, + ) + assert res.status_code == 200 + assert res.json()["connected"] is True + + +@pytest.mark.asyncio +async def test_missing_token_is_rejected(client: AsyncClient, headers: dict) -> None: + res = await client.post( + "/api/v1/proxmox/test-connection", + json={"host": "pve", "port": 8006}, + headers=headers, + ) + assert res.status_code == 400 + + +@pytest.mark.asyncio +async def test_import_pending_creates_scan_run(client: AsyncClient, headers: dict) -> None: + with patch("app.api.routes.proxmox._background_proxmox_import", new_callable=AsyncMock): + res = await client.post( + "/api/v1/proxmox/import-pending", + json={"host": "pve", "port": 8006, "token_id": "u@pam!t", "token_secret": "s"}, + headers=headers, + ) + assert res.status_code == 200 + data = res.json() + assert data["kind"] == "proxmox" + assert data["status"] == "running" + + +@pytest.mark.asyncio +async def test_requires_auth(client: AsyncClient) -> None: + res = await client.post("/api/v1/proxmox/import-pending", json={"host": "pve"}) + assert res.status_code == 401 + + +@pytest.mark.asyncio +async def test_config_omits_token(client: AsyncClient, headers: dict) -> None: + settings.proxmox_token_id = "u@pam!t" + settings.proxmox_token_secret = "supersecret" + res = await client.get("/api/v1/proxmox/config", headers=headers) + assert res.status_code == 200 + body = res.text + assert "supersecret" not in body + assert res.json()["token_configured"] is True + + +@pytest.mark.asyncio +async def test_enable_sync_without_token_rejected(client: AsyncClient, headers: dict) -> None: + res = await client.post( + "/api/v1/proxmox/config", + json={"host": "pve", "port": 8006, "verify_tls": True, "sync_enabled": True, "sync_interval": 600}, + headers=headers, + ) + assert res.status_code == 400 + + +# --- persistence / dedupe -------------------------------------------------- + +@pytest.mark.asyncio +async def test_persist_creates_pending(db_session) -> None: + nodes = [_host_node(), _guest_node(101, "10.0.0.5")] + edges = [{"source": "pve-node-pve1", "target": "pve-pve1-101"}] + result = await _persist_pending_import(db_session, nodes, edges) + assert result.pending_created == 2 + assert result.links_recorded == 1 + rows = (await db_session.execute(select(PendingDevice))).scalars().all() + assert {r.suggested_type for r in rows} == {"proxmox", "vm"} + # Specs carried as properties. + vm = next(r for r in rows if r.suggested_type == "vm") + assert any(p["key"] == "CPU Cores" for p in vm.properties) + + +@pytest.mark.asyncio +async def test_persist_merges_existing_scanned_node_by_ip(db_session) -> None: + # A device previously found by an IP scan (no ieee, no specs). + scanned = Node( + id=str(uuid.uuid4()), type="generic", label="10.0.0.5", + ip="10.0.0.5", status="online", pos_x=0, pos_y=0, + ) + db_session.add(scanned) + await db_session.commit() + + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) + + # No duplicate node; identity + specs merged onto the existing one. + nodes = (await db_session.execute(select(Node).where(Node.ip == "10.0.0.5"))).scalars().all() + assert len(nodes) == 1 + merged = nodes[0] + assert merged.ieee_address == "pve-pve1-101" + assert merged.cpu_count == 2 + assert any(p["key"] == "CPU Cores" for p in (merged.properties or [])) + # Inventory row exists as approved (already on canvas). + inv = (await db_session.execute(select(PendingDevice).where(PendingDevice.ieee_address == "pve-pve1-101"))).scalar_one() + assert inv.status == "approved" + + +@pytest.mark.asyncio +async def test_persist_resync_updates_in_place(db_session) -> None: + nodes = [_guest_node(101, "10.0.0.5")] + await _persist_pending_import(db_session, nodes, []) + # Second sync: same device, new IP. Should update, not duplicate. + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.9")], []) + rows = (await db_session.execute(select(PendingDevice).where(PendingDevice.ieee_address == "pve-pve1-101"))).scalars().all() + assert len(rows) == 1 + assert rows[0].ip == "10.0.0.9" + + +@pytest.mark.asyncio +async def test_persist_keeps_hidden_hidden(db_session) -> None: + db_session.add(PendingDevice( + id=str(uuid.uuid4()), ieee_address="pve-pve1-101", ip="10.0.0.5", + suggested_type="vm", status="hidden", discovery_source="proxmox", + )) + await db_session.commit() + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) + row = (await db_session.execute(select(PendingDevice).where(PendingDevice.ieee_address == "pve-pve1-101"))).scalar_one() + assert row.status == "hidden" + + +@pytest.mark.asyncio +async def test_persist_never_deletes(db_session) -> None: + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) + # A later sync that no longer includes vm101 must not remove it. + await _persist_pending_import(db_session, [_guest_node(202, "10.0.0.6")], []) + rows = (await db_session.execute(select(PendingDevice))).scalars().all() + ieees = {r.ieee_address for r in rows} + assert "pve-pve1-101" in ieees and "pve-pve1-202" in ieees diff --git a/backend/tests/test_proxmox_service.py b/backend/tests/test_proxmox_service.py new file mode 100644 index 0000000..ca2913e --- /dev/null +++ b/backend/tests/test_proxmox_service.py @@ -0,0 +1,123 @@ +"""Unit tests for the Proxmox VE import service (parsing, props, sanitizer).""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +from app.services import proxmox_service as svc + + +def test_gb_conversion() -> None: + assert svc._gb(1024 ** 3) == 1.0 + assert svc._gb(2 * 1024 ** 3) == 2.0 + assert svc._gb(0) is None + assert svc._gb(None) is None + assert svc._gb("nope") is None + + +def test_extract_qemu_ip_skips_loopback() -> None: + payload = { + "result": [ + {"name": "lo", "ip-addresses": [{"ip-address-type": "ipv4", "ip-address": "127.0.0.1"}]}, + {"name": "eth0", "ip-addresses": [{"ip-address-type": "ipv4", "ip-address": "192.168.1.20"}]}, + ] + } + assert svc._extract_qemu_ip(payload) == "192.168.1.20" + assert svc._extract_qemu_ip(None) is None + assert svc._extract_qemu_ip({"result": []}) is None + + +def test_extract_lxc_ip_parses_net0_static() -> None: + cfg = {"net0": "name=eth0,bridge=vmbr0,ip=192.168.1.30/24,gw=192.168.1.1"} + assert svc._extract_lxc_ip(cfg) == "192.168.1.30" + # DHCP β†’ no static IP + assert svc._extract_lxc_ip({"net0": "name=eth0,bridge=vmbr0,ip=dhcp"}) is None + assert svc._extract_lxc_ip(None) is None + + +def test_host_and_guest_node_mapping() -> None: + host = svc._host_node({"node": "pve1", "status": "online", "maxcpu": 8, "maxmem": 16 * 1024 ** 3, "maxdisk": 500 * 1024 ** 3}) + assert host["type"] == "proxmox" + assert host["ieee_address"] == "pve-node-pve1" + assert host["cpu_count"] == 8 + assert host["ram_gb"] == 16.0 + assert host["status"] == "online" + assert host["parent_ieee"] is None + + vm = svc._guest_node({"vmid": 101, "name": "web", "status": "running", "maxcpu": 2, "maxmem": 2 * 1024 ** 3, "maxdisk": 32 * 1024 ** 3}, "pve1", "qemu", "10.0.0.5") + assert vm["type"] == "vm" + assert vm["ieee_address"] == "pve-pve1-101" + assert vm["ip"] == "10.0.0.5" + assert vm["status"] == "online" + assert vm["parent_ieee"] == "pve-node-pve1" + + ct = svc._guest_node({"vmid": 200, "name": "db", "status": "stopped"}, "pve1", "lxc", None) + assert ct["type"] == "lxc" + assert ct["status"] == "offline" + + +def test_build_properties_includes_specs() -> None: + node = {"vmid": 101, "model": "QEMU", "cpu_count": 4, "ram_gb": 8.0, "disk_gb": 40.0} + props = svc.build_proxmox_properties(node) + keys = {p["key"] for p in props} + assert {"VMID", "CPU Cores", "RAM", "Disk", "Source"} <= keys + assert all(p["visible"] is False for p in props) + + +def test_parse_inventory_builds_host_guest_edges() -> None: + hosts = [{"node": "pve1", "status": "online", "maxcpu": 4}] + guests = {"pve1": [svc._guest_node({"vmid": 101, "name": "web", "status": "running"}, "pve1", "qemu", None)]} + nodes, edges = svc._parse_inventory(hosts, guests) + assert len(nodes) == 2 + assert edges == [{"source": "pve-node-pve1", "target": "pve-pve1-101"}] + + +def test_sanitize_error_hides_credentials() -> None: + exc = httpx.HTTPStatusError( + "boom", request=httpx.Request("GET", "https://h/api2/json"), + response=httpx.Response(401), + ) + msg = svc._sanitize_proxmox_error(exc) + assert "token" not in msg.lower() or "check the api token" in msg.lower() + assert "Authentication failed" in msg + + +@pytest.mark.asyncio +async def test_fetch_inventory_happy_path() -> None: + async def fake_get_json(client, path: str): + if path == "/nodes": + return [{"node": "pve1", "status": "online", "maxcpu": 8, "maxmem": 16 * 1024 ** 3}] + if path == "/nodes/pve1/qemu": + return [{"vmid": 101, "name": "web", "status": "running", "maxmem": 2 * 1024 ** 3}] + if path == "/nodes/pve1/lxc": + return [{"vmid": 200, "name": "db", "status": "stopped"}] + if path.endswith("/agent/network-get-interfaces"): + return {"result": [{"name": "eth0", "ip-addresses": [{"ip-address-type": "ipv4", "ip-address": "10.0.0.5"}]}]} + if path.endswith("/config"): + return {"net0": "name=eth0,ip=10.0.0.6/24"} + return None + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): + nodes, edges = await svc.fetch_proxmox_inventory("h", 8006, "u@pam!t", "sec") + + by_type = {n["type"] for n in nodes} + assert by_type == {"proxmox", "vm", "lxc"} + vm = next(n for n in nodes if n["type"] == "vm") + assert vm["ip"] == "10.0.0.5" + ct = next(n for n in nodes if n["type"] == "lxc") + assert ct["ip"] == "10.0.0.6" + assert len(edges) == 2 + + +@pytest.mark.asyncio +async def test_test_connection_returns_message() -> None: + async def fake_get_json(client, path: str): + return {"version": "8.2.2"} + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): + ok, msg = await svc.test_proxmox_connection("h", 8006, "u@pam!t", "sec") + assert ok is True + assert "8.2.2" in msg diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 078ea1a..12eb8a5 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -6,8 +6,11 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.core.scheduler import ( + _run_proxmox_sync, _run_service_checks, _run_status_checks, + reschedule_proxmox_sync, + set_proxmox_sync_enabled, set_service_checks_enabled, start_scheduler, stop_scheduler, @@ -148,6 +151,7 @@ def test_scheduler_uses_settings_interval(): patch("app.core.scheduler.AsyncIOScheduler", return_value=mock_sched): mock_settings.status_checker_interval = 45 mock_settings.service_check_enabled = False + mock_settings.proxmox_sync_enabled = False start_scheduler() _, kwargs = mock_sched.add_job.call_args assert kwargs["seconds"] == 45 @@ -156,7 +160,11 @@ def test_scheduler_uses_settings_interval(): def test_start_and_stop_scheduler(): """Scheduler can be started and stopped without errors.""" mock_sched = MagicMock() - with patch("app.core.scheduler.AsyncIOScheduler", return_value=mock_sched): + with patch("app.core.scheduler.AsyncIOScheduler", return_value=mock_sched), \ + patch("app.core.scheduler.settings") as mock_settings: + mock_settings.status_checker_interval = 60 + mock_settings.service_check_enabled = False + mock_settings.proxmox_sync_enabled = False start_scheduler() stop_scheduler() mock_sched.add_job.assert_called_once() @@ -245,7 +253,49 @@ def test_start_scheduler_adds_service_job_when_enabled(): mock_settings.status_checker_interval = 60 mock_settings.service_check_enabled = True mock_settings.service_check_interval = 300 + mock_settings.proxmox_sync_enabled = False start_scheduler() job_ids = [kw.get("id") for _, kw in mock_sched.add_job.call_args_list] assert "status_checks" in job_ids assert "service_checks" in job_ids + + +# --------------------------------------------------------------------------- +# Proxmox auto-sync job +# --------------------------------------------------------------------------- + +def test_set_proxmox_sync_enabled_adds_and_removes_job(): + mock_sched = MagicMock() + mock_sched.running = True + with patch("app.core.scheduler.scheduler", mock_sched), \ + patch("app.core.scheduler.settings") as mock_settings: + mock_settings.proxmox_sync_interval = 3600 + mock_sched.get_job.return_value = None + set_proxmox_sync_enabled(True) + mock_sched.add_job.assert_called_once() + mock_sched.get_job.return_value = MagicMock() + set_proxmox_sync_enabled(False) + mock_sched.remove_job.assert_called_once_with("proxmox_sync") + + +def test_reschedule_proxmox_sync_rejects_short_interval(): + with pytest.raises(ValueError): + reschedule_proxmox_sync(60) + + +@pytest.mark.asyncio +async def test_run_proxmox_sync_skips_when_disabled(): + with patch("app.core.scheduler.settings") as mock_settings: + mock_settings.proxmox_sync_enabled = False + # Must return before importing/fetching anything. + await _run_proxmox_sync() + + +@pytest.mark.asyncio +async def test_run_proxmox_sync_skips_when_no_token(): + with patch("app.core.scheduler.settings") as mock_settings: + mock_settings.proxmox_sync_enabled = True + mock_settings.proxmox_host = "pve" + mock_settings.proxmox_token_id = "" + mock_settings.proxmox_token_secret = "" + await _run_proxmox_sync() # no exception, no fetch diff --git a/docs/proxmox-import.md b/docs/proxmox-import.md new file mode 100644 index 0000000..25d9146 --- /dev/null +++ b/docs/proxmox-import.md @@ -0,0 +1,188 @@ +# Proxmox VE Import + +This feature connects Homelable to your Proxmox VE host, reads the inventory over +the Proxmox REST API, and drops your hosts, VMs and LXC containers onto the canvas +as typed nodes β€” with names, run state and hardware specs. It can also **sync** +on a schedule so the canvas keeps up with your cluster, and it **merges** with +devices already discovered by a network scan (a VM whose guest IP was already +scanned is updated in place, not duplicated). + +> πŸ”’ **Server-dependent feature** β€” requires the Homelable backend. It is hidden +> in the no-backend standalone/demo build. + +--- + +## Feature Overview + +- **API-based discovery** β€” Reads `/api2/json` from Proxmox VE using a read-only API token. +- **Typed nodes** β€” Devices map to existing Homelable node types: + - `proxmox` β€” a Proxmox host / cluster member (becomes a parent) + - `vm` β€” a QEMU/KVM virtual machine + - `lxc` β€” an LXC container +- **Hierarchy** β€” Each host is linked to its VMs/LXC with a `virtual` edge. +- **Hardware specs** β€” vCPU count, RAM and disk size are imported as node properties (CPU Cores, RAM, Disk), hidden by default β€” toggle them on from the right panel. +- **Guest IPs** β€” QEMU IPs come from the guest agent (when installed); LXC IPs are parsed from the container's static `net0` config. +- **Merge / sync** β€” Re-importing updates existing devices in place and never deletes anything. A guest IP matching a previously scanned node merges onto it. +- **Auto-sync** β€” Optional scheduled re-import into the pending inventory. + +--- + +## Prerequisites + +1. A reachable **Proxmox VE** host (default API port `8006`). +2. A **Proxmox API token** with a read-only role (see below). +3. (Optional, for QEMU guest IPs) the **QEMU guest agent** installed in your VMs. + +### Create an API token + +In the Proxmox web UI: + +1. **Datacenter β†’ Permissions β†’ API Tokens β†’ Add**. +2. Pick a **User** (e.g. `root@pam`, or better a dedicated `homelable@pve` user). +3. Give the token an **ID** (e.g. `homelable`). The full token id is then + `user@realm!tokenid` β€” for example `root@pam!homelable`. +4. Leave **Privilege Separation** checked (recommended) and click **Add**. +5. **Copy the secret now** β€” Proxmox shows it only once. + +Grant the token (or its user) a **read-only** role: + +1. **Datacenter β†’ Permissions β†’ Add β†’ API Token Permission**. +2. Path `/`, select your token, Role **`PVEAuditor`**, enable **Propagate**. + +`PVEAuditor` is read-only β€” Homelable never needs write access. + +### Where the token is stored + +The token is a real credential and is treated as one: + +- For a **one-off import**, type the token into the import dialog. It is sent with + that request only and is **never stored**. +- For **auto-sync** (which runs with no user present), configure the token on the + **server** via environment variables (below). It is read from `.env`, kept in + memory, **never written to disk by the app**, and **never returned by any API** + endpoint. Connection errors are sanitized so the token can't leak in a message. + +```env +# backend/.env +PROXMOX_TOKEN_ID=root@pam!homelable +PROXMOX_TOKEN_SECRET=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +PROXMOX_HOST=192.168.1.10 # optional default for auto-sync +PROXMOX_PORT=8006 +PROXMOX_VERIFY_TLS=true # set false only for self-signed certs +``` + +--- + +## Step-by-step Usage + +### 1. Open the Proxmox Import dialog + +Click **Proxmox Import** in the left sidebar (below "Z-Wave Import"). + +### 2. Configure the connection + +| Field | Default | Description | +|---|---|---| +| Proxmox Host | β€” | IP or hostname of the Proxmox host | +| Port | 8006 | Proxmox API port | +| Token ID | _(optional)_ | `user@realm!tokenid`; leave blank to use the server token | +| Token Secret | _(optional)_ | The token secret; leave blank to use the server token | +| Verify TLS | on | Uncheck for self-signed certificates | + +### 3. Test the connection (optional) + +Click **Test Connection**. A green indicator confirms reachability + a valid token; +red shows a sanitized error. + +### 4. Choose an import target + +- **Pending section** β€” Devices are queued in the Device Inventory for review + (and tracked as a scan run in Scan History). Approve, hide, or delete each. +- **Canvas directly** β€” Devices are fetched and shown grouped in the dialog so you + can pick which ones to add immediately. + +### 5. Fetch inventory + +Click **Import to Pending** (or **Fetch Inventory** in canvas mode). Homelable will: +1. Query `/nodes` for hosts +2. Query `/qemu` and `/lxc` per host +3. Resolve guest IPs (agent for QEMU, `net0` for LXC) best-effort +4. Return hosts + guests grouped by type + +### 6. Select and add to canvas + +(Canvas mode) Devices are grouped by type (Hosts / Virtual Machines / LXC +Containers). Use the checkboxes to pick which to add, then **Add N to Canvas**. + +### 7. Arrange on the canvas + +Nodes are placed in a grid; hostβ†’guest `virtual` edges connect them. Use +**Auto Layout** or drag nodes manually. + +--- + +## Node Type Mapping + +| Proxmox (`/api2/json`) | Homelable type | Notes | +|---|---|---| +| `/nodes` (host) | `proxmox` | Becomes the parent, linked to its guests | +| `/nodes/{node}/qemu/{vmid}` | `vm` | Guest-agent IP when available | +| `/nodes/{node}/lxc/{vmid}` | `lxc` | Static `net0` IP when set | +| `status` running/stopped | node status online/offline | | +| `maxcpu` | CPU Cores property | hidden by default | +| `maxmem` | RAM property (GB) | hidden by default | +| `maxdisk` | Disk property (GB) | hidden by default | +| VMID + host | synthetic identity (`pve-{host}-{vmid}`) | stable across re-imports | + +Guest hierarchy is rendered as `virtual` edges (host ↔ VM/LXC). + +--- + +## Auto-sync configuration + +1. Configure `PROXMOX_TOKEN_ID` / `PROXMOX_TOKEN_SECRET` (and optionally + `PROXMOX_HOST`) in `backend/.env` and restart the backend. +2. Open **Settings** β€” a **Proxmox auto-sync** section appears once a server token + is configured. +3. Toggle **Auto-sync Proxmox inventory** and set the interval (min 300 s). + +On each run, Homelable re-imports the inventory into the pending section: +- New VMs/LXC appear as **pending** for review. +- Existing devices are **updated in place** (status, specs, IP). +- Nothing is ever deleted β€” a VM removed from Proxmox is left on your canvas. + +--- + +## Security notes + +- Use a dedicated user + token with the read-only **`PVEAuditor`** role. +- The token is **never** persisted to disk by Homelable and **never** included in + any API response; only a boolean "token configured" flag is exposed. +- Keep `PROXMOX_VERIFY_TLS=true` in production; disable only for self-signed labs. +- Error messages are sanitized so credentials cannot leak. + +--- + +## Supported Versions + +Works with the Proxmox VE 7.x / 8.x REST API (`/api2/json`). No extra Proxmox +plugins are required. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| "Authentication failed" | Bad token id/secret or missing role | Re-check the token; grant `PVEAuditor` at `/` | +| "TLS verification failed" | Self-signed certificate | Uncheck **Verify TLS** (labs only) | +| "Proxmox host could not be resolved" | DNS/hostname wrong | Use the IP or a resolvable name | +| "No API token provided and none configured" | No token in the form and none in `.env` | Enter a token or set the server env vars | +| VMs have no IP | No guest agent (QEMU) / DHCP-only LXC | Install the QEMU guest agent; static IPs are read from `net0` | +| Duplicate-looking node | Same guest under a different identity | Re-import merges by IP/VMID; report if it persists | + +--- + +## Screenshots + +_(Screenshots will be added in a future release)_ diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 925e99f..b157aac 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -25,6 +25,7 @@ import { ScanConfigModal } from '@/components/modals/ScanConfigModal' import { SettingsModal } from '@/components/modals/SettingsModal' import { ZigbeeImportModal } from '@/components/zigbee/ZigbeeImportModal' import { ZwaveImportModal } from '@/components/zwave/ZwaveImportModal' +import { ProxmoxImportModal } from '@/components/proxmox/ProxmoxImportModal' import { GroupRectModal, type GroupRectFormData } from '@/components/modals/GroupRectModal' import { TextModal, type TextFormData } from '@/components/modals/TextModal' import { ThemeModal } from '@/components/modals/ThemeModal' @@ -45,6 +46,7 @@ import { useStatusPolling } from '@/hooks/useStatusPolling' import type { NodeData, EdgeData, CustomStyleDef, FloorMapConfig, NodeType } from '@/types' import type { ZigbeeNode, ZigbeeEdge } from '@/components/zigbee/types' import type { ZwaveNode, ZwaveEdge } from '@/components/zwave/types' +import type { ProxmoxNode, ProxmoxEdge } from '@/components/proxmox/types' const STANDALONE = import.meta.env.VITE_STANDALONE === 'true' @@ -84,6 +86,7 @@ export default function App() { const [exportModalOpen, setExportModalOpen] = useState(false) const [zigbeeImportOpen, setZigbeeImportOpen] = useState(false) const [zwaveImportOpen, setZwaveImportOpen] = useState(false) + const [proxmoxImportOpen, setProxmoxImportOpen] = useState(false) // Declare handleSave before the Ctrl+S effect so it is in scope. // Returns true on success, false on failure β€” the design-switch effect relies @@ -639,6 +642,52 @@ export default function App() { markUnsaved() }, [addNode, onConnect, snapshotHistory, markUnsaved]) + const handleProxmoxAddToCanvas = useCallback((pmNodes: ProxmoxNode[], pmEdges: ProxmoxEdge[]) => { + snapshotHistory() + const COLS = 4 + const SPACING_X = 190 + const SPACING_Y = 110 + const cols = Math.min(COLS, pmNodes.length) + const rows = Math.ceil(pmNodes.length / COLS) + const origin = getCenteredPosition(cols * SPACING_X, rows * SPACING_Y) + pmNodes.forEach((pn, i) => { + const col = i % COLS + const row = Math.floor(i / COLS) + const position = { x: origin.x + col * SPACING_X, y: origin.y + row * SPACING_Y } + const newNode: import('@xyflow/react').Node = { + id: pn.id, + type: pn.type, + position, + data: { + label: pn.label, + type: pn.type as NodeData['type'], + status: (pn.status === 'online' ? 'online' : 'unknown') as NodeData['status'], + services: [], + ...(pn.ip ? { ip: pn.ip } : {}), + ...(pn.hostname ? { hostname: pn.hostname } : {}), + }, + } + addNode(newNode) + }) + // Host β†’ guest links render as 'virtual' edges (VM/LXC ↔ host). + pmEdges.forEach((pe) => { + onConnect({ + source: pe.source, + sourceHandle: 'bottom', + target: pe.target, + targetHandle: 'top-t', + type: 'virtual', + } as unknown as import('@xyflow/react').Connection) + }) + const importedIds = new Set(pmNodes.map((pn) => pn.id)) + useCanvasStore.setState((state) => ({ + nodes: state.nodes.map((n) => ({ ...n, selected: importedIds.has(n.id) })), + selectedNodeIds: Array.from(importedIds), + selectedNodeId: importedIds.size === 1 ? Array.from(importedIds)[0] : null, + })) + markUnsaved() + }, [addNode, onConnect, snapshotHistory, markUnsaved]) + const handleEdgeConnect = useCallback((connection: Connection) => { setPendingConnection(connection) }, []) @@ -714,6 +763,7 @@ export default function App() { onScan={() => setScanConfigOpen(true)} onZigbeeImport={() => setZigbeeImportOpen(true)} onZwaveImport={() => setZwaveImportOpen(true)} + onProxmoxImport={() => setProxmoxImportOpen(true)} onSave={handleSave} onOpenSettings={() => setSettingsOpen(true)} onOpenHistory={() => setScanHistoryOpen(true)} @@ -840,6 +890,17 @@ export default function App() { /> )} + {!STANDALONE && ( + setProxmoxImportOpen(false)} + onAddToCanvas={handleProxmoxAddToCanvas} + onPendingImported={() => { + toast.success('Proxmox import started β€” check Scan History for results') + }} + /> + )} + {!STANDALONE && ( { mod.zwaveApi.importToPending(cfg) expect(api.post).toHaveBeenCalledWith('/zwave/import-pending', cfg) }) + + it('proxmoxApi.testConnection/importNetwork/importToPending', () => { + const cfg = { host: 'pve', port: 8006, token_id: 'u@pam!t', token_secret: 's', verify_tls: true } + mod.proxmoxApi.testConnection(cfg) + expect(api.post).toHaveBeenCalledWith('/proxmox/test-connection', cfg) + mod.proxmoxApi.importNetwork(cfg) + expect(api.post).toHaveBeenCalledWith('/proxmox/import', cfg) + mod.proxmoxApi.importToPending(cfg) + expect(api.post).toHaveBeenCalledWith('/proxmox/import-pending', cfg) + }) + + it('proxmoxApi.getConfig/saveConfig hit /proxmox/config', () => { + mod.proxmoxApi.getConfig() + expect(api.get).toHaveBeenCalledWith('/proxmox/config') + const conf = { host: 'pve', port: 8006, verify_tls: true, sync_enabled: false, sync_interval: 3600 } + mod.proxmoxApi.saveConfig(conf) + expect(api.post).toHaveBeenCalledWith('/proxmox/config', conf) + }) }) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index c063313..3c0aecb 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -120,6 +120,51 @@ export const settingsApi = { save: (data: AppSettings) => api.post('/settings', data), } +export interface ProxmoxConnection { + host: string + port: number + token_id?: string + token_secret?: string + verify_tls?: boolean +} + +export interface ProxmoxConfigData { + host: string + port: number + verify_tls: boolean + sync_enabled: boolean + sync_interval: number + token_configured: boolean +} + +export const proxmoxApi = { + testConnection: (data: ProxmoxConnection) => + api.post<{ connected: boolean; message: string }>('/proxmox/test-connection', data), + + importNetwork: (data: ProxmoxConnection) => + api.post<{ + nodes: import('@/components/proxmox/types').ProxmoxNode[] + edges: import('@/components/proxmox/types').ProxmoxEdge[] + device_count: number + }>('/proxmox/import', data), + + importToPending: (data: ProxmoxConnection) => + api.post<{ + id: string + status: string + kind: string + ranges: string[] + devices_found: number + started_at: string + finished_at: string | null + error: string | null + }>('/proxmox/import-pending', data), + + getConfig: () => api.get('/proxmox/config'), + saveConfig: (data: Omit) => + api.post('/proxmox/config', data), +} + export const designsApi = { list: () => api.get('/designs'), create: (data: { name: string; icon?: string; design_type?: string }) => diff --git a/frontend/src/components/modals/PendingDeviceModal.tsx b/frontend/src/components/modals/PendingDeviceModal.tsx index d511dea..3d9170b 100644 --- a/frontend/src/components/modals/PendingDeviceModal.tsx +++ b/frontend/src/components/modals/PendingDeviceModal.tsx @@ -1,6 +1,7 @@ import { Globe, Router, Server, Layers, Box, Container, HardDrive, Cpu, Wifi, Circle, Network } from 'lucide-react' import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' +import type { NodeProperty } from '@/types' interface Service { port: number @@ -26,6 +27,8 @@ export interface PendingDevice { model?: string | null vendor?: string | null lqi?: number | null + // Display properties carried from discovery (e.g. Proxmox specs). + properties?: NodeProperty[] discovered_at: string // How many canvases (designs) this device already appears on. Computed server-side. canvas_count?: number diff --git a/frontend/src/components/modals/PendingDevicesModal.tsx b/frontend/src/components/modals/PendingDevicesModal.tsx index 6d3af0d..8868074 100644 --- a/frontend/src/components/modals/PendingDevicesModal.tsx +++ b/frontend/src/components/modals/PendingDevicesModal.tsx @@ -73,13 +73,15 @@ const TYPE_ICONS: Record = { generic: Circle, } -type SourceFilter = 'all' | 'ip' | 'zigbee' | 'zwave' +type SourceFilter = 'all' | 'ip' | 'zigbee' | 'zwave' | 'proxmox' type StatusFilter = 'pending' | 'hidden' -function inferSource(d: PendingDevice): 'zigbee' | 'zwave' | 'ip' { +function inferSource(d: PendingDevice): 'zigbee' | 'zwave' | 'proxmox' | 'ip' { if (d.discovery_source === 'zwave') return 'zwave' if (d.discovery_source === 'zigbee') return 'zigbee' - if (d.ieee_address) return 'zigbee' + if (d.discovery_source === 'proxmox') return 'proxmox' + // Proxmox devices carry a synthetic 'pve-' ieee but are IP hosts, not mesh. + if (d.ieee_address && !d.ieee_address.startsWith('pve-')) return 'zigbee' return 'ip' } @@ -279,7 +281,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus ? buildZwaveProperties(device) : isZigbeeType(type) ? buildZigbeeProperties(device) - : buildMacProperty(device.mac) + : [...(device.properties ?? []), ...buildMacProperty(device.mac)] const nodeData = { label: fallbackLabel, type, @@ -364,7 +366,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus ? buildZwaveProperties(d) : isZigbeeType(type) ? buildZigbeeProperties(d) - : buildMacProperty(d.mac), + : [...(d.properties ?? []), ...buildMacProperty(d.mac)], }, }) }) @@ -503,6 +505,12 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus > Z-Wave + setPmSyncEnabled(e.target.checked)} + className="cursor-pointer accent-[#e57000]" + aria-label="Toggle Proxmox auto-sync" + /> + +
+ +
+ { const v = Number(e.target.value); if (!isNaN(v)) setPmInterval(v) }} + className="w-24 px-2 py-1 rounded-md text-xs font-mono bg-[#0d1117] border border-border text-foreground focus:outline-none focus:border-[#e57000]" + aria-label="Proxmox sync interval" + /> + seconds +
+

+ Re-imports hosts/VMs/LXC into the pending inventory. Min 300s (5 min). +

+
+ + )} + + )} + {/* Canvas */}
Canvas diff --git a/frontend/src/components/modals/__tests__/PendingDevicesModal.test.tsx b/frontend/src/components/modals/__tests__/PendingDevicesModal.test.tsx index ae322fc..4e7d230 100644 --- a/frontend/src/components/modals/__tests__/PendingDevicesModal.test.tsx +++ b/frontend/src/components/modals/__tests__/PendingDevicesModal.test.tsx @@ -84,6 +84,24 @@ const DEVICE_ZWAVE = { discovered_at: '2026-01-03T00:00:00Z', } +const DEVICE_PROXMOX = { + id: 'dev-d', + ip: '10.0.0.5', + hostname: 'web', + mac: null, + os: null, + services: [], + suggested_type: 'vm', + status: 'pending', + discovery_source: 'proxmox', + ieee_address: 'pve-pve1-101', + friendly_name: 'web', + vendor: 'Proxmox VE', + model: 'QEMU', + properties: [{ key: 'CPU Cores', value: '2', icon: 'Cpu', visible: false }], + discovered_at: '2026-01-04T00:00:00Z', +} + beforeEach(() => { vi.clearAllMocks() vi.mocked(useCanvasStore).mockReturnValue({ @@ -174,6 +192,23 @@ describe('PendingDevicesModal', () => { expect(screen.getByTestId('pending-card-dev-c')).toBeInTheDocument() }) + it('shows source chip PROXMOX for a proxmox device (not zigbee despite ieee)', async () => { + mockPending.mockResolvedValue({ data: [DEVICE_PROXMOX] }) + render() + await waitFor(() => expect(screen.getByTestId('pending-card-dev-d')).toBeInTheDocument()) + expect(screen.getByText('PROXMOX')).toBeInTheDocument() + expect(screen.queryByText('ZIGBEE')).not.toBeInTheDocument() + }) + + it('filters by source (proxmox only)', async () => { + mockPending.mockResolvedValue({ data: [DEVICE_IP, DEVICE_PROXMOX] }) + render() + await waitFor(() => expect(screen.getByTestId('pending-card-dev-a')).toBeInTheDocument()) + fireEvent.click(screen.getByRole('button', { name: 'Proxmox' })) + expect(screen.queryByTestId('pending-card-dev-a')).not.toBeInTheDocument() + expect(screen.getByTestId('pending-card-dev-d')).toBeInTheDocument() + }) + it('filters by suggested type', async () => { render() await waitFor(() => expect(screen.getByTestId('pending-card-dev-a')).toBeInTheDocument()) diff --git a/frontend/src/components/modals/__tests__/SettingsModal.test.tsx b/frontend/src/components/modals/__tests__/SettingsModal.test.tsx index 773e4b0..6426d39 100644 --- a/frontend/src/components/modals/__tests__/SettingsModal.test.tsx +++ b/frontend/src/components/modals/__tests__/SettingsModal.test.tsx @@ -8,9 +8,13 @@ vi.mock('@/api/client', () => ({ get: vi.fn(), save: vi.fn(), }, + proxmoxApi: { + getConfig: vi.fn(), + saveConfig: vi.fn(), + }, })) -import { settingsApi } from '@/api/client' +import { settingsApi, proxmoxApi } from '@/api/client' import { toast } from 'sonner' import { useCanvasStore } from '@/stores/canvasStore' @@ -19,6 +23,8 @@ describe('SettingsModal', () => { vi.clearAllMocks() vi.mocked(settingsApi.get).mockResolvedValue({ data: { interval_seconds: 60, service_check_enabled: false, service_check_interval: 300 } } as never) vi.mocked(settingsApi.save).mockResolvedValue({ data: { interval_seconds: 60, service_check_enabled: false, service_check_interval: 300 } } as never) + vi.mocked(proxmoxApi.getConfig).mockRejectedValue(new Error('not configured')) + vi.mocked(proxmoxApi.saveConfig).mockResolvedValue({ data: {} } as never) vi.mocked(toast.success).mockReset() vi.mocked(toast.error).mockReset() }) diff --git a/frontend/src/components/panels/Sidebar.tsx b/frontend/src/components/panels/Sidebar.tsx index 758ec67..2d48cda 100644 --- a/frontend/src/components/panels/Sidebar.tsx +++ b/frontend/src/components/panels/Sidebar.tsx @@ -1,5 +1,5 @@ import { useState, useCallback, useEffect } from 'react' -import { Plus, Save, ScanLine, ChevronLeft, ChevronRight, LayoutDashboard, Clock, EyeOff, Square, Settings, LogOut, Network, RadioTower, Type, PlusCircle, Pencil, Trash2 } from 'lucide-react' +import { Plus, Save, ScanLine, ChevronLeft, ChevronRight, LayoutDashboard, Clock, EyeOff, Square, Settings, LogOut, Network, RadioTower, Server, Type, PlusCircle, Pencil, Trash2 } from 'lucide-react' import { Logo } from '@/components/ui/Logo' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useCanvasStore } from '@/stores/canvasStore' @@ -27,13 +27,14 @@ interface SidebarProps { onScan: () => void onZigbeeImport: () => void onZwaveImport: () => void + onProxmoxImport: () => void onSave: () => void onOpenSettings: () => void onOpenHistory: () => void onOpenPending: (deviceId?: string, status?: 'pending' | 'hidden') => void } -export function Sidebar({ onAddNode, onAddGroupRect, onAddText, onScan, onZigbeeImport, onZwaveImport, onSave, onOpenSettings, onOpenHistory, onOpenPending }: SidebarProps) { +export function Sidebar({ onAddNode, onAddGroupRect, onAddText, onScan, onZigbeeImport, onZwaveImport, onProxmoxImport, onSave, onOpenSettings, onOpenHistory, onOpenPending }: SidebarProps) { const [collapsed, setCollapsed] = useState(false) const logout = useAuthStore((s) => s.logout) const { designs, activeDesignId, setActiveDesign, addDesign, updateDesign, removeDesign } = useDesignStore() @@ -265,6 +266,7 @@ export function Sidebar({ onAddNode, onAddGroupRect, onAddText, onScan, onZigbee {!STANDALONE && } {!STANDALONE && } {!STANDALONE && } + {!STANDALONE && } void + onAddToCanvas: (nodes: ProxmoxNode[], edges: ProxmoxEdge[]) => void + onPendingImported?: () => void +} + +type ImportMode = 'pending' | 'canvas' + +const ACCENT = '#e57000' + +interface ConnectionForm { + host: string + port: string + token_id: string + token_secret: string + verify_tls: boolean +} + +const DEFAULT_FORM: ConnectionForm = { + host: '', + port: '8006', + token_id: '', + token_secret: '', + verify_tls: true, +} + +const DEVICE_TYPE_ICON: Record = { + proxmox: Server, + vm: Box, + lxc: Container, +} + +const DEVICE_TYPE_LABEL: Record = { + proxmox: 'Hosts', + vm: 'Virtual Machines', + lxc: 'LXC Containers', +} + +const DEVICE_TYPE_COLOR: Record = { + proxmox: '#e57000', + vm: '#00d4ff', + lxc: '#39d353', +} + +export function ProxmoxImportModal({ open, onClose, onAddToCanvas, onPendingImported }: ProxmoxImportModalProps) { + const [form, setForm] = useState(DEFAULT_FORM) + const [connectionStatus, setConnectionStatus] = useState<'idle' | 'testing' | 'ok' | 'fail'>('idle') + const [connectionMsg, setConnectionMsg] = useState('') + const [loading, setLoading] = useState(false) + const [devices, setDevices] = useState([]) + const [edges, setEdges] = useState([]) + const [checked, setChecked] = useState>(new Set()) + const [importMode, setImportMode] = useState('pending') + + const updateField = (field: keyof ConnectionForm, value: string) => + setForm((f) => ({ ...f, [field]: value })) + + const buildPayload = (): ProxmoxConnection => ({ + host: form.host.trim(), + port: Number(form.port) || 8006, + token_id: form.token_id.trim() || undefined, + token_secret: form.token_secret || undefined, + verify_tls: form.verify_tls, + }) + + const extractError = (err: unknown): string | undefined => { + if (err && typeof err === 'object' && 'response' in err) { + return (err as { response?: { data?: { detail?: string } } }).response?.data?.detail + } + return undefined + } + + const handleTestConnection = async () => { + if (!form.host.trim()) { toast.error('Enter a Proxmox host'); return } + setConnectionStatus('testing') + try { + const res = await proxmoxApi.testConnection(buildPayload()) + setConnectionStatus(res.data.connected ? 'ok' : 'fail') + setConnectionMsg(res.data.message) + } catch (err) { + setConnectionStatus('fail') + setConnectionMsg(extractError(err) ?? 'Request failed β€” check host address') + } + } + + const handleFetchDevices = async () => { + if (!form.host.trim()) { toast.error('Enter a Proxmox host'); return } + setLoading(true) + try { + if (importMode === 'pending') { + await proxmoxApi.importToPending(buildPayload()) + toast.success('Proxmox import started β€” track progress in Scan History') + onPendingImported?.() + handleClose() + } else { + const res = await proxmoxApi.importNetwork(buildPayload()) + setDevices(res.data.nodes) + setEdges(res.data.edges) + setChecked(new Set(res.data.nodes.map((n) => n.id))) + if (res.data.device_count === 0) { + toast.info('No Proxmox guests found') + } else { + toast.success(`Found ${res.data.device_count} device${res.data.device_count !== 1 ? 's' : ''}`) + } + } + } catch (err: unknown) { + toast.error(extractError(err) ?? 'Failed to fetch Proxmox inventory') + } finally { + setLoading(false) + } + } + + const toggleCheck = (id: string) => + setChecked((prev) => { + const next = new Set(prev) + if (next.has(id)) next.delete(id); else next.add(id) + return next + }) + + const toggleAll = () => { + setChecked(checked.size === devices.length ? new Set() : new Set(devices.map((d) => d.id))) + } + + const handleAddToCanvas = () => { + const selectedDevices = devices.filter((d) => checked.has(d.id)) + const selectedIds = new Set(selectedDevices.map((d) => d.id)) + const selectedEdges = edges.filter((e) => selectedIds.has(e.source) && selectedIds.has(e.target)) + onAddToCanvas(selectedDevices, selectedEdges) + toast.success(`Added ${selectedDevices.length} device${selectedDevices.length !== 1 ? 's' : ''} to canvas`) + onClose() + } + + const handleClose = () => { + setDevices([]) + setEdges([]) + setChecked(new Set()) + setConnectionStatus('idle') + setConnectionMsg('') + setImportMode('pending') + onClose() + } + + const groupedDevices: Record = { + proxmox: devices.filter((d) => d.type === 'proxmox'), + vm: devices.filter((d) => d.type === 'vm'), + lxc: devices.filter((d) => d.type === 'lxc'), + } + + return ( + !v && handleClose()}> + + + + + Proxmox VE Import + + + +
+
+
+
+ + updateField('host', e.target.value)} + placeholder="192.168.1.x or pve.local" + className="font-mono text-sm bg-[#0d1117] border-border" + /> +
+
+ + updateField('port', e.target.value)} + placeholder="8006" + type="number" + className="font-mono text-sm bg-[#0d1117] border-border" + /> +
+
+ + updateField('token_id', e.target.value)} + placeholder="user@pam!tokenname" + className="font-mono text-sm bg-[#0d1117] border-border" + /> +
+
+ + updateField('token_secret', e.target.value)} + placeholder="β€’β€’β€’β€’β€’β€’β€’β€’-β€’β€’β€’β€’-β€’β€’β€’β€’-β€’β€’β€’β€’-β€’β€’β€’β€’β€’β€’β€’β€’β€’β€’β€’β€’" + type="password" + autoComplete="new-password" + className="text-sm bg-[#0d1117] border-border" + /> +
+
+ +
+
+ + {connectionStatus !== 'idle' && ( +
+ {connectionStatus === 'testing' && } + {connectionStatus === 'ok' && } + {connectionStatus === 'fail' && } + {connectionStatus === 'testing' ? 'Testing…' : connectionMsg} +
+ )} + +
+ Send devices to: + + +
+
+ + +
+

+ Leave the token blank to use the token configured on the server (.env). + A read-only PVEAuditor role is enough. +

+
+ + {devices.length > 0 && ( +
+
+
+ { if (el) el.indeterminate = checked.size > 0 && checked.size < devices.length }} + onChange={toggleAll} + className="w-3 h-3 cursor-pointer" + style={{ accentColor: ACCENT }} + title="Select all" + /> + + Devices ({checked.size}/{devices.length} selected) + +
+
+ + {(Object.entries(groupedDevices) as [ProxmoxNodeType, ProxmoxNode[]][]) + .filter(([, group]) => group.length > 0) + .map(([type, group]) => { + const Icon = DEVICE_TYPE_ICON[type] + const color = DEVICE_TYPE_COLOR[type] + return ( +
+
+ + + {DEVICE_TYPE_LABEL[type]} ({group.length}) + +
+ {group.map((device) => ( +
toggleCheck(device.id)} + > + toggleCheck(device.id)} + onClick={(e) => e.stopPropagation()} + className="w-3 h-3 mt-0.5 cursor-pointer shrink-0" + style={{ accentColor: ACCENT }} + /> +
+
{device.label}
+ {device.ip && ( +
{device.ip}
+ )} +
+ {[ + device.cpu_count ? `${device.cpu_count} vCPU` : null, + device.ram_gb ? `${device.ram_gb} GB RAM` : null, + device.status, + ].filter(Boolean).join(' Β· ')} +
+
+
+ ))} +
+ ) + })} +
+ )} +
+ + + + {devices.length > 0 && ( + + )} + + + + ) +} diff --git a/frontend/src/components/proxmox/__tests__/ProxmoxImportModal.test.tsx b/frontend/src/components/proxmox/__tests__/ProxmoxImportModal.test.tsx new file mode 100644 index 0000000..dd35d39 --- /dev/null +++ b/frontend/src/components/proxmox/__tests__/ProxmoxImportModal.test.tsx @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { ProxmoxImportModal } from '../ProxmoxImportModal' + +vi.mock('@/api/client', () => ({ + proxmoxApi: { + testConnection: vi.fn(), + importNetwork: vi.fn(), + importToPending: vi.fn(), + }, +})) +vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() } })) + +import { proxmoxApi } from '@/api/client' +import { toast } from 'sonner' + +const defaultProps = { + open: true, + onClose: vi.fn(), + onAddToCanvas: vi.fn(), + onPendingImported: vi.fn(), +} + +const sampleNodes = [ + { + id: 'pve-node-pve1', label: 'pve1', type: 'proxmox' as const, + ieee_address: 'pve-node-pve1', hostname: 'pve1', ip: null, status: 'online', + cpu_count: 8, ram_gb: 16, disk_gb: 500, vendor: 'Proxmox VE', model: null, parent_ieee: null, + }, + { + id: 'pve-pve1-101', label: 'web', type: 'vm' as const, + ieee_address: 'pve-pve1-101', hostname: 'web', ip: '10.0.0.5', status: 'online', + cpu_count: 2, ram_gb: 4, disk_gb: 32, vendor: 'Proxmox VE', model: 'QEMU', parent_ieee: 'pve-node-pve1', + }, +] + +describe('ProxmoxImportModal', () => { + beforeEach(() => { + vi.mocked(proxmoxApi.testConnection).mockReset() + vi.mocked(proxmoxApi.importNetwork).mockReset() + vi.mocked(proxmoxApi.importToPending).mockReset() + vi.mocked(toast.success).mockReset() + vi.mocked(toast.error).mockReset() + vi.mocked(toast.info).mockReset() + defaultProps.onClose.mockReset() + defaultProps.onAddToCanvas.mockReset() + defaultProps.onPendingImported.mockReset() + }) + + it('renders nothing when closed', () => { + const { container } = render() + expect(container.querySelector('[role="dialog"]')).toBeNull() + }) + + it('renders token fields with a masked secret input', () => { + render() + expect(screen.getByText('Proxmox VE Import')).toBeDefined() + expect(screen.getByPlaceholderText('user@pam!tokenname')).toBeDefined() + const secret = document.querySelector('input[type="password"]') + expect(secret).not.toBeNull() + }) + + it('errors when testing without a host', async () => { + render() + fireEvent.click(screen.getByRole('button', { name: /test connection/i })) + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Enter a Proxmox host')) + expect(proxmoxApi.testConnection).not.toHaveBeenCalled() + }) + + it('shows connection message on successful test', async () => { + vi.mocked(proxmoxApi.testConnection).mockResolvedValue({ + data: { connected: true, message: 'Connected to Proxmox VE 8.2.2' }, + } as never) + render() + fireEvent.change(screen.getByPlaceholderText('192.168.1.x or pve.local'), { target: { value: 'pve' } }) + fireEvent.click(screen.getByRole('button', { name: /test connection/i })) + await waitFor(() => expect(screen.getByText('Connected to Proxmox VE 8.2.2')).toBeDefined()) + }) + + it('imports to pending by default and notifies parent', async () => { + vi.mocked(proxmoxApi.importToPending).mockResolvedValue({ + data: { id: 'run-1', status: 'running', kind: 'proxmox', ranges: [], devices_found: 0, started_at: '', finished_at: null, error: null }, + } as never) + render() + fireEvent.change(screen.getByPlaceholderText('192.168.1.x or pve.local'), { target: { value: 'pve' } }) + fireEvent.click(screen.getByRole('button', { name: /import to pending/i })) + await waitFor(() => expect(proxmoxApi.importToPending).toHaveBeenCalled()) + expect(defaultProps.onPendingImported).toHaveBeenCalled() + expect(proxmoxApi.importNetwork).not.toHaveBeenCalled() + }) + + it('fetches inventory in canvas mode and groups by type', async () => { + vi.mocked(proxmoxApi.importNetwork).mockResolvedValue({ + data: { nodes: sampleNodes, edges: [{ source: 'pve-node-pve1', target: 'pve-pve1-101' }], device_count: 2 }, + } as never) + render() + fireEvent.click(screen.getByRole('radio', { name: /canvas directly/i })) + fireEvent.change(screen.getByPlaceholderText('192.168.1.x or pve.local'), { target: { value: 'pve' } }) + fireEvent.click(screen.getByRole('button', { name: /fetch inventory/i })) + await waitFor(() => { + expect(screen.getByText('pve1')).toBeDefined() + expect(screen.getByText('web')).toBeDefined() + }) + expect(toast.success).toHaveBeenCalledWith('Found 2 devices') + }) + + it('sends the token from the form in the payload', async () => { + vi.mocked(proxmoxApi.importNetwork).mockResolvedValue({ + data: { nodes: [], edges: [], device_count: 0 }, + } as never) + render() + fireEvent.click(screen.getByRole('radio', { name: /canvas directly/i })) + fireEvent.change(screen.getByPlaceholderText('192.168.1.x or pve.local'), { target: { value: 'pve' } }) + fireEvent.change(screen.getByPlaceholderText('user@pam!tokenname'), { target: { value: 'root@pam!hl' } }) + fireEvent.click(screen.getByRole('button', { name: /fetch inventory/i })) + await waitFor(() => expect(proxmoxApi.importNetwork).toHaveBeenCalled()) + const payload = vi.mocked(proxmoxApi.importNetwork).mock.calls[0][0] + expect(payload.token_id).toBe('root@pam!hl') + expect(payload.port).toBe(8006) + }) +}) diff --git a/frontend/src/components/proxmox/types.ts b/frontend/src/components/proxmox/types.ts new file mode 100644 index 0000000..b0a4b8c --- /dev/null +++ b/frontend/src/components/proxmox/types.ts @@ -0,0 +1,30 @@ +/** Shared Proxmox VE import type definitions for the frontend. */ + +export type ProxmoxNodeType = 'proxmox' | 'vm' | 'lxc' + +export interface ProxmoxNode { + id: string + label: string + type: ProxmoxNodeType + ieee_address: string + hostname?: string | null + ip?: string | null + status: string + cpu_count?: number | null + ram_gb?: number | null + disk_gb?: number | null + vendor?: string | null + model?: string | null + parent_ieee?: string | null +} + +export interface ProxmoxEdge { + source: string + target: string +} + +export interface ProxmoxImportResponse { + nodes: ProxmoxNode[] + edges: ProxmoxEdge[] + device_count: number +} From abefc42fdf7a4363a0df68a8b7b1d58c3be908ab Mon Sep 17 00:00:00 2001 From: Pouzor Date: Mon, 6 Jul 2026 00:01:27 +0200 Subject: [PATCH 2/6] fix: tolerate legacy NULL properties on pending devices The pending_devices.properties column is added by an idempotent migration, so existing rows have properties = NULL. PendingDeviceResponse typed it as a list, so GET /scan/pending 500'd on any pre-existing device. - Coerce NULL/non-list properties to [] in PendingDeviceResponse. - Backfill existing NULL rows to '[]' in init_db migrations. - Regression test: /scan/pending returns 200 with a legacy NULL-properties row. ha-relevant: maybe --- backend/app/db/database.py | 2 ++ backend/app/schemas/scan.py | 8 +++++++- backend/tests/test_proxmox_router.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/backend/app/db/database.py b/backend/app/db/database.py index 76470d6..e7177b8 100644 --- a/backend/app/db/database.py +++ b/backend/app/db/database.py @@ -117,6 +117,8 @@ async def init_db() -> None: await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_source TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN properties JSON") + with suppress(OperationalError): + await conn.exec_driver_sql("UPDATE pending_devices SET properties = '[]' WHERE properties IS NULL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE scan_runs ADD COLUMN kind TEXT NOT NULL DEFAULT 'ip'") # --- Zigbee schema migrations (logged variant per CLAUDE.md feedback) --- diff --git a/backend/app/schemas/scan.py b/backend/app/schemas/scan.py index d64bd79..0b0827c 100644 --- a/backend/app/schemas/scan.py +++ b/backend/app/schemas/scan.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, field_validator class PendingDeviceResponse(BaseModel): @@ -35,6 +35,12 @@ class PendingDeviceResponse(BaseModel): node_last_modified: datetime | None = None node_last_seen: datetime | None = None + @field_validator("properties", mode="before") + @classmethod + def _coerce_properties(cls, v: Any) -> list[Any]: + # Legacy rows (column added by migration) have properties = NULL. + return v if isinstance(v, list) else [] + model_config = {"from_attributes": True} diff --git a/backend/tests/test_proxmox_router.py b/backend/tests/test_proxmox_router.py index 2303430..115d03d 100644 --- a/backend/tests/test_proxmox_router.py +++ b/backend/tests/test_proxmox_router.py @@ -178,6 +178,21 @@ async def test_persist_keeps_hidden_hidden(db_session) -> None: assert row.status == "hidden" +@pytest.mark.asyncio +async def test_pending_endpoint_tolerates_legacy_null_properties(client: AsyncClient, headers: dict, db_session) -> None: + # Legacy row: properties column NULL (added by migration on older DBs). + dev = PendingDevice( + id=str(uuid.uuid4()), ip="192.168.1.9", suggested_type="server", + status="pending", discovery_source="arp", + ) + dev.properties = None + db_session.add(dev) + await db_session.commit() + res = await client.get("/api/v1/scan/pending", headers=headers) + assert res.status_code == 200 + assert res.json()[0]["properties"] == [] + + @pytest.mark.asyncio async def test_persist_never_deletes(db_session) -> None: await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) From 9670d0a86aaa902f06d63be09f89deae26cbdbed Mon Sep 17 00:00:00 2001 From: Pouzor Date: Mon, 6 Jul 2026 10:26:52 +0200 Subject: [PATCH 3/6] feat: proxmox import diagnostics, node style fix, and cluster edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scan History: - Proxmox is now a first-class scan kind (badge, filter chip, Server icon, completion toast) instead of being mislabeled as an IP scan. - A done run carrying a non-fatal advisory renders amber (info) with a warning toast, distinct from red failures. Import diagnostics: - test-connection probes /access/permissions and warns when the API token has no ACL (VMs/LXC would be invisible) β€” points at the PVEAuditor grant. - import surfaces an advisory when hosts import but no guests are visible (privilege-separated token whose rights are the intersection with the user), rather than a silent "done". Node style: - Proxmox container mode is now opt-in (container_mode === true), matching the rest of the codebase (App.tsx nesting logic). Imported proxmox nodes leave the flag unset and render like a manually-created node instead of an empty group container. Cluster edges: - Hosts from one import are chained with 'cluster' edges via left/right handles, distinct from the vertical host->guest 'virtual' edges. Wired for both the direct "Add to Canvas" path and the pending -> approve path (host<->host proxmox_cluster links, resolved to cluster edges on approve; cluster hosts get left/right handles). Tests added on both sides. ha-relevant: maybe --- backend/app/api/routes/proxmox.py | 76 ++++++++++++++---- backend/app/api/routes/scan.py | 47 +++++++++-- backend/app/services/proxmox_service.py | 42 +++++++++- backend/tests/test_proxmox_router.py | 79 ++++++++++++++++++- backend/tests/test_proxmox_service.py | 56 +++++++++++++ frontend/src/App.tsx | 18 +++++ .../canvas/nodes/ProxmoxGroupNode.tsx | 7 +- .../nodes/__tests__/ProxmoxGroupNode.test.tsx | 22 ++++-- .../components/modals/ScanHistoryModal.tsx | 37 ++++++--- .../__tests__/ScanHistoryModal.test.tsx | 41 +++++++++- .../proxmox/__tests__/clusterEdges.test.ts | 45 +++++++++++ .../src/components/proxmox/clusterEdges.ts | 46 +++++++++++ 12 files changed, 475 insertions(+), 41 deletions(-) create mode 100644 frontend/src/components/proxmox/__tests__/clusterEdges.test.ts create mode 100644 frontend/src/components/proxmox/clusterEdges.ts diff --git a/backend/app/api/routes/proxmox.py b/backend/app/api/routes/proxmox.py index 6abf8c5..2986777 100644 --- a/backend/app/api/routes/proxmox.py +++ b/backend/app/api/routes/proxmox.py @@ -34,6 +34,7 @@ from app.schemas.proxmox import ( from app.schemas.scan import ScanRunResponse from app.services.node_dedupe import dedupe_nodes_by_ieee from app.services.proxmox_service import ( + build_proxmox_cluster_links, build_proxmox_properties, fetch_proxmox_inventory, merge_proxmox_properties, @@ -43,6 +44,11 @@ from app.services.proxmox_service import ( logger = logging.getLogger(__name__) router = APIRouter() +# Discovery sources for the two proxmox link shapes. Hostβ†’guest links render as +# 'virtual' edges; host↔host cluster links render as 'cluster' edges. +_PROXMOX_GUEST_SOURCE = "proxmox" +_PROXMOX_CLUSTER_SOURCE = "proxmox_cluster" + def _resolve_credentials(payload: ProxmoxConnectionRequest) -> tuple[str, str]: """Pick the API token: request body first, else server env config. @@ -156,6 +162,7 @@ async def _background_proxmox_import( run.status = "done" run.devices_found = result.device_count run.finished_at = datetime.now(timezone.utc) + run.error = _guest_visibility_advisory(nodes_raw) await db.commit() except Exception as exc: logger.exception("Proxmox import %s failed", run_id) @@ -168,6 +175,27 @@ async def _background_proxmox_import( await db.commit() +def _guest_visibility_advisory(nodes_raw: list[dict[str, Any]]) -> str | None: + """Non-fatal advisory when hosts import but no VMs/LXC were visible. + + A Proxmox API token that lacks ``VM.Audit`` sees empty ``qemu``/``lxc`` lists + (HTTP 200, no error), so only host nodes come through. The usual cause is a + privilege-separated token whose effective rights are the *intersection* with + the user's rights β€” granting PVEAuditor to the token alone is not enough when + the user has none. Surface that instead of a silent success. + """ + hosts = sum(1 for n in nodes_raw if n.get("type") == "proxmox") + guests = len(nodes_raw) - hosts + if hosts and not guests: + return ( + f"Imported {hosts} host(s) but no VMs or LXC were visible to the API " + "token. Grant the PVEAuditor role at path '/' to BOTH the token and " + "the user (privilege-separated tokens get the intersection of token " + "and user rights), then re-import." + ) + return None + + async def _persist_pending_import( db: AsyncSession, nodes_raw: list[dict[str, Any]], @@ -184,6 +212,9 @@ async def _persist_pending_import( """ await dedupe_nodes_by_ieee(db) + cluster_pairs = build_proxmox_cluster_links(nodes_raw) + cluster_members = {ieee for pair in cluster_pairs for ieee in pair} + pending_created = 0 pending_updated = 0 @@ -215,6 +246,11 @@ async def _persist_pending_import( en.cpu_count = en.cpu_count or n.get("cpu_count") en.ram_gb = en.ram_gb or n.get("ram_gb") en.disk_gb = en.disk_gb or n.get("disk_gb") + # A cluster host needs one left + one right handle for the + # cluster edge endpoints (both default to 0). + if ieee in cluster_members: + en.left_handles = max(en.left_handles or 0, 1) + en.right_handles = max(en.right_handles or 0, 1) await _ensure_inventory_row(db, ieee, ip, n, props, approved=True) pending_updated += 1 continue @@ -228,7 +264,7 @@ async def _persist_pending_import( _refresh_pending(pending, ieee, ip, n, props) pending_updated += 1 - links_recorded = await _replace_links(db, edges_raw) + links_recorded = await _replace_links(db, edges_raw, cluster_pairs) await db.commit() return ProxmoxImportPendingResponse( @@ -309,27 +345,37 @@ async def _ensure_inventory_row( inv.properties = merge_proxmox_properties(list(inv.properties or []), props) -async def _replace_links(db: AsyncSession, edges_raw: list[dict[str, Any]]) -> int: - """Wipe all proxmox-source links and re-insert the freshly discovered set.""" +async def _replace_links( + db: AsyncSession, + edges_raw: list[dict[str, Any]], + cluster_pairs: list[tuple[str, str]], +) -> int: + """Wipe all proxmox-source links and re-insert the freshly discovered set. + + Two link shapes: hostβ†’guest (``proxmox`` β†’ 'virtual' edges) and host↔host + (``proxmox_cluster`` β†’ 'cluster' edges). + """ await db.execute( - sa_delete(PendingDeviceLink).where(PendingDeviceLink.discovery_source == "proxmox") + sa_delete(PendingDeviceLink).where( + PendingDeviceLink.discovery_source.in_([_PROXMOX_GUEST_SOURCE, _PROXMOX_CLUSTER_SOURCE]) + ) ) recorded = 0 seen: set[tuple[str, str]] = set() - for e in edges_raw: - src = e.get("source") - tgt = e.get("target") + + def _add(src: str | None, tgt: str | None, source: str) -> None: + nonlocal recorded if not src or not tgt or (src, tgt) in seen: - continue + return seen.add((src, tgt)) - db.add( - PendingDeviceLink( - source_ieee=src, - target_ieee=tgt, - discovery_source="proxmox", - ) - ) + db.add(PendingDeviceLink(source_ieee=src, target_ieee=tgt, discovery_source=source)) recorded += 1 + + for e in edges_raw: + _add(e.get("source"), e.get("target"), _PROXMOX_GUEST_SOURCE) + for src, tgt in cluster_pairs: + _add(src, tgt, _PROXMOX_CLUSTER_SOURCE) + return recorded diff --git a/backend/app/api/routes/scan.py b/backend/app/api/routes/scan.py index 501d95e..219a26b 100644 --- a/backend/app/api/routes/scan.py +++ b/backend/app/api/routes/scan.py @@ -356,6 +356,7 @@ async def bulk_approve_devices( device.status = "approved" node_type = device.suggested_type or "generic" is_wireless = _is_wireless(node_type) + cluster_host = await _is_proxmox_cluster_member(db, device.ieee_address) node = Node( label=device.hostname or device.friendly_name or device.ip or "device", type=node_type, @@ -371,6 +372,9 @@ async def bulk_approve_devices( # Default to ping so the status checker actually polls the new node. # Without this the scheduler skips it (check_method NULL β†’ no check). check_method="none" if is_wireless else ("ping" if device.ip else None), + # Cluster hosts get side handles for their host↔host cluster edge. + left_handles=1 if cluster_host else 0, + right_handles=1 if cluster_host else 0, design_id=default_design_id, ) db.add(node) @@ -508,6 +512,7 @@ async def approve_device( # Prefer the MAC discovered during the scan (stored on the pending device); # fall back to whatever the approve payload carried. _mac = device.mac or node_data.mac + cluster_host = await _is_proxmox_cluster_member(db, device.ieee_address) node = Node( label=node_data.label, type=node_data.type, @@ -525,6 +530,9 @@ async def approve_device( ), check_method="none" if wireless else (node_data.check_method or ("ping" if node_data.ip else None)), check_target=None if wireless else node_data.check_target, + # Cluster hosts get side handles for their host↔host cluster edge. + left_handles=1 if cluster_host else 0, + right_handles=1 if cluster_host else 0, design_id=node_design_id, ) db.add(node) @@ -542,6 +550,28 @@ async def approve_device( } +async def _is_proxmox_cluster_member(db: AsyncSession, ieee: str | None) -> bool: + """True if ``ieee`` participates in a proxmox_cluster link (host↔host). + + Such a host needs one left + one right handle for the cluster edge endpoints + (both default to 0). Checked at approve time, before the link is consumed by + ``_resolve_pending_links_for_ieee``. + """ + if not ieee: + return False + found = ( + await db.execute( + select(PendingDeviceLink.id) + .where( + PendingDeviceLink.discovery_source == "proxmox_cluster", + (PendingDeviceLink.source_ieee == ieee) | (PendingDeviceLink.target_ieee == ieee), + ) + .limit(1) + ) + ).scalar() + return found is not None + + async def _resolve_pending_links_for_ieee( db: AsyncSession, ieee: str | None ) -> list[dict[str, str]]: @@ -612,15 +642,22 @@ async def _resolve_pending_links_for_ieee( if edge_design_id is None: first = (await db.execute(select(Design).order_by(Design.created_at).limit(1))).scalar() edge_design_id = first.id if first else None - # Proxmox hostβ†’guest links render as 'virtual' (VM/LXC ↔ host); mesh - # links stay 'iot'. Default to iot for any legacy/other source. - edge_type = "virtual" if link.discovery_source == "proxmox" else "iot" + # Edge shape by link source: + # proxmox β†’ 'virtual' hostβ†’guest, vertical (bottom β†’ top) + # proxmox_cluster β†’ 'cluster' host↔host, horizontal (right β†’ left) + # anything else β†’ 'iot' mesh link, vertical + if link.discovery_source == "proxmox": + edge_type, src_handle, tgt_handle = "virtual", "bottom", "top-t" + elif link.discovery_source == "proxmox_cluster": + edge_type, src_handle, tgt_handle = "cluster", "right", "left-t" + else: + edge_type, src_handle, tgt_handle = "iot", "bottom", "top-t" edge = Edge( source=src_id, target=tgt_id, type=edge_type, - source_handle="bottom", - target_handle="top-t", + source_handle=src_handle, + target_handle=tgt_handle, design_id=edge_design_id, ) db.add(edge) diff --git a/backend/app/services/proxmox_service.py b/backend/app/services/proxmox_service.py index 423be85..e246961 100644 --- a/backend/app/services/proxmox_service.py +++ b/backend/app/services/proxmox_service.py @@ -192,6 +192,21 @@ def build_proxmox_properties(node: dict[str, Any]) -> list[dict[str, Any]]: return props +def build_proxmox_cluster_links(nodes: list[dict[str, Any]]) -> list[tuple[str, str]]: + """Chain host nodes (``type == 'proxmox'``) into cluster links. + + Hosts from one import belong to the same cluster, so they are linked + host↔host (rendered as ``cluster`` edges via left/right handles, distinct + from the vertical hostβ†’guest ``virtual`` edges). Returns consecutive + ``(source_ieee, target_ieee)`` pairs, or ``[]`` for a single host. Mirrors + the frontend ``buildProxmoxClusterEdges``. + """ + hosts = [n["ieee_address"] for n in nodes if n.get("type") == "proxmox" and n.get("ieee_address")] + if len(hosts) < 2: + return [] + return [(hosts[i], hosts[i + 1]) for i in range(len(hosts) - 1)] + + def _parse_inventory( hosts_raw: list[dict[str, Any]], guests_by_host: dict[str, list[dict[str, Any]]], @@ -231,6 +246,23 @@ async def _get_json(client: httpx.AsyncClient, path: str) -> Any: return resp.json().get("data") +async def _token_has_permissions(client: httpx.AsyncClient) -> bool: + """True if the API token holds *any* ACL. + + Proxmox list endpoints (``/qemu``, ``/lxc``) silently return an empty + ``200`` when the token lacks ``VM.Audit`` β€” indistinguishable from a host + that genuinely has no guests. ``GET /access/permissions`` returns ``{}`` for + a token with no ACL at all, which is the common misconfiguration (a + privilege-separated token created without its own permission). Best-effort: + on any error assume permissions exist so we never block a valid import. + """ + try: + perms = await _get_json(client, "/access/permissions") + except httpx.HTTPError: + return True + return bool(perms) if isinstance(perms, dict) else True + + async def fetch_proxmox_inventory( host: str, port: int, @@ -339,8 +371,16 @@ async def test_proxmox_connection( timeout=timeout, ) as client: data = await _get_json(client, "/version") + has_perms = await _token_has_permissions(client) version = (data or {}).get("version", "?") if isinstance(data, dict) else "?" - return True, f"Connected to Proxmox VE {version}" + message = f"Connected to Proxmox VE {version}" + if not has_perms: + message += ( + " β€” warning: this API token has no permissions, so VMs and LXC " + "will not be visible. Assign the PVEAuditor role at path '/' to the " + "token in Proxmox (Datacenter β†’ Permissions β†’ API Token Permission)." + ) + return True, message except httpx.HTTPError as exc: return False, _sanitize_proxmox_error(exc) except Exception as exc: # noqa: BLE001 β€” surface a safe message, log the rest diff --git a/backend/tests/test_proxmox_router.py b/backend/tests/test_proxmox_router.py index 115d03d..7d6c135 100644 --- a/backend/tests/test_proxmox_router.py +++ b/backend/tests/test_proxmox_router.py @@ -9,9 +9,10 @@ import pytest from httpx import AsyncClient from sqlalchemy import select -from app.api.routes.proxmox import _persist_pending_import +from app.api.routes.proxmox import _guest_visibility_advisory, _persist_pending_import +from app.api.routes.scan import _is_proxmox_cluster_member, _resolve_pending_links_for_ieee from app.core.config import settings -from app.db.models import Node, PendingDevice +from app.db.models import Design, Edge, Node, PendingDevice, PendingDeviceLink @pytest.fixture @@ -115,6 +116,22 @@ async def test_enable_sync_without_token_rejected(client: AsyncClient, headers: assert res.status_code == 400 +# --- guest-visibility advisory --------------------------------------------- + +def test_advisory_when_hosts_only() -> None: + msg = _guest_visibility_advisory([_host_node()]) + assert msg is not None + assert "PVEAuditor" in msg + + +def test_no_advisory_when_guests_present() -> None: + assert _guest_visibility_advisory([_host_node(), _guest_node(101, "10.0.0.5")]) is None + + +def test_no_advisory_when_nothing_imported() -> None: + assert _guest_visibility_advisory([]) is None + + # --- persistence / dedupe -------------------------------------------------- @pytest.mark.asyncio @@ -193,6 +210,64 @@ async def test_pending_endpoint_tolerates_legacy_null_properties(client: AsyncCl assert res.json()[0]["properties"] == [] +# --- cluster links --------------------------------------------------------- + +@pytest.mark.asyncio +async def test_persist_records_cluster_links_between_hosts(db_session) -> None: + # Two hosts + a guest β†’ one host↔host cluster link, one hostβ†’guest link. + nodes = [ + {**_host_node(), "id": "pve-node-a", "ieee_address": "pve-node-a", "hostname": "a", "label": "a"}, + {**_host_node(), "id": "pve-node-b", "ieee_address": "pve-node-b", "hostname": "b", "label": "b"}, + _guest_node(101, "10.0.0.5"), + ] + edges = [{"source": "pve-node-pve1", "target": "pve-pve1-101"}] + await _persist_pending_import(db_session, nodes, edges) + + links = (await db_session.execute(select(PendingDeviceLink))).scalars().all() + cluster = [ln for ln in links if ln.discovery_source == "proxmox_cluster"] + assert len(cluster) == 1 + assert (cluster[0].source_ieee, cluster[0].target_ieee) == ("pve-node-a", "pve-node-b") + # Membership helper sees both endpoints. + assert await _is_proxmox_cluster_member(db_session, "pve-node-a") is True + assert await _is_proxmox_cluster_member(db_session, "pve-node-b") is True + assert await _is_proxmox_cluster_member(db_session, "pve-pve1-101") is False + + +@pytest.mark.asyncio +async def test_single_host_records_no_cluster_link(db_session) -> None: + await _persist_pending_import(db_session, [_host_node()], []) + links = (await db_session.execute( + select(PendingDeviceLink).where(PendingDeviceLink.discovery_source == "proxmox_cluster") + )).scalars().all() + assert links == [] + + +@pytest.mark.asyncio +async def test_cluster_link_resolves_to_cluster_edge(db_session) -> None: + # Two host nodes already on a canvas + a pending cluster link between them. + design = Design(id=str(uuid.uuid4()), name="d") + db_session.add(design) + a = Node(id=str(uuid.uuid4()), type="proxmox", label="a", ieee_address="pve-node-a", + status="online", pos_x=0, pos_y=0, design_id=design.id, + left_handles=1, right_handles=1) + b = Node(id=str(uuid.uuid4()), type="proxmox", label="b", ieee_address="pve-node-b", + status="online", pos_x=0, pos_y=0, design_id=design.id, + left_handles=1, right_handles=1) + db_session.add_all([a, b]) + db_session.add(PendingDeviceLink( + id=str(uuid.uuid4()), source_ieee="pve-node-a", target_ieee="pve-node-b", + discovery_source="proxmox_cluster", + )) + await db_session.commit() + + await _resolve_pending_links_for_ieee(db_session, "pve-node-a") + + edge = (await db_session.execute(select(Edge))).scalars().one() + assert edge.type == "cluster" + assert edge.source_handle == "right" + assert edge.target_handle == "left-t" + + @pytest.mark.asyncio async def test_persist_never_deletes(db_session) -> None: await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) diff --git a/backend/tests/test_proxmox_service.py b/backend/tests/test_proxmox_service.py index ca2913e..afda931 100644 --- a/backend/tests/test_proxmox_service.py +++ b/backend/tests/test_proxmox_service.py @@ -75,6 +75,25 @@ def test_parse_inventory_builds_host_guest_edges() -> None: assert edges == [{"source": "pve-node-pve1", "target": "pve-pve1-101"}] +def test_build_cluster_links_chains_hosts() -> None: + nodes = [ + svc._host_node({"node": "pve-a", "status": "online"}), + svc._guest_node({"vmid": 101, "status": "running"}, "pve-a", "qemu", None), + svc._host_node({"node": "pve-b", "status": "online"}), + svc._host_node({"node": "pve-c", "status": "online"}), + ] + pairs = svc.build_proxmox_cluster_links(nodes) + assert pairs == [("pve-node-pve-a", "pve-node-pve-b"), ("pve-node-pve-b", "pve-node-pve-c")] + + +def test_build_cluster_links_single_host_is_not_a_cluster() -> None: + nodes = [svc._host_node({"node": "pve-a", "status": "online"})] + assert svc.build_proxmox_cluster_links(nodes) == [] + # Guests alone never form a cluster. + guest = svc._guest_node({"vmid": 1, "status": "running"}, "pve-a", "qemu", None) + assert svc.build_proxmox_cluster_links([guest]) == [] + + def test_sanitize_error_hides_credentials() -> None: exc = httpx.HTTPStatusError( "boom", request=httpx.Request("GET", "https://h/api2/json"), @@ -115,9 +134,46 @@ async def test_fetch_inventory_happy_path() -> None: @pytest.mark.asyncio async def test_test_connection_returns_message() -> None: async def fake_get_json(client, path: str): + if path == "/access/permissions": + return {"/": {"VM.Audit": 1}} # token has an ACL return {"version": "8.2.2"} with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): ok, msg = await svc.test_proxmox_connection("h", 8006, "u@pam!t", "sec") assert ok is True assert "8.2.2" in msg + assert "warning" not in msg.lower() + + +@pytest.mark.asyncio +async def test_test_connection_warns_when_token_has_no_permissions() -> None: + async def fake_get_json(client, path: str): + if path == "/access/permissions": + return {} # privilege-separated token with no effective ACL + return {"version": "8.4.19"} + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): + ok, msg = await svc.test_proxmox_connection("h", 8006, "u@pam!t", "sec") + # Auth still succeeds; the message flags the permission gap. + assert ok is True + assert "8.4.19" in msg + assert "PVEAuditor" in msg + + +@pytest.mark.asyncio +async def test_token_has_permissions_treats_empty_as_no_perms() -> None: + async def fake_get_json(client, path: str): + return {} + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): + assert await svc._token_has_permissions(object()) is False + + +@pytest.mark.asyncio +async def test_token_has_permissions_assumes_ok_on_error() -> None: + async def boom(client, path: str): + raise httpx.ConnectError("nope") + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=boom)): + # Never block a valid import on a permissions-probe failure. + assert await svc._token_has_permissions(object()) is True diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b157aac..14a28e9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -47,6 +47,7 @@ import type { NodeData, EdgeData, CustomStyleDef, FloorMapConfig, NodeType } fro import type { ZigbeeNode, ZigbeeEdge } from '@/components/zigbee/types' import type { ZwaveNode, ZwaveEdge } from '@/components/zwave/types' import type { ProxmoxNode, ProxmoxEdge } from '@/components/proxmox/types' +import { buildProxmoxClusterEdges } from '@/components/proxmox/clusterEdges' const STANDALONE = import.meta.env.VITE_STANDALONE === 'true' @@ -650,10 +651,16 @@ export default function App() { const cols = Math.min(COLS, pmNodes.length) const rows = Math.ceil(pmNodes.length / COLS) const origin = getCenteredPosition(cols * SPACING_X, rows * SPACING_Y) + // Multiple hosts from one import = a cluster β†’ chain them via left/right + // 'cluster' edges. Those endpoints need one left + one right handle each + // (both default to 0), so grant them to the host nodes up front. + const clusterEdges = buildProxmoxClusterEdges(pmNodes) + const cluster = clusterEdges.length > 0 pmNodes.forEach((pn, i) => { const col = i % COLS const row = Math.floor(i / COLS) const position = { x: origin.x + col * SPACING_X, y: origin.y + row * SPACING_Y } + const isClusterHost = cluster && pn.type === 'proxmox' const newNode: import('@xyflow/react').Node = { id: pn.id, type: pn.type, @@ -665,6 +672,7 @@ export default function App() { services: [], ...(pn.ip ? { ip: pn.ip } : {}), ...(pn.hostname ? { hostname: pn.hostname } : {}), + ...(isClusterHost ? { left_handles: 1, right_handles: 1 } : {}), }, } addNode(newNode) @@ -679,6 +687,16 @@ export default function App() { type: 'virtual', } as unknown as import('@xyflow/react').Connection) }) + // Host ↔ host links render as 'cluster' edges (left β†’ right chain). + clusterEdges.forEach((ce) => { + onConnect({ + source: ce.source, + sourceHandle: ce.sourceHandle, + target: ce.target, + targetHandle: ce.targetHandle, + type: 'cluster', + } as unknown as import('@xyflow/react').Connection) + }) const importedIds = new Set(pmNodes.map((pn) => pn.id)) useCanvasStore.setState((state) => ({ nodes: state.nodes.map((n) => ({ ...n, selected: importedIds.has(n.id) })), diff --git a/frontend/src/components/canvas/nodes/ProxmoxGroupNode.tsx b/frontend/src/components/canvas/nodes/ProxmoxGroupNode.tsx index 8d30797..3725e93 100644 --- a/frontend/src/components/canvas/nodes/ProxmoxGroupNode.tsx +++ b/frontend/src/components/canvas/nodes/ProxmoxGroupNode.tsx @@ -23,9 +23,12 @@ export function ProxmoxGroupNode(props: NodeProps>) { const theme = THEMES[activeTheme] const colors = resolveNodeColors(data, activeTheme) - // Render as a regular node when container mode is disabled. Cluster links now + // Container mode is opt-in β€” a proxmox node renders as a regular card unless + // it is explicitly a container (matches the rest of the codebase, which gates + // nesting on `container_mode === true`; see App.tsx). Imported nodes leave the + // flag unset and so render like a manually-created proxmox node. Cluster links // use the configurable per-side connection points (see BaseNode / SideHandles). - if (data.container_mode === false) { + if (data.container_mode !== true) { return } diff --git a/frontend/src/components/canvas/nodes/__tests__/ProxmoxGroupNode.test.tsx b/frontend/src/components/canvas/nodes/__tests__/ProxmoxGroupNode.test.tsx index d54ecab..5fc9763 100644 --- a/frontend/src/components/canvas/nodes/__tests__/ProxmoxGroupNode.test.tsx +++ b/frontend/src/components/canvas/nodes/__tests__/ProxmoxGroupNode.test.tsx @@ -13,6 +13,9 @@ function renderNode(data: Partial = {}, selected = false) { type: 'proxmox', status: 'online', services: [], + // Default tests to the container/group path (the branch this file covers); + // individual tests override with container_mode: false / unset as needed. + container_mode: true, ...data, } const props = { @@ -51,7 +54,7 @@ describe('ProxmoxGroupNode', () => { }) it('renders the node label', () => { - const { getByText } = renderNode({ label: 'My Proxmox' }) + const { getByText } = renderNode({ label: 'My Proxmox', container_mode: true }) expect(getByText('My Proxmox')).toBeDefined() }) @@ -90,14 +93,21 @@ describe('ProxmoxGroupNode', () => { expect(dot).not.toBeNull() }) - it('container_mode === false renders as BaseNode (no resizer group border)', () => { + it('container_mode === false renders as BaseNode (no group border)', () => { const { container } = renderNode({ container_mode: false }) - // NodeResizer should not be present when not group-rendered - expect(container.querySelector('.react-flow__resize-control')).toBeNull() + // The group container uses rounded-xl border-2; BaseNode does not. + expect(container.querySelector('.rounded-xl.border-2')).toBeNull() }) - it('container_mode default renders the group border container', () => { - const { container } = renderNode({}) + it('container mode is opt-in: default (unset) renders as a regular BaseNode', () => { + // Imported proxmox nodes leave container_mode unset and must look like a + // manually-created node (BaseNode), not an empty group container. + const { container } = renderNode({ container_mode: undefined }) + expect(container.querySelector('.rounded-xl.border-2')).toBeNull() + }) + + it('container_mode === true renders the group border container', () => { + const { container } = renderNode({ container_mode: true }) // Group border div has rounded-xl border-2 classes expect(container.querySelector('.rounded-xl.border-2')).not.toBeNull() }) diff --git a/frontend/src/components/modals/ScanHistoryModal.tsx b/frontend/src/components/modals/ScanHistoryModal.tsx index dcbb348..8dabc09 100644 --- a/frontend/src/components/modals/ScanHistoryModal.tsx +++ b/frontend/src/components/modals/ScanHistoryModal.tsx @@ -1,5 +1,5 @@ import { useState, useEffect, useCallback, useRef } from 'react' -import { RefreshCw, X, Loader2, StopCircle, Clock, ScanLine, Network, RadioTower, Inbox } from 'lucide-react' +import { RefreshCw, X, Loader2, StopCircle, Clock, ScanLine, Network, RadioTower, Server, Inbox } from 'lucide-react' import { Dialog, DialogClose, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { scanApi } from '@/api/client' @@ -22,17 +22,21 @@ interface ScanHistoryModalProps { onClose: () => void } -type KindFilter = 'all' | 'ip' | 'zigbee' | 'zwave' +type KindFilter = 'all' | 'ip' | 'zigbee' | 'zwave' | 'proxmox' /** Normalise a ScanRun.kind into one of the known display kinds. */ -function runKind(kind: string | undefined): 'ip' | 'zigbee' | 'zwave' { - return kind === 'zigbee' ? 'zigbee' : kind === 'zwave' ? 'zwave' : 'ip' +function runKind(kind: string | undefined): 'ip' | 'zigbee' | 'zwave' | 'proxmox' { + return kind === 'zigbee' ? 'zigbee' + : kind === 'zwave' ? 'zwave' + : kind === 'proxmox' ? 'proxmox' + : 'ip' } const KIND_META = { ip: { label: 'IP', color: '#a855f7' }, zigbee: { label: 'Zigbee', color: '#00d4ff' }, zwave: { label: 'Z-Wave', color: '#ff6e00' }, + proxmox: { label: 'Proxmox', color: '#e57000' }, } as const type StatusFilter = 'all' | 'running' | 'done' | 'error' | 'cancelled' @@ -49,6 +53,7 @@ const KIND_FILTERS: { key: KindFilter; label: string }[] = [ { key: 'ip', label: 'IP' }, { key: 'zigbee', label: 'Zigbee' }, { key: 'zwave', label: 'Z-Wave' }, + { key: 'proxmox', label: 'Proxmox' }, ] function statusColor(s: string): string { @@ -102,9 +107,15 @@ export function ScanHistoryModal({ open, onClose }: ScanHistoryModalProps) { toast.error(`Scan failed: ${run.error ?? 'unknown error'}`) } if (prev?.status === 'running' && run.status === 'done') { - if (run.kind === 'zigbee' || run.kind === 'zwave') { - const label = run.kind === 'zwave' ? 'Z-Wave' : 'Zigbee' - toast.success(`${label} import done β€” ${run.devices_found} device${run.devices_found !== 1 ? 's' : ''}`) + if (run.kind === 'zigbee' || run.kind === 'zwave' || run.kind === 'proxmox') { + const label = run.kind === 'zwave' ? 'Z-Wave' : run.kind === 'proxmox' ? 'Proxmox' : 'Zigbee' + // A done run can still carry a non-fatal advisory (e.g. Proxmox + // imported hosts but the token couldn't see any VMs/LXC). + if (run.error) { + toast.warning(`${label} import: ${run.error}`) + } else { + toast.success(`${label} import done β€” ${run.devices_found} device${run.devices_found !== 1 ? 's' : ''}`) + } } useCanvasStore.getState().notifyScanDeviceFound() } @@ -231,7 +242,7 @@ export function ScanHistoryModal({ open, onClose }: ScanHistoryModalProps) { {filtered.map((r) => { const kind = runKind(r.kind) const meta = KIND_META[kind] - const KindIcon = kind === 'zigbee' ? Network : kind === 'zwave' ? RadioTower : ScanLine + const KindIcon = kind === 'zigbee' ? Network : kind === 'zwave' ? RadioTower : kind === 'proxmox' ? Server : ScanLine return (
@@ -286,7 +297,15 @@ export function ScanHistoryModal({ open, onClose }: ScanHistoryModalProps) { )} {r.error && ( -
+ // A 'done' run with a message is a non-fatal advisory β†’ amber, + // not the red used for a genuine failure. +
{r.error}
)} diff --git a/frontend/src/components/modals/__tests__/ScanHistoryModal.test.tsx b/frontend/src/components/modals/__tests__/ScanHistoryModal.test.tsx index 229eea7..72e1a67 100644 --- a/frontend/src/components/modals/__tests__/ScanHistoryModal.test.tsx +++ b/frontend/src/components/modals/__tests__/ScanHistoryModal.test.tsx @@ -3,7 +3,7 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react' import { ScanHistoryModal } from '../ScanHistoryModal' import { TooltipProvider } from '@/components/ui/tooltip' -vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })) +vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn(), warning: vi.fn() } })) vi.mock('@/stores/canvasStore', () => ({ useCanvasStore: { getState: () => ({ notifyScanDeviceFound: vi.fn() }) }, })) @@ -72,6 +72,17 @@ const ZWAVE_RUN = { error: null, } +const PROXMOX_RUN = { + id: 'run-6', + status: 'done', + kind: 'proxmox', + ranges: ['pve:8006'], + devices_found: 9, + started_at: new Date().toISOString(), + finished_at: new Date().toISOString(), + error: null, +} + function renderModal() { return render( @@ -84,6 +95,7 @@ describe('ScanHistoryModal', () => { beforeEach(() => { vi.mocked(toast.success).mockReset() vi.mocked(toast.error).mockReset() + vi.mocked(toast.warning).mockReset() vi.mocked(scanApi.stop).mockReset() vi.mocked(scanApi.runs).mockResolvedValue({ data: [] } as never) }) @@ -176,4 +188,31 @@ describe('ScanHistoryModal', () => { expect(screen.getByText('5 found')).toBeDefined() expect(screen.queryByText('3 found')).toBeNull() }) + + it('renders a done proxmox run with an advisory as info, not a failure', async () => { + const ADVISORY_RUN = { + ...PROXMOX_RUN, + id: 'run-7', + devices_found: 3, + error: 'Imported 3 host(s) but no VMs or LXC were visible to the API token. Grant PVEAuditor…', + } + vi.mocked(scanApi.runs).mockResolvedValue({ data: [ADVISORY_RUN] } as never) + renderModal() + // Status stays "done" (success), yet the advisory text is surfaced. + await waitFor(() => expect(screen.getByText('done')).toBeDefined()) + expect(screen.getByText(/no VMs or LXC were visible/)).toBeDefined() + }) + + it('shows a proxmox run under its own kind, not IP', async () => { + vi.mocked(scanApi.runs).mockResolvedValue({ data: [DONE_RUN, PROXMOX_RUN] } as never) + renderModal() + await waitFor(() => expect(screen.getAllByText('done').length).toBe(2)) + // A dedicated Proxmox badge is rendered on the run (would be mislabeled "IP" + // before the fix). Both the filter chip and the run badge carry the label. + expect(screen.getAllByText('Proxmox').length).toBeGreaterThanOrEqual(2) + // Filtering to Proxmox keeps only the proxmox run (9 found), drops the IP run. + fireEvent.click(screen.getByRole('button', { name: 'Proxmox' })) + expect(screen.getByText('9 found')).toBeDefined() + expect(screen.queryByText('3 found')).toBeNull() + }) }) diff --git a/frontend/src/components/proxmox/__tests__/clusterEdges.test.ts b/frontend/src/components/proxmox/__tests__/clusterEdges.test.ts new file mode 100644 index 0000000..025c1f4 --- /dev/null +++ b/frontend/src/components/proxmox/__tests__/clusterEdges.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest' +import { + buildProxmoxClusterEdges, + isProxmoxCluster, + CLUSTER_SOURCE_HANDLE, + CLUSTER_TARGET_HANDLE, +} from '../clusterEdges' +import type { ProxmoxNode } from '../types' + +function host(id: string): ProxmoxNode { + return { id, label: id, type: 'proxmox', ieee_address: id, status: 'online' } +} +function guest(id: string, type: 'vm' | 'lxc' = 'vm'): ProxmoxNode { + return { id, label: id, type, ieee_address: id, status: 'online' } +} + +describe('buildProxmoxClusterEdges', () => { + it('chains multiple hosts leftβ†’right, ignoring guests', () => { + const nodes = [host('pve-a'), guest('vm-1'), host('pve-b'), host('pve-c'), guest('ct-1', 'lxc')] + const edges = buildProxmoxClusterEdges(nodes) + expect(edges.map((e) => [e.source, e.target])).toEqual([ + ['pve-a', 'pve-b'], + ['pve-b', 'pve-c'], + ]) + // Endpoints use the left/right handles. + for (const e of edges) { + expect(e.sourceHandle).toBe(CLUSTER_SOURCE_HANDLE) + expect(e.targetHandle).toBe(CLUSTER_TARGET_HANDLE) + } + }) + + it('returns no edges for a single host', () => { + expect(buildProxmoxClusterEdges([host('pve-a'), guest('vm-1')])).toEqual([]) + }) + + it('returns no edges when there are no hosts', () => { + expect(buildProxmoxClusterEdges([guest('vm-1'), guest('ct-1', 'lxc')])).toEqual([]) + }) + + it('isProxmoxCluster is true only with 2+ hosts', () => { + expect(isProxmoxCluster([host('a')])).toBe(false) + expect(isProxmoxCluster([host('a'), host('b')])).toBe(true) + expect(isProxmoxCluster([host('a'), guest('vm-1')])).toBe(false) + }) +}) diff --git a/frontend/src/components/proxmox/clusterEdges.ts b/frontend/src/components/proxmox/clusterEdges.ts new file mode 100644 index 0000000..5a6337a --- /dev/null +++ b/frontend/src/components/proxmox/clusterEdges.ts @@ -0,0 +1,46 @@ +/** Cluster-edge wiring for a Proxmox import. + * + * Proxmox host nodes discovered in the same import belong to one cluster, so we + * chain them together with `cluster` edges. The chain uses the left/right + * connection points (right source β†’ left target) to keep them visually distinct + * from the vertical hostβ†’guest `virtual` edges (bottom β†’ top). Left/right + * handles default to 0, so the hosts must opt into one handle per side for the + * edge endpoints to exist (see `sideDefault` in handleUtils). + */ +import type { ProxmoxNode } from './types' + +/** Source/target handle IDs for a cluster link (see handleUtils.handleId). */ +export const CLUSTER_SOURCE_HANDLE = 'right' +export const CLUSTER_TARGET_HANDLE = 'left-t' + +export interface ClusterEdgeSpec { + source: string + target: string + sourceHandle: typeof CLUSTER_SOURCE_HANDLE + targetHandle: typeof CLUSTER_TARGET_HANDLE +} + +/** + * Chain all Proxmox host nodes (`type === 'proxmox'`) from one import into a + * leftβ†’right cluster line. Returns `[]` when fewer than two hosts are present + * (a single host is not a cluster). Guests (vm/lxc) are ignored. + */ +export function buildProxmoxClusterEdges(nodes: ProxmoxNode[]): ClusterEdgeSpec[] { + const hosts = nodes.filter((n) => n.type === 'proxmox') + if (hosts.length < 2) return [] + const edges: ClusterEdgeSpec[] = [] + for (let i = 0; i < hosts.length - 1; i++) { + edges.push({ + source: hosts[i].id, + target: hosts[i + 1].id, + sourceHandle: CLUSTER_SOURCE_HANDLE, + targetHandle: CLUSTER_TARGET_HANDLE, + }) + } + return edges +} + +/** True when the import contains a Proxmox cluster (β‰₯2 host nodes). */ +export function isProxmoxCluster(nodes: ProxmoxNode[]): boolean { + return nodes.filter((n) => n.type === 'proxmox').length >= 2 +} From 05ef746f227c05910fe621bb0c483d6bb30c1e15 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Mon, 6 Jul 2026 11:13:32 +0200 Subject: [PATCH 4/6] fix: cluster edges render on left/right handles from approve flow Cluster edges created via the pending -> approve path rendered on the top handle instead of left/right, because the edge and its endpoints lost their handle information on the way to the canvas. - Approve resolver (scan.py) now returns each edge's type + source/target handle. Handle IDs are the bare slot-0 side names ('right'/'left'), the canonical stored form React Flow resolves to the correct side; a '-t' target id fails to resolve and falls back to the top handle. - Frontend injectAutoEdges no longer hardcodes iot/bottom/top-t. It injects each edge with its real type + handles and bumps the referenced nodes' left/right handle counts (which default to 0) so the cluster endpoints exist. Logic extracted to a pure, tested util (applyAutoEdges). - clusterEdges direct-import path uses the bare 'left' target to match. Tests: new autoEdges unit tests; updated backend handle assertions. ha-relevant: maybe --- backend/app/api/routes/scan.py | 23 ++++-- backend/tests/test_proxmox_router.py | 4 +- backend/tests/test_scan.py | 3 +- frontend/src/api/client.ts | 4 +- .../components/modals/PendingDevicesModal.tsx | 16 +--- .../src/components/proxmox/clusterEdges.ts | 7 +- .../src/utils/__tests__/autoEdges.test.ts | 65 ++++++++++++++++ frontend/src/utils/autoEdges.ts | 78 +++++++++++++++++++ 8 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 frontend/src/utils/__tests__/autoEdges.test.ts create mode 100644 frontend/src/utils/autoEdges.ts diff --git a/backend/app/api/routes/scan.py b/backend/app/api/routes/scan.py index 219a26b..8198c3c 100644 --- a/backend/app/api/routes/scan.py +++ b/backend/app/api/routes/scan.py @@ -642,16 +642,20 @@ async def _resolve_pending_links_for_ieee( if edge_design_id is None: first = (await db.execute(select(Design).order_by(Design.created_at).limit(1))).scalar() edge_design_id = first.id if first else None - # Edge shape by link source: + # Edge shape by link source. Handle IDs are the *bare* slot-0 side names + # (the canonical stored form β€” the save path normalizes '-t' β†’ the + # bare source id, and React Flow resolves the bare id to that side). A + # '-t' target id does not resolve here and RF falls back to the top + # handle, so never emit one. # proxmox β†’ 'virtual' hostβ†’guest, vertical (bottom β†’ top) # proxmox_cluster β†’ 'cluster' host↔host, horizontal (right β†’ left) # anything else β†’ 'iot' mesh link, vertical if link.discovery_source == "proxmox": - edge_type, src_handle, tgt_handle = "virtual", "bottom", "top-t" + edge_type, src_handle, tgt_handle = "virtual", "bottom", "top" elif link.discovery_source == "proxmox_cluster": - edge_type, src_handle, tgt_handle = "cluster", "right", "left-t" + edge_type, src_handle, tgt_handle = "cluster", "right", "left" else: - edge_type, src_handle, tgt_handle = "iot", "bottom", "top-t" + edge_type, src_handle, tgt_handle = "iot", "bottom", "top" edge = Edge( source=src_id, target=tgt_id, @@ -663,7 +667,16 @@ async def _resolve_pending_links_for_ieee( db.add(edge) await db.flush() existing_pairs.add((src_id, tgt_id)) - created.append({"id": edge.id, "source": src_id, "target": tgt_id}) + # Return the edge's type + handles so the client injects it faithfully + # (a cluster edge must keep its rightβ†’left handles, not the iot default). + created.append({ + "id": edge.id, + "source": src_id, + "target": tgt_id, + "type": edge_type, + "source_handle": src_handle, + "target_handle": tgt_handle, + }) await db.delete(link) return created diff --git a/backend/tests/test_proxmox_router.py b/backend/tests/test_proxmox_router.py index 7d6c135..a7a9bb7 100644 --- a/backend/tests/test_proxmox_router.py +++ b/backend/tests/test_proxmox_router.py @@ -265,7 +265,9 @@ async def test_cluster_link_resolves_to_cluster_edge(db_session) -> None: edge = (await db_session.execute(select(Edge))).scalars().one() assert edge.type == "cluster" assert edge.source_handle == "right" - assert edge.target_handle == "left-t" + # Bare side name (canonical) so React Flow resolves it to the left side; + # a '-t' target would fall back to the top handle. + assert edge.target_handle == "left" @pytest.mark.asyncio diff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.py index 8087e20..8e7834d 100644 --- a/backend/tests/test_scan.py +++ b/backend/tests/test_scan.py @@ -1281,7 +1281,8 @@ async def test_approve_zigbee_creates_edge_when_other_endpoint_is_node( assert edges[0].source == coord.id assert edges[0].target == data["node_id"] assert edges[0].source_handle == "bottom" - assert edges[0].target_handle == "top-t" + # Bare side name (canonical stored form); renders at the top like before. + assert edges[0].target_handle == "top" assert edges[0].type == "iot" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 3c0aecb..9d64483 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -88,7 +88,7 @@ export const scanApi = { approved: boolean node_id: string edges_created: number - edges: { id: string; source: string; target: string }[] + edges: { id: string; source: string; target: string; type?: string; source_handle?: string | null; target_handle?: string | null }[] }>(`/scan/pending/${id}/approve`, nodeData), hide: (id: string) => api.post(`/scan/pending/${id}/hide`), ignore: (id: string) => api.post(`/scan/pending/${id}/ignore`), @@ -98,7 +98,7 @@ export const scanApi = { node_ids: string[] device_ids: string[] edges_created: number - edges: { id: string; source: string; target: string }[] + edges: { id: string; source: string; target: string; type?: string; source_handle?: string | null; target_handle?: string | null }[] skipped: number }>('/scan/pending/bulk-approve', { device_ids: ids, design_id: designId ?? undefined }), bulkHide: (ids: string[]) => api.post<{ hidden: number; skipped: number }>('/scan/pending/bulk-hide', { device_ids: ids }), diff --git a/frontend/src/components/modals/PendingDevicesModal.tsx b/frontend/src/components/modals/PendingDevicesModal.tsx index 8868074..338384e 100644 --- a/frontend/src/components/modals/PendingDevicesModal.tsx +++ b/frontend/src/components/modals/PendingDevicesModal.tsx @@ -12,6 +12,7 @@ import { resolveNodeColors } from '@/utils/nodeColors' import { toast } from 'sonner' import { PendingDeviceModal, type PendingDevice } from '@/components/modals/PendingDeviceModal' import type { NodeType, ServiceInfo } from '@/types' +import { applyAutoEdges, type AutoEdge } from '@/utils/autoEdges' import { buildZigbeeProperties, isZigbeeType } from '@/utils/zigbeeProperties' import { buildZwaveProperties, isZwaveType } from '@/utils/zwaveProperties' import { buildMacProperty } from '@/utils/macProperty' @@ -100,21 +101,10 @@ function deviceLabel(d: PendingDevice): string { return d.friendly_name ?? d.hostname ?? specialServiceName(d) ?? d.ip ?? d.ieee_address ?? 'device' } -function injectAutoEdges(edges: { id: string; source: string; target: string }[] | undefined) { +function injectAutoEdges(edges: AutoEdge[] | undefined) { if (!edges || edges.length === 0) return useCanvasStore.setState((state) => ({ - edges: [ - ...state.edges, - ...edges.map((e) => ({ - id: e.id, - source: e.source, - target: e.target, - sourceHandle: 'bottom', - targetHandle: 'top-t', - type: 'iot', - data: { type: 'iot' as const }, - })), - ], + ...applyAutoEdges(state.nodes, state.edges, edges), hasUnsavedChanges: true, })) } diff --git a/frontend/src/components/proxmox/clusterEdges.ts b/frontend/src/components/proxmox/clusterEdges.ts index 5a6337a..98dbbc4 100644 --- a/frontend/src/components/proxmox/clusterEdges.ts +++ b/frontend/src/components/proxmox/clusterEdges.ts @@ -9,9 +9,12 @@ */ import type { ProxmoxNode } from './types' -/** Source/target handle IDs for a cluster link (see handleUtils.handleId). */ +/** Source/target handle IDs for a cluster link (see handleUtils.handleId). + * Both are the bare slot-0 side names β€” the canonical stored form. React Flow + * resolves a bare id to that side; a '-t' target id fails to resolve and falls + * back to the top handle (which is why the target must be 'left', not 'left-t'). */ export const CLUSTER_SOURCE_HANDLE = 'right' -export const CLUSTER_TARGET_HANDLE = 'left-t' +export const CLUSTER_TARGET_HANDLE = 'left' export interface ClusterEdgeSpec { source: string diff --git a/frontend/src/utils/__tests__/autoEdges.test.ts b/frontend/src/utils/__tests__/autoEdges.test.ts new file mode 100644 index 0000000..4f4c8f6 --- /dev/null +++ b/frontend/src/utils/__tests__/autoEdges.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest' +import { applyAutoEdges, handleSide, type AutoEdge } from '../autoEdges' +import type { Edge, Node } from '@xyflow/react' +import type { EdgeData, NodeData } from '@/types' + +function node(id: string, data: Partial = {}): Node { + return { + id, + position: { x: 0, y: 0 }, + data: { label: id, type: 'proxmox', status: 'online', services: [], ...data }, + } +} + +describe('handleSide', () => { + it('maps handle ids (with -t / slot suffixes) to their side', () => { + expect(handleSide('left-t')).toBe('left') + expect(handleSide('right')).toBe('right') + expect(handleSide('bottom-2')).toBe('bottom') + expect(handleSide('top')).toBe('top') + expect(handleSide(null)).toBeNull() + }) +}) + +describe('applyAutoEdges', () => { + it('injects a cluster edge with its handles and grants left/right handles', () => { + const nodes = [node('a'), node('b')] + const auto: AutoEdge[] = [ + { id: 'e1', source: 'a', target: 'b', type: 'cluster', source_handle: 'right', target_handle: 'left' }, + ] + const res = applyAutoEdges(nodes, [], auto) + + // Edge keeps its cluster type + rightβ†’left handles (not the iot default). + expect(res.edges).toHaveLength(1) + expect(res.edges[0]).toMatchObject({ type: 'cluster', sourceHandle: 'right', targetHandle: 'left' }) + // Source node gained a right handle; target node gained a left handle. + expect(res.nodes.find((n) => n.id === 'a')!.data.right_handles).toBe(1) + expect(res.nodes.find((n) => n.id === 'b')!.data.left_handles).toBe(1) + }) + + it('does not lower an existing higher handle count', () => { + const nodes = [node('a', { right_handles: 3 }), node('b')] + const auto: AutoEdge[] = [ + { id: 'e1', source: 'a', target: 'b', type: 'cluster', source_handle: 'right', target_handle: 'left' }, + ] + const res = applyAutoEdges(nodes, [], auto) + expect(res.nodes.find((n) => n.id === 'a')!.data.right_handles).toBe(3) + }) + + it('defaults to an iot bottomβ†’top edge and bumps no handles', () => { + const nodes = [node('a'), node('b')] + const auto: AutoEdge[] = [{ id: 'e1', source: 'a', target: 'b' }] + const res = applyAutoEdges(nodes, [], auto) + expect(res.edges[0]).toMatchObject({ type: 'iot', sourceHandle: 'bottom', targetHandle: 'top' }) + // top/bottom always exist β€” nodes are returned untouched. + expect(res.nodes).toBe(nodes) + }) + + it('appends to existing edges rather than replacing them', () => { + const existing = [{ id: 'x', source: 'a', target: 'b' }] as Edge[] + const res = applyAutoEdges([node('a'), node('b')], existing, [ + { id: 'e1', source: 'a', target: 'b', type: 'iot' }, + ]) + expect(res.edges.map((e) => e.id)).toEqual(['x', 'e1']) + }) +}) diff --git a/frontend/src/utils/autoEdges.ts b/frontend/src/utils/autoEdges.ts new file mode 100644 index 0000000..bd9c941 --- /dev/null +++ b/frontend/src/utils/autoEdges.ts @@ -0,0 +1,78 @@ +/** Apply server-created "auto" edges (from scan/import approve) to the canvas. + * + * The approve endpoints create edges server-side and return them with their + * type + handle IDs. We must inject them faithfully: a Proxmox cluster edge + * keeps its rightβ†’left handles, mesh links stay iot bottomβ†’top. Left/right + * handles default to 0, so a node referenced on its left/right side must be + * granted that side's connection point or the edge endpoint won't exist and + * React Flow falls back to the top handle. + */ +import type { Edge, Node } from '@xyflow/react' +import type { EdgeData, EdgeType, NodeData } from '@/types' +import { normalizeHandle, handleCountField, type Side } from '@/utils/handleUtils' + +export interface AutoEdge { + id: string + source: string + target: string + type?: string + source_handle?: string | null + target_handle?: string | null +} + +/** Side a handle id sits on ('left-t' β†’ 'left', 'bottom-2' β†’ 'bottom'). */ +export function handleSide(h: string | null | undefined): Side | null { + const bare = normalizeHandle(h) + const m = bare?.match(/^(top|bottom|left|right)/) + return m ? (m[1] as Side) : null +} + +/** + * Pure transform: given current nodes + edges and the server auto-edges, + * return the next nodes (with left/right handle counts bumped where an edge + * needs them) and edges (with the injected edges appended). + */ +export function applyAutoEdges( + nodes: Node[], + edges: Edge[], + autoEdges: AutoEdge[], +): { nodes: Node[]; edges: Edge[] } { + // node id β†’ left/right sides that need at least one handle. + const bumps = new Map>() + const mark = (id: string, side: Side | null) => { + if (!side || side === 'top' || side === 'bottom') return // always exist + const set = bumps.get(id) ?? new Set() + set.add(side) + bumps.set(id, set) + } + + const injected: Edge[] = autoEdges.map((e) => { + const type = (e.type ?? 'iot') as EdgeType + const sourceHandle = e.source_handle ?? 'bottom' + const targetHandle = e.target_handle ?? 'top' + mark(e.source, handleSide(sourceHandle)) + mark(e.target, handleSide(targetHandle)) + return { + id: e.id, + source: e.source, + target: e.target, + sourceHandle, + targetHandle, + type, + data: { type } as EdgeData, + } + }) + + const nextNodes = bumps.size === 0 ? nodes : nodes.map((n) => { + const sides = bumps.get(n.id) + if (!sides) return n + const data = { ...n.data } + for (const side of sides) { + const field = handleCountField(side) + data[field] = Math.max((data[field] as number | undefined) ?? 0, 1) + } + return { ...n, data } + }) + + return { nodes: nextNodes, edges: [...edges, ...injected] } +} From adf82f8f01a08d549027ae099d030de74d336da8 Mon Sep 17 00:00:00 2001 From: Pouzor Date: Mon, 6 Jul 2026 20:11:03 +0200 Subject: [PATCH 5/6] feat: merge IP-scanned and Proxmox-imported devices by MAC Reconcile the same physical device discovered by both the nmap IP scan and the Proxmox importer into a single inventory row, keyed on MAC. Previously each path only deduped by IP, and the importer captured no MAC (and no IP for stopped guests), so most guests double-listed. Backend: - mac_utils.normalize_mac: canonical MAC (lowercase, ':'-separated), the cross-source join key. Normalized on write and on compare. - proxmox_service: capture the guest NIC MAC agent-free from the net0 config (qemu virtio=, lxc hwaddr=); works for stopped guests. Resolver now returns (ip, mac). - proxmox persist: match existing Node/PendingDevice by ieee OR ip OR MAC; fill mac, keep the vm/lxc type, union sources. - scanner persist: match PendingDevice by ip OR MAC; fill the IP a Proxmox import lacked, keep a pve row's type, union the scan source. Stamp query matches raw + normalized MAC (legacy-safe). - Multi-source tags: new PendingDevice.discovery_sources JSON column so a merged device shows under both the IP and Proxmox filters. Idempotent migration backfills from discovery_source (legacy NULL-scalar rows with an IP become ["arp"]). _sources_after_merge preserves a scanned row's IP origin through the merge without tagging a pure Proxmox guest. - Import now broadcasts a scan update on completion so an open inventory reloads without a manual refresh. Frontend: - pendingSources: sourceBuckets/orderedSources map discovery_sources to filter buckets; a device with ["arp","proxmox"] matches both filters and renders both badges. PendingDevicesModal filter + badges use them. Tests: MAC normalization, config MAC capture, cross-source merge both directions, legacy-row IP-tag preservation, no-false-IP-tag guard, refresh broadcast, and the frontend bucket mapping. ha-relevant: maybe --- backend/app/api/routes/proxmox.py | 71 +++++++-- backend/app/db/database.py | 31 ++++ backend/app/db/models.py | 6 + backend/app/schemas/scan.py | 9 +- backend/app/services/discovery_sources.py | 19 +++ backend/app/services/mac_utils.py | 18 +++ backend/app/services/proxmox_service.py | 70 ++++++--- backend/app/services/scanner.py | 42 ++++-- backend/tests/test_mac_utils.py | 27 ++++ backend/tests/test_proxmox_router.py | 135 +++++++++++++++++- backend/tests/test_proxmox_service.py | 41 ++++++ backend/tests/test_scanner.py | 38 +++++ .../components/modals/PendingDeviceModal.tsx | 4 + .../components/modals/PendingDevicesModal.tsx | 41 ++---- .../utils/__tests__/pendingSources.test.ts | 49 +++++++ frontend/src/utils/pendingSources.ts | 47 ++++++ 16 files changed, 572 insertions(+), 76 deletions(-) create mode 100644 backend/app/services/discovery_sources.py create mode 100644 backend/app/services/mac_utils.py create mode 100644 backend/tests/test_mac_utils.py create mode 100644 frontend/src/utils/__tests__/pendingSources.test.ts create mode 100644 frontend/src/utils/pendingSources.ts diff --git a/backend/app/api/routes/proxmox.py b/backend/app/api/routes/proxmox.py index 2986777..e261708 100644 --- a/backend/app/api/routes/proxmox.py +++ b/backend/app/api/routes/proxmox.py @@ -32,6 +32,8 @@ from app.schemas.proxmox import ( ProxmoxTestConnectionResponse, ) from app.schemas.scan import ScanRunResponse +from app.services.discovery_sources import add_source +from app.services.mac_utils import normalize_mac from app.services.node_dedupe import dedupe_nodes_by_ieee from app.services.proxmox_service import ( build_proxmox_cluster_links, @@ -164,6 +166,10 @@ async def _background_proxmox_import( run.finished_at = datetime.now(timezone.utc) run.error = _guest_visibility_advisory(nodes_raw) await db.commit() + # Nudge the frontend to reload the inventory (same signal the IP scan + # emits) so imported/merged devices appear without a manual refresh. + from app.api.routes.status import broadcast_scan_update + await broadcast_scan_update(run_id=run_id, devices_found=result.device_count) except Exception as exc: logger.exception("Proxmox import %s failed", run_id) await db.rollback() @@ -223,14 +229,19 @@ async def _persist_pending_import( if not ieee: continue ip = n.get("ip") + mac = normalize_mac(n.get("mac")) props = build_proxmox_properties(n) - # 1) Already on a canvas? Match by ieee OR (ip when known). Refresh in - # place: merge properties, adopt the pve identity onto a scanned node, - # backfill blank specs/hostname. Do NOT stomp user-set type/status. + # 1) Already on a canvas? Match by ieee OR ip OR mac (the cross-source + # dedup key β€” a stopped VM has no IP but its configured NIC MAC still + # matches an ARP-scanned node). Refresh in place: merge properties, adopt + # the pve identity onto a scanned node, backfill blank specs/hostname/mac. + # Do NOT stomp user-set type/status. node_filter = [Node.ieee_address == ieee] if ip: node_filter.append(Node.ip == ip) + if mac: + node_filter.append(Node.mac == mac) existing_nodes = ( await db.execute(select(Node).where(or_(*node_filter)).order_by(Node.id)) ).scalars().all() @@ -242,6 +253,8 @@ async def _persist_pending_import( en.ieee_address = ieee if ip and not en.ip: en.ip = ip + if mac and not en.mac: + en.mac = mac en.hostname = en.hostname or n.get("hostname") en.cpu_count = en.cpu_count or n.get("cpu_count") en.ram_gb = en.ram_gb or n.get("ram_gb") @@ -251,17 +264,17 @@ async def _persist_pending_import( if ieee in cluster_members: en.left_handles = max(en.left_handles or 0, 1) en.right_handles = max(en.right_handles or 0, 1) - await _ensure_inventory_row(db, ieee, ip, n, props, approved=True) + await _ensure_inventory_row(db, ieee, ip, mac, n, props, approved=True) pending_updated += 1 continue # 2) Not on canvas β€” upsert the pending inventory row. - pending = await _find_pending(db, ieee, ip) + pending = await _find_pending(db, ieee, ip, mac) if pending is None: - db.add(_new_pending(ieee, ip, n, props, status="pending")) + db.add(_new_pending(ieee, ip, mac, n, props, status="pending")) pending_created += 1 else: - _refresh_pending(pending, ieee, ip, n, props) + _refresh_pending(pending, ieee, ip, mac, n, props) pending_updated += 1 links_recorded = await _replace_links(db, edges_raw, cluster_pairs) @@ -276,22 +289,30 @@ async def _persist_pending_import( async def _find_pending( - db: AsyncSession, ieee: str, ip: str | None + db: AsyncSession, ieee: str, ip: str | None, mac: str | None ) -> PendingDevice | None: filters = [PendingDevice.ieee_address == ieee] if ip: filters.append(PendingDevice.ip == ip) + if mac: + filters.append(PendingDevice.mac == mac) return ( await db.execute(select(PendingDevice).where(or_(*filters))) ).scalars().first() def _new_pending( - ieee: str, ip: str | None, n: dict[str, Any], props: list[dict[str, Any]], status: str + ieee: str, + ip: str | None, + mac: str | None, + n: dict[str, Any], + props: list[dict[str, Any]], + status: str, ) -> PendingDevice: return PendingDevice( ieee_address=ieee, ip=ip, + mac=mac, hostname=n.get("hostname"), friendly_name=n.get("label"), suggested_type=n.get("type"), @@ -299,19 +320,41 @@ def _new_pending( model=n.get("model"), properties=props, status=status, - discovery_source="proxmox", + discovery_source=_PROXMOX_GUEST_SOURCE, + discovery_sources=[_PROXMOX_GUEST_SOURCE], ) +def _sources_after_merge(row: PendingDevice) -> list[str]: + """Discovery sources for an inventory row after a Proxmox import merges in. + + Must run BEFORE the ``pve-`` ieee is adopted onto the row, so it can tell + whether the row was originally a scanned device. Preserves the prior scan + origin β€” including legacy rows created before ``discovery_sources`` existed + (empty list) and possibly with a NULL ``discovery_source`` β€” so the IP tag + survives the merge. A row that carries an IP but was not itself a Proxmox + device (no ``pve-`` ieee) was found by a scan; keep an IP-scan source. + """ + sources = add_source(row.discovery_sources, row.discovery_source) + was_scanned = not (row.ieee_address or "").startswith("pve-") + if was_scanned and row.ip and not any(s in ("arp", "mdns") for s in sources): + sources = add_source(sources, "arp") + return add_source(sources, _PROXMOX_GUEST_SOURCE) + + def _refresh_pending( pending: PendingDevice, ieee: str, ip: str | None, + mac: str | None, n: dict[str, Any], props: list[dict[str, Any]], ) -> None: + # Compute sources before adopting the pve ieee (needs the pre-merge origin). + pending.discovery_sources = _sources_after_merge(pending) pending.ieee_address = pending.ieee_address or ieee pending.ip = ip or pending.ip + pending.mac = pending.mac or mac pending.hostname = n.get("hostname") or pending.hostname pending.friendly_name = n.get("label") or pending.friendly_name pending.suggested_type = n.get("type") or pending.suggested_type @@ -328,18 +371,22 @@ async def _ensure_inventory_row( db: AsyncSession, ieee: str, ip: str | None, + mac: str | None, n: dict[str, Any], props: list[dict[str, Any]], approved: bool, ) -> None: """Ensure an inventory row exists for a device already on a canvas, so it shows in the inventory with an 'In N canvas' badge. Never changes status.""" - inv = await _find_pending(db, ieee, ip) + inv = await _find_pending(db, ieee, ip, mac) if inv is None: - db.add(_new_pending(ieee, ip, n, props, status="approved" if approved else "pending")) + db.add(_new_pending(ieee, ip, mac, n, props, status="approved" if approved else "pending")) else: + # Compute sources before adopting the pve ieee (needs the pre-merge origin). + inv.discovery_sources = _sources_after_merge(inv) inv.ieee_address = inv.ieee_address or ieee inv.ip = ip or inv.ip + inv.mac = inv.mac or mac inv.hostname = n.get("hostname") or inv.hostname inv.suggested_type = n.get("type") or inv.suggested_type inv.properties = merge_proxmox_properties(list(inv.properties or []), props) diff --git a/backend/app/db/database.py b/backend/app/db/database.py index e7177b8..79088f4 100644 --- a/backend/app/db/database.py +++ b/backend/app/db/database.py @@ -321,6 +321,37 @@ async def init_db() -> None: with suppress(OperationalError): sql = "UPDATE edges SET animated = 'none' WHERE animated = '0' OR animated = 0 OR animated IS NULL" await conn.exec_driver_sql(sql) + # Multi-source discovery tags: a device found by both an IP scan and a + # Proxmox import carries every source. Backfill from the legacy single + # discovery_source so existing rows show under their filter. + with suppress(OperationalError): + await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_sources JSON") + with suppress(OperationalError): + await conn.exec_driver_sql( + "UPDATE pending_devices SET discovery_sources = json_array(discovery_source) " + "WHERE discovery_sources IS NULL AND discovery_source IS NOT NULL" + ) + # Legacy IP-scanned rows predating discovery_source have a NULL scalar + # but a real IP β€” treat them as an ARP scan so they keep the IP tag. + with suppress(OperationalError): + await conn.exec_driver_sql( + "UPDATE pending_devices SET discovery_sources = json_array('arp') " + "WHERE discovery_sources IS NULL AND discovery_source IS NULL AND ip IS NOT NULL" + ) + with suppress(OperationalError): + await conn.exec_driver_sql( + "UPDATE pending_devices SET discovery_sources = '[]' WHERE discovery_sources IS NULL" + ) + # Canonicalize stored MACs (lowercase, ':' separators) so cross-source + # dedup can match a Proxmox NIC MAC against an ARP-scanned one by equality. + with suppress(OperationalError): + await conn.exec_driver_sql( + "UPDATE pending_devices SET mac = lower(replace(mac, '-', ':')) WHERE mac IS NOT NULL" + ) + with suppress(OperationalError): + await conn.exec_driver_sql( + "UPDATE nodes SET mac = lower(replace(mac, '-', ':')) WHERE mac IS NOT NULL" + ) async def get_db() -> AsyncGenerator[AsyncSession, None]: diff --git a/backend/app/db/models.py b/backend/app/db/models.py index ac99f89..c656202 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -119,7 +119,13 @@ class PendingDevice(Base): services: Mapped[list[Any]] = mapped_column(JSON, default=list) suggested_type: Mapped[str | None] = mapped_column(String) status: Mapped[str] = mapped_column(String, default="pending") + # Origin/primary source (first discovery): "arp"/"mdns"/"zigbee"/"zwave"/ + # "proxmox". Kept for back-compat; `discovery_sources` is the full set. discovery_source: Mapped[str | None] = mapped_column(String) + # All sources that have observed this device. A device found by both an IP + # scan and a Proxmox import carries e.g. ["arp", "proxmox"] and shows under + # both inventory filters. Source of truth for the frontend source badges. + discovery_sources: Mapped[list[Any]] = mapped_column(JSON, default=list) ieee_address: Mapped[str | None] = mapped_column(String, index=True, nullable=True, unique=True) friendly_name: Mapped[str | None] = mapped_column(String, nullable=True) device_subtype: Mapped[str | None] = mapped_column(String, nullable=True) diff --git a/backend/app/schemas/scan.py b/backend/app/schemas/scan.py index 0b0827c..8660e64 100644 --- a/backend/app/schemas/scan.py +++ b/backend/app/schemas/scan.py @@ -14,6 +14,9 @@ class PendingDeviceResponse(BaseModel): suggested_type: str | None status: str discovery_source: str | None + # All sources that have observed this device (e.g. ["arp", "proxmox"]). Drives + # the inventory source filter + badges; falls back to [discovery_source]. + discovery_sources: list[str] = [] ieee_address: str | None = None friendly_name: str | None = None device_subtype: str | None = None @@ -35,10 +38,10 @@ class PendingDeviceResponse(BaseModel): node_last_modified: datetime | None = None node_last_seen: datetime | None = None - @field_validator("properties", mode="before") + @field_validator("properties", "discovery_sources", mode="before") @classmethod - def _coerce_properties(cls, v: Any) -> list[Any]: - # Legacy rows (column added by migration) have properties = NULL. + def _coerce_list(cls, v: Any) -> list[Any]: + # Legacy rows (columns added by migration) have these = NULL. return v if isinstance(v, list) else [] model_config = {"from_attributes": True} diff --git a/backend/app/services/discovery_sources.py b/backend/app/services/discovery_sources.py new file mode 100644 index 0000000..e0e5344 --- /dev/null +++ b/backend/app/services/discovery_sources.py @@ -0,0 +1,19 @@ +"""Helpers for the multi-valued ``PendingDevice.discovery_sources`` set. + +A device discovered by more than one path (e.g. an IP scan *and* a Proxmox +import) accumulates every source that has seen it, so it surfaces under each +matching inventory filter. Order is preserved (origin first) and duplicates are +dropped. +""" + +from __future__ import annotations + +from collections.abc import Iterable + + +def add_source(sources: Iterable[str] | None, source: str | None) -> list[str]: + """Return ``sources`` with ``source`` appended if not already present.""" + out = [s for s in (sources or []) if s] + if source and source not in out: + out.append(source) + return out diff --git a/backend/app/services/mac_utils.py b/backend/app/services/mac_utils.py new file mode 100644 index 0000000..b1ed5c3 --- /dev/null +++ b/backend/app/services/mac_utils.py @@ -0,0 +1,18 @@ +"""MAC-address normalization, shared by the scan + Proxmox persist paths. + +Different discovery sources emit MACs in different casing/separators (ARP is +lowercase ``bc:24:11:..``, Proxmox config is often uppercase ``BC:24:11:..``). +Canonicalizing on write *and* on compare lets cross-source dedup match a device +by MAC with a plain ``==`` β€” the join key for merging an IP-scanned row with a +Proxmox-imported one. +""" + +from __future__ import annotations + + +def normalize_mac(mac: str | None) -> str | None: + """Canonical MAC: lowercase, ``-`` β†’ ``:``, stripped. Blank/None β†’ None.""" + if not mac: + return None + normalized = mac.strip().lower().replace("-", ":") + return normalized or None diff --git a/backend/app/services/proxmox_service.py b/backend/app/services/proxmox_service.py index e246961..9f62662 100644 --- a/backend/app/services/proxmox_service.py +++ b/backend/app/services/proxmox_service.py @@ -18,6 +18,7 @@ from typing import Any import httpx +from app.services.mac_utils import normalize_mac from app.services.zigbee_service import merge_zigbee_properties logger = logging.getLogger(__name__) @@ -32,6 +33,9 @@ _BYTES_PER_GB = 1024 ** 3 # net0 config line: "name=eth0,bridge=vmbr0,ip=192.168.1.5/24,gw=..." _LXC_IP_RE = re.compile(r"(?:^|,)ip=([0-9]{1,3}(?:\.[0-9]{1,3}){3})(?:/\d+)?") +# NIC MAC inside a net0 string β€” qemu "virtio=BC:24:11:..,bridge=.." or lxc +# "..,hwaddr=BC:24:11:..,..". A bare 6-octet MAC match works for both forms. +_NET_MAC_RE = re.compile(r"([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})") def _sanitize_proxmox_error(exc: BaseException) -> str: @@ -121,6 +125,21 @@ def _extract_lxc_ip(config_payload: dict[str, Any] | None) -> str | None: return match.group(1) if match else None +def _extract_net_mac(config_payload: dict[str, Any] | None) -> str | None: + """Parse the NIC MAC from a qemu/lxc ``net0`` config string (normalized). + + Works agent-free for both guest kinds and for stopped guests β€” the sole + identity we can reliably cross-match against an ARP-scanned device. + """ + if not config_payload: + return None + net0 = config_payload.get("net0") + if not isinstance(net0, str): + return None + match = _NET_MAC_RE.search(net0) + return normalize_mac(match.group(1)) if match else None + + def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None: """Build a homelable ``proxmox`` host node from a ``/nodes`` entry.""" name = raw.get("node") @@ -144,7 +163,9 @@ def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None: } -def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None) -> dict[str, Any] | None: +def _guest_node( + raw: dict[str, Any], host_name: str, kind: str, ip: str | None, mac: str | None = None +) -> dict[str, Any] | None: """Build a homelable vm/lxc node from a ``/qemu`` or ``/lxc`` list entry.""" vmid = raw.get("vmid") if vmid is None: @@ -159,6 +180,7 @@ def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None) "ieee_address": ieee, "hostname": name, "ip": ip, + "mac": mac, "status": "online" if raw.get("status") == "running" else "offline", "cpu_count": _int_or_none(raw.get("maxcpu") or raw.get("cpus")), "ram_gb": _gb(raw.get("maxmem")), @@ -320,34 +342,48 @@ async def _fetch_host_guests( if not isinstance(entries, list): continue for raw in entries: - ip = await _resolve_guest_ip(client, host_name, kind, raw) - node = _guest_node(raw, host_name, kind, ip) + ip, mac = await _resolve_guest_net(client, host_name, kind, raw) + node = _guest_node(raw, host_name, kind, ip, mac) if node: guests.append(node) return guests -async def _resolve_guest_ip( +async def _resolve_guest_net( client: httpx.AsyncClient, host_name: str, kind: str, raw: dict[str, Any] -) -> str | None: - """Best-effort guest IP. qemu β†’ guest agent, lxc β†’ net0 config. Never raises.""" +) -> tuple[str | None, str | None]: + """Best-effort (ip, mac) for a guest. Never raises. + + - MAC comes from the guest ``/config`` net0 line for both kinds (agent-free, + works for stopped guests) β€” the cross-source dedup key. + - IP: qemu β†’ guest agent (running only); lxc β†’ static net0 config. + Partial results are returned even if a later call fails (e.g. config MAC is + kept when the qemu agent call errors). + """ vmid = raw.get("vmid") if vmid is None: - return None + return None, None + ip: str | None = None + mac: str | None = None try: if kind == "qemu": - if raw.get("status") != "running": - return None - data = await _get_json( - client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces" - ) - return _extract_qemu_ip(data) - data = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config") - return _extract_lxc_ip(data) + config = await _get_json(client, f"/nodes/{host_name}/qemu/{vmid}/config") + mac = _extract_net_mac(config) + if raw.get("status") == "running": + data = await _get_json( + client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces" + ) + ip = _extract_qemu_ip(data) + else: + config = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config") + ip = _extract_lxc_ip(config) + mac = _extract_net_mac(config) except httpx.HTTPError: - # Guest agent not installed / container stopped / no perms β†’ no IP. Fine. - return None + # Guest agent not installed / container stopped / no perms. Keep whatever + # was resolved before the failure. + pass + return ip, mac async def test_proxmox_connection( diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.py index 2287773..b616f93 100644 --- a/backend/app/services/scanner.py +++ b/backend/app/services/scanner.py @@ -15,8 +15,10 @@ from sqlalchemy import or_, select from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import Node, PendingDevice, ScanRun +from app.services.discovery_sources import add_source from app.services.fingerprint import fingerprint_ports, suggest_node_type from app.services.http_probe import probe_open_ports +from app.services.mac_utils import normalize_mac logger = logging.getLogger(__name__) @@ -522,16 +524,21 @@ async def run_scan( ip, open_ports, verify_tls=deep_scan.verify_tls ) + norm_mac = normalize_mac(host.get("mac")) services = fingerprint_ports(open_ports) - suggested_type = suggest_node_type(open_ports, host.get("mac")) + suggested_type = suggest_node_type(open_ports, norm_mac) - # One inventory row per device (by IP). Match across pending AND - # approved so a re-scan of an already-approved device refreshes its - # row instead of spawning a fresh "pending" duplicate. Hidden rows - # are already skipped above. + # One inventory row per device. Match by IP OR MAC across pending AND + # approved so a re-scan refreshes the existing row instead of spawning + # a duplicate β€” and so a device previously imported from Proxmox (which + # may have no IP but a known NIC MAC) reconciles with this scan instead + # of doubling up. Hidden rows are already skipped above. + match_cond = [PendingDevice.ip == ip] + if norm_mac: + match_cond.append(PendingDevice.mac == norm_mac) existing_rows = (await db.execute( select(PendingDevice) - .where(PendingDevice.ip == ip, PendingDevice.status != "hidden") + .where(or_(*match_cond), PendingDevice.status != "hidden") .order_by(PendingDevice.discovered_at) )).scalars().all() @@ -543,32 +550,41 @@ async def run_scan( for dup in existing_rows: if dup is not keep: await db.delete(dup) - keep.mac = host.get("mac") or keep.mac + keep.ip = keep.ip or ip # fill an IP a Proxmox import lacked + keep.mac = norm_mac or keep.mac keep.hostname = host.get("hostname") or keep.hostname keep.os = host.get("os") or keep.os keep.services = services - keep.suggested_type = suggested_type + # Don't downgrade a Proxmox-typed guest (vm/lxc) to the generic + # scan guess; the importer knows the true type. + if not (keep.ieee_address or "").startswith("pve-"): + keep.suggested_type = suggested_type + # Merged row carries both sources (e.g. ["proxmox", "arp"]). + keep.discovery_sources = add_source(keep.discovery_sources, discovery_source) # status preserved β€” an approved device stays approved. else: db.add(PendingDevice( ip=ip, - mac=host.get("mac"), + mac=norm_mac, hostname=host.get("hostname"), os=host.get("os"), services=services, suggested_type=suggested_type, status="pending", discovery_source=discovery_source, + discovery_sources=[discovery_source], )) devices_found += 1 # Stamp last_scan on any canvas node that matches this device by IP # (or MAC, when known) so the inventory shows when the scanner last - # observed it. Matches across designs. - host_mac = host.get("mac") + # observed it. Match both the normalized and raw MAC so a legacy + # canvas node whose mac predates normalization still matches. Across + # designs. node_match = [Node.ip == ip] - if host_mac: - node_match.append(Node.mac == host_mac) + for m in {norm_mac, host.get("mac")}: + if m: + node_match.append(Node.mac == m) matching_nodes = (await db.execute( select(Node).where(or_(*node_match)) )).scalars().all() diff --git a/backend/tests/test_mac_utils.py b/backend/tests/test_mac_utils.py new file mode 100644 index 0000000..465b157 --- /dev/null +++ b/backend/tests/test_mac_utils.py @@ -0,0 +1,27 @@ +"""Unit tests for MAC normalization (the cross-source dedup key).""" + +from __future__ import annotations + +from app.services.discovery_sources import add_source +from app.services.mac_utils import normalize_mac + + +def test_normalize_mac_lowercases_and_unifies_separators() -> None: + assert normalize_mac("BC:24:11:AA:BB:CC") == "bc:24:11:aa:bb:cc" + assert normalize_mac("bc-24-11-aa-bb-cc") == "bc:24:11:aa:bb:cc" + assert normalize_mac(" BC:24:11:AA:BB:CC ") == "bc:24:11:aa:bb:cc" + + +def test_normalize_mac_blank_is_none() -> None: + assert normalize_mac(None) is None + assert normalize_mac("") is None + assert normalize_mac(" ") is None + + +def test_add_source_unions_without_duplicates() -> None: + assert add_source(None, "arp") == ["arp"] + assert add_source(["arp"], "proxmox") == ["arp", "proxmox"] + assert add_source(["arp", "proxmox"], "proxmox") == ["arp", "proxmox"] + assert add_source(["arp"], None) == ["arp"] + # Drops falsy members already present. + assert add_source(["arp", ""], "proxmox") == ["arp", "proxmox"] diff --git a/backend/tests/test_proxmox_router.py b/backend/tests/test_proxmox_router.py index a7a9bb7..b9fe684 100644 --- a/backend/tests/test_proxmox_router.py +++ b/backend/tests/test_proxmox_router.py @@ -3,13 +3,18 @@ from __future__ import annotations import uuid -from unittest.mock import AsyncMock, patch +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch import pytest from httpx import AsyncClient from sqlalchemy import select -from app.api.routes.proxmox import _guest_visibility_advisory, _persist_pending_import +from app.api.routes.proxmox import ( + _background_proxmox_import, + _guest_visibility_advisory, + _persist_pending_import, +) from app.api.routes.scan import _is_proxmox_cluster_member, _resolve_pending_links_for_ieee from app.core.config import settings from app.db.models import Design, Edge, Node, PendingDevice, PendingDeviceLink @@ -41,10 +46,11 @@ def _host_node() -> dict: } -def _guest_node(vmid: int, ip: str | None, status: str = "online") -> dict: +def _guest_node(vmid: int, ip: str | None, status: str = "online", mac: str | None = None) -> dict: return { "id": f"pve-pve1-{vmid}", "label": f"vm{vmid}", "type": "vm", "ieee_address": f"pve-pve1-{vmid}", "hostname": f"vm{vmid}", "ip": ip, + "mac": mac, "status": status, "cpu_count": 2, "ram_gb": 4.0, "disk_gb": 32.0, "vendor": "Proxmox VE", "model": "QEMU", "vmid": vmid, "parent_ieee": "pve-node-pve1", @@ -116,6 +122,27 @@ async def test_enable_sync_without_token_rejected(client: AsyncClient, headers: assert res.status_code == 400 +@pytest.mark.asyncio +async def test_background_import_broadcasts_refresh() -> None: + """After persisting, the import emits a scan update so an open inventory + reloads without a manual refresh (same signal the IP scan uses).""" + fake_db = AsyncMock() + fake_db.get = AsyncMock(return_value=None) # no ScanRun row β†’ skip status update + cm = AsyncMock() + cm.__aenter__.return_value = fake_db + cm.__aexit__.return_value = False + + with patch("app.api.routes.proxmox.AsyncSessionLocal", MagicMock(return_value=cm)), \ + patch("app.api.routes.proxmox.fetch_proxmox_inventory", new=AsyncMock(return_value=([], []))), \ + patch("app.api.routes.proxmox._persist_pending_import", + new=AsyncMock(return_value=SimpleNamespace(device_count=3))), \ + patch("app.api.routes.status.broadcast_scan_update", new=AsyncMock()) as bcast: + await _background_proxmox_import("run1", "h", 8006, "u@pam!t", "s", True) + + bcast.assert_awaited_once() + assert bcast.await_args.kwargs["devices_found"] == 3 + + # --- guest-visibility advisory --------------------------------------------- def test_advisory_when_hosts_only() -> None: @@ -172,6 +199,108 @@ async def test_persist_merges_existing_scanned_node_by_ip(db_session) -> None: assert inv.status == "approved" +@pytest.mark.asyncio +async def test_persist_merges_pending_scan_row_by_mac(db_session) -> None: + # Device previously found by an IP scan: arp source, MAC known, no ieee. + db_session.add(PendingDevice( + id=str(uuid.uuid4()), ip="10.0.0.5", mac="bc:24:11:aa:bb:cc", + suggested_type="generic", status="pending", + discovery_source="arp", discovery_sources=["arp"], + )) + await db_session.commit() + + # Proxmox import of the same box (stopped VM β†’ no IP) but same NIC MAC in a + # different casing. Must merge, not duplicate. + await _persist_pending_import(db_session, [_guest_node(101, None, mac="BC:24:11:AA:BB:CC")], []) + + rows = (await db_session.execute(select(PendingDevice))).scalars().all() + assert len(rows) == 1 + row = rows[0] + assert row.ieee_address == "pve-pve1-101" # adopted proxmox identity + assert row.suggested_type == "vm" # kept proxmox type + assert set(row.discovery_sources) == {"arp", "proxmox"} # shows in both filters + + +@pytest.mark.asyncio +async def test_persist_preserves_ip_tag_for_legacy_null_source_row(db_session) -> None: + # Legacy inventory row from an old IP scan, before discovery_source(s) were + # recorded: scalar NULL, sources empty β€” but it has an IP + MAC. + db_session.add(PendingDevice( + id=str(uuid.uuid4()), ip="192.168.1.108", mac="bc:24:11:6c:96:52", + suggested_type="lxc", status="pending", + discovery_source=None, discovery_sources=[], + )) + await db_session.commit() + + await _persist_pending_import( + db_session, [_guest_node(108, "192.168.1.108", mac="BC:24:11:6C:96:52")], [] + ) + + rows = (await db_session.execute(select(PendingDevice))).scalars().all() + assert len(rows) == 1 + # The IP tag must survive the proxmox merge even with no recorded origin. + assert set(rows[0].discovery_sources) == {"arp", "proxmox"} + + +@pytest.mark.asyncio +async def test_persist_preserves_ip_tag_on_canvas_merge_legacy_row(db_session) -> None: + # The immich case: an on-canvas node from an old scan, with a legacy + # inventory row (NULL source). Merge must keep the IP tag on the row. + node = Node( + id=str(uuid.uuid4()), type="lxc", label="immich", + ip="192.168.1.108", mac="bc:24:11:6c:96:52", + status="online", pos_x=0, pos_y=0, + ) + inv = PendingDevice( + id=str(uuid.uuid4()), ip="192.168.1.108", mac="bc:24:11:6c:96:52", + suggested_type="lxc", status="approved", + discovery_source=None, discovery_sources=[], + ) + db_session.add_all([node, inv]) + await db_session.commit() + + await _persist_pending_import( + db_session, [_guest_node(108, "192.168.1.108", mac="BC:24:11:6C:96:52")], [] + ) + + inv_row = (await db_session.execute( + select(PendingDevice).where(PendingDevice.mac == "bc:24:11:6c:96:52") + )).scalar_one() + assert set(inv_row.discovery_sources) == {"arp", "proxmox"} + + +@pytest.mark.asyncio +async def test_persist_does_not_add_ip_tag_to_pure_proxmox_guest(db_session) -> None: + # A guest first seen via Proxmox (agent IP, pve ieee) must NOT gain a spurious + # IP tag on re-sync β€” it was never IP-scanned. + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) + await _persist_pending_import(db_session, [_guest_node(101, "10.0.0.5")], []) # re-sync + row = (await db_session.execute( + select(PendingDevice).where(PendingDevice.ieee_address == "pve-pve1-101") + )).scalar_one() + assert set(row.discovery_sources) == {"proxmox"} + + +@pytest.mark.asyncio +async def test_persist_merges_canvas_node_by_mac(db_session) -> None: + # A scanned canvas node with a MAC but no IP recorded for the guest. + scanned = Node( + id=str(uuid.uuid4()), type="generic", label="box", + mac="bc:24:11:aa:bb:cc", status="online", pos_x=0, pos_y=0, + ) + db_session.add(scanned) + await db_session.commit() + + await _persist_pending_import(db_session, [_guest_node(101, None, mac="BC:24:11:AA:BB:CC")], []) + + nodes = (await db_session.execute(select(Node))).scalars().all() + assert len(nodes) == 1 # no duplicate node + merged = nodes[0] + assert merged.ieee_address == "pve-pve1-101" + assert merged.mac == "bc:24:11:aa:bb:cc" + assert merged.cpu_count == 2 # specs backfilled + + @pytest.mark.asyncio async def test_persist_resync_updates_in_place(db_session) -> None: nodes = [_guest_node(101, "10.0.0.5")] diff --git a/backend/tests/test_proxmox_service.py b/backend/tests/test_proxmox_service.py index afda931..321a6be 100644 --- a/backend/tests/test_proxmox_service.py +++ b/backend/tests/test_proxmox_service.py @@ -38,6 +38,47 @@ def test_extract_lxc_ip_parses_net0_static() -> None: assert svc._extract_lxc_ip(None) is None +def test_extract_net_mac_parses_qemu_and_lxc_forms() -> None: + # qemu: "virtio=,bridge=.." + assert svc._extract_net_mac({"net0": "virtio=BC:24:11:AA:BB:CC,bridge=vmbr0"}) == "bc:24:11:aa:bb:cc" + # lxc: "..,hwaddr=,.." + assert ( + svc._extract_net_mac({"net0": "name=eth0,bridge=vmbr0,hwaddr=BC:24:11:11:22:33,ip=dhcp"}) + == "bc:24:11:11:22:33" + ) + # No MAC / missing net0 / None β†’ None + assert svc._extract_net_mac({"net0": "name=eth0,ip=dhcp"}) is None + assert svc._extract_net_mac({}) is None + assert svc._extract_net_mac(None) is None + + +@pytest.mark.asyncio +async def test_fetch_inventory_captures_guest_mac_from_config() -> None: + """Guests carry a normalized NIC MAC read agent-free from their config.""" + async def fake_get_json(client, path: str): + if path == "/nodes": + return [{"node": "pve1", "status": "online", "maxcpu": 8}] + if path == "/nodes/pve1/qemu": + return [{"vmid": 101, "name": "web", "status": "stopped"}] # stopped: no agent IP + if path == "/nodes/pve1/lxc": + return [{"vmid": 200, "name": "db", "status": "running"}] + if path == "/nodes/pve1/qemu/101/config": + return {"net0": "virtio=AA:BB:CC:DD:EE:FF,bridge=vmbr0"} + if path == "/nodes/pve1/lxc/200/config": + return {"net0": "name=eth0,bridge=vmbr0,hwaddr=11:22:33:44:55:66,ip=10.0.0.6/24"} + return None + + with patch.object(svc, "_get_json", new=AsyncMock(side_effect=fake_get_json)): + nodes, _ = await svc.fetch_proxmox_inventory("h", 8006, "u@pam!t", "sec") + + vm = next(n for n in nodes if n["type"] == "vm") + assert vm["mac"] == "aa:bb:cc:dd:ee:ff" # captured even though VM is stopped (no IP) + assert vm["ip"] is None + ct = next(n for n in nodes if n["type"] == "lxc") + assert ct["mac"] == "11:22:33:44:55:66" + assert ct["ip"] == "10.0.0.6" + + def test_host_and_guest_node_mapping() -> None: host = svc._host_node({"node": "pve1", "status": "online", "maxcpu": 8, "maxmem": 16 * 1024 ** 3, "maxdisk": 500 * 1024 ** 3}) assert host["type"] == "proxmox" diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py index 5153282..2b296ef 100644 --- a/backend/tests/test_scanner.py +++ b/backend/tests/test_scanner.py @@ -577,6 +577,44 @@ async def test_run_scan_mdns_only_device_added(mem_db): assert device.discovery_source == "mdns" +@pytest.mark.asyncio +async def test_run_scan_merges_proxmox_row_by_mac(mem_db): + """A scan reconciles a prior Proxmox-imported row by MAC: fills the IP, + unions the source, keeps the vm type, and does not duplicate.""" + from app.services.scanner import run_scan + + run_id = _make_run_id() + async with mem_db() as session: + session.add(_make_scan_run(run_id)) + # Previously imported from Proxmox: no IP, known NIC MAC, vm type. + session.add(PendingDevice( + id="pve-row", ieee_address="pve-pve1-101", ip=None, + mac="bc:24:11:aa:bb:cc", suggested_type="vm", status="pending", + discovery_source="proxmox", discovery_sources=["proxmox"], + )) + await session.commit() + + # Scan sees the same box (same MAC, different casing) with a live IP. + nmap_hosts = [{"ip": "192.168.1.50", "hostname": "web.lan", + "mac": "BC:24:11:AA:BB:CC", "os": None, "open_ports": []}] + + async with mem_db() as session: + with patch("app.services.scanner._nmap_scan", return_value=nmap_hosts), \ + patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \ + patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock): + await run_scan(["192.168.1.0/24"], session, run_id) + + async with mem_db() as session: + rows = (await session.execute(sa_select(PendingDevice))).scalars().all() + + assert len(rows) == 1 # merged, not duplicated + row = rows[0] + assert row.ip == "192.168.1.50" # scan filled the IP + assert row.mac == "bc:24:11:aa:bb:cc" # normalized + assert row.suggested_type == "vm" # kept proxmox type + assert set(row.discovery_sources) == {"proxmox", "arp"} # both filters + + @pytest.mark.asyncio async def test_run_scan_mdns_skipped_if_already_in_nmap(mem_db): """If nmap and mDNS both find the same IP, it should not be double-counted.""" diff --git a/frontend/src/components/modals/PendingDeviceModal.tsx b/frontend/src/components/modals/PendingDeviceModal.tsx index 3d9170b..8f98975 100644 --- a/frontend/src/components/modals/PendingDeviceModal.tsx +++ b/frontend/src/components/modals/PendingDeviceModal.tsx @@ -21,6 +21,10 @@ export interface PendingDevice { suggested_type: string | null status: string discovery_source: string | null + // All sources that have observed this device (e.g. ["arp", "proxmox"]). A + // merged device shows under every matching filter. Falls back to + // [discovery_source] when absent (older rows). + discovery_sources?: string[] ieee_address?: string | null friendly_name?: string | null device_subtype?: string | null diff --git a/frontend/src/components/modals/PendingDevicesModal.tsx b/frontend/src/components/modals/PendingDevicesModal.tsx index 338384e..5aeadc9 100644 --- a/frontend/src/components/modals/PendingDevicesModal.tsx +++ b/frontend/src/components/modals/PendingDevicesModal.tsx @@ -18,6 +18,7 @@ import { buildZwaveProperties, isZwaveType } from '@/utils/zwaveProperties' import { buildMacProperty } from '@/utils/macProperty' import { formatRelative, formatTimestamp } from '@/utils/timeFormat' import { getCenteredPosition } from '@/utils/viewportCenter' +import { sourceBuckets, orderedSources, SOURCE_META, type SourceBucket } from '@/utils/pendingSources' interface PendingDevicesModalProps { open: boolean @@ -74,18 +75,9 @@ const TYPE_ICONS: Record = { generic: Circle, } -type SourceFilter = 'all' | 'ip' | 'zigbee' | 'zwave' | 'proxmox' +type SourceFilter = 'all' | SourceBucket type StatusFilter = 'pending' | 'hidden' -function inferSource(d: PendingDevice): 'zigbee' | 'zwave' | 'proxmox' | 'ip' { - if (d.discovery_source === 'zwave') return 'zwave' - if (d.discovery_source === 'zigbee') return 'zigbee' - if (d.discovery_source === 'proxmox') return 'proxmox' - // Proxmox devices carry a synthetic 'pve-' ieee but are IP hosts, not mesh. - if (d.ieee_address && !d.ieee_address.startsWith('pve-')) return 'zigbee' - return 'ip' -} - const COMMON_PORTS = new Set([22, 80, 443]) function specialServiceName(d: PendingDevice): string | undefined { @@ -162,7 +154,7 @@ export function PendingDevicesModal({ open, onClose, highlightId, initialStatus const filtered = useMemo(() => { const q = search.trim().toLowerCase() return devices.filter((d) => { - if (sourceFilter !== 'all' && inferSource(d) !== sourceFilter) return false + if (sourceFilter !== 'all' && !sourceBuckets(d).has(sourceFilter)) return false if (typeFilter !== 'all' && d.suggested_type !== typeFilter) return false // Inventory-only: optionally hide devices already placed on a canvas. if (statusFilter === 'pending' && !showOnCanvas && (d.canvas_count ?? 0) > 0) return false @@ -656,7 +648,7 @@ interface DeviceCardProps { } function DeviceCard({ device, selected, selectMode, highlighted, onClick, cardRef }: DeviceCardProps) { - const source = inferSource(device) + const sources = orderedSources(device) const roleType = (device.suggested_type ?? 'generic') as NodeType const Icon = TYPE_ICONS[roleType] ?? Circle const activeTheme = useThemeStore((s) => s.activeTheme) @@ -664,16 +656,6 @@ function DeviceCard({ device, selected, selectMode, highlighted, onClick, cardRe // (from the active theme / style section), instead of a flat grey. const roleColor = resolveNodeColors({ type: roleType, custom_colors: undefined }, activeTheme).border const label = deviceLabel(device) - const sourceColor = - source === 'zigbee' ? '#00d4ff' - : source === 'zwave' ? '#ff6e00' - : source === 'proxmox' ? '#e57000' - : '#a855f7' - const sourceLabel = - source === 'zigbee' ? 'ZIGBEE' - : source === 'zwave' ? 'Z-WAVE' - : source === 'proxmox' ? 'PROXMOX' - : (device.discovery_source ?? 'IP').toUpperCase() const services = device.services ?? [] const visibleServices = services.slice(0, 4) const moreServices = services.length - visibleServices.length @@ -737,12 +719,15 @@ function DeviceCard({ device, selected, selectMode, highlighted, onClick, cardRe
{label}
- - {sourceLabel} - + {sources.map((s) => ( + + {SOURCE_META[s].label} + + ))} {device.suggested_type && ( = {}): PendingDevice { + return { + id: 'd1', + ip: null, + mac: null, + hostname: null, + os: null, + services: [], + suggested_type: null, + status: 'pending', + discovery_source: null, + discovered_at: '2026-07-05T00:00:00Z', + ...overrides, + } +} + +describe('sourceBuckets', () => { + it('returns both IP and Proxmox for a merged device', () => { + const buckets = sourceBuckets(device({ discovery_sources: ['arp', 'proxmox'] })) + expect([...buckets].sort()).toEqual(['ip', 'proxmox']) + }) + + it('maps arp and mdns to the single ip bucket', () => { + expect([...sourceBuckets(device({ discovery_sources: ['arp'] }))]).toEqual(['ip']) + expect([...sourceBuckets(device({ discovery_sources: ['mdns'] }))]).toEqual(['ip']) + }) + + it('falls back to legacy discovery_source when discovery_sources is empty', () => { + expect([...sourceBuckets(device({ discovery_source: 'zigbee' }))]).toEqual(['zigbee']) + expect([...sourceBuckets(device({ discovery_source: 'proxmox' }))]).toEqual(['proxmox']) + }) + + it('uses the ieee heuristic when no source is recorded', () => { + // Mesh device (non-pve ieee) with no discovery_source β†’ zigbee. + expect([...sourceBuckets(device({ ieee_address: '0x00124b00' }))]).toEqual(['zigbee']) + // Nothing at all β†’ ip. + expect([...sourceBuckets(device())]).toEqual(['ip']) + }) +}) + +describe('orderedSources', () => { + it('renders IP before Proxmox regardless of input order', () => { + expect(orderedSources(device({ discovery_sources: ['proxmox', 'arp'] }))).toEqual(['ip', 'proxmox']) + }) +}) diff --git a/frontend/src/utils/pendingSources.ts b/frontend/src/utils/pendingSources.ts new file mode 100644 index 0000000..a2e4f1b --- /dev/null +++ b/frontend/src/utils/pendingSources.ts @@ -0,0 +1,47 @@ +/** Discovery-source bucketing for pending inventory devices. + * + * A device may be observed by more than one discovery path (e.g. an IP scan and + * a Proxmox import); `discovery_sources` holds every one. These helpers map that + * raw list to the UI's filter/badge buckets so a merged device shows under each + * matching filter and renders one badge per source. + */ +import type { PendingDevice } from '@/components/modals/PendingDeviceModal' + +export type SourceBucket = 'ip' | 'zigbee' | 'zwave' | 'proxmox' + +export const SOURCE_META: Record = { + zigbee: { color: '#00d4ff', label: 'ZIGBEE' }, + zwave: { color: '#ff6e00', label: 'Z-WAVE' }, + proxmox: { color: '#e57000', label: 'PROXMOX' }, + ip: { color: '#a855f7', label: 'IP' }, +} + +// Stable badge order (IP first β€” it's the primary discovery path). +const SOURCE_ORDER: SourceBucket[] = ['ip', 'proxmox', 'zigbee', 'zwave'] + +/** Every source bucket that has observed this device. A device found by both an + * IP scan and a Proxmox import returns {ip, proxmox}. */ +export function sourceBuckets(d: PendingDevice): Set { + const raw = d.discovery_sources && d.discovery_sources.length > 0 + ? d.discovery_sources + : d.discovery_source ? [d.discovery_source] : [] + const buckets = new Set() + for (const s of raw) { + if (s === 'zwave') buckets.add('zwave') + else if (s === 'zigbee') buckets.add('zigbee') + else if (s === 'proxmox') buckets.add('proxmox') + else buckets.add('ip') // arp / mdns / anything else β†’ IP scan + } + if (buckets.size === 0) { + // No source recorded β€” legacy heuristic (mesh rows carry a non-pve ieee). + if (d.ieee_address && !d.ieee_address.startsWith('pve-')) buckets.add('zigbee') + else buckets.add('ip') + } + return buckets +} + +/** Ordered bucket list for badge rendering. */ +export function orderedSources(d: PendingDevice): SourceBucket[] { + const buckets = sourceBuckets(d) + return SOURCE_ORDER.filter((b) => buckets.has(b)) +} From 3d25fcaae2ae9ba52ac9e793a7e6d4dd361650bc Mon Sep 17 00:00:00 2001 From: Pouzor Date: Tue, 7 Jul 2026 00:53:06 +0200 Subject: [PATCH 6/6] update doc --- docs/proxmox-import.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/proxmox-import.md b/docs/proxmox-import.md index 25d9146..cc1b685 100644 --- a/docs/proxmox-import.md +++ b/docs/proxmox-import.md @@ -59,8 +59,7 @@ The token is a real credential and is treated as one: that request only and is **never stored**. - For **auto-sync** (which runs with no user present), configure the token on the **server** via environment variables (below). It is read from `.env`, kept in - memory, **never written to disk by the app**, and **never returned by any API** - endpoint. Connection errors are sanitized so the token can't leak in a message. + memory. ```env # backend/.env @@ -151,15 +150,6 @@ On each run, Homelable re-imports the inventory into the pending section: - Existing devices are **updated in place** (status, specs, IP). - Nothing is ever deleted β€” a VM removed from Proxmox is left on your canvas. ---- - -## Security notes - -- Use a dedicated user + token with the read-only **`PVEAuditor`** role. -- The token is **never** persisted to disk by Homelable and **never** included in - any API response; only a boolean "token configured" flag is exposed. -- Keep `PROXMOX_VERIFY_TLS=true` in production; disable only for self-signed labs. -- Error messages are sanitized so credentials cannot leak. --- @@ -183,6 +173,4 @@ plugins are required. --- -## Screenshots -_(Screenshots will be added in a future release)_