feat: import hosts/VMs/LXC from Proxmox VE with optional auto-sync

Add a Proxmox VE importer that reads the /api2/json REST API with a read-only
API token and drops hosts (proxmox), VMs (vm) and LXC containers (lxc) onto the
canvas as typed nodes with run state and hardware specs (vCPU/RAM/disk).

- Backend: proxmox_service (httpx) + proxmox routes (test-connection, import,
  import-pending, config). Two-tier dedupe — merge onto an existing scanned node
  by IP, else synthetic pve-{host}-{vmid} identity. Update-in-place, never
  deletes. Host->guest rendered as a 'virtual' edge via the pending-link flow.
- Security: token is env-only (PROXMOX_TOKEN_*), never written to disk by the
  app, never returned by any endpoint; errors are credential-sanitized.
- Auto-sync: optional scheduled re-import into pending (APScheduler job).
- PendingDevice.properties carries specs through approve (+ migration).
- Frontend: ProxmoxImportModal, sidebar entry, pending inventory source filter,
  Settings auto-sync section, proxmoxApi client.
- Docs: docs/proxmox-import.md, README + FEATURES sections, .env.example keys.
- Tests: backend service/router/scheduler, frontend modal/client/pending.

ha-relevant: maybe
This commit is contained in:
Pouzor
2026-07-05 18:58:12 +02:00
parent 1d40d70150
commit ab36ba6f81
29 changed files with 2349 additions and 29 deletions
+379
View File
@@ -0,0 +1,379 @@
"""FastAPI router for Proxmox VE import + auto-sync config.
Fetches hosts/VMs/LXC from the Proxmox REST API and upserts them into the
pending inventory (same review→approve flow as scans and mesh imports).
Credentials: the API token comes from the request body when provided, else
falls back to the server-configured env token (``settings.proxmox_token_*``).
The token is never persisted by the app and never returned by any endpoint.
"""
import logging
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from sqlalchemy import delete as sa_delete
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_current_user
from app.core.config import settings
from app.core.scheduler import reschedule_proxmox_sync, set_proxmox_sync_enabled
from app.db.database import AsyncSessionLocal, get_db
from app.db.models import Node, PendingDevice, PendingDeviceLink, ScanRun
from app.schemas.proxmox import (
ProxmoxConfig,
ProxmoxConnectionRequest,
ProxmoxEdgeOut,
ProxmoxImportPendingResponse,
ProxmoxImportResponse,
ProxmoxNodeOut,
ProxmoxTestConnectionResponse,
)
from app.schemas.scan import ScanRunResponse
from app.services.node_dedupe import dedupe_nodes_by_ieee
from app.services.proxmox_service import (
build_proxmox_properties,
fetch_proxmox_inventory,
merge_proxmox_properties,
test_proxmox_connection,
)
logger = logging.getLogger(__name__)
router = APIRouter()
def _resolve_credentials(payload: ProxmoxConnectionRequest) -> tuple[str, str]:
"""Pick the API token: request body first, else server env config.
Raises HTTP 400 when neither carries a token.
"""
token_id = payload.token_id or settings.proxmox_token_id
token_secret = payload.token_secret or settings.proxmox_token_secret
if not token_id or not token_secret:
raise HTTPException(
status_code=400,
detail="No Proxmox API token provided and none configured on the server.",
)
return token_id, token_secret
@router.post("/test-connection", response_model=ProxmoxTestConnectionResponse)
async def test_connection_endpoint(
payload: ProxmoxConnectionRequest,
_: str = Depends(get_current_user),
) -> ProxmoxTestConnectionResponse:
"""Validate host reachability + token before importing."""
token_id, token_secret = _resolve_credentials(payload)
connected, message = await test_proxmox_connection(
host=payload.host,
port=payload.port,
token_id=token_id,
token_secret=token_secret,
verify_tls=payload.verify_tls,
)
return ProxmoxTestConnectionResponse(connected=connected, message=message)
@router.post("/import", response_model=ProxmoxImportResponse)
async def import_proxmox(
payload: ProxmoxConnectionRequest,
_: str = Depends(get_current_user),
) -> ProxmoxImportResponse:
"""Fetch the inventory and return nodes + edges ready for canvas drop."""
token_id, token_secret = _resolve_credentials(payload)
try:
nodes_raw, edges_raw = await fetch_proxmox_inventory(
host=payload.host,
port=payload.port,
token_id=token_id,
token_secret=token_secret,
verify_tls=payload.verify_tls,
)
except ConnectionError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Unexpected error during Proxmox import")
raise HTTPException(status_code=500, detail="Unexpected error during Proxmox import") from exc
nodes = [ProxmoxNodeOut(**n) for n in nodes_raw]
edges = [ProxmoxEdgeOut(**e) for e in edges_raw]
return ProxmoxImportResponse(nodes=nodes, edges=edges, device_count=len(nodes))
@router.post("/import-pending", response_model=ScanRunResponse)
async def import_proxmox_to_pending(
payload: ProxmoxConnectionRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
_: str = Depends(get_current_user),
) -> ScanRun:
"""Queue a Proxmox pending import as a background scan run (kind=proxmox)."""
token_id, token_secret = _resolve_credentials(payload)
run = ScanRun(
status="running",
kind="proxmox",
ranges=[f"{payload.host}:{payload.port}"],
)
db.add(run)
await db.commit()
await db.refresh(run)
background_tasks.add_task(
_background_proxmox_import,
run.id,
payload.host,
payload.port,
token_id,
token_secret,
payload.verify_tls,
)
return run
async def _background_proxmox_import(
run_id: str,
host: str,
port: int,
token_id: str,
token_secret: str,
verify_tls: bool,
) -> None:
async with AsyncSessionLocal() as db:
try:
nodes_raw, edges_raw = await fetch_proxmox_inventory(
host=host,
port=port,
token_id=token_id,
token_secret=token_secret,
verify_tls=verify_tls,
)
result = await _persist_pending_import(db, nodes_raw, edges_raw)
run = await db.get(ScanRun, run_id)
if run:
run.status = "done"
run.devices_found = result.device_count
run.finished_at = datetime.now(timezone.utc)
await db.commit()
except Exception as exc:
logger.exception("Proxmox import %s failed", run_id)
await db.rollback()
run = await db.get(ScanRun, run_id)
if run:
run.status = "error"
run.error = str(exc)[:500]
run.finished_at = datetime.now(timezone.utc)
await db.commit()
async def _persist_pending_import(
db: AsyncSession,
nodes_raw: list[dict[str, Any]],
edges_raw: list[dict[str, Any]],
) -> ProxmoxImportPendingResponse:
"""Upsert Proxmox nodes/edges into pending_devices + pending_device_links.
Two-tier identity (order matters):
1. Match an existing canvas Node or pending row by **IP** (merge into a
device previously found by a scan) — never duplicate.
2. Else match by synthetic ``ieee_address`` (``pve-...``).
Update-in-place only. Nothing is ever deleted; hidden rows stay hidden.
"""
await dedupe_nodes_by_ieee(db)
pending_created = 0
pending_updated = 0
for n in nodes_raw:
ieee = n.get("ieee_address")
if not ieee:
continue
ip = n.get("ip")
props = build_proxmox_properties(n)
# 1) Already on a canvas? Match by ieee OR (ip when known). Refresh in
# place: merge properties, adopt the pve identity onto a scanned node,
# backfill blank specs/hostname. Do NOT stomp user-set type/status.
node_filter = [Node.ieee_address == ieee]
if ip:
node_filter.append(Node.ip == ip)
existing_nodes = (
await db.execute(select(Node).where(or_(*node_filter)).order_by(Node.id))
).scalars().all()
if existing_nodes:
for en in existing_nodes:
en.properties = merge_proxmox_properties(en.properties, props)
if not en.ieee_address:
en.ieee_address = ieee
if ip and not en.ip:
en.ip = ip
en.hostname = en.hostname or n.get("hostname")
en.cpu_count = en.cpu_count or n.get("cpu_count")
en.ram_gb = en.ram_gb or n.get("ram_gb")
en.disk_gb = en.disk_gb or n.get("disk_gb")
await _ensure_inventory_row(db, ieee, ip, n, props, approved=True)
pending_updated += 1
continue
# 2) Not on canvas — upsert the pending inventory row.
pending = await _find_pending(db, ieee, ip)
if pending is None:
db.add(_new_pending(ieee, ip, n, props, status="pending"))
pending_created += 1
else:
_refresh_pending(pending, ieee, ip, n, props)
pending_updated += 1
links_recorded = await _replace_links(db, edges_raw)
await db.commit()
return ProxmoxImportPendingResponse(
pending_created=pending_created,
pending_updated=pending_updated,
links_recorded=links_recorded,
device_count=len(nodes_raw),
)
async def _find_pending(
db: AsyncSession, ieee: str, ip: str | None
) -> PendingDevice | None:
filters = [PendingDevice.ieee_address == ieee]
if ip:
filters.append(PendingDevice.ip == ip)
return (
await db.execute(select(PendingDevice).where(or_(*filters)))
).scalars().first()
def _new_pending(
ieee: str, ip: str | None, n: dict[str, Any], props: list[dict[str, Any]], status: str
) -> PendingDevice:
return PendingDevice(
ieee_address=ieee,
ip=ip,
hostname=n.get("hostname"),
friendly_name=n.get("label"),
suggested_type=n.get("type"),
vendor=n.get("vendor"),
model=n.get("model"),
properties=props,
status=status,
discovery_source="proxmox",
)
def _refresh_pending(
pending: PendingDevice,
ieee: str,
ip: str | None,
n: dict[str, Any],
props: list[dict[str, Any]],
) -> None:
pending.ieee_address = pending.ieee_address or ieee
pending.ip = ip or pending.ip
pending.hostname = n.get("hostname") or pending.hostname
pending.friendly_name = n.get("label") or pending.friendly_name
pending.suggested_type = n.get("type") or pending.suggested_type
pending.vendor = n.get("vendor") or pending.vendor
pending.model = n.get("model") or pending.model
pending.properties = merge_proxmox_properties(list(pending.properties or []), props)
if pending.status == "approved":
# Approved earlier but the canvas Node is gone — revive so it reappears.
pending.status = "pending"
# hidden stays hidden.
async def _ensure_inventory_row(
db: AsyncSession,
ieee: str,
ip: str | None,
n: dict[str, Any],
props: list[dict[str, Any]],
approved: bool,
) -> None:
"""Ensure an inventory row exists for a device already on a canvas, so it
shows in the inventory with an 'In N canvas' badge. Never changes status."""
inv = await _find_pending(db, ieee, ip)
if inv is None:
db.add(_new_pending(ieee, ip, n, props, status="approved" if approved else "pending"))
else:
inv.ieee_address = inv.ieee_address or ieee
inv.ip = ip or inv.ip
inv.hostname = n.get("hostname") or inv.hostname
inv.suggested_type = n.get("type") or inv.suggested_type
inv.properties = merge_proxmox_properties(list(inv.properties or []), props)
async def _replace_links(db: AsyncSession, edges_raw: list[dict[str, Any]]) -> int:
"""Wipe all proxmox-source links and re-insert the freshly discovered set."""
await db.execute(
sa_delete(PendingDeviceLink).where(PendingDeviceLink.discovery_source == "proxmox")
)
recorded = 0
seen: set[tuple[str, str]] = set()
for e in edges_raw:
src = e.get("source")
tgt = e.get("target")
if not src or not tgt or (src, tgt) in seen:
continue
seen.add((src, tgt))
db.add(
PendingDeviceLink(
source_ieee=src,
target_ieee=tgt,
discovery_source="proxmox",
)
)
recorded += 1
return recorded
@router.get("/config", response_model=ProxmoxConfig)
async def get_proxmox_config(_: str = Depends(get_current_user)) -> ProxmoxConfig:
"""Return non-secret Proxmox config. Never includes the token — only whether
one is configured on the server."""
return ProxmoxConfig(
host=settings.proxmox_host,
port=settings.proxmox_port,
verify_tls=settings.proxmox_verify_tls,
sync_enabled=settings.proxmox_sync_enabled,
sync_interval=settings.proxmox_sync_interval,
token_configured=bool(settings.proxmox_token_id and settings.proxmox_token_secret),
)
@router.post("/config", response_model=ProxmoxConfig)
async def save_proxmox_config(
payload: ProxmoxConfig,
_: str = Depends(get_current_user),
) -> ProxmoxConfig:
"""Persist non-secret Proxmox config and apply the auto-sync schedule live.
The token is NOT accepted here — it is env-only by design.
"""
if payload.sync_enabled and not (settings.proxmox_token_id and settings.proxmox_token_secret):
raise HTTPException(
status_code=400,
detail="Cannot enable auto-sync: no Proxmox API token configured on the server.",
)
try:
settings.proxmox_host = payload.host
settings.proxmox_port = payload.port
settings.proxmox_verify_tls = payload.verify_tls
settings.proxmox_sync_enabled = payload.sync_enabled
settings.proxmox_sync_interval = payload.sync_interval
settings.save_overrides()
set_proxmox_sync_enabled(payload.sync_enabled)
if payload.sync_enabled:
reschedule_proxmox_sync(payload.sync_interval)
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return await get_proxmox_config()
+9 -3
View File
@@ -367,7 +367,7 @@ async def bulk_approve_devices(
ieee_address=device.ieee_address,
properties=_wireless_properties(
node_type, device.ieee_address, device.vendor, device.model, device.lqi
) if is_wireless else build_mac_property(device.mac),
) if is_wireless else merge_mac_property(list(device.properties or []), device.mac),
# Default to ping so the status checker actually polls the new node.
# Without this the scheduler skips it (check_method NULL → no check).
check_method="none" if is_wireless else ("ping" if device.ip else None),
@@ -519,7 +519,10 @@ async def approve_device(
ieee_address=device.ieee_address,
properties=_wireless_properties(
node_data.type, device.ieee_address, device.vendor, device.model, device.lqi
) if wireless else merge_mac_property(node_data.properties, _mac),
) if wireless else merge_mac_property(
merge_zigbee_properties(list(device.properties or []), node_data.properties or []),
_mac,
),
check_method="none" if wireless else (node_data.check_method or ("ping" if node_data.ip else None)),
check_target=None if wireless else node_data.check_target,
design_id=node_design_id,
@@ -609,10 +612,13 @@ async def _resolve_pending_links_for_ieee(
if edge_design_id is None:
first = (await db.execute(select(Design).order_by(Design.created_at).limit(1))).scalar()
edge_design_id = first.id if first else None
# Proxmox host→guest links render as 'virtual' (VM/LXC ↔ host); mesh
# links stay 'iot'. Default to iot for any legacy/other source.
edge_type = "virtual" if link.discovery_source == "proxmox" else "iot"
edge = Edge(
source=src_id,
target=tgt_id,
type="iot",
type=edge_type,
source_handle="bottom",
target_handle="top-t",
design_id=edge_design_id,
+31
View File
@@ -80,6 +80,19 @@ class Settings(BaseSettings):
# Leave empty to keep the feature disabled (default).
homepage_api_key: str = ""
# Proxmox VE import.
# Token = a real credential → env/.env ONLY, never persisted by the app to
# scan_config.json and never returned by the API. token_id is
# 'user@realm!tokenname'; use a read-only PVEAuditor role.
proxmox_token_id: str = ""
proxmox_token_secret: str = ""
# Non-secret connection + auto-sync config (persisted via save_overrides).
proxmox_host: str = ""
proxmox_port: int = 8006
proxmox_verify_tls: bool = True
proxmox_sync_enabled: bool = False
proxmox_sync_interval: int = 3600 # seconds (floor 300 enforced on write)
def _override_path(self) -> Path:
return Path(self.sqlite_path).parent / "scan_config.json"
@@ -108,6 +121,17 @@ class Settings(BaseSettings):
self.scanner_http_probe_enabled = bool(data["scanner_http_probe_enabled"])
if "scanner_http_verify_tls" in data:
self.scanner_http_verify_tls = bool(data["scanner_http_verify_tls"])
# Proxmox non-secret config (token stays env-only, never here).
if "proxmox_host" in data:
self.proxmox_host = str(data["proxmox_host"])
if "proxmox_port" in data:
self.proxmox_port = int(data["proxmox_port"])
if "proxmox_verify_tls" in data:
self.proxmox_verify_tls = bool(data["proxmox_verify_tls"])
if "proxmox_sync_enabled" in data:
self.proxmox_sync_enabled = bool(data["proxmox_sync_enabled"])
if "proxmox_sync_interval" in data:
self.proxmox_sync_interval = int(data["proxmox_sync_interval"])
except Exception:
pass
@@ -122,6 +146,13 @@ class Settings(BaseSettings):
"scanner_http_ranges": self.scanner_http_ranges,
"scanner_http_probe_enabled": self.scanner_http_probe_enabled,
"scanner_http_verify_tls": self.scanner_http_verify_tls,
# Proxmox: only non-secret config. Token fields are intentionally
# excluded — they must never be written to disk by the app.
"proxmox_host": self.proxmox_host,
"proxmox_port": self.proxmox_port,
"proxmox_verify_tls": self.proxmox_verify_tls,
"proxmox_sync_enabled": self.proxmox_sync_enabled,
"proxmox_sync_interval": self.proxmox_sync_interval,
}))
+67
View File
@@ -106,6 +106,35 @@ async def _run_service_checks() -> None:
logger.error("Service checks failed for node %s: %s", node_id, exc)
async def _run_proxmox_sync() -> None:
"""Fetch the Proxmox inventory and upsert it into pending (auto-sync)."""
if not settings.proxmox_sync_enabled:
return
if not (settings.proxmox_host and settings.proxmox_token_id and settings.proxmox_token_secret):
logger.warning("Proxmox auto-sync enabled but host/token not configured — skipping")
return
# Lazy import to avoid a circular import at module load.
from app.api.routes.proxmox import _persist_pending_import
from app.services.proxmox_service import fetch_proxmox_inventory
try:
nodes_raw, edges_raw = await fetch_proxmox_inventory(
host=settings.proxmox_host,
port=settings.proxmox_port,
token_id=settings.proxmox_token_id,
token_secret=settings.proxmox_token_secret,
verify_tls=settings.proxmox_verify_tls,
)
async with AsyncSessionLocal() as db:
result = await _persist_pending_import(db, nodes_raw, edges_raw)
logger.info(
"Proxmox auto-sync: %d devices (%d new, %d updated)",
result.device_count, result.pending_created, result.pending_updated,
)
except Exception as exc:
logger.error("Proxmox auto-sync failed: %s", exc)
def _add_service_check_job() -> None:
scheduler.add_job(
_run_service_checks,
@@ -117,6 +146,17 @@ def _add_service_check_job() -> None:
)
def _add_proxmox_sync_job() -> None:
scheduler.add_job(
_run_proxmox_sync,
"interval",
seconds=settings.proxmox_sync_interval,
id="proxmox_sync",
max_instances=1,
coalesce=True,
)
def start_scheduler() -> None:
global scheduler
if scheduler.running:
@@ -135,6 +175,8 @@ def start_scheduler() -> None:
)
if settings.service_check_enabled:
_add_service_check_job()
if settings.proxmox_sync_enabled:
_add_proxmox_sync_job()
scheduler.start()
logger.info("Scheduler started — status checks every %ds", settings.status_checker_interval)
@@ -175,6 +217,31 @@ def set_service_checks_enabled(enabled: bool) -> None:
logger.info("Service checks disabled")
def reschedule_proxmox_sync(interval_seconds: int) -> None:
"""Update the Proxmox auto-sync interval on the running scheduler (if enabled)."""
if interval_seconds < 300:
raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
if not scheduler.running:
logger.warning("Scheduler not running, skipping reschedule")
return
if scheduler.get_job("proxmox_sync"):
scheduler.reschedule_job("proxmox_sync", trigger="interval", seconds=interval_seconds)
logger.info("Proxmox auto-sync rescheduled to every %ds", interval_seconds)
def set_proxmox_sync_enabled(enabled: bool) -> None:
"""Add or remove the Proxmox auto-sync job on the running scheduler."""
if not scheduler.running:
return
job = scheduler.get_job("proxmox_sync")
if enabled and not job:
_add_proxmox_sync_job()
logger.info("Proxmox auto-sync enabled — every %ds", settings.proxmox_sync_interval)
elif not enabled and job:
scheduler.remove_job("proxmox_sync")
logger.info("Proxmox auto-sync disabled")
def stop_scheduler() -> None:
if scheduler.running:
scheduler.shutdown(wait=False)
+3
View File
@@ -115,6 +115,8 @@ async def init_db() -> None:
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN right_handles INTEGER NOT NULL DEFAULT 0")
with suppress(OperationalError):
await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_source TEXT")
with suppress(OperationalError):
await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN properties JSON")
with suppress(OperationalError):
await conn.exec_driver_sql("ALTER TABLE scan_runs ADD COLUMN kind TEXT NOT NULL DEFAULT 'ip'")
# --- Zigbee schema migrations (logged variant per CLAUDE.md feedback) ---
@@ -162,6 +164,7 @@ async def init_db() -> None:
"model VARCHAR,"
"vendor VARCHAR,"
"lqi INTEGER,"
"properties JSON,"
"discovered_at DATETIME"
")"
)
+4
View File
@@ -126,6 +126,10 @@ class PendingDevice(Base):
model: Mapped[str | None] = mapped_column(String, nullable=True)
vendor: Mapped[str | None] = mapped_column(String, nullable=True)
lqi: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Display properties carried from discovery (e.g. Proxmox specs: CPU/RAM/Disk,
# VMID). Generic NodeProperty shape {key,value,icon,visible}; merged into the
# Node's properties on approve. Empty for scan/mesh sources that don't set it.
properties: Mapped[list[Any]] = mapped_column(JSON, default=list)
discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
# Transient (not persisted): populated per-request by the scan routes to report
+16 -1
View File
@@ -7,7 +7,21 @@ from typing import Any
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.api.routes import auth, canvas, designs, edges, liveview, media, nodes, scan, stats, status, zigbee, zwave
from app.api.routes import (
auth,
canvas,
designs,
edges,
liveview,
media,
nodes,
proxmox,
scan,
stats,
status,
zigbee,
zwave,
)
from app.api.routes import settings as settings_routes
from app.core.config import settings
from app.core.scheduler import start_scheduler, stop_scheduler
@@ -58,6 +72,7 @@ app.include_router(settings_routes.router, prefix="/api/v1/settings", tags=["set
app.include_router(liveview.router, prefix="/api/v1/liveview", tags=["liveview"])
app.include_router(zigbee.router, prefix="/api/v1/zigbee", tags=["zigbee"])
app.include_router(zwave.router, prefix="/api/v1/zwave", tags=["zwave"])
app.include_router(proxmox.router, prefix="/api/v1/proxmox", tags=["proxmox"])
app.include_router(stats.router, prefix="/api/v1/stats", tags=["stats"])
app.include_router(media.router, prefix="/api/v1/media", tags=["media"])
+75
View File
@@ -0,0 +1,75 @@
"""Pydantic v2 schemas for Proxmox VE import.
Token fields are accepted on requests only and are optional — when omitted the
backend falls back to the server-configured token (env). No response schema ever
carries a token; secrets are kept out of responses by structural omission.
"""
from pydantic import BaseModel, Field
class ProxmoxConnectionRequest(BaseModel):
host: str = Field(..., description="Proxmox VE host or IP")
port: int = Field(8006, ge=1, le=65535, description="Proxmox API port")
token_id: str | None = Field(
None, description="API token id 'user@realm!tokenname' (falls back to server env)"
)
token_secret: str | None = Field(
None, description="API token secret (falls back to server env)"
)
verify_tls: bool = Field(True, description="Verify the Proxmox TLS certificate")
class ProxmoxTestConnectionResponse(BaseModel):
connected: bool
message: str
class ProxmoxNodeOut(BaseModel):
"""A homelable-ready node representation of a Proxmox host / VM / LXC."""
id: str
label: str
type: str # proxmox | vm | lxc
ieee_address: str
hostname: str | None = None
ip: str | None = None
status: str
cpu_count: int | None = None
ram_gb: float | None = None
disk_gb: float | None = None
vendor: str | None = None
model: str | None = None
parent_ieee: str | None = None
class ProxmoxEdgeOut(BaseModel):
source: str
target: str
class ProxmoxImportResponse(BaseModel):
nodes: list[ProxmoxNodeOut]
edges: list[ProxmoxEdgeOut]
device_count: int
class ProxmoxImportPendingResponse(BaseModel):
"""Result of importing a Proxmox inventory into the pending section."""
pending_created: int
pending_updated: int
links_recorded: int
device_count: int
class ProxmoxConfig(BaseModel):
"""Non-secret Proxmox connection + auto-sync config. Never carries a token —
``token_configured`` reflects whether a server-side token is present."""
host: str = ""
port: int = Field(8006, ge=1, le=65535)
verify_tls: bool = True
sync_enabled: bool = False
sync_interval: int = Field(3600, ge=300)
token_configured: bool = False
+3
View File
@@ -20,6 +20,9 @@ class PendingDeviceResponse(BaseModel):
model: str | None = None
vendor: str | None = None
lqi: int | None = None
# Display properties carried from discovery (e.g. Proxmox specs). Merged into
# the node on approve; empty for scan/mesh sources that don't set them.
properties: list[Any] = []
discovered_at: datetime
# Number of distinct canvases (designs) this device already appears on,
# correlated by ip / ieee_address against existing nodes. Computed per-request.
+348
View File
@@ -0,0 +1,348 @@
"""Proxmox VE inventory service: fetch hosts + VMs + LXC via the PVE REST API.
Mirrors the Zigbee/Z-Wave import pipeline, but talks to the Proxmox VE REST API
(``/api2/json``) over HTTPS with an API token instead of MQTT. It returns plain
homelable node dicts + parent→child edge hints; DB persistence lives in the
route layer (``app.api.routes.proxmox``).
Auth uses a Proxmox **API token** (never a password):
``Authorization: PVEAPIToken=<token_id>=<secret>`` where ``token_id`` looks like
``user@realm!tokenname``. A read-only ``PVEAuditor`` role is all that is needed.
"""
from __future__ import annotations
import logging
import re
from typing import Any
import httpx
from app.services.zigbee_service import merge_zigbee_properties
logger = logging.getLogger(__name__)
# Reuse the zigbee property-merge contract verbatim (same NodeProperty shape +
# visibility-preservation rules) for re-sync updates.
merge_proxmox_properties = merge_zigbee_properties
_CONNECT_TIMEOUT = 8.0
_READ_TIMEOUT = 20.0
_BYTES_PER_GB = 1024 ** 3
# net0 config line: "name=eth0,bridge=vmbr0,ip=192.168.1.5/24,gw=..."
_LXC_IP_RE = re.compile(r"(?:^|,)ip=([0-9]{1,3}(?:\.[0-9]{1,3}){3})(?:/\d+)?")
def _sanitize_proxmox_error(exc: BaseException) -> str:
"""Return a generic, credential-free message for a Proxmox/HTTP error.
Raw httpx errors can echo the request URL and, worse, an
``Authorization: PVEAPIToken=...=<secret>`` header in some stacks. Map known
patterns to coarse categories so the token never reaches an API client. The
original exception is logged at WARNING for operators.
"""
logger.warning("Proxmox error (sanitized for client): %r", exc)
if isinstance(exc, httpx.HTTPStatusError):
code = exc.response.status_code
if code in (401, 403):
return "Authentication failed — check the API token and its permissions"
if code == 404:
return "Proxmox API path not found — is this a Proxmox VE host?"
return f"Proxmox API returned HTTP {code}"
raw = str(exc).lower()
if "name or service not known" in raw or "getaddrinfo" in raw or "nodename nor servname" in raw:
return "Proxmox host could not be resolved"
if "refused" in raw:
return "Connection refused by Proxmox host"
if "certificate" in raw or "ssl" in raw or "tls" in raw:
return "TLS verification failed — enable 'skip TLS verify' for self-signed certs"
if "timed out" in raw or "timeout" in raw:
return "Connection to Proxmox host timed out"
return "Proxmox connection failed"
def _auth_header(token_id: str, token_secret: str) -> dict[str, str]:
return {"Authorization": f"PVEAPIToken={token_id}={token_secret}"}
def _gb(value: Any) -> float | None:
"""Convert a byte count to GB (1 decimal). None/0 → None."""
try:
num = float(value)
except (TypeError, ValueError):
return None
if num <= 0:
return None
return round(num / _BYTES_PER_GB, 1)
def _int_or_none(value: Any) -> int | None:
try:
return int(value)
except (TypeError, ValueError):
return None
def _guest_type_to_homelable(kind: str) -> str:
"""qemu → vm, lxc → lxc (both existing homelable node types)."""
return "vm" if kind == "qemu" else "lxc"
def _extract_qemu_ip(agent_payload: dict[str, Any] | None) -> str | None:
"""Pull the first non-loopback IPv4 from a qemu guest-agent interfaces reply."""
if not agent_payload:
return None
result = agent_payload.get("result")
if not isinstance(result, list):
return None
for iface in result:
if not isinstance(iface, dict):
continue
for addr in iface.get("ip-addresses") or []:
if not isinstance(addr, dict):
continue
if addr.get("ip-address-type") != "ipv4":
continue
ip = addr.get("ip-address")
if isinstance(ip, str) and ip and not ip.startswith("127."):
return ip
return None
def _extract_lxc_ip(config_payload: dict[str, Any] | None) -> str | None:
"""Parse a static IPv4 from an LXC ``net0`` config string (skip dhcp)."""
if not config_payload:
return None
net0 = config_payload.get("net0")
if not isinstance(net0, str):
return None
match = _LXC_IP_RE.search(net0)
return match.group(1) if match else None
def _host_node(raw: dict[str, Any]) -> dict[str, Any] | None:
"""Build a homelable ``proxmox`` host node from a ``/nodes`` entry."""
name = raw.get("node")
if not name:
return None
ieee = f"pve-node-{name}"
return {
"id": ieee,
"label": name,
"type": "proxmox",
"ieee_address": ieee,
"hostname": name,
"ip": raw.get("ip") or None,
"status": "online" if raw.get("status") == "online" else "offline",
"cpu_count": _int_or_none(raw.get("maxcpu")),
"ram_gb": _gb(raw.get("maxmem")),
"disk_gb": _gb(raw.get("maxdisk")),
"vendor": "Proxmox VE",
"model": None,
"parent_ieee": None,
}
def _guest_node(raw: dict[str, Any], host_name: str, kind: str, ip: str | None) -> dict[str, Any] | None:
"""Build a homelable vm/lxc node from a ``/qemu`` or ``/lxc`` list entry."""
vmid = raw.get("vmid")
if vmid is None:
return None
ieee = f"pve-{host_name}-{vmid}"
node_type = _guest_type_to_homelable(kind)
name = raw.get("name") or f"{node_type}-{vmid}"
return {
"id": ieee,
"label": name,
"type": node_type,
"ieee_address": ieee,
"hostname": name,
"ip": ip,
"status": "online" if raw.get("status") == "running" else "offline",
"cpu_count": _int_or_none(raw.get("maxcpu") or raw.get("cpus")),
"ram_gb": _gb(raw.get("maxmem")),
"disk_gb": _gb(raw.get("maxdisk")),
"vendor": "Proxmox VE",
"model": kind.upper(),
"vmid": vmid,
"parent_ieee": f"pve-node-{host_name}",
}
def build_proxmox_properties(node: dict[str, Any]) -> list[dict[str, Any]]:
"""Build a NodeProperty list for a Proxmox device (specs + identity).
Icons match the existing hardware-property convention (``Cpu`` /
``MemoryStick`` / ``HardDrive``). All rows default ``visible=False`` — the
user opts in from the right panel, same as the mesh importers."""
props: list[dict[str, Any]] = []
vmid = node.get("vmid")
if vmid is not None:
props.append({"key": "VMID", "value": str(vmid), "icon": None, "visible": False})
if node.get("model"):
props.append({"key": "Kind", "value": node["model"], "icon": None, "visible": False})
if node.get("cpu_count") is not None:
props.append({"key": "CPU Cores", "value": str(node["cpu_count"]), "icon": "Cpu", "visible": False})
if node.get("ram_gb") is not None:
props.append({"key": "RAM", "value": f"{node['ram_gb']} GB", "icon": "MemoryStick", "visible": False})
if node.get("disk_gb") is not None:
props.append({"key": "Disk", "value": f"{node['disk_gb']} GB", "icon": "HardDrive", "visible": False})
props.append({"key": "Source", "value": "Proxmox VE", "icon": None, "visible": False})
return props
def _parse_inventory(
hosts_raw: list[dict[str, Any]],
guests_by_host: dict[str, list[dict[str, Any]]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""Assemble (nodes, edges) from fetched host + guest data.
``guests_by_host`` maps host name → list of already-normalized guest node
dicts. Edges are one host→guest link per guest (materialized as canvas
edges on approval, mirroring the zigbee/zwave link mechanism).
"""
nodes: list[dict[str, Any]] = []
edges: list[dict[str, Any]] = []
seen: set[str] = set()
for raw in hosts_raw:
host = _host_node(raw)
if host is None or host["id"] in seen:
continue
seen.add(host["id"])
nodes.append(host)
for host_name, guests in guests_by_host.items():
host_ieee = f"pve-node-{host_name}"
for guest in guests:
if guest["id"] in seen:
continue
seen.add(guest["id"])
nodes.append(guest)
edges.append({"source": host_ieee, "target": guest["id"]})
return nodes, edges
async def _get_json(client: httpx.AsyncClient, path: str) -> Any:
resp = await client.get(path)
resp.raise_for_status()
return resp.json().get("data")
async def fetch_proxmox_inventory(
host: str,
port: int,
token_id: str,
token_secret: str,
verify_tls: bool = True,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""Fetch hosts + VMs + LXC from a Proxmox VE host, return (nodes, edges).
Raises:
ConnectionError: transport/DNS/TLS failures (sanitized message).
ValueError: malformed API response.
"""
base_url = f"https://{host}:{port}/api2/json"
timeout = httpx.Timeout(_READ_TIMEOUT, connect=_CONNECT_TIMEOUT)
guests_by_host: dict[str, list[dict[str, Any]]] = {}
try:
async with httpx.AsyncClient(
base_url=base_url,
headers=_auth_header(token_id, token_secret),
verify=verify_tls,
timeout=timeout,
) as client:
hosts_raw = await _get_json(client, "/nodes")
if not isinstance(hosts_raw, list):
raise ValueError("Malformed /nodes response")
for host_entry in hosts_raw:
name = host_entry.get("node")
if not name or host_entry.get("status") != "online":
# Offline nodes can't be queried for guests; still shown as host.
continue
guests_by_host[name] = await _fetch_host_guests(client, name)
except httpx.HTTPStatusError as exc:
raise ConnectionError(_sanitize_proxmox_error(exc)) from exc
except httpx.HTTPError as exc:
raise ConnectionError(_sanitize_proxmox_error(exc)) from exc
return _parse_inventory(hosts_raw, guests_by_host)
async def _fetch_host_guests(
client: httpx.AsyncClient, host_name: str
) -> list[dict[str, Any]]:
"""Fetch qemu + lxc guests for one host, resolving guest IPs best-effort."""
guests: list[dict[str, Any]] = []
for kind in ("qemu", "lxc"):
try:
entries = await _get_json(client, f"/nodes/{host_name}/{kind}")
except httpx.HTTPError as exc:
logger.warning("Proxmox %s list failed for %s: %s", kind, host_name, exc)
continue
if not isinstance(entries, list):
continue
for raw in entries:
ip = await _resolve_guest_ip(client, host_name, kind, raw)
node = _guest_node(raw, host_name, kind, ip)
if node:
guests.append(node)
return guests
async def _resolve_guest_ip(
client: httpx.AsyncClient, host_name: str, kind: str, raw: dict[str, Any]
) -> str | None:
"""Best-effort guest IP. qemu → guest agent, lxc → net0 config. Never raises."""
vmid = raw.get("vmid")
if vmid is None:
return None
try:
if kind == "qemu":
if raw.get("status") != "running":
return None
data = await _get_json(
client, f"/nodes/{host_name}/qemu/{vmid}/agent/network-get-interfaces"
)
return _extract_qemu_ip(data)
data = await _get_json(client, f"/nodes/{host_name}/lxc/{vmid}/config")
return _extract_lxc_ip(data)
except httpx.HTTPError:
# Guest agent not installed / container stopped / no perms → no IP. Fine.
return None
async def test_proxmox_connection(
host: str,
port: int,
token_id: str,
token_secret: str,
verify_tls: bool = True,
) -> tuple[bool, str]:
"""Quick reachability + auth check via ``GET /version``.
Returns (connected, message). Never raises credentials outward.
"""
base_url = f"https://{host}:{port}/api2/json"
timeout = httpx.Timeout(_READ_TIMEOUT, connect=_CONNECT_TIMEOUT)
try:
async with httpx.AsyncClient(
base_url=base_url,
headers=_auth_header(token_id, token_secret),
verify=verify_tls,
timeout=timeout,
) as client:
data = await _get_json(client, "/version")
version = (data or {}).get("version", "?") if isinstance(data, dict) else "?"
return True, f"Connected to Proxmox VE {version}"
except httpx.HTTPError as exc:
return False, _sanitize_proxmox_error(exc)
except Exception as exc: # noqa: BLE001 — surface a safe message, log the rest
logger.exception("Unexpected error during Proxmox connection test")
return False, _sanitize_proxmox_error(exc)