Pouzor
ab36ba6f81
Add a Proxmox VE importer that reads the /api2/json REST API with a read-only
API token and drops hosts (proxmox), VMs (vm) and LXC containers (lxc) onto the
canvas as typed nodes with run state and hardware specs (vCPU/RAM/disk).
- Backend: proxmox_service (httpx) + proxmox routes (test-connection, import,
import-pending, config). Two-tier dedupe — merge onto an existing scanned node
by IP, else synthetic pve-{host}-{vmid} identity. Update-in-place, never
deletes. Host->guest rendered as a 'virtual' edge via the pending-link flow.
- Security: token is env-only (PROXMOX_TOKEN_*), never written to disk by the
app, never returned by any endpoint; errors are credential-sanitized.
- Auto-sync: optional scheduled re-import into pending (APScheduler job).
- PendingDevice.properties carries specs through approve (+ migration).
- Frontend: ProxmoxImportModal, sidebar entry, pending inventory source filter,
Settings auto-sync section, proxmoxApi client.
- Docs: docs/proxmox-import.md, README + FEATURES sections, .env.example keys.
- Tests: backend service/router/scheduler, frontend modal/client/pending.
ha-relevant: maybe
327 lines
17 KiB
Python
327 lines
17 KiB
Python
import json as _json
|
|
import logging
|
|
import shutil
|
|
import uuid as _uuid_mod
|
|
from collections.abc import AsyncGenerator
|
|
from contextlib import suppress
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy.exc import OperationalError
|
|
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession, async_sessionmaker, create_async_engine
|
|
from sqlalchemy.orm import DeclarativeBase
|
|
|
|
from app.core.config import APP_VERSION, settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def _try_migrate(conn: AsyncConnection, sql: str, *, label: str) -> None:
|
|
"""Run an idempotent migration statement, logging any error.
|
|
|
|
Distinguishes 'already applied' errors (debug) from genuine failures
|
|
(warning) so silent corruption is avoided. Used for new in-commit
|
|
migrations; existing legacy ALTERs above remain wrapped in suppress.
|
|
"""
|
|
try:
|
|
await conn.exec_driver_sql(sql)
|
|
except OperationalError as exc:
|
|
msg = str(exc).lower()
|
|
if "duplicate column" in msg or "already exists" in msg:
|
|
logger.debug("Migration %s skipped (already applied): %s", label, exc)
|
|
else:
|
|
logger.warning("Migration %s failed: %s", label, exc)
|
|
|
|
# Ensure the data directory exists before SQLite tries to open the file
|
|
Path(settings.sqlite_path).parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
engine = create_async_engine(
|
|
f"sqlite+aiosqlite:///{settings.sqlite_path}",
|
|
echo=False,
|
|
)
|
|
|
|
AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
def _backup_db() -> None:
|
|
db_path = Path(settings.sqlite_path)
|
|
if not db_path.exists():
|
|
return
|
|
backup_path = db_path.with_suffix(f".db.back-{APP_VERSION}")
|
|
if backup_path.exists():
|
|
return
|
|
try:
|
|
shutil.copy2(db_path, backup_path)
|
|
logger.info("DB backup created: %s", backup_path.name)
|
|
except OSError:
|
|
logger.warning("Could not create DB backup at %s", backup_path)
|
|
|
|
|
|
async def init_db() -> None:
|
|
_backup_db()
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
# Add columns introduced after initial schema (idempotent)
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN container_mode BOOLEAN NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN custom_colors JSON")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN custom_color TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN path_style TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN custom_icon TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN source_handle TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN target_handle TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN animated BOOLEAN NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN marker_start TEXT NOT NULL DEFAULT 'none'")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN marker_end TEXT NOT NULL DEFAULT 'none'")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN line_style TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN width_mult REAL")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN cpu_count INTEGER")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN cpu_model TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN ram_gb REAL")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN disk_gb REAL")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN show_hardware BOOLEAN NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN show_port_numbers BOOLEAN NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN width REAL")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN height REAL")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN bottom_handles INTEGER NOT NULL DEFAULT 1")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN top_handles INTEGER NOT NULL DEFAULT 1")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN left_handles INTEGER NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN right_handles INTEGER NOT NULL DEFAULT 0")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_source TEXT")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN properties JSON")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE scan_runs ADD COLUMN kind TEXT NOT NULL DEFAULT 'ip'")
|
|
# --- Zigbee schema migrations (logged variant per CLAUDE.md feedback) ---
|
|
zigbee_migrations: list[tuple[str, str]] = [
|
|
("nodes.ieee_address", "ALTER TABLE nodes ADD COLUMN ieee_address TEXT"),
|
|
(
|
|
"nodes.ieee_address.index",
|
|
"CREATE INDEX IF NOT EXISTS ix_nodes_ieee_address ON nodes(ieee_address)",
|
|
),
|
|
("pending_devices.ieee_address", "ALTER TABLE pending_devices ADD COLUMN ieee_address TEXT"),
|
|
(
|
|
"pending_devices.ieee_address.index",
|
|
"CREATE INDEX IF NOT EXISTS ix_pending_devices_ieee_address "
|
|
"ON pending_devices(ieee_address)",
|
|
),
|
|
("pending_devices.friendly_name", "ALTER TABLE pending_devices ADD COLUMN friendly_name TEXT"),
|
|
("pending_devices.device_subtype", "ALTER TABLE pending_devices ADD COLUMN device_subtype TEXT"),
|
|
("pending_devices.model", "ALTER TABLE pending_devices ADD COLUMN model TEXT"),
|
|
("pending_devices.vendor", "ALTER TABLE pending_devices ADD COLUMN vendor TEXT"),
|
|
("pending_devices.lqi", "ALTER TABLE pending_devices ADD COLUMN lqi INTEGER"),
|
|
]
|
|
for label, sql in zigbee_migrations:
|
|
await _try_migrate(conn, sql, label=label)
|
|
# Drop NOT NULL on pending_devices.ip (Zigbee devices have no IP).
|
|
# SQLite can't ALTER column nullability — rebuild the table if needed.
|
|
try:
|
|
info = await conn.exec_driver_sql("PRAGMA table_info(pending_devices)")
|
|
cols = info.fetchall()
|
|
ip_col = next((c for c in cols if c[1] == "ip"), None)
|
|
# PRAGMA table_info row layout: (cid, name, type, notnull, dflt, pk)
|
|
if ip_col and ip_col[3] == 1:
|
|
logger.info("Migrating pending_devices: dropping NOT NULL on ip column")
|
|
await conn.exec_driver_sql("PRAGMA foreign_keys = OFF")
|
|
await conn.exec_driver_sql(
|
|
"CREATE TABLE pending_devices_new ("
|
|
"id VARCHAR PRIMARY KEY,"
|
|
"ip VARCHAR,"
|
|
"mac VARCHAR, hostname VARCHAR, os VARCHAR, services JSON,"
|
|
"suggested_type VARCHAR,"
|
|
"status VARCHAR,"
|
|
"discovery_source VARCHAR,"
|
|
"ieee_address VARCHAR,"
|
|
"friendly_name VARCHAR,"
|
|
"device_subtype VARCHAR,"
|
|
"model VARCHAR,"
|
|
"vendor VARCHAR,"
|
|
"lqi INTEGER,"
|
|
"properties JSON,"
|
|
"discovered_at DATETIME"
|
|
")"
|
|
)
|
|
await conn.exec_driver_sql(
|
|
"INSERT INTO pending_devices_new "
|
|
"(id, ip, mac, hostname, os, services, suggested_type, status, "
|
|
"discovery_source, ieee_address, friendly_name, device_subtype, "
|
|
"model, vendor, lqi, discovered_at) "
|
|
"SELECT id, ip, mac, hostname, os, services, suggested_type, status, "
|
|
"discovery_source, ieee_address, friendly_name, device_subtype, "
|
|
"model, vendor, lqi, discovered_at FROM pending_devices"
|
|
)
|
|
await conn.exec_driver_sql("DROP TABLE pending_devices")
|
|
await conn.exec_driver_sql(
|
|
"ALTER TABLE pending_devices_new RENAME TO pending_devices"
|
|
)
|
|
await conn.exec_driver_sql(
|
|
"CREATE INDEX IF NOT EXISTS ix_pending_devices_ieee_address "
|
|
"ON pending_devices(ieee_address)"
|
|
)
|
|
await conn.exec_driver_sql("PRAGMA foreign_keys = ON")
|
|
except OperationalError as exc:
|
|
logger.warning("pending_devices ip-nullable rebuild failed: %s", exc)
|
|
# --- end Zigbee schema migrations -------------------------------------
|
|
# --- Electrical designs schema migrations -----------------------------
|
|
# Create designs table (idempotent)
|
|
await _try_migrate(
|
|
conn,
|
|
"CREATE TABLE IF NOT EXISTS designs ("
|
|
"id VARCHAR PRIMARY KEY,"
|
|
"name VARCHAR NOT NULL,"
|
|
"design_type VARCHAR NOT NULL DEFAULT 'network',"
|
|
"created_at DATETIME,"
|
|
"updated_at DATETIME"
|
|
")",
|
|
label="designs.table",
|
|
)
|
|
# Add user-chosen icon to designs (idempotent), then backfill existing rows
|
|
# so legacy designs keep a sensible icon based on their original type.
|
|
await _try_migrate(
|
|
conn, "ALTER TABLE designs ADD COLUMN icon VARCHAR", label="designs.icon",
|
|
)
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql(
|
|
"UPDATE designs SET icon = 'zap' WHERE icon IS NULL AND design_type = 'electrical'"
|
|
)
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql(
|
|
"UPDATE designs SET icon = 'dashboard' WHERE icon IS NULL"
|
|
)
|
|
# Seed default Network Topology design if designs table is empty
|
|
_default_design_id = str(_uuid_mod.uuid4())
|
|
row = await conn.exec_driver_sql("SELECT COUNT(*) FROM designs")
|
|
count_row = row.fetchone()
|
|
count = count_row[0] if count_row else 0
|
|
if count == 0:
|
|
await conn.exec_driver_sql(
|
|
"INSERT INTO designs (id, name, design_type, icon, created_at, updated_at) "
|
|
"VALUES (?, 'Network Topology', 'network', 'dashboard', datetime('now'), datetime('now'))",
|
|
(_default_design_id,),
|
|
)
|
|
else:
|
|
row2 = await conn.exec_driver_sql("SELECT id FROM designs WHERE design_type = 'network' LIMIT 1")
|
|
default = row2.fetchone()
|
|
_default_design_id = default[0] if default else _default_design_id
|
|
|
|
# Add design_id to nodes
|
|
await _try_migrate(
|
|
conn, "ALTER TABLE nodes ADD COLUMN design_id VARCHAR REFERENCES designs(id)",
|
|
label="nodes.design_id",
|
|
)
|
|
# Assign existing nodes to default design
|
|
await conn.exec_driver_sql(
|
|
"UPDATE nodes SET design_id = ? WHERE design_id IS NULL", (_default_design_id,),
|
|
)
|
|
|
|
# Add design_id to edges
|
|
await _try_migrate(
|
|
conn, "ALTER TABLE edges ADD COLUMN design_id VARCHAR REFERENCES designs(id)",
|
|
label="edges.design_id",
|
|
)
|
|
# Assign existing edges to default design
|
|
await conn.exec_driver_sql(
|
|
"UPDATE edges SET design_id = ? WHERE design_id IS NULL", (_default_design_id,),
|
|
)
|
|
|
|
# Migrate canvas_state from id=1 to design_id PK (SQLite rebuild)
|
|
try:
|
|
info = await conn.exec_driver_sql("PRAGMA table_info(canvas_state)")
|
|
cols = info.fetchall()
|
|
has_design_id = any(c[1] == "design_id" for c in cols)
|
|
if not has_design_id:
|
|
logger.info("Migrating canvas_state: switching to design_id primary key")
|
|
await conn.exec_driver_sql("PRAGMA foreign_keys = OFF")
|
|
await conn.exec_driver_sql(
|
|
"CREATE TABLE canvas_state_new ("
|
|
"design_id VARCHAR PRIMARY KEY REFERENCES designs(id) ON DELETE CASCADE,"
|
|
"viewport JSON,"
|
|
"custom_style JSON,"
|
|
"saved_at DATETIME"
|
|
")"
|
|
)
|
|
# Copy existing row(s), mapping id=1 to default design_id
|
|
old_rows = await conn.exec_driver_sql("SELECT id, viewport, custom_style, saved_at FROM canvas_state")
|
|
for old in old_rows.fetchall():
|
|
cs_id, viewport, custom_style, saved_at = old
|
|
target_design = _default_design_id
|
|
await conn.exec_driver_sql(
|
|
"INSERT INTO canvas_state_new (design_id, viewport, custom_style, saved_at) "
|
|
"VALUES (?, ?, ?, ?)",
|
|
(target_design, viewport, custom_style, saved_at),
|
|
)
|
|
await conn.exec_driver_sql("DROP TABLE canvas_state")
|
|
await conn.exec_driver_sql("ALTER TABLE canvas_state_new RENAME TO canvas_state")
|
|
await conn.exec_driver_sql("PRAGMA foreign_keys = ON")
|
|
except OperationalError as exc:
|
|
logger.warning("canvas_state migration failed: %s", exc)
|
|
# --- end Electrical designs schema migrations --------------------------
|
|
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN waypoints JSON")
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN properties JSON")
|
|
# Migrate hardware columns → properties JSON (idempotent: only runs on nodes where properties IS NULL)
|
|
with suppress(OperationalError):
|
|
rows = await conn.exec_driver_sql(
|
|
"SELECT id, cpu_model, cpu_count, ram_gb, disk_gb, show_hardware "
|
|
"FROM nodes WHERE properties IS NULL"
|
|
)
|
|
for r in rows.fetchall():
|
|
node_id, cpu_model, cpu_count, ram_gb, disk_gb, show_hardware = r
|
|
props = []
|
|
visible = bool(show_hardware)
|
|
if cpu_model:
|
|
props.append({"key": "CPU Model", "value": str(cpu_model), "icon": "Cpu", "visible": visible})
|
|
if cpu_count is not None:
|
|
props.append({"key": "CPU Cores", "value": str(cpu_count), "icon": "Cpu", "visible": visible})
|
|
if ram_gb is not None:
|
|
props.append({"key": "RAM", "value": f"{ram_gb} GB", "icon": "MemoryStick", "visible": visible})
|
|
if disk_gb is not None:
|
|
props.append({"key": "Disk", "value": f"{disk_gb} GB", "icon": "HardDrive", "visible": visible})
|
|
await conn.exec_driver_sql(
|
|
"UPDATE nodes SET properties = ? WHERE id = ?",
|
|
(_json.dumps(props), node_id),
|
|
)
|
|
# Inventory timestamp: last time a scan observed this node (idempotent)
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN last_scan DATETIME")
|
|
# Migrate animated column from boolean (0/1) to string ('none'/'snake')
|
|
with suppress(OperationalError):
|
|
await conn.exec_driver_sql("UPDATE edges SET animated = 'snake' WHERE animated = '1' OR animated = 1")
|
|
with suppress(OperationalError):
|
|
sql = "UPDATE edges SET animated = 'none' WHERE animated = '0' OR animated = 0 OR animated IS NULL"
|
|
await conn.exec_driver_sql(sql)
|
|
|
|
|
|
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
|
async with AsyncSessionLocal() as session:
|
|
yield session
|