import json as _json import logging import shutil import uuid as _uuid_mod from collections.abc import AsyncGenerator from contextlib import suppress from pathlib import Path from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import DeclarativeBase from app.core.config import APP_VERSION, settings logger = logging.getLogger(__name__) async def _try_migrate(conn: AsyncConnection, sql: str, *, label: str) -> None: """Run an idempotent migration statement, logging any error. Distinguishes 'already applied' errors (debug) from genuine failures (warning) so silent corruption is avoided. Used for new in-commit migrations; existing legacy ALTERs above remain wrapped in suppress. """ try: await conn.exec_driver_sql(sql) except OperationalError as exc: msg = str(exc).lower() if "duplicate column" in msg or "already exists" in msg: logger.debug("Migration %s skipped (already applied): %s", label, exc) else: logger.warning("Migration %s failed: %s", label, exc) # Ensure the data directory exists before SQLite tries to open the file Path(settings.sqlite_path).parent.mkdir(parents=True, exist_ok=True) engine = create_async_engine( f"sqlite+aiosqlite:///{settings.sqlite_path}", echo=False, ) AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) class Base(DeclarativeBase): pass def _backup_db() -> None: db_path = Path(settings.sqlite_path) if not db_path.exists(): return backup_path = db_path.with_suffix(f".db.back-{APP_VERSION}") if backup_path.exists(): return try: shutil.copy2(db_path, backup_path) logger.info("DB backup created: %s", backup_path.name) except OSError: logger.warning("Could not create DB backup at %s", backup_path) async def init_db() -> None: _backup_db() async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) # Add columns introduced after initial schema (idempotent) with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN container_mode BOOLEAN NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN custom_colors JSON") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN custom_color TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN path_style TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN custom_icon TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN source_handle TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN target_handle TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN animated BOOLEAN NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN marker_start TEXT NOT NULL DEFAULT 'none'") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN marker_end TEXT NOT NULL DEFAULT 'none'") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN line_style TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN width_mult REAL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN cpu_count INTEGER") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN cpu_model TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN ram_gb REAL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN disk_gb REAL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN show_hardware BOOLEAN NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN show_port_numbers BOOLEAN NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN width REAL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN height REAL") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN bottom_handles INTEGER NOT NULL DEFAULT 1") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN top_handles INTEGER NOT NULL DEFAULT 1") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN left_handles INTEGER NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN right_handles INTEGER NOT NULL DEFAULT 0") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN discovery_source TEXT") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE pending_devices ADD COLUMN properties JSON") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE scan_runs ADD COLUMN kind TEXT NOT NULL DEFAULT 'ip'") # --- Zigbee schema migrations (logged variant per CLAUDE.md feedback) --- zigbee_migrations: list[tuple[str, str]] = [ ("nodes.ieee_address", "ALTER TABLE nodes ADD COLUMN ieee_address TEXT"), ( "nodes.ieee_address.index", "CREATE INDEX IF NOT EXISTS ix_nodes_ieee_address ON nodes(ieee_address)", ), ("pending_devices.ieee_address", "ALTER TABLE pending_devices ADD COLUMN ieee_address TEXT"), ( "pending_devices.ieee_address.index", "CREATE INDEX IF NOT EXISTS ix_pending_devices_ieee_address " "ON pending_devices(ieee_address)", ), ("pending_devices.friendly_name", "ALTER TABLE pending_devices ADD COLUMN friendly_name TEXT"), ("pending_devices.device_subtype", "ALTER TABLE pending_devices ADD COLUMN device_subtype TEXT"), ("pending_devices.model", "ALTER TABLE pending_devices ADD COLUMN model TEXT"), ("pending_devices.vendor", "ALTER TABLE pending_devices ADD COLUMN vendor TEXT"), ("pending_devices.lqi", "ALTER TABLE pending_devices ADD COLUMN lqi INTEGER"), ] for label, sql in zigbee_migrations: await _try_migrate(conn, sql, label=label) # Drop NOT NULL on pending_devices.ip (Zigbee devices have no IP). # SQLite can't ALTER column nullability — rebuild the table if needed. try: info = await conn.exec_driver_sql("PRAGMA table_info(pending_devices)") cols = info.fetchall() ip_col = next((c for c in cols if c[1] == "ip"), None) # PRAGMA table_info row layout: (cid, name, type, notnull, dflt, pk) if ip_col and ip_col[3] == 1: logger.info("Migrating pending_devices: dropping NOT NULL on ip column") await conn.exec_driver_sql("PRAGMA foreign_keys = OFF") await conn.exec_driver_sql( "CREATE TABLE pending_devices_new (" "id VARCHAR PRIMARY KEY," "ip VARCHAR," "mac VARCHAR, hostname VARCHAR, os VARCHAR, services JSON," "suggested_type VARCHAR," "status VARCHAR," "discovery_source VARCHAR," "ieee_address VARCHAR," "friendly_name VARCHAR," "device_subtype VARCHAR," "model VARCHAR," "vendor VARCHAR," "lqi INTEGER," "properties JSON," "discovered_at DATETIME" ")" ) await conn.exec_driver_sql( "INSERT INTO pending_devices_new " "(id, ip, mac, hostname, os, services, suggested_type, status, " "discovery_source, ieee_address, friendly_name, device_subtype, " "model, vendor, lqi, discovered_at) " "SELECT id, ip, mac, hostname, os, services, suggested_type, status, " "discovery_source, ieee_address, friendly_name, device_subtype, " "model, vendor, lqi, discovered_at FROM pending_devices" ) await conn.exec_driver_sql("DROP TABLE pending_devices") await conn.exec_driver_sql( "ALTER TABLE pending_devices_new RENAME TO pending_devices" ) await conn.exec_driver_sql( "CREATE INDEX IF NOT EXISTS ix_pending_devices_ieee_address " "ON pending_devices(ieee_address)" ) await conn.exec_driver_sql("PRAGMA foreign_keys = ON") except OperationalError as exc: logger.warning("pending_devices ip-nullable rebuild failed: %s", exc) # --- end Zigbee schema migrations ------------------------------------- # --- Electrical designs schema migrations ----------------------------- # Create designs table (idempotent) await _try_migrate( conn, "CREATE TABLE IF NOT EXISTS designs (" "id VARCHAR PRIMARY KEY," "name VARCHAR NOT NULL," "design_type VARCHAR NOT NULL DEFAULT 'network'," "created_at DATETIME," "updated_at DATETIME" ")", label="designs.table", ) # Add user-chosen icon to designs (idempotent), then backfill existing rows # so legacy designs keep a sensible icon based on their original type. await _try_migrate( conn, "ALTER TABLE designs ADD COLUMN icon VARCHAR", label="designs.icon", ) with suppress(OperationalError): await conn.exec_driver_sql( "UPDATE designs SET icon = 'zap' WHERE icon IS NULL AND design_type = 'electrical'" ) with suppress(OperationalError): await conn.exec_driver_sql( "UPDATE designs SET icon = 'dashboard' WHERE icon IS NULL" ) # Seed default Network Topology design if designs table is empty _default_design_id = str(_uuid_mod.uuid4()) row = await conn.exec_driver_sql("SELECT COUNT(*) FROM designs") count_row = row.fetchone() count = count_row[0] if count_row else 0 if count == 0: await conn.exec_driver_sql( "INSERT INTO designs (id, name, design_type, icon, created_at, updated_at) " "VALUES (?, 'Network Topology', 'network', 'dashboard', datetime('now'), datetime('now'))", (_default_design_id,), ) else: row2 = await conn.exec_driver_sql("SELECT id FROM designs WHERE design_type = 'network' LIMIT 1") default = row2.fetchone() _default_design_id = default[0] if default else _default_design_id # Add design_id to nodes await _try_migrate( conn, "ALTER TABLE nodes ADD COLUMN design_id VARCHAR REFERENCES designs(id)", label="nodes.design_id", ) # Assign existing nodes to default design await conn.exec_driver_sql( "UPDATE nodes SET design_id = ? WHERE design_id IS NULL", (_default_design_id,), ) # Add design_id to edges await _try_migrate( conn, "ALTER TABLE edges ADD COLUMN design_id VARCHAR REFERENCES designs(id)", label="edges.design_id", ) # Assign existing edges to default design await conn.exec_driver_sql( "UPDATE edges SET design_id = ? WHERE design_id IS NULL", (_default_design_id,), ) # Migrate canvas_state from id=1 to design_id PK (SQLite rebuild) try: info = await conn.exec_driver_sql("PRAGMA table_info(canvas_state)") cols = info.fetchall() has_design_id = any(c[1] == "design_id" for c in cols) if not has_design_id: logger.info("Migrating canvas_state: switching to design_id primary key") await conn.exec_driver_sql("PRAGMA foreign_keys = OFF") await conn.exec_driver_sql( "CREATE TABLE canvas_state_new (" "design_id VARCHAR PRIMARY KEY REFERENCES designs(id) ON DELETE CASCADE," "viewport JSON," "custom_style JSON," "saved_at DATETIME" ")" ) # Copy existing row(s), mapping id=1 to default design_id old_rows = await conn.exec_driver_sql("SELECT id, viewport, custom_style, saved_at FROM canvas_state") for old in old_rows.fetchall(): cs_id, viewport, custom_style, saved_at = old target_design = _default_design_id await conn.exec_driver_sql( "INSERT INTO canvas_state_new (design_id, viewport, custom_style, saved_at) " "VALUES (?, ?, ?, ?)", (target_design, viewport, custom_style, saved_at), ) await conn.exec_driver_sql("DROP TABLE canvas_state") await conn.exec_driver_sql("ALTER TABLE canvas_state_new RENAME TO canvas_state") await conn.exec_driver_sql("PRAGMA foreign_keys = ON") except OperationalError as exc: logger.warning("canvas_state migration failed: %s", exc) # --- end Electrical designs schema migrations -------------------------- with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE edges ADD COLUMN waypoints JSON") with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN properties JSON") # Migrate hardware columns → properties JSON (idempotent: only runs on nodes where properties IS NULL) with suppress(OperationalError): rows = await conn.exec_driver_sql( "SELECT id, cpu_model, cpu_count, ram_gb, disk_gb, show_hardware " "FROM nodes WHERE properties IS NULL" ) for r in rows.fetchall(): node_id, cpu_model, cpu_count, ram_gb, disk_gb, show_hardware = r props = [] visible = bool(show_hardware) if cpu_model: props.append({"key": "CPU Model", "value": str(cpu_model), "icon": "Cpu", "visible": visible}) if cpu_count is not None: props.append({"key": "CPU Cores", "value": str(cpu_count), "icon": "Cpu", "visible": visible}) if ram_gb is not None: props.append({"key": "RAM", "value": f"{ram_gb} GB", "icon": "MemoryStick", "visible": visible}) if disk_gb is not None: props.append({"key": "Disk", "value": f"{disk_gb} GB", "icon": "HardDrive", "visible": visible}) await conn.exec_driver_sql( "UPDATE nodes SET properties = ? WHERE id = ?", (_json.dumps(props), node_id), ) # Inventory timestamp: last time a scan observed this node (idempotent) with suppress(OperationalError): await conn.exec_driver_sql("ALTER TABLE nodes ADD COLUMN last_scan DATETIME") # Migrate animated column from boolean (0/1) to string ('none'/'snake') with suppress(OperationalError): await conn.exec_driver_sql("UPDATE edges SET animated = 'snake' WHERE animated = '1' OR animated = 1") with suppress(OperationalError): sql = "UPDATE edges SET animated = 'none' WHERE animated = '0' OR animated = 0 OR animated IS NULL" await conn.exec_driver_sql(sql) async def get_db() -> AsyncGenerator[AsyncSession, None]: async with AsyncSessionLocal() as session: yield session