perf: run status checks concurrently to prevent scheduler overrun

Replace sequential node checks with asyncio.gather so all nodes are
checked in parallel. Add coalesce=True + max_instances=1 to suppress
APScheduler "maximum instances reached" log spam.
This commit is contained in:
Pouzor
2026-04-02 11:41:08 +02:00
parent d5069d9b96
commit e4bfab7e58
+40 -23
View File
@@ -1,4 +1,5 @@
"""APScheduler setup for background scan and status check jobs.""" """APScheduler setup for background scan and status check jobs."""
import asyncio
import logging import logging
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -15,34 +16,43 @@ logger = logging.getLogger(__name__)
scheduler: AsyncIOScheduler = AsyncIOScheduler() scheduler: AsyncIOScheduler = AsyncIOScheduler()
async def _run_status_checks() -> None: async def _check_single_node(node: Node) -> tuple[str, dict | None]:
"""Check all nodes and broadcast results via WebSocket.""" """Run a single node check; returns (node_id, result_or_None)."""
from app.api.routes.status import broadcast_status # avoid circular import from app.api.routes.status import broadcast_status # avoid circular import
try:
check_result = await check_node(node.check_method, node.check_target, node.ip)
async with AsyncSessionLocal() as db:
n = await db.get(Node, node.id)
if n:
n.status = check_result["status"]
n.response_time_ms = check_result["response_time_ms"]
if check_result["status"] == "online":
n.last_seen = datetime.now(timezone.utc)
await db.commit()
await broadcast_status(
node_id=node.id,
status=check_result["status"],
checked_at=datetime.now(timezone.utc).isoformat(),
response_time_ms=check_result["response_time_ms"],
)
return node.id, check_result
except Exception as exc:
logger.error("Status check failed for node %s: %s", node.id, exc)
return node.id, None
async def _run_status_checks() -> None:
"""Check all nodes concurrently and broadcast results via WebSocket."""
async with AsyncSessionLocal() as db: async with AsyncSessionLocal() as db:
result = await db.execute(select(Node)) result = await db.execute(select(Node))
nodes = result.scalars().all() nodes = result.scalars().all()
for node in nodes: checkable = [n for n in nodes if n.check_method]
if not node.check_method: if not checkable:
continue return
try:
check_result = await check_node(node.check_method, node.check_target, node.ip) await asyncio.gather(*[_check_single_node(n) for n in checkable])
async with AsyncSessionLocal() as db:
n = await db.get(Node, node.id)
if n:
n.status = check_result["status"]
n.response_time_ms = check_result["response_time_ms"]
n.last_seen = datetime.now(timezone.utc) if check_result["status"] == "online" else n.last_seen
await db.commit()
await broadcast_status(
node_id=node.id,
status=check_result["status"],
checked_at=datetime.now(timezone.utc).isoformat(),
response_time_ms=check_result["response_time_ms"],
)
except Exception as exc:
logger.error("Status check failed for node %s: %s", node.id, exc)
def start_scheduler() -> None: def start_scheduler() -> None:
@@ -50,7 +60,14 @@ def start_scheduler() -> None:
if scheduler.running: if scheduler.running:
scheduler.shutdown(wait=False) scheduler.shutdown(wait=False)
scheduler = AsyncIOScheduler() scheduler = AsyncIOScheduler()
scheduler.add_job(_run_status_checks, "interval", seconds=settings.status_checker_interval, id="status_checks") scheduler.add_job(
_run_status_checks,
"interval",
seconds=settings.status_checker_interval,
id="status_checks",
max_instances=1,
coalesce=True,
)
scheduler.start() scheduler.start()
logger.info("Scheduler started — status checks every %ds", settings.status_checker_interval) logger.info("Scheduler started — status checks every %ds", settings.status_checker_interval)