From f72ee6532de2cbeb1157ece2f5dc5a12f162d4fe Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Tue, 14 Jul 2026 22:05:38 -0500 Subject: [PATCH 1/6] resource_plan: count physical cores on Windows (GetLogicalProcessorInformationEx) os.cpu_count() returns logical processors, so on SMT machines the plan sets OMP_NUM_THREADS to 2 threads/core, which thrashes the AVX-512 units during expert matmul (9950X3D: 32 logical vs 16 physical). Count RelationProcessorCore records instead, with the existing lscpu/cpu_count fallbacks intact. Co-Authored-By: Claude Fable 5 --- c/resource_plan.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/c/resource_plan.py b/c/resource_plan.py index 2fe4c3c..7258180 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -167,6 +167,30 @@ def discover_gpus(): def physical_cpu_count(): + if sys.platform == "win32": + # os.cpu_count() conta i processori logici (SMT): 2 thread/core saturano + # le unita' AVX-512 e peggiorano il matmul. Contiamo i core fisici veri + # con GetLogicalProcessorInformationEx(RelationProcessorCore). + try: + import ctypes + k32 = ctypes.windll.kernel32 + need = ctypes.c_ulong(0) + k32.GetLogicalProcessorInformationEx(0, None, ctypes.byref(need)) + buf = (ctypes.c_char * need.value)() + if k32.GetLogicalProcessorInformationEx(0, buf, ctypes.byref(need)): + raw, cores, off = bytes(buf), 0, 0 + while off + 8 <= need.value: + relationship = int.from_bytes(raw[off:off + 4], "little") + size = int.from_bytes(raw[off + 4:off + 8], "little") + if size <= 0: + break + if relationship == 0: # RelationProcessorCore + cores += 1 + off += size + if cores: + return cores + except (OSError, ValueError, AttributeError): + pass try: result = subprocess.run(["lscpu", "-p=core,socket"], text=True, capture_output=True, check=True, timeout=5) From 51dd4805afc43f5f5db250bc543a1787db240ff0 Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Tue, 14 Jul 2026 22:23:44 -0500 Subject: [PATCH 2/6] benchmark_cuda_fixture: accept the current PROFILE service/wait format profile_print now emits 'expert-disk N.NNNs service / N.NNNs wait' but the harness regex still expected the old single expert-disk number, so every run died with 'benchmark output missing'. Accept both formats and report disk = service + wait. Co-Authored-By: Claude Fable 5 --- c/tools/benchmark_cuda_fixture.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index 1b44205..bced18e 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -10,8 +10,10 @@ from pathlib import Path SPEED_RE = re.compile(r"REPLAY decode:.*\| ([0-9.]+) tok/s") +# accetta sia il formato storico "expert-disk 0.123s |" sia quello attuale +# "expert-disk 0.123s service / 0.045s wait |" (glm.c profile_print) PROFILE_RE = re.compile( - r"PROFILE: expert-disk ([0-9.]+)s \| expert-matmul ([0-9.]+)s " + r"PROFILE: expert-disk ([0-9.]+)s(?: service / ([0-9.]+)s wait)? \| expert-matmul ([0-9.]+)s " r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s" ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") @@ -23,7 +25,9 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: profile = PROFILE_RE.search(stdout) if not speed or not profile: raise RuntimeError(f"benchmark output missing\nstdout:\n{stdout}\nstderr:\n{stderr}") - return float(speed.group(1)), [float(value) for value in profile.groups()] + service, wait, *rest = profile.groups() + disk = float(service) + (float(wait) if wait else 0.0) + return float(speed.group(1)), [disk] + [float(value) for value in rest] def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]: From 4059e107617974026bdf7bd8c7437b544a972d74 Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Wed, 15 Jul 2026 01:21:15 -0500 Subject: [PATCH 3/6] =?UTF-8?q?docs(readme):=20mechanical=20fixes=20?= =?UTF-8?q?=E2=80=94=20expert=20count,=20IO=5FTHREADS,=20line=20counts,=20?= =?UTF-8?q?desktop/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Expert count 21,504 -> 19,456 (75x256 + MTP head), matching the rest of the repo. - Replace phantom IO_THREADS with PIPE_WORKERS (default 8); state the pool only engages under PIPE=1. - Drop rotting precise line-count claims (glm.c ~2,400; web ~390) — keep the prose. - Add the desktop/ directory to the repo-layout section. Co-Authored-By: Claude Fable 5 --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 86c3009..b3ceecb 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,9 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hov A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So: - the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB); -- the **21,504 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2. +- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2. -The engine is a single C file (`c/glm.c`, ~2,400 lines) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below). +The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below). ## What's implemented @@ -455,8 +455,8 @@ CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. ### Web interface -`web/` contains a community-contributed browser UI (React + TypeScript, ~390 -lines of source, a pure API client — it never touches the engine directly): +`web/` contains a community-contributed browser UI (React + TypeScript, a pure +API client — it never touches the engine directly): ```bash cd web @@ -493,10 +493,10 @@ Disk is an immutable recovery source, not a normal decode target. If the plan leaves cold expert bytes on disk, speed depends on cache hit rate; output quality does not. -Cold expert reads use a deferred pipeline: resident RAM/VRAM experts execute +Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute while missing experts are loaded in a bounded background I/O pool, then the -cold results join before the layer completes. `IO_THREADS=n` overrides the -default eight loader threads when foreground work exists. Profiling reports +cold results join before the layer completes. The pool engages only under +`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports both disk service time and the smaller foreground-visible wait time so overlap is explicit rather than credited as unexplained speedup. @@ -657,6 +657,7 @@ c/ ├── scripts/ long-running conversion helpers └── tests/ dependency-free C and Python tests web/ browser UI (pure OpenAI-API client, community-maintained) +desktop/ Tauri v2 desktop shell wrapping the web UI ``` The runtime path intentionally stays flat and readable: `glm.c` plus its small From 1a243cfc3e940bfe5477ed9fab7055e4bdd07c69 Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Wed, 15 Jul 2026 12:06:48 -0500 Subject: [PATCH 4/6] =?UTF-8?q?coli:=20measured=20Windows=20launcher=20def?= =?UTF-8?q?aults=20=E2=80=94=20OMP=20tuning=20parity=20+=20DIRECT/PIPE/PIL?= =?UTF-8?q?OT=5FREAL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows the engine self-exec OMP tuning never runs (Linux/FreeBSD-only) and posix_fadvise readahead is a compat.h no-op, so a stock Windows run leaves large measured wins on the table. The launcher now setdefaults, on win32 only, each independently overridable by setting the variable: - OMP_WAIT_POLICY=active, GOMP_SPINCOUNT=200000, OMP_DYNAMIC=FALSE, OMP_NUM_THREADS= (parity with the glm.c self-exec block; COLI_NO_OMP_TUNE disables exactly this block, presence-based like the engine). OMP_PROC_BIND/OMP_PLACES deliberately omitted and also removed from environment_for_plan on win32: MinGW libgomp has no affinity support ("Affinity not supported on this configuration"). - DIRECT=1: unbuffered expert reads. Measured on a 9950X3D + Samsung 9100 PRO Gen5 + Win11: iobench 10.68 GB/s O_DIRECT vs 9.03 buffered (warm); end-to-end REPLAY 0.48 -> 1.02 tok/s. Matches #162 (1.47x same class). - PIPE=1: load/matmul overlap, byte-identical output; +8% on top of DIRECT (PIPE_WORKERS untouched at 8 - 4/8/16 swept flat on Gen5). - PILOT_REAL=1: real cross-layer prefetch, the only working prefetch on Windows; +11% and expert hit rate +19 points. Full ladder methodology and numbers: 96-token greedy REPLAY, one lever per step, medians of 3-4 runs (see the fork tuning doc referenced in the PR). tests/test_env_defaults.py covers the defaults, explicit-override-wins, the kill-switch scope, and the non-win32 no-op. Co-Authored-By: Claude Fable 5 --- c/coli | 28 +++++++++++++++ c/resource_plan.py | 7 ++-- c/tests/test_env_defaults.py | 68 +++++++++++++++++++++++++++++++++++ c/tests/test_resource_plan.py | 9 +++-- 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 c/tests/test_env_defaults.py diff --git a/c/coli b/c/coli index 8c89caf..9408281 100755 --- a/c/coli +++ b/c/coli @@ -161,6 +161,34 @@ def resource_request(a, env): def env_for(a): e = dict(os.environ, SNAP=a.model) + if sys.platform == "win32": + # COLI_NO_OMP_TUNE spegne SOLO il blocco OMP (stesso perimetro del + # self-exec di glm.c; presence-based come nel motore: impostarla a + # qualsiasi valore, anche 0, disattiva). I default I/O piu' sotto + # restano attivi: kill-switch dedicati = le var stesse (DIRECT=0 ecc.) + if not e.get("COLI_NO_OMP_TUNE"): + # parita' col tuning OMP self-exec di glm.c (solo Linux/FreeBSD, e + # comunque saltato sotto CUDA/Metal): libgomp legge queste variabili + # prima di main, quindi su Windows vanno nell'ambiente del figlio. + # niente OMP_PROC_BIND/OMP_PLACES: la libgomp di MinGW non supporta + # l'affinity su Windows ("Affinity not supported on this configuration") + from resource_plan import physical_cpu_count + for k, v in (("OMP_WAIT_POLICY", "active"), + ("GOMP_SPINCOUNT", "200000"), + ("OMP_DYNAMIC", "FALSE"), + ("OMP_NUM_THREADS", str(physical_cpu_count()))): + e.setdefault(k, v) + # Default Windows misurati sul box di riferimento (docs/tuning-9950x3d-5090.md), + # tutti lossless e tutti setdefault (un override esplicito vince sempre): + # - DIRECT=1: 10.7 GB/s O_DIRECT vs 9.0 buffered (iobench, anche a cache + # calda); nel motore 0.48 -> 1.02 tok/s. Upstream #162: 1.47x. + # - PIPE=1: overlap load/matmul, +8% sopra DIRECT (byte-identico, riordina + # solo l'I/O). PIPE_WORKERS resta al default 8 (sweep 4/8/16 piatto). + # - PILOT_REAL=1: prefetch cross-layer con load veri (unico prefetch + # funzionante su Windows: fadvise e' no-op), +11%, hit rate +19 punti. + e.setdefault("DIRECT", "1") + e.setdefault("PIPE", "1") + e.setdefault("PILOT_REAL", "1") e["COLI_POLICY"]=a.policy if a.ram: e["RAM_GB"]=str(a.ram) if a.ngen: e["NGEN"]=str(a.ngen) diff --git a/c/resource_plan.py b/c/resource_plan.py index 7258180..6b3f2fe 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -313,8 +313,11 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): result = dict(env or {}) result.setdefault("COLI_POLICY", plan["policy"]["name"]) result.setdefault("OMP_NUM_THREADS", str(plan["cpu"]["physical_cores"])) - result.setdefault("OMP_PROC_BIND", "spread") - result.setdefault("OMP_PLACES", "cores") + if sys.platform != "win32": + # la libgomp di MinGW non supporta l'affinity su Windows + # ("Affinity not supported on this configuration"): non impostarle li'. + result.setdefault("OMP_PROC_BIND", "spread") + result.setdefault("OMP_PLACES", "cores") if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.py new file mode 100644 index 0000000..12a3c99 --- /dev/null +++ b/c/tests/test_env_defaults.py @@ -0,0 +1,68 @@ +"""env_for: default Windows misurati (DIRECT/PIPE/PILOT_REAL + blocco OMP). + +Carica `coli` come modulo (ha la guardia __main__) e verifica il contratto: +- win32: i tre default I/O e il blocco OMP sono setdefault +- un override esplicito dell'utente vince sempre +- COLI_NO_OMP_TUNE spegne SOLO il blocco OMP, non i default I/O +- non-win32: env_for non tocca nulla di tutto questo +""" +import importlib.machinery +import importlib.util +import os +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(HERE)) + +_loader = importlib.machinery.SourceFileLoader("coli_cli", str(HERE / "coli")) +_spec = importlib.util.spec_from_loader("coli_cli", _loader) +coli = importlib.util.module_from_spec(_spec) +_loader.exec_module(coli) + + +def args(**over): + base = dict(model="X", policy="quality", ram=0, ngen=0, topp=0, topk=0, + temp=None, repin=0, ctx=0, auto_tier=False, gpu=None, vram=0) + base.update(over) + return types.SimpleNamespace(**base) + + +class EnvDefaultsTest(unittest.TestCase): + def env_for_with(self, environ, platform): + with mock.patch.dict(os.environ, environ, clear=True), \ + mock.patch.object(sys, "platform", platform): + return coli.env_for(args()) + + def test_win32_sets_measured_defaults(self): + e = self.env_for_with({}, "win32") + self.assertEqual(e["DIRECT"], "1") + self.assertEqual(e["PIPE"], "1") + self.assertEqual(e["PILOT_REAL"], "1") + self.assertEqual(e["OMP_WAIT_POLICY"], "active") + self.assertNotIn("OMP_PROC_BIND", e) # MinGW libgomp: niente affinity + + def test_explicit_override_wins(self): + e = self.env_for_with({"DIRECT": "0", "PIPE": "0"}, "win32") + self.assertEqual(e["DIRECT"], "0") + self.assertEqual(e["PIPE"], "0") + self.assertEqual(e["PILOT_REAL"], "1") # non overridden -> default + + def test_kill_switch_scope_is_omp_only(self): + e = self.env_for_with({"COLI_NO_OMP_TUNE": "1"}, "win32") + self.assertNotIn("OMP_WAIT_POLICY", e) + self.assertNotIn("OMP_NUM_THREADS", e) + self.assertEqual(e["DIRECT"], "1") # i default I/O restano attivi + self.assertEqual(e["PIPE"], "1") + + def test_non_windows_untouched(self): + e = self.env_for_with({}, "linux") + for k in ("DIRECT", "PIPE", "PILOT_REAL", "OMP_WAIT_POLICY"): + self.assertNotIn(k, e) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py index 5a89773..023cb93 100644 --- a/c/tests/test_resource_plan.py +++ b/c/tests/test_resource_plan.py @@ -112,8 +112,13 @@ class ResourcePlanTest(unittest.TestCase): self.assertEqual(env["COLI_CUDA"], "1") self.assertEqual(env["COLI_GPUS"], "1") self.assertEqual(env["OMP_NUM_THREADS"], str(plan["cpu"]["physical_cores"])) - self.assertEqual(env["OMP_PROC_BIND"], "spread") - self.assertEqual(env["OMP_PLACES"], "cores") + if sys.platform == "win32": + # MinGW libgomp: niente affinity su Windows, le chiavi non vanno emesse + self.assertNotIn("OMP_PROC_BIND", env) + self.assertNotIn("OMP_PLACES", env) + else: + self.assertEqual(env["OMP_PROC_BIND"], "spread") + self.assertEqual(env["OMP_PLACES"], "cores") self.assertEqual(env["PIN_GB"], env["CUDA_EXPERT_GB"]) explicit_threads = environment_for_plan(plan, {"OMP_NUM_THREADS": "7", From 939b689a10bf290ea69fabc0271598a06b529b3e Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Wed, 15 Jul 2026 12:14:13 -0500 Subject: [PATCH 5/6] =?UTF-8?q?glm:=20hwinfo=5Femit=20Windows=20support=20?= =?UTF-8?q?=E2=80=94=20CPU=20brand,=20cores,=20RAM=20were=20hardcoded-zero?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HWINFO snapshot read /proc/cpuinfo, /proc/meminfo and sysconf(_SC_NPROCESSORS_ONLN) — all absent on native Windows — so the dashboard runtime panel showed "0 GB RAM / 0 cores" while the CUDA branch populated VRAM fine. Now: CPUID brand string (0x80000002..4, no new deps), GetSystemInfo for logical cores, and the existing compat_meminfo (GlobalMemoryStatusEx) for RAM. Verified live: panel reads "AMD Ryzen 9 9950X3D 16-Core Processor / 66 GB RAM 24 GB free / 32 cores". Co-Authored-By: Claude Fable 5 --- c/glm.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/c/glm.c b/c/glm.c index fb93e59..da92c2b 100644 --- a/c/glm.c +++ b/c/glm.c @@ -36,6 +36,9 @@ #include /* mlock: inchioda le pagine in RAM / wire pages into RAM */ #include /* fstat per mmap degli shard (COLI_MMAP) */ #endif +#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) +#include /* hwinfo_emit: CPU brand string senza /proc */ +#endif #include "st.h" #include "tok.h" #include "tier.h" @@ -4453,25 +4456,45 @@ static void ehit_mark(Model *m, int layer, int eid){ static void hwinfo_emit(Model *m){ Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ /* CPU */ - char cpu[256]=""; FILE *ci=fopen("/proc/cpuinfo","r"); + char cpu[256]=""; +#ifdef _WIN32 + /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero + * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' + * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,sizeof(cpu),"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); if(ci){ char ln[256]; while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; snprintf(cpu,sizeof(cpu),"%s",p); } break; } fclose(ci); } +#endif int cores=0; -#ifdef _SC_NPROCESSORS_ONLN +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) cores=(int)sysconf(_SC_NPROCESSORS_ONLN); #endif /* RAM */ double ram_total=0,ram_avail=0; +#ifdef _WIN32 + compat_meminfo(&ram_total,&ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ +#else FILE *mi=fopen("/proc/meminfo","r"); if(mi){ char ln[256]; double mt=0,ma=0; while(fgets(ln,sizeof(ln),mi)){ if(sscanf(ln,"MemTotal: %lf",&mt)==1) ram_total=mt/1e6; if(sscanf(ln,"MemAvailable: %lf",&ma)==1) ram_avail=ma/1e6; } fclose(mi); } +#endif /* GPU */ int ngpu=0; double vram_total=0; char gpu_name[128]=""; From 97fa52698f92c510173da1bc140014a3b15faeff Mon Sep 17 00:00:00 2001 From: NeuralNotwerk Date: Wed, 15 Jul 2026 22:12:31 +0000 Subject: [PATCH 6/6] glm: make MLOCK=1 actually wire pinned experts under COLI_MMAP pin_wire() only locked the legacy private-slab path (s->slab / s->fslab), which stay NULL under COLI_MMAP -- so MLOCK=1 on an mmap build silently wired 0 bytes and the 'pinned' set was just warm page cache, evictable under memory pressure (measured: hundreds of MB/s of re-reads from disk mid-generation on a 503 GB host once the cache tightened). Three pieces: - qt_wire_mmap(): mlock each pinned expert's weight + scale ranges inside the file mappings. Skips cuda_eligible QTs: VRAM-tier experts compute from device memory, and expert_host_release() early-returns for mmap experts (no slab) WITHOUT nulling q8/q4, so a host-pointer check alone wires the whole VRAM tier too (~137 GB of never-touched locked pages on a 6x32GB-class rig -- enough to starve the kernel into thrashing). - qt_unwire_mmap(): REPIN gpu_swap promotions drop the promoted expert's host lock instead of leaking it on every swap. - expert_load() deliberately does NOT wire (it also runs for the transient VRAM-staging pass); wiring happens once in pin_wire() on the final set. Measured on GLM-5.2 int4, 4x RTX 5090 + 1x RTX 4090, 503 GB RAM, PIN_GB=all: wired goes 0 -> 226 GB (exactly the RAM tier), decode-time disk reads drop to zero once warm. Co-Authored-By: Claude Fable 5 --- c/glm.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/c/glm.c b/c/glm.c index fb93e59..e9903fc 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1365,6 +1365,14 @@ static void embed_row(Model *m, int tok, float *x){ static int g_mmap=0; static struct { int fd; void *base; size_t len; } g_maps[512]; static int g_nmaps; static pthread_mutex_t g_map_mtx = PTHREAD_MUTEX_INITIALIZER; /* expert_load e' OMP-parallel */ +/* forward decls: mem_should_wire/mem_wire live near pin_wire() further down, but + * qt_wire_mmap() (also further down, used by pin_wire()'s COLI_MMAP path) needs + * them declared before its own definition. Real mlock-ing of mmap'd pinned + * experts happens there, not in expert_load() -- see qt_wire_mmap() for why. */ +static int mem_should_wire(void); +static int mem_wire(void *addr, size_t len); +static void qt_unwire_mmap(QT *t); /* def. presso pin_wire / defined near pin_wire */ +static int64_t g_mmap_wired=0; static long g_mmap_wire_failed=0; static void *map_of_fd(int fd){ pthread_mutex_lock(&g_map_mtx); for(int i=0;ioff; size_t nq=(size_t)tq[k]->nbytes; for(size_t i=0;islab, always NULL under mmap, so wiring here would + * leak locked pages for every GPU-tier expert). See pin_wire() below: it wires + * the final resident set only, after GPU release has already nulled out the + * pointers for anything that isn't genuinely RAM-tier. */ } s->eid=eid; return 0; } @@ -3918,6 +3933,9 @@ static void repin_pass_limit(Model *m,int limit){ if(!qt_cuda_update(hq[k])) ok=0; } if(!ok){ fprintf(stderr,"[REPIN] refresh VRAM fallito\n"); exit(1); } + /* promoted expert now computes from VRAM: drop its host mlock + * (mmap path; no-op otherwise) or every swap leaks locked pages */ + qt_unwire_mmap(&hot->g); qt_unwire_mmap(&hot->u); qt_unwire_mmap(&hot->d); if(g_cuda_release_host) expert_host_release(m,hot); gpu_swaps++; if(getenv("REPIN_VERBOSE")) fprintf(stderr, @@ -4626,8 +4644,57 @@ static int mem_wire(void *addr, size_t len){ } /* Inchioda tutti gli slab degli expert pinnati (pesi + scale). Non fatale se fallisce. * EN: wire all pinned-expert slabs (weights + scales). Non-fatal on failure. */ +/* mlock a single mmap'd QT's weight + scale ranges. Skips VRAM-tier QTs + * (cuda_eligible): their compute runs from device memory, so wiring the host + * mmap range would pin ~137 GB of never-touched file pages. NOTE the q8/q4 + * NULL check alone is NOT enough here: expert_host_release() early-returns + * for mmap experts (no slab) without nulling the host pointers, so GPU-tier + * slots keep live-looking q8/q4 forever -- that was the bug that wired 363 GB + * instead of 231 GB and starved the kernel into page-cache thrashing. + * wired/failed are accumulated into the caller's counters. */ +/* undo qt_wire_mmap for one QT: used when a REPIN gpu_swap promotes a wired + * RAM-tier expert into VRAM -- without this every promotion leaks its locked + * host range and the dead-weight lock re-grows over a long session. */ +static void qt_unwire_mmap(QT *t){ + if(!g_mmap || !mem_should_wire()) return; + if(!t->q8 && !t->q4) return; + int64_t scale_b=(int64_t)t->O*4; + int64_t weight_b=qt_bytes(t)-scale_b; + void *wp=t->q8?(void*)t->q8:(void*)t->q4; +#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) + if(weight_b>0 && !munlock(wp,(size_t)weight_b)) g_mmap_wired-=weight_b; + if(t->s && scale_b>0 && !munlock(t->s,(size_t)scale_b)) g_mmap_wired-=scale_b; +#elif defined(_WIN32) + if(weight_b>0 && !compat_munlock(wp,(size_t)weight_b)) g_mmap_wired-=weight_b; + if(t->s && scale_b>0 && !compat_munlock(t->s,(size_t)scale_b)) g_mmap_wired-=scale_b; +#endif +} +static void qt_wire_mmap(QT *t, int64_t *wired, long *failed){ + if(!t->q8 && !t->q4) return; + if(t->cuda_eligible) return; /* resident in VRAM; host range is dead weight */ + int64_t scale_b=(int64_t)t->O*4; + int64_t weight_b=qt_bytes(t)-scale_b; + void *wp=t->q8?(void*)t->q8:(void*)t->q4; + if(weight_b>0){ if(mem_wire(wp,(size_t)weight_b)==0) *wired+=weight_b; else (*failed)++; } + if(t->s && scale_b>0){ if(mem_wire(t->s,(size_t)scale_b)==0) *wired+=scale_b; else (*failed)++; } +} static void pin_wire(Model *m){ if(!mem_should_wire()) return; + if(g_mmap){ + /* Wire the FINAL resident set only, after pin_load's GPU-upload pass + * has already run -- qt_wire_mmap() skips cuda_eligible (VRAM-tier) + * slots, so only the genuinely RAM-tier experts get locked. */ + Cfg *c=&m->c; double t0=now_s(); + for(int i=0;in_layers;i++) for(int z=0;znpin[i];z++){ + ESlot *s=&m->pin[i][z]; + qt_wire_mmap(&s->g,&g_mmap_wired,&g_mmap_wire_failed); + qt_wire_mmap(&s->u,&g_mmap_wired,&g_mmap_wire_failed); + qt_wire_mmap(&s->d,&g_mmap_wired,&g_mmap_wire_failed); + } + fprintf(stderr,"[PIN] mlock (mmap): %.1f GB wired in physical RAM%s in %.0fs\n", + g_mmap_wired/1e9, g_mmap_wire_failed?" (some allocations failed -- raise: ulimit -l unlimited)":"", now_s()-t0); + return; + } Cfg *c=&m->c; double t0=now_s(); int64_t wired=0; long failed=0; for(int i=0;in_layers;i++) for(int z=0;znpin[i];z++){ ESlot *s=&m->pin[i][z];