This commit is contained in:
JustVugg
@@ -48,9 +48,9 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hov
|
||||
A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So:
|
||||
|
||||
- the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB);
|
||||
- the **21,504 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2.
|
||||
- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2.
|
||||
|
||||
The engine is a single C file (`c/glm.c`, ~2,400 lines) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below).
|
||||
The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below).
|
||||
|
||||
## What's implemented
|
||||
|
||||
@@ -455,8 +455,8 @@ CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
|
||||
|
||||
### Web interface
|
||||
|
||||
`web/` contains a community-contributed browser UI (React + TypeScript, ~390
|
||||
lines of source, a pure API client — it never touches the engine directly):
|
||||
`web/` contains a community-contributed browser UI (React + TypeScript, a pure
|
||||
API client — it never touches the engine directly):
|
||||
|
||||
```bash
|
||||
cd web
|
||||
@@ -493,10 +493,10 @@ Disk is an immutable recovery source, not a normal decode target. If the plan
|
||||
leaves cold expert bytes on disk, speed depends on cache hit rate; output
|
||||
quality does not.
|
||||
|
||||
Cold expert reads use a deferred pipeline: resident RAM/VRAM experts execute
|
||||
Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute
|
||||
while missing experts are loaded in a bounded background I/O pool, then the
|
||||
cold results join before the layer completes. `IO_THREADS=n` overrides the
|
||||
default eight loader threads when foreground work exists. Profiling reports
|
||||
cold results join before the layer completes. The pool engages only under
|
||||
`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports
|
||||
both disk service time and the smaller foreground-visible wait time so overlap
|
||||
is explicit rather than credited as unexplained speedup.
|
||||
|
||||
@@ -657,6 +657,7 @@ c/
|
||||
├── scripts/ long-running conversion helpers
|
||||
└── tests/ dependency-free C and Python tests
|
||||
web/ browser UI (pure OpenAI-API client, community-maintained)
|
||||
desktop/ Tauri v2 desktop shell wrapping the web UI
|
||||
```
|
||||
|
||||
The runtime path intentionally stays flat and readable: `glm.c` plus its small
|
||||
|
||||
@@ -161,6 +161,34 @@ def resource_request(a, env):
|
||||
|
||||
def env_for(a):
|
||||
e = dict(os.environ, SNAP=a.model)
|
||||
if sys.platform == "win32":
|
||||
# COLI_NO_OMP_TUNE spegne SOLO il blocco OMP (stesso perimetro del
|
||||
# self-exec di glm.c; presence-based come nel motore: impostarla a
|
||||
# qualsiasi valore, anche 0, disattiva). I default I/O piu' sotto
|
||||
# restano attivi: kill-switch dedicati = le var stesse (DIRECT=0 ecc.)
|
||||
if not e.get("COLI_NO_OMP_TUNE"):
|
||||
# parita' col tuning OMP self-exec di glm.c (solo Linux/FreeBSD, e
|
||||
# comunque saltato sotto CUDA/Metal): libgomp legge queste variabili
|
||||
# prima di main, quindi su Windows vanno nell'ambiente del figlio.
|
||||
# niente OMP_PROC_BIND/OMP_PLACES: la libgomp di MinGW non supporta
|
||||
# l'affinity su Windows ("Affinity not supported on this configuration")
|
||||
from resource_plan import physical_cpu_count
|
||||
for k, v in (("OMP_WAIT_POLICY", "active"),
|
||||
("GOMP_SPINCOUNT", "200000"),
|
||||
("OMP_DYNAMIC", "FALSE"),
|
||||
("OMP_NUM_THREADS", str(physical_cpu_count()))):
|
||||
e.setdefault(k, v)
|
||||
# Default Windows misurati sul box di riferimento (docs/tuning-9950x3d-5090.md),
|
||||
# tutti lossless e tutti setdefault (un override esplicito vince sempre):
|
||||
# - DIRECT=1: 10.7 GB/s O_DIRECT vs 9.0 buffered (iobench, anche a cache
|
||||
# calda); nel motore 0.48 -> 1.02 tok/s. Upstream #162: 1.47x.
|
||||
# - PIPE=1: overlap load/matmul, +8% sopra DIRECT (byte-identico, riordina
|
||||
# solo l'I/O). PIPE_WORKERS resta al default 8 (sweep 4/8/16 piatto).
|
||||
# - PILOT_REAL=1: prefetch cross-layer con load veri (unico prefetch
|
||||
# funzionante su Windows: fadvise e' no-op), +11%, hit rate +19 punti.
|
||||
e.setdefault("DIRECT", "1")
|
||||
e.setdefault("PIPE", "1")
|
||||
e.setdefault("PILOT_REAL", "1")
|
||||
e["COLI_POLICY"]=a.policy
|
||||
if a.ram: e["RAM_GB"]=str(a.ram)
|
||||
if a.ngen: e["NGEN"]=str(a.ngen)
|
||||
|
||||
@@ -37,6 +37,9 @@
|
||||
#include <sys/stat.h> /* fstat per mmap degli shard (COLI_MMAP) */
|
||||
#include <signal.h> /* SIGINT = stop morbido del turno in serve mode */
|
||||
#endif
|
||||
#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
|
||||
#include <cpuid.h> /* hwinfo_emit: CPU brand string senza /proc */
|
||||
#endif
|
||||
#include "st.h"
|
||||
#ifdef __linux__
|
||||
#include "uring.h"
|
||||
@@ -1578,6 +1581,14 @@ static void embed_row(Model *m, int tok, float *x){
|
||||
static int g_mmap=0;
|
||||
static struct { int fd; void *base; size_t len; } g_maps[512]; static int g_nmaps;
|
||||
static pthread_mutex_t g_map_mtx = PTHREAD_MUTEX_INITIALIZER; /* expert_load e' OMP-parallel */
|
||||
/* forward decls: mem_should_wire/mem_wire live near pin_wire() further down, but
|
||||
* qt_wire_mmap() (also further down, used by pin_wire()'s COLI_MMAP path) needs
|
||||
* them declared before its own definition. Real mlock-ing of mmap'd pinned
|
||||
* experts happens there, not in expert_load() -- see qt_wire_mmap() for why. */
|
||||
static int mem_should_wire(void);
|
||||
static int mem_wire(void *addr, size_t len);
|
||||
static void qt_unwire_mmap(QT *t); /* def. presso pin_wire / defined near pin_wire */
|
||||
static int64_t g_mmap_wired=0; static long g_mmap_wire_failed=0;
|
||||
static void *map_of_fd(int fd){
|
||||
pthread_mutex_lock(&g_map_mtx);
|
||||
for(int i=0;i<g_nmaps;i++) if(g_maps[i].fd==fd){ void *b=g_maps[i].base; pthread_mutex_unlock(&g_map_mtx); return b; }
|
||||
@@ -1691,6 +1702,13 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){
|
||||
acc+=p[n-1]; (void)acc;
|
||||
char *q=(char*)bq[k]+tq[k]->off; size_t nq=(size_t)tq[k]->nbytes;
|
||||
for(size_t i=0;i<nq;i+=4096) acc+=q[i];
|
||||
/* mlock deliberately NOT done here: this fires for every expert_load call,
|
||||
* including the transient VRAM-staging pass in pin_load (host copy loaded,
|
||||
* uploaded to GPU, then "released" via expert_host_release -- which only
|
||||
* knows how to munlock s->slab, always NULL under mmap, so wiring here would
|
||||
* leak locked pages for every GPU-tier expert). See pin_wire() below: it wires
|
||||
* the final resident set only, after GPU release has already nulled out the
|
||||
* pointers for anything that isn't genuinely RAM-tier. */
|
||||
}
|
||||
s->eid=eid; return 0;
|
||||
}
|
||||
@@ -4562,6 +4580,9 @@ static void repin_pass_limit(Model *m,int limit){
|
||||
if(!qt_cuda_update(hq[k])) ok=0;
|
||||
}
|
||||
if(!ok){ fprintf(stderr,"[REPIN] refresh VRAM fallito\n"); exit(1); }
|
||||
/* promoted expert now computes from VRAM: drop its host mlock
|
||||
* (mmap path; no-op otherwise) or every swap leaks locked pages */
|
||||
qt_unwire_mmap(&hot->g); qt_unwire_mmap(&hot->u); qt_unwire_mmap(&hot->d);
|
||||
if(g_cuda_release_host) expert_host_release(m,hot);
|
||||
gpu_swaps++;
|
||||
if(getenv("REPIN_VERBOSE")) fprintf(stderr,
|
||||
@@ -5144,25 +5165,45 @@ static void ehit_mark(Model *m, int layer, int eid){
|
||||
static void hwinfo_emit(Model *m){
|
||||
Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */
|
||||
/* CPU */
|
||||
char cpu[256]=""; FILE *ci=fopen("/proc/cpuinfo","r");
|
||||
char cpu[256]="";
|
||||
#ifdef _WIN32
|
||||
/* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero
|
||||
* dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche'
|
||||
* tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
{ unsigned int r[12]={0}; unsigned int *w=r;
|
||||
for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4)
|
||||
__get_cpuid(f,&w[0],&w[1],&w[2],&w[3]);
|
||||
char *b=(char*)r; b[47]=0; while(*b==' ')b++;
|
||||
snprintf(cpu,sizeof(cpu),"%s",b); }
|
||||
#endif
|
||||
#else
|
||||
FILE *ci=fopen("/proc/cpuinfo","r");
|
||||
if(ci){ char ln[256];
|
||||
while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){
|
||||
char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++;
|
||||
int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0;
|
||||
snprintf(cpu,sizeof(cpu),"%s",p); } break; }
|
||||
fclose(ci); }
|
||||
#endif
|
||||
int cores=0;
|
||||
#ifdef _SC_NPROCESSORS_ONLN
|
||||
#ifdef _WIN32
|
||||
{ SYSTEM_INFO si; GetSystemInfo(&si); cores=(int)si.dwNumberOfProcessors; }
|
||||
#elif defined(_SC_NPROCESSORS_ONLN)
|
||||
cores=(int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
#endif
|
||||
/* RAM */
|
||||
double ram_total=0,ram_avail=0;
|
||||
#ifdef _WIN32
|
||||
compat_meminfo(&ram_total,&ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */
|
||||
#else
|
||||
FILE *mi=fopen("/proc/meminfo","r");
|
||||
if(mi){ char ln[256]; double mt=0,ma=0;
|
||||
while(fgets(ln,sizeof(ln),mi)){
|
||||
if(sscanf(ln,"MemTotal: %lf",&mt)==1) ram_total=mt/1e6;
|
||||
if(sscanf(ln,"MemAvailable: %lf",&ma)==1) ram_avail=ma/1e6;
|
||||
} fclose(mi); }
|
||||
#endif
|
||||
/* GPU */
|
||||
int ngpu=0; double vram_total=0;
|
||||
char gpu_name[128]="";
|
||||
@@ -5317,8 +5358,57 @@ static int mem_wire(void *addr, size_t len){
|
||||
}
|
||||
/* Inchioda tutti gli slab degli expert pinnati (pesi + scale). Non fatale se fallisce.
|
||||
* EN: wire all pinned-expert slabs (weights + scales). Non-fatal on failure. */
|
||||
/* mlock a single mmap'd QT's weight + scale ranges. Skips VRAM-tier QTs
|
||||
* (cuda_eligible): their compute runs from device memory, so wiring the host
|
||||
* mmap range would pin ~137 GB of never-touched file pages. NOTE the q8/q4
|
||||
* NULL check alone is NOT enough here: expert_host_release() early-returns
|
||||
* for mmap experts (no slab) without nulling the host pointers, so GPU-tier
|
||||
* slots keep live-looking q8/q4 forever -- that was the bug that wired 363 GB
|
||||
* instead of 231 GB and starved the kernel into page-cache thrashing.
|
||||
* wired/failed are accumulated into the caller's counters. */
|
||||
/* undo qt_wire_mmap for one QT: used when a REPIN gpu_swap promotes a wired
|
||||
* RAM-tier expert into VRAM -- without this every promotion leaks its locked
|
||||
* host range and the dead-weight lock re-grows over a long session. */
|
||||
static void qt_unwire_mmap(QT *t){
|
||||
if(!g_mmap || !mem_should_wire()) return;
|
||||
if(!t->q8 && !t->q4) return;
|
||||
int64_t scale_b=(int64_t)t->O*4;
|
||||
int64_t weight_b=qt_bytes(t)-scale_b;
|
||||
void *wp=t->q8?(void*)t->q8:(void*)t->q4;
|
||||
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
|
||||
if(weight_b>0 && !munlock(wp,(size_t)weight_b)) g_mmap_wired-=weight_b;
|
||||
if(t->s && scale_b>0 && !munlock(t->s,(size_t)scale_b)) g_mmap_wired-=scale_b;
|
||||
#elif defined(_WIN32)
|
||||
if(weight_b>0 && !compat_munlock(wp,(size_t)weight_b)) g_mmap_wired-=weight_b;
|
||||
if(t->s && scale_b>0 && !compat_munlock(t->s,(size_t)scale_b)) g_mmap_wired-=scale_b;
|
||||
#endif
|
||||
}
|
||||
static void qt_wire_mmap(QT *t, int64_t *wired, long *failed){
|
||||
if(!t->q8 && !t->q4) return;
|
||||
if(t->cuda_eligible) return; /* resident in VRAM; host range is dead weight */
|
||||
int64_t scale_b=(int64_t)t->O*4;
|
||||
int64_t weight_b=qt_bytes(t)-scale_b;
|
||||
void *wp=t->q8?(void*)t->q8:(void*)t->q4;
|
||||
if(weight_b>0){ if(mem_wire(wp,(size_t)weight_b)==0) *wired+=weight_b; else (*failed)++; }
|
||||
if(t->s && scale_b>0){ if(mem_wire(t->s,(size_t)scale_b)==0) *wired+=scale_b; else (*failed)++; }
|
||||
}
|
||||
static void pin_wire(Model *m){
|
||||
if(!mem_should_wire()) return;
|
||||
if(g_mmap){
|
||||
/* Wire the FINAL resident set only, after pin_load's GPU-upload pass
|
||||
* has already run -- qt_wire_mmap() skips cuda_eligible (VRAM-tier)
|
||||
* slots, so only the genuinely RAM-tier experts get locked. */
|
||||
Cfg *c=&m->c; double t0=now_s();
|
||||
for(int i=0;i<c->n_layers;i++) for(int z=0;z<m->npin[i];z++){
|
||||
ESlot *s=&m->pin[i][z];
|
||||
qt_wire_mmap(&s->g,&g_mmap_wired,&g_mmap_wire_failed);
|
||||
qt_wire_mmap(&s->u,&g_mmap_wired,&g_mmap_wire_failed);
|
||||
qt_wire_mmap(&s->d,&g_mmap_wired,&g_mmap_wire_failed);
|
||||
}
|
||||
fprintf(stderr,"[PIN] mlock (mmap): %.1f GB wired in physical RAM%s in %.0fs\n",
|
||||
g_mmap_wired/1e9, g_mmap_wire_failed?" (some allocations failed -- raise: ulimit -l unlimited)":"", now_s()-t0);
|
||||
return;
|
||||
}
|
||||
Cfg *c=&m->c; double t0=now_s(); int64_t wired=0; long failed=0;
|
||||
for(int i=0;i<c->n_layers;i++) for(int z=0;z<m->npin[i];z++){
|
||||
ESlot *s=&m->pin[i][z];
|
||||
|
||||
+29
-2
@@ -167,6 +167,30 @@ def discover_gpus():
|
||||
|
||||
|
||||
def physical_cpu_count():
|
||||
if sys.platform == "win32":
|
||||
# os.cpu_count() conta i processori logici (SMT): 2 thread/core saturano
|
||||
# le unita' AVX-512 e peggiorano il matmul. Contiamo i core fisici veri
|
||||
# con GetLogicalProcessorInformationEx(RelationProcessorCore).
|
||||
try:
|
||||
import ctypes
|
||||
k32 = ctypes.windll.kernel32
|
||||
need = ctypes.c_ulong(0)
|
||||
k32.GetLogicalProcessorInformationEx(0, None, ctypes.byref(need))
|
||||
buf = (ctypes.c_char * need.value)()
|
||||
if k32.GetLogicalProcessorInformationEx(0, buf, ctypes.byref(need)):
|
||||
raw, cores, off = bytes(buf), 0, 0
|
||||
while off + 8 <= need.value:
|
||||
relationship = int.from_bytes(raw[off:off + 4], "little")
|
||||
size = int.from_bytes(raw[off + 4:off + 8], "little")
|
||||
if size <= 0:
|
||||
break
|
||||
if relationship == 0: # RelationProcessorCore
|
||||
cores += 1
|
||||
off += size
|
||||
if cores:
|
||||
return cores
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass
|
||||
try:
|
||||
result = subprocess.run(["lscpu", "-p=core,socket"], text=True,
|
||||
capture_output=True, check=True, timeout=5)
|
||||
@@ -289,8 +313,11 @@ def environment_for_plan(plan, env=None, cuda_enabled=True):
|
||||
result = dict(env or {})
|
||||
result.setdefault("COLI_POLICY", plan["policy"]["name"])
|
||||
result.setdefault("OMP_NUM_THREADS", str(plan["cpu"]["physical_cores"]))
|
||||
result.setdefault("OMP_PROC_BIND", "spread")
|
||||
result.setdefault("OMP_PLACES", "cores")
|
||||
if sys.platform != "win32":
|
||||
# la libgomp di MinGW non supporta l'affinity su Windows
|
||||
# ("Affinity not supported on this configuration"): non impostarle li'.
|
||||
result.setdefault("OMP_PROC_BIND", "spread")
|
||||
result.setdefault("OMP_PLACES", "cores")
|
||||
if plan["policy"]["name"] == "balanced":
|
||||
result.setdefault("REPIN", "64")
|
||||
ram = plan["tiers"]["ram"]
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""env_for: default Windows misurati (DIRECT/PIPE/PILOT_REAL + blocco OMP).
|
||||
|
||||
Carica `coli` come modulo (ha la guardia __main__) e verifica il contratto:
|
||||
- win32: i tre default I/O e il blocco OMP sono setdefault
|
||||
- un override esplicito dell'utente vince sempre
|
||||
- COLI_NO_OMP_TUNE spegne SOLO il blocco OMP, non i default I/O
|
||||
- non-win32: env_for non tocca nulla di tutto questo
|
||||
"""
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
HERE = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
_loader = importlib.machinery.SourceFileLoader("coli_cli", str(HERE / "coli"))
|
||||
_spec = importlib.util.spec_from_loader("coli_cli", _loader)
|
||||
coli = importlib.util.module_from_spec(_spec)
|
||||
_loader.exec_module(coli)
|
||||
|
||||
|
||||
def args(**over):
|
||||
base = dict(model="X", policy="quality", ram=0, ngen=0, topp=0, topk=0,
|
||||
temp=None, repin=0, ctx=0, auto_tier=False, gpu=None, vram=0)
|
||||
base.update(over)
|
||||
return types.SimpleNamespace(**base)
|
||||
|
||||
|
||||
class EnvDefaultsTest(unittest.TestCase):
|
||||
def env_for_with(self, environ, platform):
|
||||
with mock.patch.dict(os.environ, environ, clear=True), \
|
||||
mock.patch.object(sys, "platform", platform):
|
||||
return coli.env_for(args())
|
||||
|
||||
def test_win32_sets_measured_defaults(self):
|
||||
e = self.env_for_with({}, "win32")
|
||||
self.assertEqual(e["DIRECT"], "1")
|
||||
self.assertEqual(e["PIPE"], "1")
|
||||
self.assertEqual(e["PILOT_REAL"], "1")
|
||||
self.assertEqual(e["OMP_WAIT_POLICY"], "active")
|
||||
self.assertNotIn("OMP_PROC_BIND", e) # MinGW libgomp: niente affinity
|
||||
|
||||
def test_explicit_override_wins(self):
|
||||
e = self.env_for_with({"DIRECT": "0", "PIPE": "0"}, "win32")
|
||||
self.assertEqual(e["DIRECT"], "0")
|
||||
self.assertEqual(e["PIPE"], "0")
|
||||
self.assertEqual(e["PILOT_REAL"], "1") # non overridden -> default
|
||||
|
||||
def test_kill_switch_scope_is_omp_only(self):
|
||||
e = self.env_for_with({"COLI_NO_OMP_TUNE": "1"}, "win32")
|
||||
self.assertNotIn("OMP_WAIT_POLICY", e)
|
||||
self.assertNotIn("OMP_NUM_THREADS", e)
|
||||
self.assertEqual(e["DIRECT"], "1") # i default I/O restano attivi
|
||||
self.assertEqual(e["PIPE"], "1")
|
||||
|
||||
def test_non_windows_untouched(self):
|
||||
e = self.env_for_with({}, "linux")
|
||||
for k in ("DIRECT", "PIPE", "PILOT_REAL", "OMP_WAIT_POLICY"):
|
||||
self.assertNotIn(k, e)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -112,8 +112,13 @@ class ResourcePlanTest(unittest.TestCase):
|
||||
self.assertEqual(env["COLI_CUDA"], "1")
|
||||
self.assertEqual(env["COLI_GPUS"], "1")
|
||||
self.assertEqual(env["OMP_NUM_THREADS"], str(plan["cpu"]["physical_cores"]))
|
||||
self.assertEqual(env["OMP_PROC_BIND"], "spread")
|
||||
self.assertEqual(env["OMP_PLACES"], "cores")
|
||||
if sys.platform == "win32":
|
||||
# MinGW libgomp: niente affinity su Windows, le chiavi non vanno emesse
|
||||
self.assertNotIn("OMP_PROC_BIND", env)
|
||||
self.assertNotIn("OMP_PLACES", env)
|
||||
else:
|
||||
self.assertEqual(env["OMP_PROC_BIND"], "spread")
|
||||
self.assertEqual(env["OMP_PLACES"], "cores")
|
||||
self.assertEqual(env["PIN_GB"], env["CUDA_EXPERT_GB"])
|
||||
|
||||
explicit_threads = environment_for_plan(plan, {"OMP_NUM_THREADS": "7",
|
||||
|
||||
@@ -10,8 +10,10 @@ from pathlib import Path
|
||||
|
||||
|
||||
SPEED_RE = re.compile(r"REPLAY decode:.*\| ([0-9.]+) tok/s")
|
||||
# accetta sia il formato storico "expert-disk 0.123s |" sia quello attuale
|
||||
# "expert-disk 0.123s service / 0.045s wait |" (glm.c profile_print)
|
||||
PROFILE_RE = re.compile(
|
||||
r"PROFILE: expert-disk ([0-9.]+)s \| expert-matmul ([0-9.]+)s "
|
||||
r"PROFILE: expert-disk ([0-9.]+)s(?: service / ([0-9.]+)s wait)? \| expert-matmul ([0-9.]+)s "
|
||||
r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s"
|
||||
)
|
||||
PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other")
|
||||
@@ -23,7 +25,9 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
|
||||
profile = PROFILE_RE.search(stdout)
|
||||
if not speed or not profile:
|
||||
raise RuntimeError(f"benchmark output missing\nstdout:\n{stdout}\nstderr:\n{stderr}")
|
||||
return float(speed.group(1)), [float(value) for value in profile.groups()]
|
||||
service, wait, *rest = profile.groups()
|
||||
disk = float(service) + (float(wait) if wait else 0.0)
|
||||
return float(speed.group(1)), [disk] + [float(value) for value in rest]
|
||||
|
||||
|
||||
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]:
|
||||
|
||||
Reference in New Issue
Block a user