url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #400 from JustVugg/dev

Browse Source 
Release v1.0.0
This commit is contained in:
Vincenzo Fornaro
2026-07-19 12:25:43 +02:00
committed by GitHub
parent 72d3d37231 e25aeecc4c
commit 819941ee2c
66 changed files with 5250 additions and 755 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/check.yml
+49
View File
@@ -0,0 +1,49 @@
# CI: run the repo's own dependency-free gate (`make check` = clean + portable
# CPU build + C unit suites + Python stdlib tests) on the three claimed
# platforms. No model downloads, no CUDA, no external deps — by design (#140).
name: check
on:
push:
branches: [main, dev]
pull_request:
branches: [main, dev]
jobs:
linux:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: make check
run: make -C c check
windows:
# The job that would have caught #68/#137 pre-merge: native MinGW-w64
# (MSYS2/UCRT64), the exact toolchain the README's Windows port targets.
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@v4
- uses: msys2/setup-msys2@v2
with:
msystem: UCRT64
update: false
install: >-
make
mingw-w64-ucrt-x86_64-gcc
mingw-w64-ucrt-x86_64-python
- name: make check
run: make -C c check
macos:
# clang; libomp for the threaded path (Makefile falls back to
# single-threaded automatically if it's ever missing).
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: install libomp
run: brew install libomp
- name: make check
run: make -C c check
.github/workflows/ci.yml
+72
View File
@@ -41,6 +41,78 @@ jobs:
-Xcompiler=-Wall,-Wextra
echo "CUDA syntax check passed"
windows-cuda-build:
# The Windows CUDA path has no coverage anywhere: `engine-cuda-syntax` above
# compiles with nvcc's *GCC* host on Linux, and check.yml's windows job is
# MinGW/UCRT64 CPU-only by design (#140). But nvcc on Windows requires MSVC
# as its host compiler — it does not accept MinGW — so `make cuda-dll` runs a
# toolchain nothing else in CI touches. That gap is not theoretical: every
# bug in #158 was a *build* failure on this path (MSVC rejects the GCC-style
# -Xcompiler=-Wall,-Wextra with "D8021 invalid numeric argument '/Wextra'",
# unresolvable CUDA_HOME/NVCC defaults, POSIX setenv in the kernel test), and
# #314 was CUDA_HOME with spaces — the layout the CUDA installer ships by
# default. Both classes are compile-time and need no GPU to catch.
#
# Build-only ON PURPOSE: GitHub's hosted runners have no NVIDIA device, so
# this job proves the Windows+MSVC CUDA build stays buildable, NOT that the
# kernels or the DLL loader behave on real silicon. That still needs hardware
# (see #157). Claiming otherwise would be the false confidence the
# engine-cuda-syntax comment above already warns about.
name: CUDA build (Windows, MSVC host)
# windows-2022, NOT windows-latest: the latest image now ships Visual Studio
# 18 (MSVC 14.5x), and CUDA's crt/host_config.h hard-errors on any host newer
# than VS 2022 ("Only the versions between 2017 and 2022 (inclusive) are
# supported"). That is a real constraint for every CUDA user on Windows, not
# a CI quirk — pinning tracks what the toolkit actually supports. Revisit when
# a CUDA release accepts VS 18; -allow-unsupported-compiler would only mask it.
runs-on: windows-2022
steps:
- uses: actions/checkout@v4
- name: MSVC environment (puts cl.exe on PATH for nvcc -ccbin)
uses: ilammy/msvc-dev-cmd@v1
- name: Install CUDA toolkit (compiler only)
uses: Jimver/cuda-toolkit@v0.2.19
with:
# Same pin as engine-cuda-syntax: v0.2.19's version table stops at
# 12.6.2. This installs to the default "C:\Program Files\NVIDIA GPU
# Computing Toolkit\..." path, so CUDA_HOME is space-bearing here —
# which is exactly the #314 regression this job would have caught.
cuda: '12.6.2'
method: network
# cudart as well as nvcc: unlike the Linux job (which only needs to
# *compile* backend_cuda.cu), cuda-dll links it, and on Windows the
# runtime headers/import lib ship as a separate installer component —
# with '["nvcc"]' alone this fails at `#include <cuda_runtime.h>`.
sub-packages: '["nvcc", "cudart"]'
- uses: msys2/setup-msys2@v2
with:
msystem: UCRT64
update: false
# inherit: cl.exe (msvc-dev-cmd) and nvcc (cuda-toolkit) are added to
# the *Windows* PATH by the steps above; without inheriting it the
# recipe's `command -v` guards fail inside the MSYS2 shell.
path-type: inherit
install: >-
make
mingw-w64-ucrt-x86_64-gcc
- name: make cuda-dll (nvcc + MSVC host)
shell: msys2 {0}
run: |
cd c
# CUDA_ARCH is pinned: the default is `native`, which asks the driver
# what card is present — there is none here, so it must be explicit.
# sm_80 matches engine-cuda-syntax and is supported by the 12.6 pin.
make cuda-dll CUDA_ARCH=sm_80
test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; }
echo "coli_cuda.dll built (MSVC host)"
- name: make glm CUDA_DLL=1 (host links backend_loader, not cudart)
shell: msys2 {0}
run: |
cd c
make glm CUDA_DLL=1
test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; }
echo "glm.exe built against the DLL loader"
web:
name: Web UI
runs-on: ubuntu-latest
.github/workflows/release.yml
+99
View File
@@ -0,0 +1,99 @@
name: Release
on:
push:
tags: ['v*']
permissions:
contents: write
jobs:
build:
strategy:
matrix:
include:
- os: ubuntu-latest
name: linux-x86_64
ext: ""
make_args: "ARCH=x86-64-v3"
- os: macos-latest
name: macos-arm64
ext: ""
make_args: ""
- os: windows-latest
name: windows-x86_64
ext: ".exe"
make_args: ""
shell: msys2
runs-on: ${{ matrix.os }}
defaults:
run:
shell: ${{ matrix.shell || 'bash' }}
steps:
- uses: actions/checkout@v4
- if: matrix.shell == 'msys2'
uses: msys2/setup-msys2@v2
with:
msystem: UCRT64
update: false
install: >-
make
mingw-w64-ucrt-x86_64-gcc
- if: matrix.os == 'macos-latest'
run: brew install libomp
- name: Build engine
run: |
cd c
make glm ${{ matrix.make_args }}
ls -lh glm${{ matrix.ext }}
- name: Package
run: |
TAG=${GITHUB_REF#refs/tags/}
mkdir -p dist
cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }}
cp c/coli dist/
cp c/version.py dist/
cp c/openai_server.py dist/
cp c/resource_plan.py dist/
cp c/doctor.py dist/
cp LICENSE dist/
cd dist
if [ "${{ matrix.ext }}" = ".exe" ]; then
7z a colibri-${TAG}-${{ matrix.name }}.zip *
else
tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz *
fi
- uses: actions/upload-artifact@v4
with:
name: colibri-${{ matrix.name }}
path: dist/colibri-*.*
release:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
path: artifacts
merge-multiple: true
- name: Create GitHub Release
env:
GH_TOKEN: ${{ github.token }}
run: |
TAG=${GITHUB_REF#refs/tags/}
# Extract the latest version section from CHANGELOG
NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md)
if [ -z "$NOTES" ]; then
NOTES="Release ${TAG}"
fi
gh release create "$TAG" artifacts/* \
--title "colibrì ${TAG}" \
--notes "$NOTES"
CHANGELOG.md
+53
View File
@@ -0,0 +1,53 @@
# Changelog
All notable changes to colibrì are documented here.
Format follows [Keep a Changelog](https://keepachangelog.com/).
## [1.0.0] — 2026-07-19
First tagged release. The engine has been running in production since late June
2026; this tag marks the baseline for semantic versioning going forward.
### Highlights
- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk
- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning
cache that pins your workload's hottest experts automatically
- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched
ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`)
- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention
on unified memory GPU
- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel-
pinned verification (`SPEC_PIN=1`)
- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded
queue, web dashboard (`coli web`)
- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown,
expert atlas 3-D galaxy
- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three
- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives
MTP/PIPE/NUMA/PIN settings with explanations
### Engine
- Token-exact validation against `transformers` oracle (teacher-forcing 32/32)
- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across
restarts (`.coli_kv`, zero re-prefill)
- DSA sparse attention (lightning indexer), faithfully implemented
- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive)
- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`)
- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket)
- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels
- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats
### Tools
- `coli convert` — FP8→int4 one-shard-at-a-time converter
- `coli doctor` — read-only setup diagnostics
- `coli plan` — resource planner with auto-tune prescription
- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks
- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts
### Community
- 30+ hardware datapoints in the benchmark tracker
- Contributions from 20+ authors across engine, docs, tooling, and ports
README.md
+152 -587
View File
@@ -2,12 +2,16 @@
<img src="assets/colibri.svg" width="500" alt="colibrì — tiny engine, immense model">
</p>
<p align="center">
English · <a href="README.zh-TW.md">繁體中文</a>
</p>
**Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk.
Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM,
RAM, and storage as one managed memory hierarchy. Insufficient fast memory may
reduce speed, but the default policy never silently changes model precision or
router semantics.
Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM,
and storage as one managed memory hierarchy. Insufficient fast memory may reduce
speed, but the default policy **never silently changes model precision or router
semantics**.
```
$ ./coli chat
@@ -17,14 +21,14 @@ $ ./coli chat
◆ Ciao! 😊 Come posso aiutarti oggi?
```
## See it running
<p align="center">
<img src="docs/media/colibri-dashboard.png" width="900" alt="colibrì web dashboard — live metrics, hardware panel, expert tiers">
</p>
<p align="center"><em>The web dashboard (<code>./coli web</code>): a 744B model answering at 4+ tok/s end-to-end on 6× RTX 5090 —
with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.</em></p>
<p align="center"><em>The web dashboard (<code>./coli web</code>): a 744B model at <strong>4 tok/s, TTFT 1.6 s, disk 0</strong> —
full expert residency on 6× RTX 5090, with live token metrics, the per-turn time breakdown,
the VRAM/RAM/disk tier bar and the live mini-brain in the corner.</em></p>
<p align="center">
<img src="docs/media/colibri-brain.png" width="900" alt="the Brain page — 19,456 experts as a live cortex">
@@ -33,613 +37,174 @@ with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.
brightness is routing heat, and every expert routed in a turn flashes white. Hovering shows the expert's
<a href="https://github.com/JustVugg/colibri/issues/175">measured topic affinity</a>.</em></p>
## Contents
- [The idea](#the-idea)
- [See it running](#see-it-running)
- [What's implemented](#whats-implemented)
- [Honest numbers](#honest-numbers-wsl2-12-cores-25-gb-ram-nvme-via-vhdx)
- [Download the model](#download-the-model)
- [Web dashboard](#web-dashboard)
- [Got a better machine?](#got-a-better-machine-try-it--heres-what-to-expect)
<p align="center">
<img src="docs/media/colibri-atlas.png" width="900" alt="the Atlas page — the measured expert atlas as a 3-D galaxy">
</p>
<p align="center"><em>The <strong>Atlas</strong> page: the <a href="https://github.com/JustVugg/colibri/issues/175">measured expert atlas</a>
as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic
(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.</em></p>
## The idea
A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So:
A 744B Mixture-of-Experts model activates only ~40B parameters per token — and
only ~11 GB of those change from token to token (the routed experts):
- the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB);
- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2.
<p align="center">
<img src="docs/media/sparse.png" width="880" alt="only ~5.4% of parameters are active per token">
</p>
The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below).
So the model doesn't need to *fit* in fast memory — it needs to be **placed**:
## What's implemented
- the **dense part** (attention, shared experts, embeddings — ~17B params) stays
**resident in RAM at int4** (~9.9 GB);
- the **19,456 routed experts** (75 MoE layers × 256 + the MTP head, ~19 MB each
at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a
per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier.
- **Faithful GLM-5.2 (`glm_moe_dsa`) forward** — validated token-exact against a `transformers` oracle (teacher-forcing 32/32, greedy 20/20 on a tiny-random model with the real architecture).
- **MLA attention** (q/kv-LoRA, interleaved partial RoPE) with **compressed KV-cache**: 576 floats/token instead of 32,768 (57× smaller — GLM-5.2 has 64 heads and no GQA).
- **DeepSeek-V3-style sigmoid router** (noaux_tc, routed_scaling_factor), shared expert, first-3-dense layers.
- **Native MTP speculative decoding** — GLM-5.2's own multi-token-prediction head (layer 78) drafts tokens that the main model verifies in one batched forward. **The head must be int8** (the converter does this by default): at int4 draft acceptance collapses to 0–4% and speculation never engages; at int8 it's 39–59% acceptance, **2.2–2.8 tokens/forward** (community-measured, [#8](https://github.com/JustVugg/colibri/issues/8)). Lossless *in exact arithmetic* — but **not byte-identical to non-speculative greedy in practice** ([#100](https://github.com/JustVugg/colibri/issues/100)). This isn't MTP-specific: colibrì's quantized integer kernels are shape-dependent, so any batched (S>1) or GPU forward rounds slightly differently from the single-token path, and int4 GLM-5.2 sits close enough to argmax ties that such a rounding change can flip a token. MTP, the CUDA expert tier, and batched prefill are three different ways to trip the same sensitivity (community-confirmed in #100: swapping only the kernel family forks greedy output on 3/5 prompts, with **zero speculation**). Every emitted token is still the argmax of a *valid* forward — the continuation stays correct — it just isn't the same stream. For byte-exact reproducibility: `DRAFT=0` (no speculation), plus `IDOT=0 COLI_CUDA=0` if you also want kernel-family/GPU independence. Under sampling, rejection sampling keeps the distribution correct. Honest caveat from the same measurement: on a **cold** cache each verified draft routes to extra experts (~660 → ~1100 expert-loads/token), so speculation can be a net *time* loss until the cache/pin warms up.
- **Grammar-forced speculative drafts** (`GRAMMAR=file.gbnf`, [#48](https://github.com/JustVugg/colibri/issues/48)) — on constrained-output workloads (JSON/NDJSON, function calling, structured extraction) the grammar itself is a third draft source: wherever it admits exactly **one** legal byte (braces, quotes, key names, enum bodies), that forced span is tokenized and injected as pre-accepted drafts with ~1.0 acceptance — no draft head, no lookup table, and it engages even with the int4 MTP head from [#8](https://github.com/JustVugg/colibri/issues/8). It never constrains sampling: forced spans are verified in the same batch-union forward as any draft, so a wrong or out-of-sync grammar cannot change the output — worst case is rejected drafts, and an adaptive guard turns the source off below 50% acceptance. Byte-level GBNF subset (literals, char classes, `| ( ) ? * +`, comments); `GRAMMAR_DRAFT=n` caps the forced span per forward (default 24). Composes with `DRAFT`/MTP, which fill the free-text gaps between forced spans. Full reference — mechanism, measured A/Bs, when it pays, prior art: [docs/grammar-draft.md](docs/grammar-draft.md).
- **True sampling** — temperature + nucleus, defaults tuned for int4 reality (0.7 / 0.90; the official 1.0 / 0.95 samples quantization noise from the tail).
- **Integer-dot kernels** (Q8_0-style int8 activations, AVX2 `maddubs`): int8 matmuls 1.4–2.5× faster (119 GFLOP/s measured), int4 1.8× in batch — routing decided per shape by measurement (int4 single-row stays f32: it measured slower).
- **MLA weight absorption** (DeepSeek trick) for decode: no per-token k/v reconstruction — the query absorbs `kv_b`, context is projected after attention. Validated exact: TF 32/32 and generation 20/20 with absorption forced everywhere.
- **Async expert readahead**: while one block of experts is being multiplied, the kernel is already reading the next (`WILLNEED`).
- **Quantization kernels**: int8 / packed int4 / packed int2, per-row scales, AVX2, dequant-on-use. Packing validated bit-identical to the int8 container.
- **DSA sparse attention** — GLM-5.2's lightning indexer, faithful to the reference `glm_moe_dsa` modeling: per-layer top-2048 causal key selection (full/shared indexer layers), auto-detected from the `out-idx-*` weights (`--indexer` converter mode, ~189 MB extracted from the FP8 repo). Validated exact: forcing the selection to keep every key reproduces dense attention token-for-token. `DSA=0` disables, `DSA_TOPK` overrides.
- **KV-cache persistence** — conversations reopen **warm** across engine restarts: serve mode appends the compressed MLA KV to `.coli_kv` after every turn (~182 KB/token, crash-safe) and resumes it at startup with zero re-prefill. Validated byte-identical to an uninterrupted session. `KVSAVE=0` disables.
- **Router-lookahead prefetch** (`PILOT=1`, experimental) — the next layer's routing is 71.6% predictable from the current layer's post-attention state (measured); a dedicated I/O thread prefetches those experts while the current layer computes.
- **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it.
- **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges).
- **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires.
- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable.
The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python
at runtime, no GPU required.
## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX)
## How it works
Detailed GPU experiment: [GLM-5.2 on 6x RTX 5090](docs/experiments/glm52-6x5090-2026-07-12.md) — full expert residency across VRAM+RAM reaches 6.84 tok/s single-request decode.
### The per-token path
| metric | value |
|---|---|
| model on disk (int4 container) | ~370 GB |
| resident RAM (dense, int4) | 9.9 GB |
| load time | ~30 s |
| peak RSS during chat | ~20 GB (auto-capped) |
| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) |
| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold |
| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) |
<p align="center">
<img src="docs/media/token-path.png" width="880" alt="route → union → place → overlap → learn">
</p>
This is not fast. It is a 744B frontier-class model **answering correctly on a machine that costs less than one H100 fan**. Warm cache, pinned hot experts and MTP push the useful-response latency down considerably; the physics of the disk does the rest.
Every layer of every token walks the same five steps. The design goal is that
**placement only ever decides speed** — the router's decisions and the weights'
precision are the same whether an expert answered from VRAM or from disk.
### SSD note
Cold starts are heavy on random reads (~11 GB/token), but reads don't meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns under heavy use are (1) **swap traffic** if the system runs out of RAM (writes do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed to stay clear of swap) and (2) **sustained thermals**: hours at full read duty cycle will heat cheaper drives. Monitor drive temperature and health.
### One memory hierarchy instead of one memory requirement
## Download the model
<p align="center">
<img src="docs/media/tiers.png" width="880" alt="VRAM / RAM / NVMe three-tier expert residency">
</p>
A pre-converted **GLM-5.2 int4** model for colibrì is available on Hugging Face — **use the version with the int8 MTP heads** (matey-0's clone):
The same engine spans the whole range: on a 25 GB laptop everything streams from
disk (slow but correct); on a large host the entire expert set becomes resident
(`CUDA_EXPERT_GB=auto PIN_GB=all`) and disk drops out of the decode path
entirely. Between the tiers sits a **learning cache**: the engine records which
experts *your* workload routes to (`.coli_usage`, updated every turn) and pins
the hottest ones automatically — colibrì literally gets faster the more you use
it. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights across
memory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)).
### Never wait for the disk twice
Misses are expensive, so the engine spends most of its cleverness avoiding and
overlapping them: each expert's three matrices are stored adjacent and read in
one `pread`; a bounded async I/O pool (`PIPE=1`, default) loads missing experts
while resident ones compute; batched positions read each unique expert once
(**batch-union**); and a router-lookahead thread (`PILOT=1`) prefetches the next
layer's experts — routing is measurably **71.6% predictable one layer ahead**.
On GPUs, the resident pipeline (`COLI_CUDA_PIPE=2`) keeps the residual stream
on-device across layers so the CPU expert loop runs uninterrupted; on Apple
Silicon an experimental [Metal backend](docs/metal.md) does the batched expert
math on the unified-memory GPU.
### Faithful model, compressed state
The forward pass is validated **token-exact against a `transformers` oracle**
(teacher-forcing 32/32). MLA attention stores a compressed KV state — 576
floats/token instead of 32,768 (**57× smaller**) — and persists it across
restarts (`.coli_kv`): conversations reopen warm with zero re-prefill,
byte-identical to an uninterrupted session. DSA sparse attention (GLM-5.2's
lightning indexer) is implemented faithfully and validated by forcing full-key
selection to reproduce dense attention exactly.
### Speculative decoding, honestly
GLM-5.2's native MTP head drafts tokens that the main model verifies in one
batched forward — 2.2–2.8 tokens/forward when it pays. Two hard-won rules ship
as defaults: the MTP head must be **int8** (int4 heads collapse to 0–4%
acceptance, [#8](https://github.com/JustVugg/colibri/issues/8)), and draft and
verify must compute **the same function** — `SPEC_PIN=1` pins both to one
kernel family ([#163](https://github.com/JustVugg/colibri/issues/163) is the
full forensic story). Grammar-forced drafts
([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) add ~free acceptance on
constrained JSON output. Whether speculation is a net win depends on your
cache temperature — measure, and use `DRAFT=0` when it doesn't pay.
## What it achieves
<p align="center">
<img src="docs/media/ladder.png" width="880" alt="measured decode speed by hardware class">
</p>
Same engine, same int4 container — the hardware only changes where the experts
live. Highlights from the [full benchmark tables](docs/benchmarks.md):
- **6× RTX 5090, full residency:** 5.8–6.8 tok/s decode, TTFT ~13 s
([experiment log](docs/experiments/glm52-6x5090-2026-07-12.md));
- **128 GB CPU-only desktop:** ~1.8 tok/s warm ([#200](https://github.com/JustVugg/colibri/issues/200));
- **single RTX 5070 Ti laptop-class box:** 1.07 tok/s via the GPU-resident
pipeline ([#273](https://github.com/JustVugg/colibri/issues/273));
- **25 GB dev box:** 0.05–0.1 tok/s cold — the proven floor where this project
started, and still the honest baseline.
Quality is measured, not assumed: the int4 container's quantization cost and the
scale-granularity/rotation ablations live in
[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) and
[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81).
## Get started
### 1. Get the model
A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the
version with the int8 MTP heads**:
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
> ⚠️ **The MTP head must be int8.** The original mirror ([jlnsrk/GLM-5.2-colibri-int4](https://huggingface.co/jlnsrk/GLM-5.2-colibri-int4)) ships **int4** MTP heads, which give **0% draft acceptance** — speculation silently never engages and you lose the ~2× MTP lever. This is the single most common "why is MTP stuck at 0%?" report ([#8](https://github.com/JustVugg/colibri/issues/8), [#102](https://github.com/JustVugg/colibri/issues/102)). The int8 head gives the measured **39–59% acceptance**. matey-0's clone above is the original int4 model with the three `out-mtp-*` files already swapped to int8 — download that one and you're done.
>
> Check what you have: `ls -l <model>/out-mtp-*`
> · **int8 (correct):** `3527131672 / 5366238584 / 1065950496`
> · **int4 (0% acceptance):** `1765523544 / 2686077736 / 536747200` — if you see these, replace just those three files from the int8 mirror.
> ⚠️ The original mirror ships int4 MTP heads → 0% draft acceptance
> ([#8](https://github.com/JustVugg/colibri/issues/8)). Check yours:
> `ls -l <model>/out-mtp-*` — int8 (correct) is `3527131672 / 5366238584 / 1065950496`.
Download the repository and point `COLI_MODEL` to its directory:
Or convert from the FP8 source yourself — one resumable command that never needs
the full 756 GB on disk at once:
```bash
COLI_MODEL=/path/to/GLM-5.2-colibri-int4-with-int8-mtp ./coli chat
cd c && ./setup.sh # checks gcc/OpenMP, builds, self-tests
./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time)
```
This skips the FP8 → int4 conversion step entirely. Thanks to DatPat for the original mirror and matey-0 for the int8-head clone.
### Quick start
### 2. Run it
```bash
cd c
./setup.sh # checks gcc/OpenMP, builds, self-tests
# ONE command does everything model-side: downloads GLM-5.2-FP8 shard by shard
# (never needs the full 756 GB at once), converts to the int4 container, then
# converts the MTP head for speculative decoding. Resumable at any point.
# Conversion (only) needs python with: pip install torch safetensors huggingface_hub numpy
./coli convert --model /nvme/glm52_i4 # ~400 GB free on a real ext4/NVMe path
# chat — RAM budget, expert cache and MTP are all detected automatically:
COLI_MODEL=/nvme/glm52_i4 ./coli chat
COLI_MODEL=/nvme/glm52_i4 ./coli chat # RAM budget, cache and MTP auto-detected
COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk placement
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check
./coli web --model /nvme/glm52_i4 # API + web dashboard on one port
./coli serve --model /nvme/glm52_i4 # OpenAI-compatible API only
```
Inspect the planned storage hierarchy before loading the model:
The engine at runtime is pure C — python is only used by the one-time converter
and the optional API gateway.
```bash
COLI_MODEL=/nvme/glm52_i4 ./coli plan
COLI_MODEL=/nvme/glm52_i4 ./coli plan --gpu 0,1 --ram 128 --vram 48 --json
### 3. Go deeper
# apply the bounded plan to the normal runner
COLI_MODEL=/nvme/glm52_i4 ./coli chat --auto-tier
```
`coli plan` reads only safetensors headers and reports the model's exact dense/expert
footprint, runtime RAM reserve, safe expert-cache cap, and bounded VRAM hot tier. Its
versioned JSON output is intended to be shared by the CLI, API server, Web UI, and
desktop shell; it does not allocate model tensors or start inference.
`--auto-tier` applies the same plan to `chat`, `run`, `serve`, and benchmarks. It
sets the RAM budget and context immediately; the VRAM tier is enabled only when
the current `glm` binary is linked with CUDA. Explicit flags and environment
variables keep precedence over automatic values.
Before loading the model, `coli doctor` performs a read-only readiness check and
explains whether the selected Disk/RAM/VRAM placement is runnable:
```bash
COLI_MODEL=/nvme/glm52_i4 ./coli doctor
COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json
```
Doctor validates the model directory, config, tokenizer, safetensors headers,
engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the
same placement budget used by `coli plan`. It never starts `glm`, reads tensor
payloads, imports a model framework, or creates a CUDA context. The versioned JSON
report uses stable check IDs for automation. Warnings keep exit status 0; missing
requirements or an unsafe RAM projection return 1, while invalid CLI values return 2.
The engine at runtime is pure C — python is only used by the one-time converter.
### Windows 11 (native, no WSL)
colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port adds
a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the Windows API
(pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned allocation, MoveFileEx rename,
GlobalMemoryStatusEx RAM detection). All platform differences stay in `compat.h`; the
engine source is unchanged.
**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. Tested with
GCC 16.1.0 (x86_64-ucrt-posix-seh).
```powershell
# One-time toolchain install (pick one):
scoop install mingw-winlibs # portable, no shell needed
# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2
# Build (from c/ directory):
make glm.exe # GLM-5.2 engine (static, no DLL dependencies)
make olmoe.exe # OLMoE engine (same shims)
make iobench.exe # disk I/O benchmark
make test-c # run C tests
make test-python # run Python tests (requires python)
# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8
# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster
# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out;
# build for THIS machine to enable it:
make glm.exe ARCH=native # banner prints "idot: avx-vnni"
# Verify (tiny model, 2.4 MB):
pip install torch transformers safetensors huggingface_hub
python tools/make_glm_oracle.py # generate tiny oracle
SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions"
# Run with real model:
SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference
python coli chat --model D:\glm52_i4 # interactive chat
python coli serve --model D:\glm52_i4 # OpenAI-compatible API
```
**Warmup (overnight cache priming):** the engine's expert cache learns from
your workload. The included `warmup.ps1` script runs `coli run` in a loop with
diverse prompts to build the `.coli_usage` histogram unattended, so the next
real session starts with a large, accurate hot-expert pin. Each run saves usage
atomically on clean completion.
```powershell
.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress
```
**NVIDIA GPU (optional, via runtime DLL):** on Windows the engine is built with
MinGW gcc but CUDA kernels require MSVC + nvcc. The split is clean: build the
CUDA backend into a standalone `coli_cuda.dll` (nvcc + MSVC), then the host
`glm.exe` loads it at runtime via `LoadLibrary` (`c/backend_loader.c`). The host
never links cudart directly; if the DLL is absent the engine falls back to CPU
without error.
```powershell
# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH.
# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or
# "x64 Native Tools Command Prompt for VS"):
make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120
# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and
# links backend_loader.o instead of cudart):
make glm.exe CUDA_DLL=1 ARCH=native
# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM):
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8"
python coli chat --model D:\glm52_i4 --topp 0.7
```
The DLL exports 11 `extern "C"` symbols (`coli_cuda_init`, `coli_cuda_matmul`,
etc.); `backend_loader.c` resolves them via `GetProcAddress` on first use.
`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the
MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match
your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series,
`sm_89` for Ada / RTX 40-series).
**Status:** Phase 1 complete (compiles, correct, static-linked). The Windows
GPU tier (runtime `coli_cuda.dll` via `LoadLibrary`) is implemented and
verified on RTX 50-series (sm_120). O_DIRECT (Phase 2) and full-model
validation against the transformers oracle remain separate workstreams.
### OpenAI-compatible API
`coli serve` keeps one model process loaded and exposes a text-only OpenAI-compatible
HTTP API. The gateway uses only the Python standard library; inference still runs in
the same dependency-free C engine.
```bash
cd c
COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \
--host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri
curl http://127.0.0.1:8000/v1/chat/completions \
-H 'Authorization: Bearer local-secret' \
-H 'Content-Type: application/json' \
-d '{
"model": "glm-5.2-colibri",
"messages": [{"role": "user", "content": "Hello"}],
"stream": true
}'
```
Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
completion requests support JSON responses, SSE streaming, usage counts,
`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
`reasoning_effort` field also enables it unless set to `none`.
The first version is deliberately text-only and serves one generation at a time:
the 744B model stays in one persistent process, so concurrent HTTP requests queue
instead of loading duplicate model copies. Tools, image/audio input, custom stop
sequences, log probabilities, and token penalties return an explicit error rather
than being silently ignored. The default bind address is localhost; set
`COLI_API_KEY` before exposing the server beyond the machine.
Browser access from the Vite development server and Tauri local origins is enabled
by default. Repeat `--cors-origin https://your-ui.example` to allow another exact
origin, or use `--cors-origin '*'` only on a trusted local network.
The engine owns one mutable KV context, so HTTP generation uses a bounded FIFO
admission queue instead of pretending to run unsafe parallel sequences. Configure it
with `--max-queue N` (default 8) and `--queue-timeout SECONDS` (default 300), or the
`COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment variables. Saturated and timed-out
requests receive OpenAI-shaped HTTP 429 errors before streaming headers are sent.
`GET /health` exposes active/queued/completed/rejected counters, and successful
generation responses include `x-colibri-queue-wait-ms`.
### Isolated KV contexts
`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. Requests
select one with the optional integer `cache_slot` field; ordinary OpenAI clients omit
it and keep the original slot 0 behavior.
```json
{
"model": "glm-5.2-colibri",
"messages": [{"role": "user", "content": "Continue this conversation"}],
"cache_slot": 1
}
```
Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and
crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine still executes
one sequence at a time; this establishes explicit KV ownership without pretending that
threaded HTTP is continuous batching. RAM admission accounts for every configured slot.
Use `COLI_KV_SLOTS=N` as the environment equivalent. Start with a small value: at the
default 4096-token context, every slot costs hundreds of MB.
### Experimental Metal backend (Apple Silicon)
On Apple Silicon the decode profile is matmul-bound, and unified memory removes the
PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì has an
opt-in Metal backend that runs the **routed-expert SwiGLU (batched, zero-copy from
the RAM slabs)**, the **fused decode attention** (full MLA layer in one command
buffer, S≤4), and **prefill's large GEMMs** on the GPU. Token-exact vs the CPU path.
```bash
cd c
make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime)
make metal-test # standalone kernel/attention correctness vs CPU reference
COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96
```
Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal **0.42 tok/s (~1.4×)**
(best config adds `DIRECT=1`; ~3× vs this machine's first cold run).
Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a loss —
everything is batched into few command buffers per layer, and the resident experts'
GPU work is submitted *before* the missed experts' disk reads so I/O and compute
overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row threshold (default 16).
Streaming, cache, MTP, DSA and the persistence formats are unchanged; every GPU
path falls back to the CPU per-block on any fault. Numerics are dequant→f32-MAC
(same as the CUDA tier); greedy outputs are byte-identical to the CPU engine.
### Experimental resident CUDA backend
colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming
experts deliberately remain on the original CPU path for now: copying an expert
from NVMe to the GPU on every use would only replace the disk bottleneck with a
PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused.
```bash
cd c
make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness
make CUDA=1
# optional dense-path experiment (hot experts are configured below)
COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4
```
Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under
`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). `CUDA_ARCH=native`
builds for the GPU in the current machine; set an explicit architecture when
cross-compiling. Requesting CUDA with a CPU-only binary, an invalid device, or
an unavailable runtime fails at startup instead of silently falling back.
The normal `make` build and runtime behavior are unchanged. CUDA defaults to an
expert-only accelerator. `CUDA_DENSE=1` additionally distributes resident
dense/attention projection tensors round-robin across the selected devices;
their projected footprint is reserved before the expert tier is placed. On six
RTX 5090s with a 150 GB expert tier, a warmed two-request/64-token GLM-5.2 run
improved from 1.650 to 2.157 aggregate tok/s (+30.8%) while retaining the full
expert tier. Treat this as an opt-in until the projected dense set and the 2 GB
per-device runtime reserve fit the target GPUs.
A measured `PIN` profile can promote its hottest experts into the persistent
VRAM tier while keeping the rest in RAM:
```bash
STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first
COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \
PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4
# multi-GPU expert tier, 150 GB total budget across six 32 GB devices
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \
CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \
SNAP=/nvme/glm52_i4 ./glm 64 4 4
# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \
CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \
SNAP=/nvme/glm52_i4 ./glm 64 4 4
```
Selected experts are uploaded during startup, so capacity failures occur before
inference and the log reports their exact tensor footprint. The budget is clamped
against free VRAM after reserving the projected dense resident set and 2 GB of
runtime headroom per selected device. With `COLI_GPUS`, `CUDA_EXPERT_GB` is a
total budget across the device set; experts are assigned whole to the
least-loaded device that can hold them. Multi-GPU runs also default to
`PIN_FILL=1`: the measured hot set is placed first, then unused VRAM is filled
with zero-heat experts. `CUDA_RELEASE_HOST=1` (the multi-GPU default) releases
the RAM copy after a successful upload and reloads it from disk only if CUDA
later fails. Set either variable to `0` to restore the conservative behavior.
When host backing is released, placement is disjoint and staged: the hottest
prefix is loaded, uploaded to VRAM, and freed before the next-ranked suffix is
loaded into RAM. `PIN_GB` therefore describes the combined ranked set rather
than duplicate RAM and VRAM copies. On a 256 GB dual-socket host, moving from a
150 GB VRAM + 130 GB RAM placement to 150 GB VRAM + 150 GB RAM raised fixed-token
replay from 1.87 to 2.16 tok/s (+15.7%), reduced expert disk wait from 5.144s to
3.948s, and kept the projected RAM peak below `RAM_GB=226`. The cache cap adjusts
down automatically (54 to 40 in that run) so the larger pinned tier does not exceed
the process budget. Start lower on hosts with less available RAM.
`CUDA_EXPERT_GB=auto` fills each selected device only up to its measured free
memory minus projected dense tensors and 2 GB of runtime headroom. `PIN_GB=all`
then loads every remaining routed expert into RAM, eliminating decode-time disk
misses when the host budget permits it. The regular `RAM_GB` guard still clamps
the per-layer working cache and rejects unsafe projections; this mode is intended
for dedicated high-memory inference hosts, not desktops running other workloads.
On a dedicated 251 GiB host with six RTX 5090s, this mode selected a 176.7 GB
VRAM expert tier and a 191.3 GB RAM tier (all 19,456 experts resident). The
mode also adapts the VRAM tier every 16 emitted tokens by swapping hot RAM
experts into existing GPU slots. A real 64-token greedy GLM-5.2 generation
measured **6.00 tok/s decode**, up from
2.20 tok/s end-to-end with the earlier 150 GB tier; expert hit rate was 100%
and disk wait was zero. Prompt prefill is reported separately. This is a
host-specific capacity result, not a portable default.
Text-mode timing reports prefill separately from decode. The decode rate starts
after the prompt KV is built, so it is comparable to `REPLAY` throughput without
hiding time-to-first-token.
MTP speculation defaults off on CUDA because cold draft routes increase expert
traffic; an explicit `DRAFT=n` still overrides the default.
On six RTX 5090 32 GB cards with GLM-5.2 int4, a 150 GB hot-first tier sustained
0.94 token/s over a 64-token varied prompt (87.8% expert hit rate), and reached
1.64 token/s on a warmed short prompt (99.3% hit rate). The same capacity filled
without routing heat managed only 0.29 token/s, so profile quality matters more
than raw VRAM capacity. These are single-run engineering measurements, not a
portable performance guarantee.
Current limitations: devices use independent contexts and synchronous
host-staged activation copies—there is no P2P/NCCL dependency yet. Independent
expert groups execute concurrently across devices, but a single expert is not
sharded. The kernels are correctness-first custom kernels rather than
cuBLAS/Tensor Core kernels.
For a reproducible backend A/B without the full checkpoint, generate the
deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay:
```bash
cd c
python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
```
The fixture has random weights and is not a language model. It exists only to
preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only
CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
### Web interface
`web/` contains a community-contributed browser UI (React + TypeScript, a pure
API client — it never touches the engine directly):
```bash
cd web
npm ci && npm run dev # then point it at an OpenAI-compatible endpoint
```
It speaks the standard OpenAI Chat Completions protocol with SSE streaming, so it
works against the colibrì OpenAI-compatible server (in review, #21) or any other
compatible endpoint. Nothing leaves the endpoint you configure. The terminal
`coli chat` remains the first-class interface.
Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `URING=1` Linux-only batched expert I/O (implies `PIPE=1`; also batches `PILOT_REAL`), `PIPE=0` disable the async expert-load pool (**default ON on Windows** — overlaps expert `pread` with the matmul so the CPU isn't idle waiting on the SSD; measured −18% disk service time), `RAM_GB=<n>` claim more RAM for the expert cache than the conservative auto-detect (e.g. `RAM_GB=31` on a 32 GB host raises the cache cap and hit rate measurably), `CAP_RAISE=0` don't auto-grow the expert cache.
### Resource policy
`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing
(disk) tiers, the reason for each placement, and the expected bottleneck. The
default `--policy quality` and `--policy balanced` modes preserve checkpoint
quantization and router decisions unless `--topk` or `--topp` is passed; those
explicit lossy overrides print a warning and proceed.
Auto-tier plans size OpenMP from physical cores and bind workers across cores.
Memory-bound quantized kernels can regress sharply when SMT siblings compete
for limited memory channels; explicit `OMP_*` settings always take precedence.
```bash
coli plan --model /models/glm52_i4 --policy quality
coli run --auto-tier --policy quality "Explain MoE offloading"
# Explicit research-only router reduction:
coli run --policy experimental-fast --topk 4 "Benchmark prompt"
```
Disk is an immutable recovery source, not a normal decode target. If the plan
leaves cold expert bytes on disk, speed depends on cache hit rate; output
quality does not.
Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute
while missing experts are loaded in a bounded background I/O pool, then the
cold results join before the layer completes. The pool engages only under
`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports
both disk service time and the smaller foreground-visible wait time so overlap
is explicit rather than credited as unexplained speedup.
`--policy balanced` enables lossless live placement (`REPIN=64`). At safe
request boundaries, a per-layer LFRU score combines decaying session frequency
with recent access and replaces at most four sufficiently colder pinned
experts. `--policy quality` leaves live replacement off by default; `REPIN=0`
always disables it. Persistent `.coli_usage` history and session-local LFRU
state remain separate.
For single-token q4 CPU experts, gate and up projections share one OpenMP
dispatch while retaining the same per-row AVX2/NEON arithmetic. This removes
one thread-team launch per RAM expert without activation requantization or a
lower-precision fallback. It is a stepping stone toward a persistent native
CPU expert pool, not a replacement for one.
**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.**
**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome.
**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it.
**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying
session heat map replaces cold pinned experts with hotter streamed experts. Replacement
loads the expert from disk into the existing RAM slot; GPU-backed slots immediately
refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier
thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed.
**Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it.
## Web dashboard
One command serves the OpenAI-compatible API **and** the web console on the same port, then opens your browser when the engine is ready:
```bash
cd web && npm install && npm run build # once
./coli web --model <model-dir>
```
What you get:
- **Chat** with live metrics: a flashing token counter while generating, then tok/s, time-to-first-token, prompt→completion counts and queue wait;
- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the scheduler, and the live expert-tier bar — how many of the 19,456 experts sit in VRAM / RAM / disk right now;
- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = tier, brightness = routing heat, and the experts routed in each turn flash white and decay — you watch the model think. Hover any cell for its tier, heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175) (specialists for code, Chinese, math, law… live in layers 11–22).
The dashboard talks to the engine over two tiny protocol lines (`TIERS`, `EMAP`/`HITS`) and plain JSON endpoints — nothing heavier than the engine itself.
## Got a better machine? Try it — here's what to expect
colibrì was built on deliberately humble hardware (12 cores, 25 GB RAM, an older DRAM-less NVMe behind a WSL2 VHDX that measured ~1 GB/s random on *this* drive — note WSL2 VHDX is not inherently slow: a community 5090 box measured 10.5 GB/s O_DIRECT through one, [#101](https://github.com/JustVugg/colibri/issues/101)). **Every one of those constraints is a knob your machine can turn up.** The engine needs: Linux (or WSL2), macOS, or **Windows 11 natively (MinGW-w64)**; gcc with OpenMP, AVX2, ≥16 GB RAM, and the ~370 GB int4 model on a local NVMe (ext4/NTFS — never a network/9p mount).
**How to test it, in order:**
```bash
cd c && ./setup.sh # build + architecture self-test (expects 32/32)
# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads):
gcc -O2 -fopenmp iobench.c -o iobench
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache)
# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box
# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and
# run it on a shard you haven't touched this session (a prior buffered run caches its pages).
# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't
# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right
# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read.
# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS):
COLI_MODEL=/path/to/glm52_i4 ./coli chat
# 3) record expert usage, then pin the hottest experts in your spare RAM:
STATS=stats.txt ./coli chat
PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
# 4) quality benchmarks (MMLU/HellaSwag/ARC):
./coli bench
```
**Back-of-envelope predictions** (decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP speculation roughly halves the effective cost *once the cache is warm*; RAM turns cold reads into free cache hits):
| machine | expected |
| topic | doc |
|---|---|
| this dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline |
| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s |
| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s |
| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels |
| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier |
These are estimates, not measurements — if you run colibrì on serious hardware, **please open an issue with your numbers**: real datapoints from better machines are exactly what this project needs next.
### Community benchmarks (measured)
Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active:
| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured |
|---|---|---|---|
| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB |
| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB |
| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB |
| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run |
| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output · fastest datapoint yet (still on the pre-rebase Metal branch) |
| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) — entry Apple Silicon on a third the RAM beats the 32-core 9950X row |
| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB |
| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB |
| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk |
| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** |
| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul |
| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% · fastest non-Apple datapoint |
| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** |
| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** (no AVX-512/VNNI on Zen3) |
| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint, port validated |
| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** · MTP-off wins disk-bound · fastest x86 datapoint yet |
| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · first Strix Halo / gfx1151 datapoint (unified memory: no discrete VRAM tier) |
| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti (unused) ([#128](https://github.com/JustVugg/colibri/issues/128)) | — | int8 MTP head · **with [#131](https://github.com/JustVugg/colibri/pull/131) (pipe + RAM fixes), warm cache, no GPU** | 0.03 cold → **0.5 tok/s** warm (~7-prompt warmup) · cache-warming on native Windows once the portability blockers are fixed — stock main hung on the `\r\n` READY sentinel before #131 |
| Dell Pro Max GB10 (DGX Spark: Grace 10×X925 + 10×A725, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · Dell OEM 4 TB NVMe · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136)) | **5.58 GB/s** O_DIRECT (NVIDIA-OEM unit in #76 was 10.74 — same platform, different SSD) | int8 MTP head · warm cache | 0.21 cold → **0.50 tok/s** warm · hit 83% · MTP 73% (3.20 tok/fw) · **matmul-bound** (matmul 130 s vs disk 58 s) — unified memory, CUDA placement tier neutral; the lever here is an i8mm compute kernel, not placement |
Takeaways: with 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so decode stays cold even on a disk 2–2.7× faster than the dev box — **on small-RAM machines the RAM cap, not the disk, is the binding constraint**, exactly as the table above predicts; `--topp 0.7` alone bought a clean 1.6× end-to-end speedup. The M5 Max datapoint lands right on the table's second row: **~1 tok/s of a 744B model on a laptop SSD** — and its 14 GB/s disk shifts the bottleneck back to RAM budget and kernels. The Framework 13 rows are the cache thesis proven end-to-end on one machine: 0.29 → 0.37 tok/s (hit 28% → 66%, speculation finally engaging at 52% acceptance) just by giving the cache its RAM — int8 MTP head + a bigger cap + the learned pin. The cap part is now automatic (cap auto-raise, 2026-07-10). The 9950X pair is the cleanest bottleneck experiment yet — same machine, same history, only the disk swapped: ×5.8 disk bandwidth bought ×2.9 tokens, and the profile **flipped from 66% disk to 57% matmul**. But the crossover depends on the CPU kernel: the 9800X3D row ([#101](https://github.com/JustVugg/colibri/issues/101)) shows that with the OMP hot-team tuning on, the AVX-512 CPU matmul is fast enough that even a **10 GB/s NVMe stays disk-bound** — and there the **CUDA expert tier buys ≈ 0%**, because the CPU already matches the 5090 on expert matmul. The GPU tier earns its VRAM only when the CPU is the weak link, not by default. (Honest correction from #101: an earlier version of that report ran with the OMP tuning off, which manufactured a false matmul-bound crossover and a false +14% for CUDA — neither survived a clean re-run.)
## Quality benchmark — help wanted
**First measurement is in** ([#108](https://github.com/JustVugg/colibri/issues/108), thanks dnnspaul): the int4 container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot log-likelihood, n=40) — below the 85–95% published for full-precision GLM-5.2, but **the gap is not yet attributable to quantization.** Two confounds sit in the way: (1) 0-shot log-likelihood MC scoring badly underserves a *reasoning* model like GLM-5.2 (it never gets to think), so a large gap is expected even at fp16; (2) n=40 is ±14pp. The **decisive experiment** is the OLMoE fp16-vs-int4 A/B under this same harness (small enough to run both precisions) — that delta *is* the quantization cost with the scoring protocol cancelled out. Until it's run, 62.5% is a datapoint, not a verdict.
The code is here and ready; one command runs it end to end (it auto-downloads the datasets on first use):
```bash
cd c
pip install tokenizers datasets # in addition to the convert deps above
./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each
./coli bench hellaswag --limit 200 # one task, more questions
./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget
```
It prints per-task accuracy (log-likelihood scoring, EleutherAI-harness style). **If you can run the OLMoE fp16-vs-int4 A/B (or a large-n GLM run), please open an issue with the numbers** — it's the measurement that turns 62.5% into either "int4 is fine, scoring artifact" or "quantization is the ceiling, grouped-scale is the priority."
| Benchmarks, community datapoints, quality measurements | [docs/benchmarks.md](docs/benchmarks.md) |
| Tuning knobs, policies, the learning cache, prefetch | [docs/tuning.md](docs/tuning.md) |
| Windows 11 native build (+ CUDA DLL) | [docs/windows.md](docs/windows.md) |
| CUDA backend, VRAM expert tier, full residency | [docs/cuda.md](docs/cuda.md) |
| Apple Silicon Metal backend | [docs/metal.md](docs/metal.md) |
| OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) |
| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) |
| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
## Supporting the project
colibrì is a one-person project, written and tested entirely on a 12-core laptop with 25 GB of RAM — the numbers above are the ceiling of what I can measure at home. If this project is useful or interesting to you and you'd like to support its development (better test hardware translates *directly* into a faster engine for everyone: real NVMe scaling data, bigger pinned caches, int2/int3 quality sweeps on real benchmarks), you can:
colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM;
today its numbers come from a community of real machines. If it's useful to you:
- ⭐ star the repo and share it;
- 🐛 open issues with benchmark numbers from your hardware;
- 💬 reach out via GitHub issues if you'd like to sponsor development or donate hardware.
Every contribution, from a datapoint to a disk, moves the ceiling.
- 🐛 open issues with benchmark numbers from your hardware — datapoints move
this project more than anything else;
- 💬 reach out via GitHub issues to sponsor development or donate hardware.
## Repo layout
@@ -656,20 +221,20 @@ c/
├── tools/ offline conversion, fixtures and benchmarks
├── scripts/ long-running conversion helpers
└── tests/ dependency-free C and Python tests
web/ browser UI (pure OpenAI-API client, community-maintained)
web/ browser UI (pure OpenAI-API client)
desktop/ Tauri v2 desktop shell wrapping the web UI
docs/ reference docs, experiments, media
```
The runtime path intentionally stays flat and readable: `glm.c` plus its small
headers. Auxiliary Python and shell tooling is grouped separately and is never a
runtime dependency of the engine.
From the repository root, `make`, `make check`, and `make clean` delegate to the
engine Makefile. Existing commands run from `c/` continue to work unchanged.
headers. From the repository root, `make`, `make check`, and `make clean`
delegate to the engine Makefile.
## Why "colibrì"
The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience.
The hummingbird weighs a few grams, hovers in place, and visits a thousand
flowers a day. This engine keeps a 744-billion-parameter giant alive on
hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience.
## License
README.zh-TW.md
+233
View File
@@ -0,0 +1,233 @@
<p align="center">
<img src="assets/colibri.svg" width="500" alt="colibrì——小巧引擎，龐大模型">
</p>
<p align="center">
<a href="README.md">English</a> · 繁體中文
</p>
**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。
Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM
與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度，
但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。
```
$ ./coli chat
🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU
✓ ready in 32s · resident 9.9 GB
› ciao!
◆ Ciao! 😊 Come posso aiutarti oggi?
```
## 實際運行畫面
<p align="center">
<img src="docs/media/colibri-dashboard.png" width="900" alt="colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級">
</p>
<p align="center"><em>網頁儀表板（<code>./coli web</code>）：744B 模型達到 <strong>4 tok/s、TTFT 1.6 秒、硬碟讀取 0</strong>——
在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、
VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。</em></p>
<p align="center">
<img src="docs/media/colibri-brain.png" width="900" alt="大腦頁面——以即時皮質呈現 19,456 個專家">
</p>
<p align="center"><em><strong>大腦（Brain）</strong>頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級，
亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其
<a href="https://github.com/JustVugg/colibri/issues/175">實測主題親和度</a>。</em></p>
<p align="center">
<img src="docs/media/colibri-atlas.png" width="900" alt="圖譜頁面——以 3D 星系呈現實測專家圖譜">
</p>
<p align="center"><em><strong>圖譜（Atlas）</strong>頁面：將<a href="https://github.com/JustVugg/colibri/issues/175">實測專家圖譜</a>
呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集
（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。</em></p>
## 核心概念
744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數——
其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）：
<p align="center">
<img src="docs/media/sparse.png" width="880" alt="每個 token 只會啟用約 5.4% 的參數">
</p>
所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**：
- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4
**常駐 RAM**（約 9.9 GB）；
- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB）
**存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、
會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。
引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS，
執行階段不需要 Python，也不需要 GPU。
## 運作方式
### 每個 token 的處理路徑
<p align="center">
<img src="docs/media/token-path.png" width="880" alt="路由 → 聯集 → 配置 → 重疊執行 → 學習">
</p>
每個 token 的每一層都會走過相同的五個步驟。設計目標是讓
**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。
### 統一記憶體階層，取代單一記憶體門檻
<p align="center">
<img src="docs/media/tiers.png" width="880" alt="VRAM／RAM／NVMe 三層專家常駐架構">
</p>
同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入
（慢，但結果正確）；在大型主機上，則可讓整組專家常駐
（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。
兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家
（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。
在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器
（[#82](https://github.com/JustVugg/colibri/issues/82)）。
### 絕不為同一次硬碟讀取等待兩遍
快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取：
每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool
（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個
不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家——
實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。
在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端，
使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的
[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。
### 忠實模型，壓縮狀態
前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致**
（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個
浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存
（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段
逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作，
並透過強制選取所有 key，驗證可精確重現稠密注意力。
### 如實呈現推測式解碼
GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證——
條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值：
MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見
[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算
**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family
（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。
文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中，
以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測，
若不划算就使用 `DRAFT=0`。
## 實際成果
<p align="center">
<img src="docs/media/ladder.png" width="880" alt="各硬體等級的實測解碼速度">
</p>
同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。
[完整 benchmark 表格](docs/benchmarks.md)中的重點如下：
- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒
（[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）；
- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s
（[#200](https://github.com/JustVugg/colibri/issues/200)）；
- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s
（[#273](https://github.com/JustVugg/colibri/issues/273)）；
- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限，
也仍是如實呈現的基準。
品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation
消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、
[#108](https://github.com/JustVugg/colibri/issues/108) 與
[#81](https://github.com/JustVugg/colibri/issues/81)。
## 開始使用
### 1. 取得模型
Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用
**含 int8 MTP head 的版本**：
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0%
>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本：
> `ls -l <model>/out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。
你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要
在硬碟上同時存放完整的 756 GB：
```bash
cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試
./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python）
```
### 2. 執行
```bash
COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP
COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查
./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板
./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API
```
引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。
### 3. 深入了解
| 主題 | 文件 |
|---|---|
| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) |
| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) |
| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) |
| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) |
| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) |
| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) |
| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) |
| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
## 支持專案
colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發；
如今它的數據來自社群中的各種真實機器。如果這個專案對你有用：
- ⭐ 為儲存庫加星並分享；
- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案；
- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。
## 儲存庫結構
```
Makefile 根目錄建置／檢查入口
c/
├── glm.c 單檔 GLM 引擎
├── st.h, tok.h, json.h 執行階段標頭檔
├── backend_cuda.* 選用的 CUDA 層級
├── Makefile 建置與本機檢查
├── coli 使用者介面 CLI
├── openai_server.py OpenAI 相容 HTTP gateway
├── setup.sh 單一指令完成本機設定
├── tools/ 離線轉換、fixtures 與 benchmarks
├── scripts/ 長時間轉換輔助工具
└── tests/ 零相依套件的 C 與 Python 測試
web/ 瀏覽器 UI（純 OpenAI API client）
desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell
docs/ 參考文件、實驗與媒體檔
```
執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。
在儲存庫根目錄執行 `make`、`make check` 與 `make clean`，
都會轉交給引擎的 Makefile。
## 為什麼叫做「colibrì」
蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。
這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉：
25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。
## 授權條款
Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。
c/.gitignore
+1
View File
@@ -2,3 +2,4 @@
glm_tiny/
olmoe_hf/
olmoe_i4/
.build-config
c/Makefile
+57 -12
View File
@@ -166,7 +166,7 @@ else
PYTHON ?= python3
endif
CUDA_OBJ =
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE)
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
@@ -212,11 +212,25 @@ all: glm$(EXE)
# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform)
glm: glm$(EXE)
glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ)
# Config stamp: make only tracks file timestamps, not flag changes. Without this,
# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and
# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it
# worked but isn't. We record the build-affecting flags in .build-config and rewrite
# it ONLY when they change (evaluated here at parse time, so the file's timestamp
# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend
# on it, so they relink on a config change and stay put otherwise. (#306)
BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL)
BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null)
ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)"
$(shell printf '%s' '$(BUILD_CONFIG)' > .build-config)
endif
.build-config: ;
glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config
$(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS)
# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.
backend_loader.o: backend_loader.c backend_cuda.h compat.h
backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config
$(CC) $(CFLAGS) -c backend_loader.c -o $@
# Windows CUDA DLL: compile backend_cuda.cu with nvcc (+MSVC cl.exe as host
@@ -225,15 +239,15 @@ backend_loader.o: backend_loader.c backend_cuda.h compat.h
# "x64 Native Tools Command Prompt"). COLI_CUDA_BUILDING_DLL enables
# __declspec(dllexport) so the 15 API symbols are exported.
cuda-dll: backend_cuda.cu backend_cuda.h
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
@command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; }
$(NVCC) $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \
"$(NVCC)" $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \
-L"$(CUDA_HOME)/lib/x64" -lcudart \
backend_cuda.cu -o coli_cuda.dll
backend_cuda.o: backend_cuda.cu backend_cuda.h
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
$(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@
backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
"$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@
backend_metal.o: backend_metal.mm backend_metal.h
$(METALXX) -c backend_metal.mm -o $@
@@ -243,13 +257,13 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h
./backend_metal_test
cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
$(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
"$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
./backend_cuda_test$(EXE)
cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
$(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE)
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
"$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE)
./backend_cuda_bench$(EXE)
olmoe$(EXE): olmoe.c st.h json.h compat.h
@@ -279,6 +293,9 @@ iobench$(EXE): iobench.c compat.h
tests/test_json$(EXE): tests/test_json.c json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
$(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS)
tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
@@ -297,15 +314,43 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test
# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp
tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),
# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select
tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
c/backend_cuda.cu
+143 -3
View File
@@ -8,12 +8,21 @@
#include <cstring>
#include <mutex>
struct RaggedKVEntry {
const void *key;
const float *host_l,*host_r;
float *latent,*rope;
int length,capacity,K,R;
};
struct ColiCudaTensor {
void *weights;
float *scales;
size_t weight_bytes;
int fmt, I, O, device;
int tracked;
RaggedKVEntry ragged[512];
int ragged_count;
};
typedef struct {
@@ -23,7 +32,7 @@ typedef struct {
size_t x_cap, y_cap, gate_cap, up_cap;
uint8_t *qx; float *qscale;
size_t qx_cap, qscale_cap;
float *host_x,*host_y; size_t host_x_cap,host_y_cap;
float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap;
float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap;
float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */
cudaStream_t stream;
@@ -338,6 +347,49 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,
ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}
}
/* Independent device-resident KV sequence per row. lengths selects the valid
* prefix; latent/rope point at paged caches updated by the host wrapper. */
__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
const float *const *latent,const float *const *rope,const int *lengths,
const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,
int V,int K,int T,float scale){
int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);
if(s>=S||nt<1||nt>T)return;
extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;
const float *qs=q+((size_t)s*H+h)*(Q+R);
const float *ls=latent[s],*rs=rope[s];
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)
a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*
(fmt?wscale[rbase+d]:1.f);qa[k]=a;}
__syncthreads();
for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;
const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];
for(int d=0;d<R;d++)a+=qs[Q+d]*rt[d];scores[t]=a*scale;}
__syncthreads();
float local=-3.402823466e+38F;for(int t=tid;t<nt;t+=blockDim.x)local=fmaxf(local,scores[t]);
red[tid]=local;__syncthreads();
for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]=fmaxf(red[tid],red[tid+n]);__syncthreads();}
float mx=red[0];local=0;for(int t=tid;t<nt;t+=blockDim.x){float e=expf(scores[t]-mx);scores[t]=e;local+=e;}
red[tid]=local;__syncthreads();
for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]+=red[tid+n];__syncthreads();}
float inv=1.f/red[0];for(int t=tid;t<nt;t+=blockDim.x)scores[t]*=inv;
__syncthreads();
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}
__syncthreads();
for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);
for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k);
ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}
}
__global__ static void ragged_kv_append(float *const *latent,float *const *rope,
const float *packed,const int *old_len,const int *add,const int *offset,int K,int R){
int s=blockIdx.x,n=add[s],base=offset[s];
for(int i=threadIdx.x;i<n*(K+R);i+=blockDim.x){
if(i<n*K)latent[s][(size_t)old_len[s]*K+i]=packed[base+i];
else rope[s][(size_t)old_len[s]*R+i-n*K]=packed[base+i];
}
}
static int reserve(float **ptr, size_t *cap, size_t bytes) {
if (*cap >= bytes) return 1;
if (*ptr) cudaFree(*ptr);
@@ -406,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) {
for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]);
if (ctx->host_x) cudaFreeHost(ctx->host_x);
if (ctx->host_y) cudaFreeHost(ctx->host_y);
if (ctx->host_kv) cudaFreeHost(ctx->host_kv);
if (ctx->stream) cudaStreamDestroy(ctx->stream);
if (ctx->group_desc) cudaFree(ctx->group_desc);
ctx->x = ctx->y = ctx->gate = ctx->up = nullptr;
ctx->qx=nullptr; ctx->qscale=nullptr;
ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr;
ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr;
ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr;
ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0;
ctx->qx_cap=ctx->qscale_cap=0;
ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0;
ctx->host_x_cap=ctx->host_y_cap=0;
ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0;
ctx->group_desc=nullptr; ctx->group_desc_cap=0;
}
g_nctx = 0;
@@ -770,6 +823,89 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso
return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale);
}
extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj,
float *out,const float *q,const void *const *keys,
const float *const *latent,const float *const *rope,
const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){
if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192||
H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)||
proj->device!=w->device||proj->I!=H*V)return 0;
DeviceContext *dc=find_ctx(w->device);
if(!select_ctx(dc))return 0;
float **dl=(float**)std::malloc((size_t)S*sizeof(*dl));
float **dr=(float**)std::malloc((size_t)S*sizeof(*dr));
int *old=(int*)std::malloc((size_t)S*sizeof(*old));
int *add=(int*)std::malloc((size_t)S*sizeof(*add));
int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0;
if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
for(int s=0;s<S;s++){
if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
RaggedKVEntry *e=nullptr;
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;}
if(!e){
if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s];
}
if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]<e->length){
if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
e->latent=e->rope=nullptr;e->length=e->capacity=0;
e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s];
}
if(lengths[s]>e->capacity){
int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr;
if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")||
!cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){
if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;
}
if(e->length){
cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
}
if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
e->latent=nl;e->rope=nr;e->capacity=cap;
}
dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length;
off[s]=packed_n;packed_n+=add[s]*(K+R);
}
size_t qb=(size_t)S*H*(Q+R)*sizeof(float);
size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float);
size_t pb=(size_t)packed_n*sizeof(float);
size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int));
int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&&
reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&&
(!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb)));
char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S;
int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;
if(ok&&pb){
for(int s=0;s<S;s++)if(add[s]){
float *p=dc->host_kv+off[s];
std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float));
std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float));
}
ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload");
}
if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&&
cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&&
cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&&
cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&&
cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&&
cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&&
cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets");
if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R);
if(ok)for(int s=0;s<S;s++){
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;}
}
std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;
size_t shared=(size_t)(2*K+T+256)*sizeof(float);
attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,
dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale);
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I));
return cuda_ok(cudaGetLastError(),"ragged attention launch")&&
cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&&
cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize");
}
extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
if (!tensor) return;
DeviceContext *ctx = find_ctx(tensor->device);
@@ -781,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
}
if (tensor->weights) cudaFree(tensor->weights);
if (tensor->scales) cudaFree(tensor->scales);
for(int i=0;i<tensor->ragged_count;i++){
if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent);
if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope);
}
std::free(tensor);
}
c/backend_cuda.h
+6 -1
View File
@@ -14,6 +14,7 @@
#define COLI_CUDA_DLLEXPORT
#endif
#ifdef __cplusplus
extern "C" {
#endif
@@ -92,6 +93,11 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C
const float *rope,int S,int H,int Q,int R,
int V,int K,int T,float attention_scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q,const void *const *keys,
const float *const *latent,const float *const *rope,
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
@@ -143,4 +149,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);
#endif
#endif
c/backend_loader.c
+15
View File
@@ -61,6 +61,10 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f
typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale);
typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale);
typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q,const void *const *keys,
const float *const *latent,const float *const *rope,
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n);
@@ -107,6 +111,7 @@ static struct {
fn_attention_absorb_batch_dev attention_absorb_batch_dev;
fn_attention_absorb_kvdev attention_absorb_kvdev;
fn_attention_project_batch attention_project_batch;
fn_attention_project_ragged attention_project_ragged;
fn_attention_project_batch_dev attention_project_batch_dev;
fn_attention_project_batch_dev_out attention_project_batch_dev_out;
fn_pipe_add pipe_add;
@@ -200,6 +205,7 @@ static int coli_cuda_load(void){
RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev)
RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev)
RESOLVE(attention_project_batch, fn_attention_project_batch)
RESOLVE(attention_project_ragged, fn_attention_project_ragged)
RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev)
RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out)
RESOLVE(pipe_add, fn_pipe_add)
@@ -342,6 +348,15 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro
return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);
}
int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q,const void *const *keys,
const float *const *latent,const float *const *rope,
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){
if(!coli_cuda_load()) return 0;
return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths,
S,H,Q,R,V,K,max_t,attention_scale);
}
int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){
if(!g_cuda.available){ return 0; }
return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);
c/coli
+220 -15
View File
@@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco
--topp P adaptive expert top-p --topk N fixed top-k
--ngen N maximum response tokens --cap N cache slots/layer
"""
import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap
import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct
# The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256.
if sys.platform != "win32":
@@ -40,6 +40,8 @@ if sys.platform == "win32":
except (AttributeError, OSError): pass
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from version import __version__ as _version
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
# support modules (resource_plan.py, doctor.py, openai_server.py) and
@@ -115,7 +117,7 @@ def sprite_lines():
def banner(sub=""):
sp=sprite_lines()
txt=[
f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}",
f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}",
f"{C.dim}tiny engine, immense model{C.r}",
f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}",
f"{C.dgray}{sub}{C.r}" if sub else "",
@@ -141,12 +143,24 @@ def need_model(model):
sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build")
def cuda_binary():
if not os.path.exists(GLM) or sys.platform != "linux": return False
try:
linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)
return any("libcudart" in line and "not found" not in line
for line in linked.stdout.splitlines())
except (OSError,subprocess.SubprocessError): return False
if not os.path.exists(GLM): return False
if sys.platform == "linux":
try:
linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)
return any("libcudart" in line and "not found" not in line
for line in linked.stdout.splitlines())
except (OSError,subprocess.SubprocessError): return False
if sys.platform == "win32":
# Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads
# coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no
# import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a
# marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require
# coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup).
try:
with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read()
except OSError: return False
return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll"))
return False
def resource_request(a, env):
ctx=a.ctx or int(env.get("CTX",4096))
@@ -480,13 +494,141 @@ def cmd_run(a):
need_model(a.model)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
banner("run")
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink)
e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|><think></think>"
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).
# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves
# <think> open so the engine emits its reasoning block; the default stays nothink.
tk="<think>" if os.environ.get("THINK","0")=="1" else "<think></think>"
e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|>{tk}"
sys.exit(subprocess.call([GLM, str(a.cap)], env=e))
def server_probe(base, api_key=None, timeout=1.5):
"""Is a coli serve alive at `base`? Returns its model_id, or None.
Probes /health then /v1/models — both cheap, neither touches the engine."""
import urllib.request, urllib.error
def get(path):
req=urllib.request.Request(base.rstrip("/")+path)
if api_key: req.add_header("Authorization", f"Bearer {api_key}")
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8","replace"))
try:
if get("/health").get("status")!="ok": return None
data=get("/v1/models").get("data") or []
return data[0]["id"] if data else None
except Exception:
return None
def chat_attached(a, base, model_id):
"""The chat REPL over HTTP against a running `coli serve`.
Why this exists (the cold-chat cost, measured): spawning a private engine
pays 34-136 s of resident load on EVERY start, and begins with an empty
expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A
resident server pays load once and keeps the LRU warm across sessions;
its KV slots reuse the conversation prefix, so a continued chat skips
re-prefill too. The engine byte-protocol stays untouched — this is plain
OpenAI SSE over localhost, stdlib only."""
import urllib.request
print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}")
print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n")
msgs=[]
w=term_w()-4
while True:
if TTY:
print(f" {C.dgray}╭{'─'*w}╮{C.r}")
try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ")
except EOFError: print(); break
print(f" {C.dgray}╰{'─'*w}╯{C.r}")
else:
try: msg=input()
except EOFError: break
msg=msg.strip()
if msg in (":q",":quit","exit"): break
if not msg: continue
if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue
msgs.append({"role":"user","content":msg})
body=json.dumps({"model":model_id,"messages":msgs,"stream":True,
"max_tokens":a.ngen}).encode()
req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body,
headers={"Content-Type":"application/json"})
if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}")
print(f"\n {C.teal}◆ colibrì{C.r}")
sp=Spinner("thinking…"); sp.start()
md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False
try:
with urllib.request.urlopen(req) as r:
for raw in r:
line=raw.decode("utf-8","replace").strip()
if not line.startswith("data: "): continue
data=line[6:]
if data=="[DONE]": break
try: ev=json.loads(data)
except ValueError: continue
for ch in ev.get("choices",[]):
d=ch.get("delta",{})
txt=d.get("content")
if not txt: continue # ping/ruolo/reasoning: non è testo
if first: sp.stop(); first=False
md.feed(txt); reply.append(txt)
except KeyboardInterrupt:
interrupted=True # il server annulla la richiesta alla disconnessione
except OSError as e:
sp.stop()
print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break
sp.stop(); md.close()
if reply: msgs.append({"role":"assistant","content":"".join(reply)})
else: msgs.pop() # turno vuoto: non sporcare la history
el=time.time()-t0
note=" · ⏹ interrupted" if interrupted else ""
print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n")
print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}")
def kv_resume_notice(model_dir):
"""SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat
started today continues a conversation from days ago, with `first=0` so the
turn is appended WITHOUT the [gMASK]<sop> prefix. The engine does announce it
on stderr — but nothing here ever shows that: the drain thread's
p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty
when the status lines are printed. The warning only appeared once the engine
DIED, which is exactly when it no longer mattered.
Measured cost of the silence: a chat inherited 670 tokens of an old Italian
session ("il mio numero preferito e 7, ricordalo!"). Every later reply came
back in Italian, and "explain fibonacci in short" was answered about the
number 7 — the model was being coherent with a context nobody could see, and
it read as a quantization bug for a day.
So say it here, in Python, from the file itself: no pipe, no thread, no
Windows deadlock risk (see the stderr comment below)."""
p=os.path.join(model_dir, ".coli_kv")
try:
with open(p,"rb") as f:
if f.read(8)!=b"COLIKV1\0": return
h=struct.unpack("<8i", f.read(32))
n=h[6]
if n<1: return
age=time.time()-os.path.getmtime(p)
when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now"
print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}")
print(f" {C.dgray} it steers tone, language and topic. :reset clears it · "
f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}")
except (OSError, struct.error): pass
def cmd_chat(a):
# ATTACH: a running `coli serve` beats a private engine every time — the load
# (34-136 s) and the cache warmth survive between sessions. Explicit --attach
# wins; otherwise probe localhost quietly and use it if it's there. --no-attach
# forces the old behaviour. The probe costs ~1 ms when nothing is listening.
if not getattr(a,"no_attach",False):
base=getattr(a,"attach",None) or "http://127.0.0.1:8000"
mid=server_probe(base, getattr(a,"api_key",None))
if mid:
banner(f"chat · {mid} · attached")
chat_attached(a, base, mid); return
if getattr(a,"attach",None):
sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model <dir>)")
need_model(a.model)
banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}")
kv_resume_notice(a.model)
errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False)
e=env_for(a); e["SERVE"]="1"
# stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the
@@ -619,12 +761,65 @@ def cmd_chat(a):
except Exception: pass
print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n")
def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid")
def cmd_serve(a):
need_model(a.model)
# pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano.
# EN: pidfile so `coli stop` can shut everything down without manual pkill.
try:
with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n")
except OSError: pass
from openai_server import serve
serve(a.model, a.host, a.port, a.model_id, a.api_key,
a.cap,a.ngen,GLM,env_for(a),a.cors_origin,
a.max_queue,a.queue_timeout,a.kv_slots)
try:
serve(a.model, a.host, a.port, a.model_id, a.api_key,
a.cap,a.ngen,GLM,env_for(a),a.cors_origin,
a.max_queue,a.queue_timeout,a.kv_slots)
finally:
try: os.unlink(serve_pidfile(a.port))
except OSError: pass
def cmd_stop(a):
"""Shut down a running `coli serve` AND its engine — one command, no pkill.
The engine re-execs itself for OMP tuning, so its process is named `exe`,
not `glm`: every `pkill -x glm` in history silently killed nothing (that is
how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the
real processes: the pidfile first, then /proc by cmdline/environ — only
processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve`
in the command line)."""
banner("stop")
targets=[] # (pid, descrizione)
pf=serve_pidfile(a.port)
try:
pid=int(open(pf).read().split()[0])
os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})"))
except (OSError,ValueError,IndexError): pass
for pd in os.listdir("/proc"):
if not pd.isdigit(): continue
pid=int(pd)
try:
cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace")
if "coli" in cmd and " serve" in cmd and pid!=os.getpid():
if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)"))
comm=open(f"/proc/{pd}/comm").read().strip()
if comm in ("glm","exe","olmoe"):
env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace")
if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)"))
except (OSError,PermissionError): continue
if not targets:
print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return
for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}")
if a.dry_run: return
for pid,_ in targets:
try: os.kill(pid, signal.SIGTERM)
except OSError: pass
time.sleep(2.0)
for pid,_ in targets:
try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)")
except OSError: pass # gia' morto: bene
try: os.unlink(pf)
except OSError: pass
print(f" {C.grn}✓ stopped{C.r} — RAM released")
def cmd_web(a):
"""serve + open the dashboard in the browser once the API answers."""
@@ -712,6 +907,7 @@ def main():
common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)
common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)
ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally")
ap.add_argument("--version", action="version", version=f"colibrì {_version}")
sub=ap.add_subparsers(dest="cmd")
sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])
pp=sub.add_parser("plan",parents=[common])
@@ -719,7 +915,14 @@ def main():
pd=sub.add_parser("doctor",parents=[common])
pd.add_argument("--json",action="store_true",help="emit a versioned JSON report")
pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")
sub.add_parser("chat", parents=[common])
pc=sub.add_parser("chat", parents=[common])
pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None,
help="chat against a running `coli serve` instead of spawning an engine "
"(keeps the model loaded and the expert cache warm across chat sessions). "
"Bare --attach probes localhost:8000.")
pc.add_argument("--no-attach", action="store_true",
help="never auto-attach, always spawn a private engine")
pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
ps=sub.add_parser("serve", parents=[common])
ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)
ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))
@@ -728,6 +931,8 @@ def main():
ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8")))
ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300")))
ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1")))
pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine")
pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true")
pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser")
for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)),
("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))),
@@ -751,7 +956,7 @@ def main():
pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)")
a=ap.parse_args()
handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench,
"convert":cmd_convert,"web":cmd_web}.get(a.cmd)
if handler: sys.exit(handler(a) or 0)
banner(); print(__doc__)
c/compat.h
+55 -1
View File
@@ -80,6 +80,7 @@ static inline int compat_open_direct(const char *path){
#endif
#include <windows.h>
#include <io.h>
#include <direct.h> /* _mkdir (for the mkdtemp shim below) */
#include <process.h>
#include <malloc.h>
#include <fcntl.h>
@@ -143,6 +144,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){
* Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking
* per letture >2 GB (anche se i tensori individuali sono nell'ordine dei
* MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */
/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante
* (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI
* fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi
* dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra
* tre persone perche' il codice vero non compariva da nessuna parte). */
static __thread DWORD compat_pread_lasterr __attribute__((unused));
static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){
intptr_t osfh = _get_osfhandle(fd);
if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; }
@@ -158,6 +165,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){
if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){
DWORD err = GetLastError();
if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */
compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */
if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF;
else errno = EIO;
return -1;
@@ -237,7 +245,9 @@ static inline int compat_rename(const char *old, const char *new){
/* --- rss_gb: getrusage -> GetProcessMemoryInfo ---
* ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */
#include <psapi.h>
#pragma comment(lib, "psapi.lib")
#ifdef _MSC_VER
#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */
#endif
struct rusage { long ru_maxrss; };
#define RUSAGE_SELF 0
static inline int getrusage(int who, struct rusage *r){
@@ -304,8 +314,52 @@ static inline int compat_setenv(const char *name, const char *value, int overwri
}
#define setenv(name,value,overwrite) compat_setenv(name,value,overwrite)
/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage ---
* Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view
* of the process environment block, not UTF-8. A parent that hands the child a
* Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess
* module, which coli uses to pass the chat prompt) round-trips correctly only
* through GetEnvironmentVariableW; going through narrow getenv() re-encodes it
* via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out
* corrupted before the byte-level tokenizer ever sees it. Read the wide value
* directly and convert straight to UTF-8, bypassing the ANSI codepage entirely.
* Returned buffer is intentionally leaked: called a handful of times at
* startup, lives for the process. */
static inline const char *compat_getenv_utf8(const char *name){
wchar_t wname[64];
if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name);
DWORD need = GetEnvironmentVariableW(wname, NULL, 0);
if(!need) return NULL;
wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t));
if(!wval) return NULL;
GetEnvironmentVariableW(wname, wval, need);
int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL);
char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL;
if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL);
free(wval);
return val;
}
#define getenv_utf8(name) compat_getenv_utf8(name)
/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) ---
* Test binaries (test_stops.c) create a scratch dir in the CWD via a
* "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The
* Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it.
* Returns the template pointer on success, NULL on failure — matching POSIX. */
static inline char *compat_mkdtemp(char *tmpl){
if(!tmpl) return NULL;
if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */
if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */
return tmpl;
}
#define mkdtemp(tmpl) compat_mkdtemp(tmpl)
#endif /* _WIN32 */
#ifndef getenv_utf8
#define getenv_utf8(name) getenv(name)
#endif
/* --- compat_aligned_free su piattaforme diverse da Windows ---
* Su Linux/macOS, posix_memalign usa free() normale. */
#ifndef compat_aligned_free
c/doctor.py
+26 -9
View File
@@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details):
def cuda_linkage(engine_path):
"""Return CUDA linkage state without loading the executable or CUDA runtime."""
if not Path(engine_path).is_file() or os.name != "posix":
engine = Path(engine_path)
if not engine.is_file():
return {"linked": False, "missing": False}
try:
result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True,
timeout=3, check=False)
except (OSError, subprocess.SubprocessError):
return {"linked": False, "missing": False}
lines = [line for line in result.stdout.splitlines() if "libcudart" in line]
return {"linked": any("not found" not in line for line in lines),
"missing": any("not found" in line for line in lines)}
if os.name == "posix":
try:
result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True,
timeout=3, check=False)
except (OSError, subprocess.SubprocessError):
return {"linked": False, "missing": False}
lines = [line for line in result.stdout.splitlines() if "libcudart" in line]
return {"linked": any("not found" not in line for line in lines),
"missing": any("not found" in line for line in lines)}
if sys.platform == "win32":
# Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads
# coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no
# import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a
# marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require
# coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup).
try:
built = b"[CUDA] mode: routed experts" in engine.read_bytes()
except OSError:
return {"linked": False, "missing": False}
if not built:
return {"linked": False, "missing": False}
dll_present = (engine.parent / "coli_cuda.dll").is_file()
return {"linked": dll_present, "missing": not dll_present}
return {"linked": False, "missing": False}
def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
c/glm.c
+616 -78
View File
File diff suppressed because it is too large Load Diff
c/olmoe.c
+44
View File
@@ -400,6 +400,37 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
}
}
/* teacher-forced NLL of full_ids[np..nfull): feed the REFERENCE token at each step
* (never the argmax), accumulate -log softmax(logits)[next_ref]. A loss meter for
* throughput experiments: same engine path as decode, so hit rate/speed stay
* comparable, but quality is measured as perplexity instead of exact-match.
* Cross-checked vs HF transformers bf16 on identical token ids: engine (int8
* experts) 12.11 ppl vs reference 12.25 (#108). Enabled by PPL=1. */
static int tf_nll(Model *m, const int *full, int nfull, int np, double *nll_out) {
Cfg *c = &m->c;
m->max_t = nfull;
m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
for (int i = 0; i < c->n_layers; i++) {
m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
}
double nll = 0; int scored = 0;
float *logit = step(m, full, np, 0); /* prefill on the prompt */
for (int i = np; i < nfull; i++) {
/* log softmax(logit)[full[i]] without materializing the softmax */
float mx = logit[0]; for (int v = 1; v < c->vocab; v++) if (logit[v] > mx) mx = logit[v];
double Z = 0; for (int v = 0; v < c->vocab; v++) Z += exp((double)logit[v] - mx);
nll += -((double)logit[full[i]] - mx - log(Z));
scored++;
free(logit); logit = NULL;
if (i == nfull - 1) break;
logit = step(m, &full[i], 1, i); /* teacher forcing */
}
if (logit) free(logit);
*nll_out = nll / scored;
return scored;
}
/* ---------- lettura ref.json ---------- */
static int *read_int_array(jval *o, const char *key, int *n_out) {
jval *a = json_get(o, key);
@@ -430,6 +461,19 @@ int main(int argc, char **argv) {
Model m; model_init(&m, snap, cap, bits);
printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
if (getenv("PPL") && atoi(getenv("PPL")) == 1) { /* loss-meter mode: teacher-forced NLL */
double nll; double t = now_s();
int scored = tf_nll(&m, full, nfull, np, &nll);
double dt = now_s() - t;
double tot = m.hits + m.miss;
printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll));
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
(unsigned long long)m.hits, (unsigned long long)m.miss);
printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());
free(buf); free(arena);
return 0;
}
int *out = malloc((np + n_new) * sizeof(int));
double t = now_s();
generate(&m, prompt, np, n_new, out);
c/openai_server.py
+35 -1
View File
@@ -27,6 +27,7 @@ HERE = Path(__file__).resolve().parent
END = b"\x01\x01END\x01\x01\n"
READY = b"\x01\x01READY\x01\x01\n"
MAX_BODY = 4 << 20
PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile
DEFAULT_CORS_ORIGINS = (
"http://127.0.0.1:8000",
"http://localhost:8000",
@@ -270,6 +271,14 @@ def parse_tool_calls(reply, tools=None):
salvaged.append(name)
calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}})
if tools and not calls and re.search(r"</?tool_call>|</?arg_key>|</?arg_value>", reply):
# Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la
# sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato).
# EN: #401 field diagnosis: tools were declared and the model attempted the syntax,
# EN: but the strict parse matched nothing (typically quantization-mangled output).
sys.stderr.write("[api] tools declared and tool-call markers present, but no call "
"parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n")
sys.stderr.flush()
text = _BOX_RE.sub("", reply)
if THINK_CLOSE in text:
text = text.split(THINK_CLOSE, 1)[1]
@@ -405,7 +414,10 @@ def generation_options(body, limit):
maximum = body.get("max_tokens")
maximum_param = "max_tokens"
if maximum is None:
maximum = min(256, limit)
# Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
# --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
# Generation still ends at EOS, so this is a cap, not a target.
maximum = limit
temperature = body.get("temperature")
top_p = body.get("top_p")
temperature = 0.7 if temperature is None else temperature
@@ -473,6 +485,8 @@ class Engine:
self.emap = None
self.hits = None
self.hits_seq = 0 # latest "TIERS" snapshot from the engine
self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings
self.profile_seq = 0
read_engine_turn(self.process.stdout, READY, lambda _: None)
self.dispatcher = threading.Thread(target=self._dispatch_stdout,
name="colibri-stdout", daemon=True)
@@ -550,6 +564,20 @@ class Engine:
elif kind == "HITS" and len(fields) == 4:
self.hits = fields[3]
self.hits_seq += 1
elif kind == "PROF" and len(fields) >= 10:
# per-turn phase timings: where the engine spent this turn's wall time
self.profile.append({
"wall_s": float(fields[1]),
"prompt_tokens": int(fields[2]),
"completion_tokens": int(fields[3]),
"expert_disk_s": float(fields[4]),
"expert_wait_s": float(fields[5]),
"expert_matmul_s": float(fields[6]),
"attention_s": float(fields[7]),
"lm_head_s": float(fields[8]),
"forwards": int(fields[9]),
})
self.profile_seq += 1
elif kind == "TIERS" and len(fields) >= 6:
self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]),
"disk": int(fields[3]), "vram_gb": float(fields[4]),
@@ -782,6 +810,12 @@ class APIHandler(BaseHTTPRequestHandler):
payload["seq"] = eng.hits_seq
self.send_json(200, payload, request_id)
return
if path == "/profile":
eng = self.server.engine
payload = {"seq": getattr(eng, "profile_seq", 0) if eng else 0,
"turns": list(getattr(eng, "profile", ()) or ()) if eng else []}
self.send_json(200, payload, request_id)
return
if self.serve_static(path):
return
self.require_auth()
c/resource_plan.py
+23 -1
View File
@@ -203,6 +203,22 @@ def physical_cpu_count():
return os.cpu_count() or 1
def cpu_socket_count():
"""Return the number of physical CPU sockets visible to this process."""
if not sys.platform.startswith("linux"):
return 1
try:
result = subprocess.run(["lscpu", "-p=socket"], text=True,
capture_output=True, check=True, timeout=5)
sockets = {int(line) for line in result.stdout.splitlines()
if line and not line.startswith("#")}
if sockets:
return len(sockets)
except (OSError, ValueError, subprocess.SubprocessError):
pass
return 1
POLICIES = {
"quality": {"preserve_quantization": True, "preserve_router": True},
"balanced": {"preserve_quantization": True, "preserve_router": True},
@@ -212,11 +228,12 @@ POLICIES = {
def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
available_memory=None, available_disk=None, gpus=None,
policy="quality", physical_cpus=None):
policy="quality", physical_cpus=None, cpu_sockets=None):
if policy not in POLICIES:
raise ValueError(f"unknown policy: {policy}")
info = analyze_model(model)
physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus
cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets
cfg = info["config"]
available_memory = memory_available() if available_memory is None else available_memory
if available_disk is None:
@@ -286,6 +303,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
"quality_preserving": policy != "experimental-fast"},
"model": {key: value for key, value in info.items() if key != "config"},
"cpu": {"physical_cores": max(1, int(physical_cpus)),
"sockets": max(1, int(cpu_sockets)),
"thread_policy": "physical-cores"},
"tiers": {
"disk": {"role": "cold-backing", "model_bytes": info["model_bytes"],
@@ -318,6 +336,10 @@ def environment_for_plan(plan, env=None, cuda_enabled=True):
# ("Affinity not supported on this configuration"): non impostarle li'.
result.setdefault("OMP_PROC_BIND", "spread")
result.setdefault("OMP_PLACES", "cores")
if sys.platform.startswith("linux") and plan["cpu"].get("sockets", 1) > 1:
# Selectively interleave large expert/dense slabs across memory controllers.
# Unlike blanket numactl interleave, this leaves CUDA staging buffers local.
result.setdefault("COLI_NUMA", "1")
if plan["policy"]["name"] == "balanced":
result.setdefault("REPIN", "64")
ram = plan["tiers"]["ram"]
c/st.h
+38 -5
View File
@@ -12,6 +12,7 @@
#include <string.h>
#include <stdint.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
@@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) {
}
/* indicizza tutti i model-*.safetensors in snap_dir */
/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux
* — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore
* ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo
* stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile
* per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and
* big bf16 tensors exceed it), EINTR retry, honest short-read errors.
* Exits on failure, like every st.h reader. */
#ifndef ST_PREAD_CHUNK
#define ST_PREAD_CHUNK (1u << 30)
#endif
static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) {
char *p = (char *)buf;
int64_t got = 0;
while (got < n) {
int64_t want = n - got;
if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK;
ssize_t r = pread(fd, p + got, (size_t)want, off + got);
if (r < 0) {
if (errno == EINTR) continue;
fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno),
(long long)off, (long long)got, (long long)n);
exit(1);
}
if (r == 0) {
fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n",
tag, (long long)off, (long long)got, (long long)n);
exit(1);
}
got += r;
}
}
static void st_init(shards *S, const char *snap_dir) {
memset(S, 0, sizeof(*S));
S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor));
@@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) {
if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); }
int64_t fsz = (int64_t)sst.st_size;
uint64_t hlen;
if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); }
st_pread_full(fd, &hlen, 8, 0, "pread hlen");
/* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di
* prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in
* overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una
@@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) {
files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); }
char *hdr = malloc(hlen + 1);
if (!hdr) { perror("malloc safetensors header"); exit(1); }
if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); }
st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr");
hdr[hlen] = 0;
int64_t data_start = 8 + (int64_t)hlen;
char *arena = NULL;
@@ -218,7 +251,7 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
void *raw = malloc(t->nbytes);
if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); }
if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); }
st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data");
if (t->dtype == 2) {
memcpy(out, raw, t->nbytes);
} else if (t->dtype == 0) {
@@ -243,7 +276,7 @@ static int64_t st_nbytes(shards *S, const char *name) {
static void st_read_raw(shards *S, const char *name, void *out, int drop) {
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); }
st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw");
if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
}
@@ -256,7 +289,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int
int esz = (t->dtype == 2) ? 4 : 2;
int64_t boff = t->off + elem_off * esz, nb = n_elems * esz;
void *raw = malloc(nb);
if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); }
st_pread_full(t->fd, raw, nb, boff, "pread slice");
if (t->dtype == 2) memcpy(out, raw, nb);
else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); }
else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); }
c/tests/bench_dsa_select.c
+165
View File
@@ -0,0 +1,165 @@
/* Microbenchmark: old (full-qsort) vs new (quickselect partial-select) DSA top-keep.
*
* This is NOT a unit test -- test_dsa_select.c proves correctness. This measures the
* headline claim of #356: that replacing the O(nk log nk) qsort over all nk context
* scores with an O(nk) partial_select_desc is materially faster per call, which is
* the win the issue was opened for -- and that the win GROWS with context length
* (because quickselect is linear average, qsort is n-log-n).
*
* It re-implements the OLD top-keep inline (qsort + threshold + scans) on a private
* buffer so the A/B runs in one process, same inputs, same warm caches -- a controlled
* comparison. It calls the REAL (new) partial_select_desc via the include-glm.c
* pattern, replicating the production threshold derivation + scans.
*
* Methodology (chosen to be honest, not to flatter the change):
* - keep = 2048 (the real GLM-5.2 index_topk), nk swept across context lengths from
* the 2049 activation boundary up to 65536 (a long conversation).
* - Three score shapes: (a) realistic peaked -- a few hot keys, long tail, the shape
* real DSA attention scores take; (b) uniform random -- no structure; (c) a plateau
* of ties, to exercise the boundary-membership path.
* - Each (shape, nk) is timed over N_REPEAT=2000 iterations, with the scores frozen
* so both algorithms do IDENTICAL work. We report median ns/call and the new/old
* ratio. A warmup pass primes caches before timing.
*
* Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS)
*/
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
#include <math.h>
#include <stdint.h>
/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */
static int cmp_pdesc_old(const void *a, const void *b){
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){
float *tmp=malloc((size_t)nk*sizeof(float));
memcpy(tmp,isc,(size_t)nk*sizeof(float));
qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old);
float thr=tmp[keep-1]; int nd=0;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
free(tmp); *nd_out=nd;
}
/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */
#define N_REPEAT 2000
static double bench_ns(void (*fn)(const float*,int,int,int*,int*),
const float *isc, int nk, int keep){
static double ts[N_REPEAT]; int *dst=malloc((size_t)nk*sizeof(int)); int nd;
for(int r=0;r<N_REPEAT;r++){
double t0=now_s();
fn(isc,nk,keep,dst,&nd);
ts[r]=(now_s()-t0)*1e9;
}
for(int a=1;a<N_REPEAT;a++){ double k=ts[a]; int b=a-1;
while(b>=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; }
free(dst);
return ts[N_REPEAT/2];
}
/* Sort an array of doubles ascending (median-of-medians aggregation below). */
static void dsort(double *a, int n){
for(int s=1;s<n;s++){ double k=a[s]; int b=s-1;
while(b>=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; }
}
/* the NEW algorithm calls the real partial_select_desc + replicates the production
* threshold derivation and position scans. */
static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){
float *tmp=malloc((size_t)nk*sizeof(float));
memcpy(tmp,isc,(size_t)nk*sizeof(float));
partial_select_desc(tmp,nk,keep);
float thr=tmp[0]; for(int t=1;t<keep;t++) if(tmp[t]<thr) thr=tmp[t];
int nd=0;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
free(tmp); *nd_out=nd;
}
/* deterministic score fill for three shapes */
static uint32_t brng = 0xA5A5A5A5u;
static void brng_seed(uint32_t s){ brng = s; }
static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5;
return (double)(brng >> 8) * (1.0 / 16777216.0); }
static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */
for(int i=0;i<nk;i++) isc[i]=(float)(-1.0 - brand()*4.0);
isc[0]=3.f; isc[nk/50<nk?nk/50:nk-1]=1.f; isc[nk/200<nk?nk/200:nk-1]=0.5f;
}
static void fill_uniform(float *isc, int nk){ /* no structure */
for(int i=0;i<nk;i++) isc[i]=(float)(brand()*1000.0);
}
static void fill_plateau(float *isc, int nk){ /* tie blocks -> boundary path */
for(int i=0;i<nk;i++) isc[i]=(float)(-(double)(i/7));
}
/* Multi-seed aggregation. Per @KingIcyCreamProjects (#357 thread): with a single frozen
* input per cell, quickselect's deterministic median-of-three pivot means one lucky input
* can spike a single nk row (an observed ~75x at nk=8192 on a 9950X3D that was really a
* ~13-40x algorithm). Two bugs compounded it: (1) the old bench drew ONE input per cell;
* (2) brng was never reset, so each cell's input depended on every prior cell's draws --
* reordering nks[] silently shifted all later inputs. Both fixed here: brng is reseeded
* per (shape, nk, seed), and we take the MEDIAN of N_SEEDS independent inputs, each itself
* a median over N_REPEAT timing reps. A lucky pivot now moves one of the N_SEEDS samples,
* not the reported number. */
int main(void){
int keep = 2048; /* GLM-5.2 index_topk */
int nks[] = {2049, 4096, 8192, 16384, 32768, 65536};
const int N_SEEDS = 11; /* odd so the median is a real sample, not interpolated */
float *isc = malloc((size_t)65536*sizeof(float));
int *dst = malloc((size_t)65536*sizeof(int)); int nd;
double *old_seeds = malloc((size_t)N_SEEDS*sizeof(double));
double *new_seeds = malloc((size_t)N_SEEDS*sizeof(double));
struct { const char *name; void (*fill)(float*,int); } shapes[] = {
{ "realistic", fill_realistic },
{ "uniform", fill_uniform },
{ "plateau", fill_plateau },
};
printf("bench_dsa_select: DSA top-keep, old (qsort) vs new (partial-select) keep=%d (median of %d seeds x %d reps)\n",
keep, N_SEEDS, N_REPEAT);
printf("%-12s %7s %14s %14s %9s\n", "shape", "nk", "old ns/call", "new ns/call", "speedup");
printf("------------------------------------------------------------------------\n");
for(size_t sh=0; sh<sizeof(shapes)/sizeof(shapes[0]); sh++){
for(size_t ni=0; ni<sizeof(nks)/sizeof(nks[0]); ni++){
int nk=nks[ni];
int bad = 0;
for(int sd=0; sd<N_SEEDS; sd++){
/* reseed per (shape,nk,seed) so each cell's input is reproducible and
* independent of cell ordering, and so lucky pivots are sampled, not fixed. */
brng_seed(0xA5A5A5A5u + (uint32_t)(sd*0x9E3779B9u));
shapes[sh].fill(isc,nk);
/* warmup both paths so caches/branch predictors are primed */
for(int w=0; w<50; w++){ keep_old(isc,nk,keep,dst,&nd); keep_new(isc,nk,keep,dst,&nd); }
/* sanity: both must keep exactly `keep` (correctness is test_dsa_select's
* job, but a count divergence here would make the timing meaningless) */
keep_old(isc,nk,keep,dst,&nd); int na=nd;
keep_new(isc,nk,keep,dst,&nd); int nb=nd;
if(na!=keep || nb!=keep){ bad++; continue; }
old_seeds[sd] = bench_ns(keep_old,isc,nk,keep);
new_seeds[sd] = bench_ns(keep_new,isc,nk,keep);
}
if(bad == N_SEEDS){
printf("%-12s %7d (BAD COUNTS on all %d seeds, skipped)\n",
shapes[sh].name, nk, bad);
continue;
}
/* report median-of-seed-medians: robust to a single lucky/unlucky pivot */
dsort(old_seeds, N_SEEDS);
dsort(new_seeds, N_SEEDS);
double t_old = old_seeds[N_SEEDS/2];
double t_new = new_seeds[N_SEEDS/2];
printf("%-12s %7d %14.0f %14.0f %8.2fx\n",
shapes[sh].name, nk, t_old, t_new, t_old/t_new);
}
printf("\n");
}
printf("bench_dsa_select: done (lower ns is better; speedup = old/new)\n");
free(isc); free(dst); free(old_seeds); free(new_seeds);
return 0;
}
c/tests/bench_topp.c
+131
View File
@@ -0,0 +1,131 @@
/* Microbenchmark: old (full-vocab qsort) vs new (heap partial-select) top-p truncation.
*
* This is NOT a unit test -- test_topp.c proves correctness. This measures the headline
* claim of #335: that replacing the O(V log V) qsort over all 151936 vocab entries with
* an O(V) heapify + k*log-V pops is materially faster per call, which is the win the issue
* was opened for.
*
* It re-implements the OLD dist_build inline (qsort + scan) on a private buffer so the A/B
* runs in one process, same inputs, same warm caches -- a controlled comparison. It calls
* the REAL (new) dist_build via the include-glm.c pattern on the global g_pbuf.
*
* Methodology (chosen to be honest, not to flatter the change):
* - V = 151936 (the actual GLM-5.2 vocab), g_nuc swept across the values that matter
* for serving: 0.5 / 0.9 (serve default) / 0.95 / 0.99.
* - Three logit shapes: (a) realistic peaked -- one hot token, long exponential tail,
* the shape real language-model logits take; (b) uniform -- worst case for the heap,
* maximum pop count; (c) a plateau of ties, to exercise the tie path.
* - Each (shape, nuc) is timed over N_REPEAT=2000 iterations, with the RNG/logits frozen
* so both algorithms do IDENTICAL work. We report median ns/call and the new/old ratio.
* - A warmup pass primes caches before timing.
*
* Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate)
*/
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
#include <math.h>
#include <stdint.h>
/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */
static float *s_pbuf; static int *s_pidx; static double *s_ref;
static int cmp_pdesc_old(const void *a, const void *b){
double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b];
return pa < pb ? 1 : pa > pb ? -1 : 0; }
static void dist_build_old(const float *lo, int V, double temp, double nuc){
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4);
for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; }
for (int i = 0; i < V; i++) s_ref[i] /= s;
if (nuc > 0 && nuc < 1.0){
for (int i = 0; i < V; i++) s_pidx[i] = i;
qsort(s_pidx, V, sizeof(int), cmp_pdesc_old);
double cum = 0; int keep = V;
for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } }
double s2 = 0;
for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0;
for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]];
for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2;
}
(void)s_pbuf;
}
/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */
#define N_REPEAT 2000
static double bench_ns(void (*fn)(const float*,int,double,double),
const float *lo, int V, double temp, double nuc){
static double ts[N_REPEAT];
for (int r = 0; r < N_REPEAT; r++){
double t0 = now_s();
fn(lo, V, temp, nuc);
ts[r] = (now_s() - t0) * 1e9;
}
/* insertion sort the N_REPEAT samples (small), take median */
for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1;
while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; }
return ts[N_REPEAT/2];
}
/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf).
* Wrap it so the bench signature matches, and set the globals it reads. */
static void dist_build_new(const float *lo, int V, double temp, double nuc){
g_temp = (float)temp; g_nuc = (float)nuc;
dist_build(lo, V);
}
/* deterministic logit fill for three shapes */
static uint32_t brng = 0xA5A5A5A5u;
static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5;
return (double)(brng >> 8) * (1.0 / 16777216.0); }
static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */
for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001);
lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f;
}
static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */
for (int i = 0; i < V; i++) lo[i] = 0.f;
}
static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */
for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50));
}
int main(void){
int V = 151936;
float *lo = malloc((size_t)V * sizeof(float));
s_ref = malloc((size_t)V * sizeof(double));
s_pidx = malloc((size_t)V * sizeof(int));
/* force the new dist_build to allocate g_pbuf/g_pidx at full V once */
g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V);
double temp = 0.7;
struct { const char *name; void (*fill)(float*,int); } shapes[] = {
{ "realistic", fill_realistic },
{ "uniform", fill_uniform },
{ "plateau", fill_plateau },
};
double nucs[] = { 0.5, 0.9, 0.95, 0.99 };
printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp);
printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep");
printf("-----------------------------------------------------------------------------\n");
for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){
shapes[sh].fill(lo, V);
for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){
double nuc = nucs[ni];
/* warmup both paths so caches/branch predictors are primed */
for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); }
double t_old = bench_ns(dist_build_old, lo, V, temp, nuc);
double t_new = bench_ns(dist_build_new, lo, V, temp, nuc);
/* keep count = non-zero entries the new path leaves (== old's keep) */
int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++;
printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n",
shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep);
}
printf("\n");
}
printf("bench_topp: done (lower ns is better; speedup = old/new)\n");
free(lo); free(s_ref); free(s_pidx);
return 0;
}
c/tests/test_benchmark_cuda_fixture.py
+5 -1
View File
@@ -1,11 +1,12 @@
import unittest
from tools.benchmark_cuda_fixture import parse_output
from tools.benchmark_cuda_fixture import parse_output, parse_p0
SAMPLE = """
REPLAY decode: 4 tokens | 12.34 tok/s
PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s
P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s
"""
@@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, "benchmark output missing"):
parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed")
def test_extracts_p0_profile(self):
self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1])
if __name__ == "__main__":
unittest.main()
c/tests/test_dsa_select.c
+223
View File
@@ -0,0 +1,223 @@
/* DSA top-keep partial-select: the quickselect rewrite (#356) must produce a
* BIT-IDENTICAL kept-position set to the old full-vocab qsort, for every score
* shape the attention indexer can see.
*
* Why this test exists (#356): attention_rows() selects the top-`keep` context
* keys (index_topk=2048 on GLM-5.2) to attend to. It previously did this by
* full-qsorting all `nk` scores (O(nk log nk)) and reading tmp[keep-1] as the
* threshold. It now does a partial_select_desc (quickselect, O(nk) average) and
* takes the threshold as the min of the selected top-keep block. The contract is
* subtle but STRONGER than test_topp's:
*
* The two position-order scans that build dst[] --
* for t: if isc[t] > thr -> keep (strictly above threshold)
* for t: if isc[t] == thr -> keep (ties, in position order)
* -- are UNCHANGED by the rewrite. So if the threshold value is identical,
* the kept-position set is identical element-by-element (not just as a
* multiset, which is all the unstable sampling heap in #335 could promise).
*
* Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern)
* and replicate the production threshold derivation + scans, then compare the
* resulting dst[] against an INDEPENDENT reference that re-implements the OLD
* algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept
* sets must be element-wise equal on every shape, including tie plateaus where
* the boundary membership is decided by the position scan.
*
* We also directly unit-test partial_select_desc's partition invariant: after
* the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really
* did land in the prefix. This catches a broken quickselect even before the
* end-to-end comparison.
*
* In-memory only (no scratch files), so it builds clean on the Windows MinGW CI
* job without the unmerged compat shim. */
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
#include <math.h>
static int g_nfails = 0;
#define FAIL(fmt, ...) do { \
fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \
label, nk, keep, shape_name, ##__VA_ARGS__); \
g_nfails++; \
return; \
} while (0)
/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */
/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */
static int cmp_ref_desc(const void *a, const void *b){
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold =
* tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */
static int keep_old(const float *isc, int nk, int keep, int *dst){
float *tmp=malloc((size_t)nk*sizeof(float));
memcpy(tmp,isc,(size_t)nk*sizeof(float));
qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc);
float thr=tmp[keep-1];
int nd=0;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
free(tmp);
return nd;
}
/* Reproduce the NEW glm.c path: partial_select desc, threshold = min of the
* selected block, same two scans. Uses the REAL partial_select_desc from glm.c. */
static int keep_new(const float *isc, int nk, int keep, int *dst){
float *tmp=malloc((size_t)nk*sizeof(float));
memcpy(tmp,isc,(size_t)nk*sizeof(float));
partial_select_desc(tmp,nk,keep);
float thr=tmp[0]; for(int t=1;t<keep;t++) if(tmp[t]<thr) thr=tmp[t];
int nd=0;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
free(tmp);
return nd;
}
/* ---- direct unit test of the partition invariant ---- */
static void check_partition(const char *label, const float *isc, int nk, int keep,
const char *shape_name){
float *tmp=malloc((size_t)nk*sizeof(float));
memcpy(tmp,isc,(size_t)nk*sizeof(float));
partial_select_desc(tmp,nk,keep);
/* invariant: every element of tmp[0..keep) is >= every element of tmp[keep..n).
* (>=, not >: equal values may sit on either side of the partition boundary,
* which is fine -- the threshold is the MIN of the prefix, and the position
* scan handles ties.) */
float top_min=INFINITY, tail_max=-INFINITY;
for(int i=0;i<keep;i++) if(tmp[i]<top_min) top_min=tmp[i];
for(int i=keep;i<nk;i++) if(tmp[i]>tail_max) tail_max=tmp[i];
if(!(top_min >= tail_max))
FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max);
free(tmp);
}
/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */
static void check_case(const char *label, int nk, int keep, const char *shape_name,
const float *isc){
int *da=malloc((size_t)nk*sizeof(int));
int *db=malloc((size_t)nk*sizeof(int));
int na=keep_old(isc,nk,keep,da);
int nb=keep_new(isc,nk,keep,db);
/* 1. both keep exactly `keep` positions (the contract: keep the top-keep by
* count). A count mismatch is a real bug, not a tie artifact. */
if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep);
if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep);
if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb);
/* 2. element-wise identical dst[]. This is the strong contract: because the
* threshold is derived identically and the position-order scans are byte-
* for-byte the same, the kept SET and its ORDER must match exactly. (This
* is what makes #356 cleaner than #335, which was multiset-only.) */
int first_diff=-1;
for(int i=0;i<na;i++){ if(da[i]!=db[i]){ first_diff=i; break; } }
if(first_diff>=0)
FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d",
first_diff, first_diff, da[first_diff], first_diff, db[first_diff]);
/* 3. also check the partition invariant directly (catches a subtly broken
* quickselect even if the threshold happened to come out right). */
check_partition(label,isc,nk,keep,shape_name);
free(da); free(db);
printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name);
}
#undef FAIL
/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */
static uint32_t rng_state = 0x12345678u;
static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17;
rng_state ^= rng_state << 5; return rng_state; }
static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */
/* fill scores for a given shape. Shapes stress the threshold boundary and the
* quickselect's median-of-three pivot differently. */
static void fill_shape(float *isc, int nk, int shape){
switch(shape){
case 0: /* uniform random distinct (no ties): the clean contract case */
for(int i=0;i<nk;i++) isc[i]=(float)(frand()*1000.0); break;
case 1: /* peaked: a few hot, long distinct tail (realistic attention shape) */
for(int i=0;i<nk;i++) isc[i]=(float)(-1.0 - frand()*4.0);
isc[0]=3.f; if(nk>3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break;
case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case
* for a naive quickselect; median-of-three must handle it */
for(int i=0;i<nk;i++) isc[i]=(float)(-0.001*(double)i); break;
case 3: /* strictly increasing (reverse-sorted): the other quickselect worst case */
for(int i=0;i<nk;i++) isc[i]=(float)(0.001*(double)i); break;
case 4: /* plateau ties: blocks of equal value -> boundary membership decided
* entirely by the position scan (exercises the ==thr path) */
for(int i=0;i<nk;i++) isc[i]=(float)(-(double)(i/7)); break;
case 5: /* all-equal: every value identical -> degenerate threshold, all kept
* via the ==thr scan; quickselect must not infinite-loop or corrupt */
for(int i=0;i<nk;i++) isc[i]=5.f; break;
}
}
int main(void){
/* Sizes around the real index_topk=2048 boundary, plus small cases that
* exercise the k>=n / k==1 / k==n edges. */
int nks[] = {1, 2, 8, 64, 2049, 4097, 8193};
int keeps[] = {1, 8, 256, 1024, 2048};
int n_shapes = 6;
int cases = 0;
for(size_t ni=0; ni<sizeof(nks)/sizeof(nks[0]); ni++){
int nk=nks[ni];
float *isc=malloc((size_t)nk*sizeof(float));
for(int shape=0; shape<n_shapes; shape++){
/* skip shapes that write out of bounds on tiny nk (fill_shape guards
* the hot-spots with nk>n, but skip the plateau/geometric edge if nk<7) */
fill_shape(isc,nk,shape);
for(size_t ki=0; ki<sizeof(keeps)/sizeof(keeps[0]); ki++){
int keep=keeps[ki];
if(keep>nk) continue; /* keep<=nk invariant of the production code */
if(keep<=0) continue;
char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape);
const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape];
check_case(label,nk,keep,sn,isc);
cases++;
}
}
free(isc);
}
/* edge: keep == nk (nothing to partition; both paths keep everything) */
{
int nk=100, keep=100; float isc[100];
for(int i=0;i<nk;i++) isc[i]=(float)frand();
int *db=malloc(sizeof(int)*nk); int nb=keep_new(isc,nk,keep,db);
if(nb!=nk){ fprintf(stderr," FAIL [keep==nk]: kept %d expected %d\n",nb,nk); g_nfails++; }
else printf(" ok [keep==nk nk=%d]\n",nk);
free(db); cases++;
}
/* edge: keep == 1 (threshold = the single max; quickselect must find it) */
{
int nk=500, keep=1; float isc[500];
for(int i=0;i<nk;i++) isc[i]=(float)frand();
int da[1],db[1]; int na=keep_old(isc,nk,keep,da), nb=keep_new(isc,nk,keep,db);
if(na!=1||nb!=1||da[0]!=db[0]){
fprintf(stderr," FAIL [keep==1]: old={%d (n=%d)} new={%d (n=%d)}\n",da[0],na,db[0],nb); g_nfails++; }
else printf(" ok [keep==1 argmax=%d]\n",db[0]); cases++;
}
/* edge: all-equal scores, keep in the middle -> every kept slot is a tie;
* the position scan must pick positions 0..keep-1 deterministically */
{
int nk=1000, keep=500; float isc[1000];
for(int i=0;i<nk;i++) isc[i]=3.14f;
int *db=malloc(sizeof(int)*nk); int nb=keep_new(isc,nk,keep,db);
int bad=0; for(int i=0;i<nb;i++) if(db[i]!=i) bad=1;
if(nb!=keep||bad){ fprintf(stderr," FAIL [all-equal keep=%d]: nb=%d bad=%d\n",keep,nb,bad); g_nfails++; }
else printf(" ok [all-equal keep=%d -> positions 0..%d]\n",keep,keep-1); cases++;
free(db);
}
printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails);
if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; }
printf("test_dsa_select: ok\n");
return 0;
}
c/tests/test_i4_grouped.c
+201
View File
@@ -0,0 +1,201 @@
/* Exactness test for the grouped-int4 kernel (fmt=4, one f32 scale per `gs`
* elements along I) against a plain-C reference that dequantizes and multiplies
* in double.
*
* Why this test exists: matmul_i4_grouped is the REFERENCE the CUDA fmt=4 path
* (#298) is expected to reproduce, and it had no test of its own. Debugging a
* backend against an unverified oracle means two moving targets. Anyone porting
* fmt=4 to a new backend can now diff against a kernel that is known exact here.
*
* Covers: I a clean multiple of gs, I with a partial last group (the `glen`
* clamp), odd I (the nibble tail), gs larger than I (single group), and the
* nibble edges 0 and 15 (which decode to -8 and +7 — an offset encoding, NOT
* two's complement; getting this backwards is silent and looks like noise).
*
* FP note: the kernel sums each group in f32 (AVX2 accumulator + scalar tail)
* while the reference sums in double, so we compare against a relative epsilon
* rather than bit-exactly. The tolerance is tight enough that a wrong scale
* index, a wrong group boundary or a swapped nibble cannot hide under it —
* those are O(1) relative errors, not O(1e-6). */
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
static uint32_t rng_state=0xC0FFEEu;
static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; }
static float frand(void){ return (float)((int)(xr()%2001)-1000)/1000.0f; }
/* Reference: dequantize nibble -> (v-8)*scale[group], accumulate in double.
* Deliberately the dumbest possible expression of the format. */
static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q4,
const float *scale, int S, int I, int O, int gs){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
for(int o=0;o<O;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*I; double a=0, m=0;
for(int i=0;i<I;i++){
uint8_t byte=w[i>>1];
int nib=(i&1)?(int)(byte>>4):(int)(byte&0xF);
double term=(double)xs[i] * (double)(nib-8) * (double)scl[i/gs];
a += term; m += fabs(term);
}
y[(int64_t)s*O+o]=a;
/* Sum of |terms|: the scale the f32 rounding error actually lives on.
* Comparing against |result| instead would flag pure cancellation --
* a dot product of signed terms can land near zero, and then a 1e-6
* absolute error reads as a 1e-3 relative one. That is the accumulator's
* precision, not a kernel defect. A wrong scale index or group boundary
* shifts the result by a fraction OF THE TERMS, so it is caught here. */
mag[(int64_t)s*O+o]=m;
}
}
}
static int check(const char *name, int S, int I, int O, int gs, int fill_edges){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
uint8_t *q4=malloc((size_t)O*rb);
float *scale=malloc((size_t)O*ng*sizeof(float));
float *x=malloc((size_t)S*I*sizeof(float));
float *y=malloc((size_t)S*O*sizeof(float));
double *yr=malloc((size_t)S*O*sizeof(double));
double *ym=malloc((size_t)S*O*sizeof(double));
if(!q4||!scale||!x||!y||!yr||!ym){ fprintf(stderr,"%s: OOM\n",name); return 1; }
for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF);
if(fill_edges){
/* nibble extremes: 0x0F -> +7, 0x00 -> -8. A two's-complement misread
* turns 15 into -1 instead of +7 and the error is data-dependent noise. */
for(size_t i=0;i<(size_t)O*rb && i<64;i++) q4[i]=(i&1)?0x00:0xFF;
}
/* scales span a few orders of magnitude: a wrong group index shows up big */
for(int i=0;i<O*ng;i++) scale[i]=(0.001f+(float)(xr()%1000)/1000.0f)*((xr()&1)?1.f:-1.f);
for(int i=0;i<S*I;i++) x[i]=frand();
matmul_i4_grouped(y,x,q4,scale,S,I,O,gs);
ref_grouped(yr,ym,x,q4,scale,S,I,O,gs);
int bad=0; double worst=0;
for(int i=0;i<S*O;i++){
double d=fabs((double)y[i]-yr[i]);
double rel = ym[i]>1e-30 ? d/ym[i] : d; /* error relative to the summed magnitude */
if(rel>worst) worst=rel;
if(rel>1e-6){
if(bad<3) fprintf(stderr,"%s: [%d] got %.9g want %.9g (|terms| %.3g, rel %.3g)\n",
name,i,(double)y[i],yr[i],ym[i],rel);
bad++;
}
}
free(q4);free(scale);free(x);free(y);free(yr);free(ym);
if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; }
printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g)\n",name,S,I,O,gs,ng,worst);
return 0;
}
/* matmul_i4_grouped_pair (fused gate+up, #298) reads x once instead of twice.
* Checked two ways, because "identical" is only true where it can be:
*
* - Correctness, always: both outputs must match the double reference within
* the same magnitude-relative epsilon as the unfused kernel.
* - Bit-exactness, only when I % gs == 0: then every group is covered by the
* AVX2 body, whose accumulation order is identical to the unfused kernel, so
* the results agree to the last bit. This is the shape the real g64
* checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path.
*
* With a PARTIAL last group the group tail falls to scalar code, and the
* compiler is free to contract/reassociate the fused body differently from the
* single-matrix one. The results then differ by ~1e-7 -- rounding, not logic
* (which of gate/up "differs" is arbitrary, the tell that it is FP luck).
* Demanding bit-exactness there would report a compiler artifact as a bug. */
#ifdef COLI_HAVE_GROUPED_PAIR
static int check_pair(const char *name, int S, int I, int O, int gs){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
uint8_t *qg=malloc((size_t)O*rb), *qu=malloc((size_t)O*rb);
float *sg=malloc((size_t)O*ng*sizeof(float)), *su=malloc((size_t)O*ng*sizeof(float));
float *x=malloc((size_t)S*I*sizeof(float));
float *yg=malloc((size_t)S*O*sizeof(float)), *yu=malloc((size_t)S*O*sizeof(float));
float *rg=malloc((size_t)S*O*sizeof(float)), *ru=malloc((size_t)S*O*sizeof(float));
double *dg=malloc((size_t)S*O*sizeof(double)), *du=malloc((size_t)S*O*sizeof(double));
double *mg=malloc((size_t)S*O*sizeof(double)), *mu=malloc((size_t)S*O*sizeof(double));
if(!qg||!qu||!sg||!su||!x||!yg||!yu||!rg||!ru||!dg||!du||!mg||!mu){ fprintf(stderr,"%s: OOM\n",name); return 1; }
for(size_t i=0;i<(size_t)O*rb;i++){ qg[i]=(uint8_t)(xr()&0xFF); qu[i]=(uint8_t)(xr()&0xFF); }
for(int i=0;i<O*ng;i++){ sg[i]=frand(); su[i]=frand(); }
for(int i=0;i<S*I;i++) x[i]=frand();
matmul_i4_grouped_pair(yg,yu,x,qg,sg,qu,su,S,I,O,gs);
matmul_i4_grouped(rg,x,qg,sg,S,I,O,gs);
matmul_i4_grouped(ru,x,qu,su,S,I,O,gs);
ref_grouped(dg,mg,x,qg,sg,S,I,O,gs);
ref_grouped(du,mu,x,qu,su,S,I,O,gs);
int bad=0, exact=1; double worst=0;
for(int i=0;i<S*O;i++){
double eg = mg[i]>1e-30 ? fabs((double)yg[i]-dg[i])/mg[i] : fabs((double)yg[i]-dg[i]);
double eu = mu[i]>1e-30 ? fabs((double)yu[i]-du[i])/mu[i] : fabs((double)yu[i]-du[i]);
if(eg>worst) worst=eg;
if(eu>worst) worst=eu;
if(eg>1e-6||eu>1e-6){
if(bad<3) fprintf(stderr,"%s: [%d] gate %.9g/%.9g up %.9g/%.9g (rel %.3g/%.3g)\n",
name,i,(double)yg[i],dg[i],(double)yu[i],du[i],eg,eu);
bad++;
}
if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;
}
/* Aligned shapes run entirely through the AVX2 body: same order as unfused,
* so bit-exactness is a real invariant there and worth asserting. */
if(I%gs==0 && !exact){
fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape "
"(no scalar tail runs here; the orders must match)\n",name);
bad++;
}
free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru);
free(dg);free(du);free(mg);free(mu);
if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; }
printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst,
I%gs==0?", bit-exact vs unfused":"");
return 0;
}
#endif
int main(void){
int fail=0;
printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n");
/* the shape the g64 checkpoints actually use */
fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0);
fail|=check("gs=64, single row single token", 1, 128, 1, 64, 0);
fail|=check("gs=64, nibble edges (0x00/0xFF)", 1, 256, 4, 64, 1);
/* partial last group: glen clamp, the classic off-by-one */
fail|=check("gs=64, partial last group (I=200)", 2, 200, 4, 64, 0);
fail|=check("gs=64, I just over a group (I=65)", 1, 65, 3, 64, 0);
fail|=check("gs=64, I one under a group (I=63)", 1, 63, 3, 64, 0);
/* odd I: the scalar nibble tail (i+1 == I) */
fail|=check("gs=64, odd I (I=201)", 2, 201, 4, 64, 0);
fail|=check("gs=16, odd I (I=33)", 1, 33, 2, 16, 0);
/* gs > I: everything in one group */
fail|=check("gs=128 > I=64 (single group)", 1, 64, 4, 128, 0);
/* the other documented group size */
fail|=check("gs=128, I multiple of gs", 2, 512, 4, 128, 0);
/* batch: S>1 exercises the per-s inner loop against a shared scale row */
fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0);
#ifdef COLI_HAVE_GROUPED_PAIR
printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n");
fail|=check_pair("pair: gs=64, I multiple of gs", 2, 512, 8, 64);
fail|=check_pair("pair: gs=64, partial last group",2, 200, 4, 64);
fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64);
fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64);
fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128);
#endif
if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; }
printf("test_i4_grouped: ok\n");
return 0;
}
c/tests/test_logit_nan.c
+70
View File
@@ -0,0 +1,70 @@
/* Regression for non-finite-logit poisoning of sampling.
*
* A single NaN or +Inf in the logits (a bad streamed expert tile, or an fp
* overflow in the matmul at a low-RAM eviction boundary) used to make softmax
* produce an all-NaN g_pbuf; dist_sample then never satisfied cum>=u and fell
* through to return token 0 — so the engine silently emitted an unbroken run of
* token 0 with no error, on the DEFAULT serve path (TEMP>0, 0<NUCLEUS<1).
*
* Fix under test: argmax_v() skips NaN (picks the max finite/+Inf entry instead
* of being pinned to index 0), and dist_build() detects a non-finite softmax sum
* and collapses to a one-hot over the finite argmax (warning once) rather than
* dividing every entry into NaN. Degrade + diagnose, never silently corrupt.
*
* No model file needed: exercises argmax_v / dist_build / dist_sample directly. */
#include <assert.h>
#include <math.h>
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
static int approx1(double x){ return x > 0.999 && x < 1.001; }
int main(void){
/* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */
{ float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f};
assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); }
{ float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f};
assert(argmax_v(lo,8)==1 && "pick the +Inf position"); }
{ float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN;
assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); }
g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */
/* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */
{ float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */
dist_build(lo,8);
double sum=0; int nan=0;
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
assert(!nan && "g_pbuf must be finite after a NaN logit");
assert(approx1(sum) && "g_pbuf must normalize to 1");
assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)");
assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); }
/* --- NaN at index 0: poisons the max scan itself (the old mx=lo[0] seed),
* the failure mode that starts before the sum (review note on #369) --- */
{ float lo[8]={NAN,1.f,0.5f,6.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (6.0) */
dist_build(lo,8);
double sum=0; int nan=0;
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
assert(!nan && "NaN at lo[0] must not poison via the mx seed");
assert(approx1(sum) && "still normalizes to 1");
assert(dist_sample(8,-1)==3 && "emits the max finite logit, not token 0"); }
/* --- all-NaN logits: worst case — must stay finite, no crash --- */
{ float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN;
dist_build(lo,8);
for(int i=0;i<8;i++) assert(g_pbuf[i]==g_pbuf[i] && "all-NaN: g_pbuf stays finite"); }
/* --- regression: clean logits still produce a valid distribution --- */
{ float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */
dist_build(lo,8);
double sum=0; int nan=0;
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
assert(!nan && "clean softmax stays finite");
assert(approx1(sum) && "clean softmax must sum to 1");
assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); }
printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n");
return 0;
}
c/tests/test_openai_server.py
+33
View File
@@ -380,6 +380,25 @@ class DispatcherTest(unittest.TestCase):
engine.close()
self.assertEqual(chunks, ["é"])
def test_records_profile_snapshots_from_prof_lines(self):
def respond(process, frame):
request_id = frame.split()[1]
process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")
process.stdout.feed(b"PROF 2.500 7 12 0.400 0.100 0.900 0.600 0.200 15\n")
process.stdout.feed(b"DONE " + request_id + b" STAT 12 4.8 0 1.0 7 0\n")
process = FakeProcess(respond)
with patch("openai_server.subprocess.Popen", return_value=process):
engine = Engine("glm", "model")
engine.generate("hello", 16, 0.7, 0.9, lambda _: None)
engine.close()
self.assertEqual(engine.profile_seq, 1)
self.assertEqual(list(engine.profile), [{
"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15,
}])
def test_cancels_generation_after_consumer_disconnects(self):
request_id = None
@@ -443,6 +462,20 @@ class HTTPTest(unittest.TestCase):
self.assertIn("queued", scheduler)
self.assertEqual(health["kv_slots"], 2)
def test_profile_reports_recent_turns_without_auth(self):
with urlopen(self.base + "/profile", timeout=2) as response:
self.assertEqual(json.load(response), {"seq": 0, "turns": []})
turn = {"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15}
self.engine.profile = [turn]
self.engine.profile_seq = 1
try:
with urlopen(self.base + "/profile", timeout=2) as response:
self.assertEqual(json.load(response), {"seq": 1, "turns": [turn]})
finally:
del self.engine.profile, self.engine.profile_seq
def test_browser_preflight(self):
request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={
"Origin": "http://localhost:5173",
c/tests/test_openai_tools_e2e.py
+182
View File
@@ -0,0 +1,182 @@
"""End-to-end tool-calling test for the OpenAI gateway (#401).
Unlike the unit tests in test_openai_server.py (which call parse_tool_calls /
render_chat directly), this suite runs openai_server.py as a real subprocess
against a mock engine that speaks the actual SERVE wire protocol
(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down
the full path a coding client exercises: tool declaration rendering, marker
suppression in streamed deltas (across chunk boundaries), tool_calls in both
response shapes, and the <|observation|><tool_response> round trip.
"""
import json
import os
import socket
import subprocess
import sys
import tempfile
import unittest
import urllib.request
from pathlib import Path
SERVER = Path(__file__).resolve().parent.parent / "openai_server.py"
MODEL_ID = "glm-5.2-colibri"
# Mock engine: replies are keyed on the prompt so one process covers every case.
# Prompts received are appended to MOCK_LOG for assertions on the rendering.
MOCK_ENGINE = r'''#!/usr/bin/env python3
import sys, os
out, inp = sys.stdout.buffer, sys.stdin.buffer
out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()
def reply(rid, text, chunks=1):
data = text.encode("utf-8")
n = max(1, len(data) // chunks)
for i in range(0, len(data), n):
part = data[i:i+n]
out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush()
out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())
out.flush()
while True:
line = inp.readline()
if not line: break
f = line.decode().strip().split()
if not f or f[0] != "SUBMIT": continue
rid, plen = f[1], int(f[3])
prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)
with open(os.environ["MOCK_LOG"], "a") as log:
log.write(prompt + "\n\x00\n")
if "<tool_response>" in prompt:
reply(rid, "25 degrees and sunny in Rome.")
elif "weather in Rome" in prompt:
reply(rid, "<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Rome</arg_value></tool_call>")
elif "weather in Milan" in prompt:
# split across many tiny DATA chunks: streamed marker suppression must
# hold even when a marker straddles a chunk boundary
reply(rid, "Checking. <tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Milan</arg_value></tool_call>", chunks=20)
else:
reply(rid, "Hello from the mock engine.")
'''
TOOLS = [{"type": "function", "function": {
"name": "get_weather",
"description": "Current weather for a city",
"parameters": {"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]}}}]
@unittest.skipUnless(os.name == "posix",
"the mock engine is a shebang script the gateway execs directly; "
"Windows CreateProcess cannot run it. The gateway logic under test "
"is platform-independent and covered by the POSIX CI jobs.")
class ToolCallingE2E(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
mock = Path(cls.tmp.name) / "mock_engine.py"
mock.write_text(MOCK_ENGINE)
mock.chmod(0o755)
cls.mock_log = Path(cls.tmp.name) / "prompts.log"
cls.mock_log.touch()
with socket.socket() as probe: # free port, then hand it to the server
probe.bind(("127.0.0.1", 0))
cls.port = probe.getsockname()[1]
env = dict(os.environ, MOCK_LOG=str(cls.mock_log))
cls.server = subprocess.Popen(
[sys.executable, str(SERVER), "--model", cls.tmp.name,
"--engine", str(mock), "--port", str(cls.port)],
env=env, stderr=subprocess.DEVNULL)
cls.base = f"http://127.0.0.1:{cls.port}/v1"
for _ in range(100):
try:
urllib.request.urlopen(cls.base + "/models", timeout=2)
return
except OSError:
if cls.server.poll() is not None:
raise RuntimeError("gateway exited during startup")
import time
time.sleep(0.1)
raise RuntimeError("gateway did not come up")
@classmethod
def tearDownClass(cls):
cls.server.terminate()
cls.server.wait(timeout=5)
cls.tmp.cleanup()
def post(self, body, stream=False):
req = urllib.request.Request(
self.base + "/chat/completions", json.dumps(body).encode(),
{"Content-Type": "application/json"})
resp = urllib.request.urlopen(req, timeout=30)
if not stream:
return json.loads(resp.read())
events = []
for raw in resp:
line = raw.decode().strip()
if line.startswith("data: ") and line != "data: [DONE]":
events.append(json.loads(line[6:]))
return events
def test_tool_call_non_stream(self):
r = self.post({"model": MODEL_ID, "tools": TOOLS,
"messages": [{"role": "user",
"content": "What is the weather in Rome?"}]})
choice = r["choices"][0]
self.assertEqual(choice["finish_reason"], "tool_calls")
calls = choice["message"]["tool_calls"]
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "get_weather")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"})
self.assertNotIn("<tool_call>", choice["message"].get("content") or "")
def test_tool_call_streamed_markers_suppressed(self):
events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True,
"messages": [{"role": "user",
"content": "What is the weather in Milan?"}]},
stream=True)
deltas = [e["choices"][0]["delta"] for e in events if e["choices"]]
text = "".join(d.get("content") or "" for d in deltas)
self.assertNotIn("<tool_call>", text)
self.assertNotIn("<arg_key>", text)
calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")]
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0][0]["function"]["name"], "get_weather")
self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]),
{"city": "Milan"})
finish = [e["choices"][0]["finish_reason"] for e in events
if e["choices"] and e["choices"][0].get("finish_reason")]
self.assertEqual(finish, ["tool_calls"])
def test_tool_result_round_trip(self):
r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [
{"role": "user", "content": "What is the weather in Rome?"},
{"role": "assistant", "content": None, "tool_calls": [
{"id": "call_x", "type": "function",
"function": {"name": "get_weather",
"arguments": "{\"city\": \"Rome\"}"}}]},
{"role": "tool", "tool_call_id": "call_x",
"content": "25 degrees, sunny"}]})
choice = r["choices"][0]
self.assertEqual(choice["finish_reason"], "stop")
self.assertFalse(choice["message"].get("tool_calls"))
self.assertIn("25 degrees", choice["message"]["content"])
rendered = self.mock_log.read_text().split("\x00")[-2]
self.assertIn("<|observation|><tool_response>25 degrees, sunny</tool_response>",
rendered)
self.assertIn("# Tools", rendered)
self.assertIn('"get_weather"', rendered)
def test_no_tools_plain_text(self):
r = self.post({"model": MODEL_ID,
"messages": [{"role": "user", "content": "Hi!"}]})
choice = r["choices"][0]
self.assertEqual(choice["finish_reason"], "stop")
self.assertIn("mock engine", choice["message"]["content"])
if __name__ == "__main__":
unittest.main()
c/tests/test_ragged_attention.cu
+39
View File
@@ -0,0 +1,39 @@
#include "../backend_cuda.h"
#include <cmath>
#include <cstdio>
#include <vector>
int main(){
int dev=0;if(!coli_cuda_init(&dev,1))return 77;
constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3;
std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));
for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;
for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;
for(size_t i=0;i<q.size();i++)q[i]=((int)(i%13)-6)*.05f;
ColiCudaTensor *tw=nullptr,*tp=nullptr;
if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||
!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;
int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);
const float *lp[S],*rp[S];
const void *keys[S];
for(int s=0;s<S;s++){
l[s].resize(n[s]*K);r[s].resize(n[s]*R);
for(size_t i=0;i<l[s].size();i++)l[s][i]=((int)((i+s*3)%9)-4)*.08f;
for(size_t i=0;i<r[s].size();i++)r[s][i]=((int)((i+s)%5)-2)*.06f;
lp[s]=l[s].data();rp[s]=r[s].data();keys[s]=&l[s];
}
float got[S*O],ref[S*O],warm[S*O];
int first[S]={1,1,1};
if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,first,
S,H,Q,R,V,K,1,.2f))return 2;
if(!coli_cuda_attention_project_ragged(tw,tp,got,q.data(),keys,lp,rp,n,S,H,Q,R,V,K,T,.2f))return 2;
for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tw,tp,ref+s*O,
q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 3;
double e=0,z=0;for(int i=0;i<S*O;i++){
double d=got[i]-ref[i];e+=d*d;z+=(double)ref[i]*ref[i];
}
double rms=std::sqrt(e/(z+1e-30));std::printf("ragged_relative_rms=%.9g\n",rms);
coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tp);coli_cuda_shutdown();
return rms<1e-6?0:4;
}
c/tests/test_resource_plan.py
+17 -2
View File
@@ -10,6 +10,7 @@ from resource_plan import (
GB,
analyze_model,
build_plan,
cpu_socket_count,
environment_for_plan,
format_plan,
memory_available,
@@ -66,15 +67,19 @@ class ResourcePlanTest(unittest.TestCase):
# 0 slots/layer. The value must be a sane positive number of bytes.
self.assertGreater(memory_available(), 0)
def test_cpu_socket_count_is_positive(self):
self.assertGreaterEqual(cpu_socket_count(), 1)
def test_builds_bounded_three_tier_plan(self):
gpus = [{"index": 0, "name": "test-gpu", "total_bytes": 12 * GB,
"free_bytes": 10 * GB}]
plan = build_plan(self.model, ram_gb=16, context=32, vram_gb=20,
available_memory=32 * GB, available_disk=100 * GB, gpus=gpus,
physical_cpus=24)
physical_cpus=24, cpu_sockets=2)
self.assertEqual(plan["version"], 2)
self.assertEqual(plan["policy"]["name"], "quality")
self.assertEqual(plan["cpu"]["physical_cores"], 24)
self.assertEqual(plan["cpu"]["sockets"], 2)
self.assertTrue(plan["policy"]["preserve_quantization"])
self.assertFalse(plan["tiers"]["vram"]["requires_host_backing"])
self.assertEqual(plan["tiers"]["ram"]["budget_bytes"], 16 * GB)
@@ -105,7 +110,7 @@ class ResourcePlanTest(unittest.TestCase):
{"index": 1, "name": "b", "total_bytes": 12 * GB, "free_bytes": 10 * GB},
]
plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,
available_disk=1, gpus=gpus)
available_disk=1, gpus=gpus, cpu_sockets=2)
env = environment_for_plan(plan, {"RAM_GB": "12", "PIN": "stats.txt",
"COLI_GPUS": "1"})
self.assertEqual(env["RAM_GB"], "12")
@@ -125,6 +130,16 @@ class ResourcePlanTest(unittest.TestCase):
"OMP_PROC_BIND": "close"})
self.assertEqual(explicit_threads["OMP_NUM_THREADS"], "7")
self.assertEqual(explicit_threads["OMP_PROC_BIND"], "close")
if sys.platform.startswith("linux"):
self.assertEqual(env["COLI_NUMA"], "1")
explicit_numa = environment_for_plan(plan, {"COLI_NUMA": "0"})
self.assertEqual(explicit_numa["COLI_NUMA"], "0")
def test_single_socket_plan_does_not_enable_numa(self):
plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,
gpus=[], physical_cpus=8, cpu_sockets=1)
self.assertNotIn("COLI_NUMA", environment_for_plan(plan))
def test_cpu_binary_does_not_apply_gpu_tier(self):
plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,
gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB,
c/tests/test_sample_nan.c
+62
View File
@@ -0,0 +1,62 @@
/* Regressione #369: un solo logit NaN o +Inf faceva emettere a dist_build/dist_sample
* il token 0 PER SEMPRE, in silenzio (il fallback `g_pbuf[i]>0` e' falso su NaN ovunque).
* Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta.
*
* Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf
* iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione
* del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
#include <stdio.h>
static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;}
static int pbuf_has_nan(int V){ for(int i=0;i<V;i++) if(!isfinite(g_pbuf[i])) return 1; return 0; }
int main(void){
int fail=0, V=2000;
float *lo=malloc(V*sizeof(float));
g_temp=0.7f; g_nuc=0.90f; /* il path serve di default */
/* (a) logit sani: il campionato deve avere prob > 0 e nessun NaN nel buffer */
for(int i=0;i<V;i++) lo[i]=(float)((int)(xr()%2000)-1000)/100.f;
int known=1337; lo[known]=50.f; /* picco netto */
dist_build(lo,V);
if(pbuf_has_nan(V)){ printf(" FAIL: NaN in g_pbuf con logit sani\n"); fail=1; }
if(g_pbuf[known]<=0.f){ printf(" FAIL: il picco ha prob 0\n"); fail=1; }
if(!fail) printf(" logit sani: distribuzione valida, picco vivo ok\n");
/* (b) NaN/+Inf iniettato in varie posizioni; il finito-argmax deve vincere */
float bad[3]; bad[0]=NAN; bad[1]=INFINITY; bad[2]=-INFINITY;
const char *bn[3]={"NaN","+Inf","-Inf"};
for(int b=0;b<3;b++){
for(int pos=0;pos<3;pos++){ /* lo[0], meta', ultimo */
for(int i=0;i<V;i++) lo[i]=(float)((int)(xr()%400)-200)/100.f;
int amax=777; lo[amax]=9.0f; /* massimo FINITO atteso */
int at = pos==0?0 : pos==1?V/2 : V-1;
if(at==amax) amax=amax+1; /* non sovrapporre */
lo[amax]=9.0f;
lo[at]=bad[b]; /* veleno */
dist_build(lo,V);
if(pbuf_has_nan(V)){ printf(" FAIL: NaN sopravvive (%s @ %d)\n",bn[b],at); fail=1; continue; }
/* con -Inf il fallback puo' non scattare (max finito resta), ma il buffer
* deve restare valido e sommare ~1: con +Inf/NaN scatta il delta su amax */
int picked=-1; float pv=-1;
for(int i=0;i<V;i++) if(g_pbuf[i]>pv){pv=g_pbuf[i];picked=i;}
if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */
printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1;
}
}
}
if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n");
/* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */
for(int i=0;i<V;i++) lo[i]=NAN;
dist_build(lo,V);
if(pbuf_has_nan(V)){ printf(" FAIL: tutti-NaN lascia NaN nel buffer\n"); fail=1; }
else printf(" tutti non-finiti: nessun crash, buffer valido ok\n");
printf(fail?"test_sample_nan: FAIL\n":"test_sample_nan: ok\n");
return fail;
}
c/tests/test_st_pread.c
+89
View File
@@ -0,0 +1,89 @@
/* st_pread_full: chunk loop + honest truncation errors.
* Built with -DST_PREAD_CHUNK=7 so a ~100-byte tensor takes many pread calls —
* exercising the loop that production only needs past 2^31 bytes (one pread
* caps there on Linux; big bf16 tensors exceed it). Also forks a child against
* a truncated shard and requires exit(1) with a "short read" message instead
* of the old perror("... : Success"). */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifndef _WIN32
#include <sys/wait.h>
#include <unistd.h>
#endif
#include "../st.h"
#define CHECK(condition) do { \
if (!(condition)) { \
fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \
return 1; \
} \
} while (0)
static void write_snap(const char *dir, int truncate_bytes) {
char path[512];
snprintf(path, sizeof(path), "%s/model.safetensors", dir);
unsigned char data[96];
for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3);
const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}";
uint64_t hlen = strlen(hdr);
FILE *f = fopen(path, "wb");
fwrite(&hlen, 8, 1, f);
fwrite(hdr, 1, hlen, f);
fwrite(data, 1, (size_t)(96 - truncate_bytes), f);
fclose(f);
}
int main(void) {
/* relative to the CWD, per test_stops: MinGW .exe files resolve Windows
* paths and "/tmp" is not one */
char dir[] = "test_st_pread_XXXXXX";
if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; }
/* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */
write_snap(dir, 0);
shards S; st_init(&S, dir);
unsigned char out[96] = {0};
st_read_raw(&S, "t", out, 0);
for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3));
#ifndef _WIN32
/* 2) shard truncated AFTER st_init (init validates static bounds, so the
* pread path only fires when the file shrinks underneath a live handle):
* child must exit(1) with an honest message, not perror's "Success" */
char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir);
struct stat sb; CHECK(stat(shard, &sb) == 0);
CHECK(truncate(shard, sb.st_size - 40) == 0);
int pipefd[2]; CHECK(pipe(pipefd) == 0);
pid_t pid = fork(); CHECK(pid >= 0);
if (pid == 0) {
dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]);
unsigned char buf[96];
st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */
_exit(42); /* reaching here = bug */
}
close(pipefd[1]);
char err[512] = {0};
ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n;
close(pipefd[0]);
int status = 0; waitpid(pid, &status, 0);
CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1);
CHECK(strstr(err, "short read") != NULL);
CHECK(strstr(err, "Success") == NULL);
#else
/* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */
printf("test_st_pread: truncation subtest skipped on Windows\n");
#endif
char cmd[600];
#ifdef _WIN32
snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir);
#else
snprintf(cmd, sizeof(cmd), "rm -rf %s", dir);
#endif
if (system(cmd)) {}
printf("test_st_pread: chunk loop + honest truncation error: ok\n");
return 0;
}
c/tests/test_stops.c
+136
View File
@@ -0,0 +1,136 @@
/* Stop-token arming: the engine must not depend on a converted checkpoint's
* config.json being complete.
*
* Why this test exists (#298, #307): GLM-5.2 declares THREE eos ids
* (<|endoftext|>, <|user|>, <|observation|>). A conversion tool that rewrites
* config.json with a reduced eos_token_id list leaves the engine stopping on
* fewer tokens than the model actually emits -- and the ones it misses get
* DETOKENIZED AND PRINTED INTO THE CHAT as literal text, with generation
* continuing past the real end of the turn. @woolcoxm hit exactly this on a
* g64 checkpoint and confirmed it by comparing token ids.
*
* Two independent defenses, one test each:
* 1. eos_token_id is unioned from generation_config.json (HuggingFace's
* authority for generation) as well as config.json.
* 2. every added-token the TOKENIZER marks "special":true is a stop, whatever
* the configs say. Those are control tokens (<|user|>, <sop>, ...) and are
* never legitimate content. <think>/<tool_call> are "special":false and
* must NOT be swept up -- they are real output.
*
* Defense 2 is what makes this robust against checkpoints we don't control:
* even with BOTH configs mutilated, a control token cannot leak into a reply. */
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
static const char *TOKJSON =
"{\"model\":{\"vocab\":{\"a\":0,\"b\":1,\"c\":2},\"merges\":[[\"a\",\"b\"]]},"
" \"added_tokens\":["
" {\"id\":100,\"content\":\"<|endoftext|>\",\"special\":true},"
" {\"id\":101,\"content\":\"<|user|>\",\"special\":true},"
" {\"id\":102,\"content\":\"<|observation|>\",\"special\":true},"
" {\"id\":103,\"content\":\"<|assistant|>\",\"special\":true},"
" {\"id\":104,\"content\":\"<sop>\",\"special\":true},"
" {\"id\":110,\"content\":\"<think>\",\"special\":false},"
" {\"id\":111,\"content\":\"<tool_call>\",\"special\":false}"
"]}";
/* minimal config.json that survives load_cfg's range validation */
static void write_cfg(const char *dir, const char *fname, const char *eos_json){
char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname);
FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }
fprintf(f,"{\"hidden_size\":64,\"num_hidden_layers\":2,\"num_attention_heads\":4,"
"\"n_routed_experts\":8,\"num_experts_per_tok\":2,\"moe_intermediate_size\":32,"
"\"intermediate_size\":64,\"first_k_dense_replace\":1,\"q_lora_rank\":0,"
"\"kv_lora_rank\":16,\"qk_nope_head_dim\":8,\"qk_rope_head_dim\":8,"
"\"v_head_dim\":8,\"n_shared_experts\":1,\"vocab_size\":200,"
"\"n_group\":1,\"topk_group\":1,\"rope_theta\":10000.0");
if(eos_json) fprintf(f,",\"eos_token_id\":%s",eos_json);
fprintf(f,"}\n"); fclose(f);
}
static void write_tok(const char *dir){
char p[512]; snprintf(p,sizeof(p),"%s/tokenizer.json",dir);
FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }
fputs(TOKJSON,f); fclose(f);
}
static void rm_file(const char *dir, const char *fname){
char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); remove(p);
}
static int expect(const char *what, int id, int want){
int got=is_stop(id);
if(got!=want){ fprintf(stderr," FAIL %s: token %d stop=%d, expected %d\n",what,id,got,want); return 1; }
return 0;
}
int main(void){
int fail=0;
/* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...".
* These binaries are built by MinGW into native Windows .exe files, which
* resolve Windows paths: "/tmp" is not one, so mkdtemp there fails ENOENT
* and the whole `make check` goes red on the windows job (and only there). */
char dir[]="test_stops_XXXXXX";
if(!mkdtemp(dir)){ perror("mkdtemp"); return 1; }
write_tok(dir);
char tkp[512]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",dir);
Tok T; tok_load(&T,tkp);
printf("test_stops: stop arming vs incomplete checkpoint metadata\n");
/* 1. the tokenizer's "special" flag must survive loading, and must NOT
* swallow <think>/<tool_call>, which are real content. */
if(!T.id_special[101]){ fprintf(stderr," FAIL <|user|> not marked special\n"); fail=1; }
if(!T.id_special[104]){ fprintf(stderr," FAIL <sop> not marked special\n"); fail=1; }
if(T.id_special[110]){ fprintf(stderr," FAIL <think> wrongly marked special\n"); fail=1; }
if(T.id_special[111]){ fprintf(stderr," FAIL <tool_call> wrongly marked special\n"); fail=1; }
if(!fail) printf(" tokenizer: special flag parsed, <think>/<tool_call> excluded ok\n");
/* 2. config.json mutilated to one eos, generation_config.json intact:
* the union must recover all three. This is @woolcoxm's case. */
{ Cfg c; memset(&c,0,sizeof c);
write_cfg(dir,"config.json","[100]");
write_cfg(dir,"generation_config.json","[100,101,102]");
load_cfg(&c,dir);
if(c.n_stop!=3){ fprintf(stderr," FAIL union: expected 3 stops, got %d\n",c.n_stop); fail=1; }
else printf(" config eos=[100] + generation_config=[100,101,102] -> 3 stops ok\n"); }
/* 3. no generation_config.json at all: must not crash, config alone wins */
{ Cfg c; memset(&c,0,sizeof c);
write_cfg(dir,"config.json","[100,101,102]");
rm_file(dir,"generation_config.json");
load_cfg(&c,dir);
if(c.n_stop!=3){ fprintf(stderr," FAIL no-genconfig: expected 3, got %d\n",c.n_stop); fail=1; }
else printf(" generation_config.json absent -> config alone, no crash ok\n"); }
/* 4. BOTH configs mutilated: the tokenizer's special set must still stop the
* turn on every control token, and must still leave <think> alone. */
{ Cfg c; memset(&c,0,sizeof c);
write_cfg(dir,"config.json","[100]");
write_cfg(dir,"generation_config.json","[100]");
load_cfg(&c,dir);
stops_arm_tok(&c,-1,&T);
fail|=expect("endoftext (config)", 100,1);
fail|=expect("<|user|> (tokenizer)", 101,1);
fail|=expect("<|observation|>", 102,1);
fail|=expect("<|assistant|>", 103,1);
fail|=expect("<sop>", 104,1);
fail|=expect("<think> must NOT stop",110,0);
fail|=expect("<tool_call> must NOT stop",111,0);
fail|=expect("plain token must NOT stop",2,0);
if(!fail) printf(" both configs mutilated -> tokenizer still stops all 5 control tokens ok\n"); }
/* 5. T=NULL (validation/oracle path): config stops only, unchanged behaviour */
{ Cfg c; memset(&c,0,sizeof c);
write_cfg(dir,"config.json","[100]");
rm_file(dir,"generation_config.json");
load_cfg(&c,dir);
stops_arm_tok(&c,-1,NULL);
fail|=expect("T=NULL: config stop",100,1);
fail|=expect("T=NULL: no tokenizer sweep",101,0);
if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); }
rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json");
rmdir(dir);
if(fail){ printf("test_stops: FAIL\n"); return 1; }
printf("test_stops: ok\n");
return 0;
}
c/tests/test_topp.c
+280
View File
@@ -0,0 +1,280 @@
/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be
* indistinguishable from the old full-vocab qsort for every shape dist_sample can see.
*
* Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry
* vocab on every sampled token to find the few-hundred-token head whose cumulative mass
* reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win
* is structural; the risk is a silent sampling-distribution change, because the contract
* is subtle:
*
* dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly.
* So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every
* truncated tail entry -- merely excluding the tail from the head would leave mass on
* it and the sampled distribution would drift with no crash and no error.
*
* Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a
* sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference
* that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a
* private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to
* the reference (the issue's stated invariant: s2 accumulates in the same descending
* order). On tie shapes, where the unstable qsort already left ordering unspecified, we
* check multiset equality instead. Every shape also checks: exact-zero tails, head sums
* to 1.0, and a sane keep-count.
*
* No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on
* the Windows MinGW CI job without the unmerged compat shim (#352). */
#define main coli_glm_main_unused
#include "../glm.c"
#undef main
#include <math.h>
static int g_nfails = 0;
/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer
* (the qsort comparator gets no user-data argument in C). */
static const double *g_ref_p = NULL;
#define FAIL(fmt, ...) do { \
fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \
label, V, nuc, shape_name, ##__VA_ARGS__); \
g_nfails++; \
return; \
} while (0)
/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */
/* Stable qsort by descending probability (ties broken by ascending index, which makes
* the reference deterministic regardless of the production comparator). */
static int cmp_ref_desc(const void *a, const void *b){
double pa = ((const double *)g_ref_p)[*(const int*)a];
double pb = ((const double *)g_ref_p)[*(const int*)b];
if (pa < pb) return 1;
if (pa > pb) return -1;
/* tie -> lower index first (stable, unlike the production comparator) */
return *(const int*)a - *(const int*)b;
}
/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the
* old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized.
* Returns the keep-count through *keep_out. */
static void ref_build(const float *lo, int V, double temp, double nuc,
double *out, int *pidx, int *keep_out){
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4);
for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; }
for (int i = 0; i < V; i++) out[i] /= s;
if (nuc > 0 && nuc < 1.0){
for (int i = 0; i < V; i++) pidx[i] = i;
qsort(pidx, V, sizeof(int), cmp_ref_desc);
double cum = 0; int keep = V;
for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } }
double s2 = 0;
for (int i = keep; i < V; i++) out[pidx[i]] = 0;
for (int i = 0; i < keep; i++) s2 += out[pidx[i]];
for (int i = 0; i < keep; i++) out[pidx[i]] /= s2;
*keep_out = keep;
} else {
*keep_out = V;
}
}
/* count how many production g_pbuf entries are non-zero == the head size */
static int head_count(int V){
int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n;
}
/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference.
* shape_name is for diagnostics only. */
static void check_case(const char *label, int V, double nuc, const char *shape_name,
const float *lo){
/* reference on a private buffer */
double *ref = malloc((size_t)V * sizeof(double));
int *ridx = malloc((size_t)V * sizeof(int));
int ref_keep = 0;
g_ref_p = ref; /* cmp_ref_desc reads this */
ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep);
/* production: drive the real dist_build (writes the global g_pbuf) */
g_nuc = (float)nuc;
dist_build(lo, V);
int got_keep = head_count(V);
/* 1. keep-count must match the reference exactly. The partial select and the old
* qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule);
* a count divergence is a real bug, not a tie artifact. */
if (got_keep != ref_keep)
FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep);
/* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set.
* A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable:
* both algorithms keep the right count but may keep different MEMBERS. So any input
* with a duplicated softmax value needs the relaxed multiset comparison below. We
* detect this on the reference softmax (pre-truncation) by sorting all V values. */
int has_ties = 0;
{
double *all = malloc((size_t)V * sizeof(double));
/* reconstruct the pre-truncation softmax the same way ref_build does */
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4);
for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; }
for (int i = 0; i < V; i++) all[i] /= s;
for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1;
while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; }
for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; }
free(all);
}
if (has_ties){
/* Multiset equality of the non-zero (head) values. Ties make membership
* interchangeable, so we compare sorted value-multisets, not id-aligned values.
* Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference
* double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */
double *got = malloc((size_t)ref_keep * sizeof(double));
int gm = 0;
for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i];
if (gm != ref_keep)
FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep);
for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1;
while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; }
double *rsort = malloc((size_t)ref_keep * sizeof(double));
int rm = 0;
for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i];
for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1;
while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; }
int mm = 0; double worst = 0;
for (int i = 0; i < gm; i++){
double d = fabs(got[i] - rsort[i]);
double rel = rsort[i] > 1e-30 ? d / rsort[i] : d;
if (rel > worst) worst = rel;
if (rel > 1e-6) mm++;
}
free(got); free(rsort);
if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)",
mm, ref_keep, worst);
} else {
/* No ties anywhere: membership is forced, so compare id-aligned head values. The
* engine computes in float (g_pbuf /= (float)s2) while the reference uses double,
* so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial
* select and qsort accumulate s2 in the same descending order, so any difference
* is pure float-rounding noise, not an ordering bug. */
int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0;
for (int i = 0; i < V; i++){
if (ref[i] == 0.0) continue; /* tail */
float want = (float)ref[i];
double d = fabs((double)g_pbuf[i] - (double)want);
double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d;
if (rel > worst) worst = rel;
if (rel > 1e-6){
bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; }
if (bad > 3) break;
}
}
if (bad)
FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)",
bad, first_id, (double)gv, (double)rv, worst);
}
/* 3. head must renormalize to 1.0 (within float epsilon) */
double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i];
if (fabs(sum - 1.0) > 1e-5)
FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep);
free(ref); free(ridx);
printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n",
V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum);
}
#undef FAIL
/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */
static uint32_t rng_state = 0x12345678u;
static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17;
rng_state ^= rng_state << 5; return rng_state; }
static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */
/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail
* boundary differently. */
static void fill_shape(float *lo, int V, int shape){
switch (shape){
case 0: /* uniform -> every token equal probability -> massive tie plateau */
for (int i = 0; i < V; i++) lo[i] = 0.f; break;
case 1: /* peaked: one dominant token, rest small and distinct (no ties).
* The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't
* write out of bounds and corrupt heap metadata on the later free(lo). */
for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0);
lo[0] = 3.f; lo[V/3<V?V/3:V-1] = 1.f; lo[V/2<V?V/2:V-1] = 0.5f; break;
case 2: /* all-equal distinct decay (no ties): geometric, strictly decreasing */
for (int i = 0; i < V; i++) lo[i] = (float)(-0.001 * (double)i); break;
case 3: /* plateau ties: blocks of equal value -> comparator tie handling */
for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */
break;
case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor).
* Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */
for (int i = 0; i < V; i++) lo[i] = -8.f;
{ int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break;
}
}
int main(void){
/* sizes: small for exhaustive tie detection up to near-production scale */
int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */
double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */
int n_shapes = 5;
/* temperature used by dist_build: pick a normal serving value */
g_temp = 0.7f;
int cases = 0;
for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){
int V = sizes[si];
/* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in
* real serving, where V is the constant model vocab). This sweep varies V, so free
* and force a reallocation per size -- otherwise a later, larger V would overflow
* the buffer sized for the first (smallest) V. */
free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL;
float *lo = malloc((size_t)V * sizeof(float));
for (int shape = 0; shape < n_shapes; shape++){
fill_shape(lo, V, shape);
for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){
char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape);
const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape];
check_case(label, V, nucs[ni], sn, lo);
cases++;
}
}
free(lo);
}
/* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */
{
int V = 256; float lo[256];
for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2);
g_nuc = 1.0f; dist_build(lo, V);
int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++;
if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; }
else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz);
cases++;
g_nuc = 0.0f; dist_build(lo, V);
nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++;
if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; }
else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz);
cases++;
}
/* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */
{
float lo[1] = {5.f};
g_nuc = 0.5f; dist_build(lo, 1);
if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){
fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++;
} else printf(" ok [V=1 keep=1]\n");
cases++;
}
printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails);
if (g_nfails){ printf("test_topp: FAIL\n"); return 1; }
printf("test_topp: ok\n");
return 0;
}
c/tok.h
+8
View File
@@ -42,6 +42,11 @@ typedef struct {
hmap vocab; /* stringa byte-level -> id */
hmap merges; /* "left\0right" -> rank */
char **id2str; int *id_added; int n_ids; /* id -> stringa; id_added=1 se added-token (output letterale) */
int *id_special; /* 1 = added-token con "special":true nel tokenizer:
* token di CONTROLLO (<|user|>, <|assistant|>, <sop>, ...),
* mai contenuto legittimo di una risposta. Distinto da
* id_added, che copre anche <think>/<tool_call> ("special"
* false), i quali sono testo vero e vanno renderizzati. */
Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */
uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3];
int16_t cp2byte[1024];
@@ -106,6 +111,7 @@ static void tok_load(Tok *T, const char *path){
T->n_ids=maxid+1;
T->id2str=calloc(T->n_ids,sizeof(char*));
T->id_added=calloc(T->n_ids,sizeof(int));
T->id_special=calloc(T->n_ids,sizeof(int));
/* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */
int vc=1; while(vc < vocab->len*2) vc<<=1;
@@ -133,6 +139,8 @@ static void tok_load(Tok *T, const char *path){
char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num;
T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id;
T->id2str[id]=content; T->id_added[id]=1;
jval *sf=json_get(a,"special"); /* "special": true/false */
if(sf && sf->t==J_BOOL && sf->boolean) T->id_special[id]=1;
}
qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */
}
c/tools/benchmark_cuda_fixture.py
+26 -3
View File
@@ -17,6 +17,12 @@ PROFILE_RE = re.compile(
r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s"
)
PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other")
P0_RE = re.compile(
r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| "
r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s"
)
P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical",
"router", "p2p", "p2p_hops", "orchestration")
def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
@@ -30,11 +36,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
return float(speed.group(1)), [disk] + [float(value) for value in rest]
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]:
def parse_p0(stdout: str) -> list[float]:
"""Extract the optional PROF=1 execution-layer breakdown."""
row = P0_RE.search(stdout)
if not row:
raise RuntimeError("benchmark output missing P0-EXEC profile")
return [float(value) for value in row.groups()]
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]:
run = subprocess.run(
[engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True
)
return parse_output(run.stdout, run.stderr)
speed, profile = parse_output(run.stdout, run.stderr)
return speed, profile, parse_p0(run.stdout)
def main() -> None:
@@ -63,6 +78,8 @@ def main() -> None:
OMP_NUM_THREADS=str(args.threads),
OMP_PROC_BIND="spread",
OMP_PLACES="cores",
DRAFT="0",
PROF="1",
)
execute(args.engine, base | {"STATS": str(stats)})
@@ -86,13 +103,15 @@ def main() -> None:
execute(args.engine, base | extra) # warm-up
speeds = {name: [] for name in modes}
profiles = {name: [] for name in modes}
p0_profiles = {name: [] for name in modes}
names = list(modes)
for run_index in range(args.runs):
order = names[run_index % len(names):] + names[:run_index % len(names)]
for name in order:
speed, profile = execute(args.engine, base | modes[name])
speed, profile, p0 = execute(args.engine, base | modes[name])
speeds[name].append(speed)
profiles[name].append(profile)
p0_profiles[name].append(p0)
result = {}
for name in names:
@@ -103,6 +122,10 @@ def main() -> None:
key: statistics.median(row[index] for row in profiles[name])
for index, key in enumerate(PROFILE_KEYS)
},
"median_p0": {
key: statistics.median(row[index] for row in p0_profiles[name])
for index, key in enumerate(P0_KEYS)
},
}
print(json.dumps(result, indent=2))
c/tools/convert_fp8_to_int4.py
+113 -7
View File
@@ -286,6 +286,17 @@ def main():
# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre
# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.
a.ebits = 8 if (a.mtp or a.indexer) else 4
if a.mtp and a.ebits < 8 and a.group_size <= 0:
# Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di
# colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera
# meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8).
# EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row
# EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot
# EN: see the input token -> ~0% acceptance (issue #8). A container converted
# EN: this way is repairable in place with tools/repair_mtp_int8.py.
print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's "
"embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, "
"or add --group-size 128 for group-scaled int4.")
if a.xbits is None: a.xbits = a.ebits
# Build per-type bits map. If a type-specific arg is set, use it; otherwise the
@@ -299,6 +310,16 @@ def main():
if bits_map:
print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))
# Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il
# default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'
# gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job
# da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.
mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"
grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \
(f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")
print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "
f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")
if a.selftest_nvfp4:
import torch
# 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.
@@ -379,14 +400,99 @@ def main():
if a.indir: # conversione locale (test)
shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors")))
from safetensors.numpy import save_file
# #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard
# che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni
# scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione
# completa come prima.
# EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY
# the shards that hold the requested tensors (3 instead of scanning all 141 —
# every empty scan still opens a 5 GB shard). Without the index: full scan as
# before.
if a.mtp or a.indexer:
idxp = os.path.join(a.indir, "model.safetensors.index.json")
if os.path.exists(idxp):
wmap = json.load(open(idxp))["weight_map"]
if a.mtp:
want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")}
else:
want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers}
keep = [sp for sp in shards if os.path.basename(sp) in want]
print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors")
shards = keep
# BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva
# out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e
# keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir
# SOVRASCRIVEVA il container gia' finito con una riconversione int8
# completa, in silenzio (137/141 shard distrutti prima di accorgersene).
# Ora il ramo locale rispecchia il download path: prefisso corretto,
# flag passate, shard vuoti saltati.
prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-"
# RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di
# input (gli shard senza tensori rilevanti non producono file), quindi "il
# file esiste" non basta per saltare il lavoro gia' fatto. Un manifest
# sidecar ricorda input -> output (o "vuoto") e con quali parametri: la
# ripresa salta solo cio' che combacia, e parametri diversi sulla stessa
# outdir vengono rifiutati invece di mescolare container (il modo #355).
# EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input
# EN: index (shards with no relevant tensors emit no file), so "the file
# EN: exists" is not enough to skip completed work. A sidecar manifest
# EN: records input -> output (or "empty") plus the conversion parameters:
# EN: resume skips only what matches, and different parameters on the same
# EN: outdir are refused instead of mixing containers (the #355 failure mode).
params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits,
"group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map}
prog_path = os.path.join(a.outdir, f".{prefix}progress.json")
prog = {}
if os.path.exists(prog_path):
try: prog = json.loads(open(prog_path).read())
except (OSError, ValueError): prog = {}
if prog and prog.get("params") != params:
print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n"
f" this run uses {params}. Refusing to mix conversions in the same "
f"outdir — use a fresh --outdir (or delete the manifest and the "
f"{prefix}*.safetensors shards to redo).")
return
done = prog.setdefault("shards", {}); prog["params"] = params
n = 0; fresh = 0; skipped = 0
for i, sp in enumerate(shards):
out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map)
save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors"))
# copia config + tokenizer
for fn in ["config.json"]:
src = os.path.join(a.indir, fn)
if os.path.exists(src): shutil.copy(src, a.outdir)
print(f"converted {len(shards)} shards -> {a.outdir}")
key = os.path.basename(sp)
prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso
if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))):
if prev: n += 1
skipped += 1
continue
out = {}
convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits,
keep_mtp=a.mtp, keep_idx=a.indexer,
group_size=a.group_size, bits_map=bits_map)
if not out: # shard senza MTP/idx: niente file (come il download path)
done[key] = ""
else:
name = f"{prefix}{n:05d}.safetensors"
save_file(out, os.path.join(a.outdir, name))
done[key] = name; n += 1; fresh += 1
tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto
with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest
os.replace(tmp_prog, prog_path)
if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped")
# metadati per la conversione principale: gli stessi quattro file del download
# path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno
# nella stessa outdir di un container gia' completo di metadati.
# EN: metadata for the main pass: the same four files as the download path —
# EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target
# EN: an outdir whose container already has its metadata.
if not a.mtp and not a.indexer:
copied, missing = [], []
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]:
src = os.path.join(a.indir, fn)
if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn)
else: missing.append(fn)
print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}")
if missing:
print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}"
+ (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else ""))
tag = "MTP" if a.mtp else "indexer" if a.indexer else "main"
print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)")
return
# reale: scarica shard per shard, converte, cancella
c/tools/convert_olmoe.py
+23 -8
View File
@@ -3,7 +3,10 @@
Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct).
Dense weights stay as-is (engine reads BF16/F16 → F32 on load).
Expert weights get row-wise int8 quantization with float32 scales.
Expert weights get row-wise symmetric quantization to --ebits bits (default 4)
with float32 scales. Storage stays one value per int8 byte regardless of bits,
matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the
values are simply confined to [-8, 7] with scales computed against qmax=7.
Usage:
python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4
@@ -29,12 +32,21 @@ except ImportError as exc:
EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight"
def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Row-wise int8 quantization. Returns (int8_weights, float32_scales)."""
def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]:
"""Row-wise symmetric quantization to `bits` (2..8).
Returns (int8_weights, float32_scales). Storage is one value per int8 byte
for every bit width — the engine dequantizes as q*scale and never assumes
the full int8 range — mirroring olmoe.c quantize_rows():
qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1)
scale = amax(|w|, row) / qmax
q = clamp(round(w / scale), -qmax-1, qmax)
"""
qmax = (1 << (bits - 1)) - 1
w_f32 = w.float()
row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
scales = row_max / 127.0
q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8)
scales = row_max / qmax
q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8)
return q, scales.squeeze(1)
@@ -50,9 +62,12 @@ def main():
src.add_argument("--model", help="Local HF checkpoint directory")
ap.add_argument("--out", required=True, help="Output directory for int4 model")
ap.add_argument("--ebits", type=int, default=4,
help="Expert quant bits (4 or 8, default 4)")
help="Expert quant bits (2..8, default 4)")
args = ap.parse_args()
if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c)
sys.exit(f"--ebits must be 2..8 (got {args.ebits})")
if args.repo:
from huggingface_hub import snapshot_download
from huggingface_hub.errors import LocalEntryNotFoundError
@@ -96,7 +111,7 @@ def main():
for name, tensor in tensors.items():
if is_expert_weight(name):
expert_count += 1
q, scales = quantize_row(tensor)
q, scales = quantize_row(tensor, args.ebits)
total_expert_f32 += tensor.numel() * tensor.element_size()
total_expert_q += q.numel() * 1 + scales.numel() * 4
out_tensors[name] = q
@@ -109,7 +124,7 @@ def main():
ratio = total_expert_q / max(total_expert_f32, 1) * 100
print(f"ok")
print(f"\nDone. {expert_count} expert tensors quantized.")
print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.")
print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)")
print(f"Model ready at: {out}")
print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16")
c/tools/make_glm_oracle.py
+40
View File
@@ -17,6 +17,45 @@ EN: computed AFTER the FP8 round-trip, so the reference matches exactly what the
EN: ingests. Default: bf16 (original oracle unchanged)."""
import json, sys, argparse
from pathlib import Path
# --- Version gate (must run BEFORE the heavy `from transformers import ...`,
# which triggers transformers' lazy-loading and can in turn reset the in-memory
# __version__ attribute; importlib.metadata reads the installed package version
# directly and is immune to that). ---
#
# GLM-5.2's MLA attention uses interleaved (DeepSeek-style) RoPE, which is what
# the C engine implements. transformers < 5.11.0 applied split-half (Llama-style)
# RoPE in GlmMoeDsa* instead; an oracle built on those versions drifts and the
# engine then scores 25/32 instead of the documented 32/32 (issue #281). Weights
# come out identical across versions — only the forward pass differs — so there
# is no safe "partial" run: a too-old transformers silently produces an invalid
# ref_glm.json. Hard-fail rather than warn. EN: same.
import transformers
from importlib.metadata import version as _pkg_version, PackageNotFoundError
_MIN_TRANSFORMERS = (5, 11)
def _tf_version_tuple():
try:
v = _pkg_version("transformers") # authoritative: installed dist metadata
except PackageNotFoundError:
v = getattr(transformers, "__version__", "0") # fallback (editable/src installs)
out = []
for part in v.split(".")[:2]: # major.minor only
out.append(int("".join(c for c in part if c.isdigit()) or "0"))
while len(out) < 2:
out.append(0)
return tuple(out[:2])
_tf_ver = _tf_version_tuple()
if _tf_ver < _MIN_TRANSFORMERS:
sys.exit(
f"\nERROR: make_glm_oracle.py requires transformers >= "
f"{'.'.join(map(str, _MIN_TRANSFORMERS))}.0 (found {_tf_ver[0]}.{_tf_ver[1]}). "
f"GLM-5.2 MLA uses interleaved RoPE; older versions apply split-half RoPE "
f"and silently produce an oracle the engine scores 25/32 against (issue #281). "
f"Upgrade: pip install -U 'transformers>=5.11'\n"
)
import torch
from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM
@@ -115,6 +154,7 @@ print("tf_pred:", tf_pred)
sd = model.state_dict()
unfuse_experts(sd)
Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves
if args.fp8:
n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")
print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "
c/tools/quant_ablation.py
+15 -4
View File
@@ -118,7 +118,7 @@ def quantize_param(w, bits, group, rot=False, e8=""):
def _grid_or_e8(x, bits, group, e8):
if e8:
return _quant_e8(x.float(), group, ball=(e8 == "-e8"))
return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8"))
return _quant_last_dim(x, bits, group)
@@ -169,8 +169,15 @@ def _e8_ball(y, r2=10.0):
return p
def _quant_e8(x, group, ball):
"""Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples."""
_E8_R2_REPORTED = set()
def _e8_radius(bits):
# E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4.
# Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there.
return 10.0 * (4.0 ** (bits - 2))
def _quant_e8(x, group, bits, ball):
"""Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.
ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal."""
if x.shape[-1] % 8:
raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})")
g = group or x.shape[-1]
@@ -183,7 +190,11 @@ def _quant_e8(x, group, ball):
for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4):
s = rms * k
yb = (xg / s).reshape(-1, g // 8, 8)
p = _e8_ball(yb) if ball else _e8_nearest(yb)
p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb)
if ball and bits not in _E8_R2_REPORTED:
_E8_R2_REPORTED.add(bits)
import sys as _sys
_sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n")
out = (p.reshape(-1, g) * s)
err = (out - xg).pow(2).sum(-1, keepdim=True)
if best_err is None:
c/tools/repair_mtp_int8.py
+229
View File
@@ -0,0 +1,229 @@
"""Repair an existing colibri int4 container whose MTP head was quantized at int4.
WHY THIS EXISTS
`model.layers.<N>.eh_proj.weight` [D, 2D] multiplies the MTP concat
[embedding_norm ; hidden_norm], and its two column halves differ in scale by
~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2).
Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half
weight lands below half a quantization step and np.rint rounds the ENTIRE
embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts
garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 —
which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8).
A container converted (or downloaded) with an int4 MTP head does not need a
full re-conversion: this script re-downloads ONLY the affected dense tensors
(~355 MB of HTTP range reads against the FP8 source repo), requantizes them
at int8 with the converter's exact math, and patches the local shards in
place. Originals are kept beside as *.bak-int4.
WHAT IT TOUCHES
The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj,
shared_experts.*): the ones that stream into RAM once and stay resident.
Routed experts (model.layers.<N>.mlp.experts.*) are NOT touched — they are
statistically like the main layers' experts and int4 is acceptable there.
The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no
engine or config change is needed. Cost: ~+133 MB on disk / resident RAM.
USAGE
python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair
python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only
--source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public
int4 containers were converted from). Requires numpy and network access;
no torch, no HF token (public repo, anonymous range reads).
"""
import argparse, glob, json, os, ssl, struct, sys, urllib.request
import numpy as np
# macOS python.org builds ship no CA bundle: use certifi when available (Linux
# system Pythons generally have working system certs and skip this).
try:
import certifi
_SSL_CTX = ssl.create_default_context(cafile=certifi.where())
except ImportError:
_SSL_CTX = ssl.create_default_context()
# ---------- HTTP range reads against the source repo ----------
def http_range(url, start, length, tries=5):
req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair",
"Range": f"bytes={start}-{start+length-1}"})
for attempt in range(tries):
try:
with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r:
data = r.read()
if len(data) == length:
return data
except KeyboardInterrupt:
raise
except Exception as ex:
if attempt == tries - 1:
raise RuntimeError(f"range read failed for {url}: {ex}")
raise RuntimeError(f"short range read for {url}")
class SourceRepo:
def __init__(self, repo, revision="main"):
self.base = f"https://huggingface.co/{repo}/resolve/{revision}/"
with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r:
self.wmap = json.loads(r.read())["weight_map"]
self._hdr = {}
def _shard_header(self, shard):
if shard not in self._hdr:
n = struct.unpack("<Q", http_range(self.base + shard, 0, 8))[0]
self._hdr[shard] = (json.loads(http_range(self.base + shard, 8, n)), 8 + n)
return self._hdr[shard]
def meta(self, name):
shard = self.wmap.get(name)
if not shard:
return None
hdr, _ = self._shard_header(shard)
return hdr[name]
def fetch_f32(self, name):
"""Download one tensor and dequantize to f32 [O, I] (BF16 or FP8+block scales)."""
shard = self.wmap[name]
hdr, base = self._shard_header(shard)
m = hdr[name]
o0, o1 = m["data_offsets"]
raw = http_range(self.base + shard, base + o0, o1 - o0)
if m["dtype"] == "BF16":
u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
return u.view(np.float32).reshape(m["shape"]).astype(np.float32)
if m["dtype"] == "F8_E4M3":
b = np.frombuffer(raw, dtype=np.uint8).astype(np.uint16)
sign = np.where(b & 0x80, -1.0, 1.0)
e = (b >> 3) & 0xF
mant = (b & 7).astype(np.float64)
v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7),
mant / 8 * np.exp2(-6.0))).reshape(m["shape"])
sn = name + "_scale_inv"
sshard = self.wmap[sn]
shdr, sbase = self._shard_header(sshard)
sm = shdr[sn]
so0, so1 = sm["data_offsets"]
sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0),
dtype=np.float32).reshape(sm["shape"])
O, I = m["shape"]
scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I]
return (v * scf).astype(np.float32)
raise ValueError(f"{name}: unsupported source dtype {m['dtype']}")
# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ----------
def quant_int8(w):
amax = np.abs(w).max(axis=1, keepdims=True)
s = np.maximum(amax / 127, 1e-8)
q = np.clip(np.rint(w / s), -128, 127).astype(np.int8)
return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32)
# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ----------
def read_shard(path):
with open(path, "rb") as fh:
n = struct.unpack("<Q", fh.read(8))[0]
hdr = json.loads(fh.read(n))
base = 8 + n
order = sorted(((k, v) for k, v in hdr.items() if k != "__metadata__"),
key=lambda kv: kv[1]["data_offsets"][0])
out = {}
for k, v in order:
fh.seek(base + v["data_offsets"][0])
out[k] = (v["dtype"], v["shape"], fh.read(v["data_offsets"][1] - v["data_offsets"][0]))
return out, hdr.get("__metadata__")
def write_shard(path, tensors, meta):
hdr = {}
off = 0
for k, (dt, shape, raw) in tensors.items():
hdr[k] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + len(raw)]}
off += len(raw)
if meta:
hdr["__metadata__"] = meta
hj = json.dumps(hdr).encode()
hj += b" " * ((8 - len(hj) % 8) % 8)
with open(path, "wb") as fh:
fh.write(struct.pack("<Q", len(hj)))
fh.write(hj)
for _, (_, _, raw) in tensors.items():
fh.write(raw)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--snap", required=True, help="local colibri int4 container directory")
ap.add_argument("--source-repo", default="zai-org/GLM-5.2-FP8")
ap.add_argument("--revision", default="main")
ap.add_argument("--dry-run", action="store_true", help="report what would change, touch nothing")
a = ap.parse_args()
cfg = json.load(open(os.path.join(a.snap, "config.json")))
L = cfg["num_hidden_layers"]
pref = f"model.layers.{L}."
src = SourceRepo(a.source_repo, a.revision)
# find the MTP layer's dense int4 tensors across the local shards
plan = {} # shard path -> [tensor names to repair]
already_ok, skipped_experts = [], 0
for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))):
with open(f, "rb") as fh:
n = struct.unpack("<Q", fh.read(8))[0]
hdr = json.loads(fh.read(n))
for name, v in hdr.items():
if not name.startswith(pref) or name.endswith(".qs") or v.get("dtype") != "U8":
continue
if ".mlp.experts." in name:
skipped_experts += 1
continue
m = src.meta(name)
if m is None:
print(f" ?? {name}: not in source repo, skipping")
continue
O, I = m["shape"]
nb = v["data_offsets"][1] - v["data_offsets"][0]
if nb == O * I:
already_ok.append(name)
elif nb == O * ((I + 1) // 2):
plan.setdefault(f, []).append((name, O, I))
else:
print(f" ?? {name}: unexpected blob size {nb} for shape {O}x{I}, skipping")
n_fix = sum(len(v) for v in plan.values())
print(f"MTP layer {L}: {n_fix} dense tensor(s) at per-row int4 to repair, "
f"{len(already_ok)} already int8, {skipped_experts} routed-expert tensors left as-is")
if not n_fix:
print("nothing to do."); return
if a.dry_run:
for f, names in plan.items():
for nm, O, I in names:
print(f" would repair {nm} [{O},{I}] in {os.path.basename(f)}")
return
for f, names in plan.items():
print(f"patching {os.path.basename(f)} ({len(names)} tensors)")
tensors, meta = read_shard(f)
for nm, O, I in names:
print(f" {nm}: fetching source + requantizing at int8...", flush=True)
w = src.fetch_f32(nm)
assert w.shape == (O, I), f"{nm}: source shape {w.shape} != container {O}x{I}"
q, s = quant_int8(w)
tensors[nm] = ("U8", [len(q)], q.tobytes())
tensors[nm + ".qs"] = ("F32", [O], s.tobytes())
# verify first row against the source
loc = q[:I].view(np.int8).astype(np.float64) * s[0]
ref = w[0].astype(np.float64)
cos = float(ref @ loc / (np.linalg.norm(ref) * np.linalg.norm(loc) + 1e-30))
print(f" row-0 cosine vs source: {cos:.5f}")
bak = f + ".bak-int4"
write_shard(f + ".new", tensors, meta)
os.replace(f, bak)
os.replace(f + ".new", f)
print(f" saved; original kept as {os.path.basename(bak)}")
print("done. Re-run with --dry-run to confirm (should report 'already int8').")
if __name__ == "__main__":
main()
c/version.py
+3
View File
@@ -0,0 +1,3 @@
"""Single source of truth for the colibrì version number."""
__version__ = "1.0.0"
docs/ENVIRONMENT.md
+37 -3
View File
@@ -2,7 +2,7 @@
Reference for the environment variables read by the colibrì engine.
**Generated from `upstream/dev @ 6d3ed7e`** by scanning every `getenv()` site in `c/glm.c`. Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes.
**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes.
## Which program reads these?
@@ -43,6 +43,7 @@ Format: `VAR` — default — effect.
| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |
| `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. |
| `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. |
| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. |
| `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |
| `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. |
| `PREFETCH` | `0` | Prefetch depth for streamed experts. |
@@ -54,15 +55,26 @@ Format: `VAR` — default — effect.
| `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. |
| `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). |
| `PILOT_K` | `6` if `PILOT_REAL` else `8` | Number of experts the pilot prefetches per step. |
| `PILOT_TWO` | `0` (off) | Two-step shared-expert-corrected router prediction for the pilot. |
| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. |
| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. |
| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. |
| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). |
| `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. |
| `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. |
| `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). |
| `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). |
| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. |
| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). |
| `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. |
| `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). |
| `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. |
| `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. |
| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. |
| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. |
| `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=<n>` (testing). |
| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). |
| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. |
---
@@ -76,7 +88,22 @@ Format: `VAR` — default — effect.
| `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. |
| `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. |
| `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. |
| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). |
| `COLI_CUDA_PROFILE` | off | Emit CUDA timing. |
| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. |
| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). |
| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. |
| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). |
| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. |
| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. |
| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. |
| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). |
| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. |
| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). |
| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). |
| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. |
| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. |
| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). |
---
@@ -88,8 +115,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda
|---|---|---|
| `SPEC` | `1` | Speculative decoding on/off. |
| `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. |
| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. |
| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. |
| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). |
| `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. |
| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). |
| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). |
| `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. |
| `DSA_FORCE` | `0` | Force the DSA path on. |
| `DSA_TOPK` | model value | Override the DSA index top-k (testing). |
@@ -100,11 +130,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda
| `PIN_FILL` | `0` | Fill the pinned store even without usage data. |
| `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. |
| `STATS` | unset | Write an expert-usage histogram to `STATS=<file>` at end of run. |
| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. |
| `SCORE` | unset | Scoring/eval mode over `SCORE=<file>`. |
| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]<sop>` to scoring contexts (GLM-family only). |
| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. |
| `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. |
| `REPLAY` | unset | Replay mode. |
| `TF` | unset | Teacher-forcing mode. |
| `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). |
| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. |
---
@@ -135,7 +169,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe
- `SNAP` — model snapshot directory (required by `glm`; set from `--model`).
- `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode.
- `PROMPT` — one-shot text mode.
- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters).
- `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing.
---
docs/api.md
+186
View File
@@ -0,0 +1,186 @@
# OpenAI-compatible API, KV contexts & web UI
## `coli serve`
`coli serve` keeps one model process loaded and exposes a text-only
OpenAI-compatible HTTP API. The gateway uses only the Python standard library;
inference still runs in the same dependency-free C engine.
```bash
cd c
COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \
--host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri
curl http://127.0.0.1:8000/v1/chat/completions \
-H 'Authorization: Bearer local-secret' \
-H 'Content-Type: application/json' \
-d '{
"model": "glm-5.2-colibri",
"messages": [{"role": "user", "content": "Hello"}],
"stream": true
}'
```
Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
completion requests support JSON responses, SSE streaming, usage counts,
`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
`reasoning_effort` field also enables it unless set to `none`.
The server is deliberately text-only and serves one generation at a time: the
744B model stays in one persistent process, so concurrent HTTP requests queue
instead of loading duplicate model copies. Tools, image/audio input, custom
stop sequences, log probabilities, and token penalties return an explicit error
rather than being silently ignored. The default bind address is localhost; set
`COLI_API_KEY` before exposing the server beyond the machine.
Browser access from the Vite development server and Tauri local origins is
enabled by default. Repeat `--cors-origin https://your-ui.example` to allow
another exact origin, or use `--cors-origin '*'` only on a trusted local
network.
The engine owns its KV contexts, so HTTP generation uses a bounded FIFO
admission queue instead of pretending to run unsafe parallel sequences.
Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS`
(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment
variables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429
errors before streaming headers are sent. `GET /health` exposes
active/queued/completed/rejected counters, and successful generation responses
include `x-colibri-queue-wait-ms`.
## Connect a coding CLI or editor
The API is OpenAI-compatible, so most coding CLIs and editor extensions work by
pointing them at Colibri as an *OpenAI-compatible* provider. Three settings:
- **Base URL** — `http://localhost:8000/v1`
- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`)
- **API key** — any non-empty string, e.g. `local`
Colibri needs **no** API key by default, but many clients refuse to start without
one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`.
Smoke-test the endpoint first (no key needed unless you set one):
```bash
curl http://127.0.0.1:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}'
```
**aider**
```bash
export OPENAI_API_BASE=http://localhost:8000/v1
export OPENAI_API_KEY=local
aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE
```
**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or
`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows):
```json
{
"$schema": "https://charm.land/crush.json",
"providers": {
"colibri": {
"name": "Colibri",
"type": "openai-compat",
"base_url": "http://localhost:8000/v1/",
"api_key": "local",
"models": [
{ "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri",
"context_window": 131072, "default_max_tokens": 1024 }
]
}
}
}
```
The `"api_key": "local"` dummy is what satisfies clients that demand a key.
`context_window` is only the client's budget display — set it to whatever your
KV configuration actually allows.
**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base
URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy
key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools).
> **Set your expectations before connecting an agentic CLI.** Two costs dominate,
> and the first one is invisible until you know it's there:
>
> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a
> large system prompt plus tool definitions — often 10–20k tokens — *before
> your first word*. Prefill on the CPU-streaming path runs at a few tokens per
> second (it is attention-bound, see #153), so a 15k-token agent preamble is
> **an hour of silent "thinking" before the first output token**. The client
> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that
> answers in about a minute, the pipeline works and what you're paying for is
> prompt size.
> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops
> (which re-pay the growing context every turn) compound the cost.
>
> Practical guidance: single surgical asks with a short context work; iterative
> agent sessions against a disk-streaming 744B model do not resemble a hosted
> API and mostly won't be worth the wait. If your client lets you trim or disable
> its system preamble and tool catalog, do it.
## Isolated KV contexts
`coli serve --kv-slots N` allocates up to 16 independent sequence contexts.
Requests select one with the optional integer `cache_slot` field; ordinary
OpenAI clients omit it and keep the original slot 0 behavior.
```json
{
"model": "glm-5.2-colibri",
"messages": [{"role": "user", "content": "Continue this conversation"}],
"cache_slot": 1
}
```
Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and
crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine matches
each request's tokenized prompt against the slot's history and reuses the common
KV prefix, so stateless HTTP turns keep their cache across requests and even
across engine restarts. Use `COLI_KV_SLOTS=N` as the environment equivalent.
Start small: at the default 4096-token context, every slot costs hundreds of MB.
## Web dashboard
One command serves the OpenAI-compatible API **and** the web console on the
same port, then opens your browser when the engine is ready:
```bash
cd web && npm install && npm run build # once
./coli web --model <model-dir>
```
What you get:
- **Chat** with live metrics: a flashing token counter while generating, then
tok/s, time-to-first-token, prompt→completion counts and queue wait;
- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the
scheduler, and the live expert-tier bar — how many of the 19,456 experts sit
in VRAM / RAM / disk right now;
- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour =
tier, brightness = routing heat, and the experts routed in each turn flash
white and decay — you watch the model think. Hover any cell for its tier,
heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175);
- **Atlas**: the measured expert atlas as a 3-D galaxy (publish `experts.json`
from `tools/expert_atlas/analyze.py --web`).
The dashboard talks to the engine over a small line protocol and plain JSON
endpoints — nothing heavier than the engine itself. `web/` is a pure OpenAI-API
client (React + TypeScript) and also works against any other compatible
endpoint; the terminal `coli chat` remains the first-class interface.
The layout is responsive down to phone widths, and the sidebar carries the full
telemetry stack — hardware, scheduler, tier bar, per-turn time breakdown, tok/s
trend and per-GPU expert counts:
<p align="center">
<img src="media/colibri-mobile.png" width="270" alt="the dashboard on a phone-sized viewport">
&nbsp;&nbsp;
<img src="media/colibri-metrics.png" width="300" alt="the telemetry sidebar">
</p>
docs/benchmarks.md
+144
View File
@@ -0,0 +1,144 @@
# Benchmarks & measured numbers
Everything on this page is a measurement, not a promise. If you run colibrì on
hardware not listed here, **please open an issue with your numbers** — real
datapoints are what move this project.
## Reference numbers (the original dev box: WSL2, 12 cores, 25 GB RAM, NVMe via VHDX)
Detailed GPU experiment: [GLM-5.2 on 6× RTX 5090](experiments/glm52-6x5090-2026-07-12.md) —
full expert residency across VRAM+RAM reaches **6.84 tok/s** single-request decode.
| metric | value |
|---|---|
| model on disk (int4 container) | ~370 GB |
| resident RAM (dense, int4) | 9.9 GB |
| load time | ~30 s |
| peak RSS during chat | ~20 GB (auto-capped) |
| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) |
| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold |
| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) |
This is not fast. It is a 744B frontier-class model **answering correctly on a
machine that costs less than one H100 fan**. Warm cache, pinned hot experts and
MTP push the useful-response latency down considerably; the physics of the disk
does the rest.
### SSD note
Cold starts are heavy on random reads (~11 GB/token), but reads don't
meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns
under heavy use are (1) **swap traffic** if the system runs out of RAM (writes
do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed
to stay clear of swap) and (2) **sustained thermals**: hours at full read duty
cycle will heat cheaper drives. Monitor drive temperature and health.
## Test your machine, in order
```bash
cd c && ./setup.sh # build + architecture self-test (expects 32/32)
# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads):
gcc -O2 -fopenmp iobench.c -o iobench
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache)
# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box
# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and
# run it on a shard you haven't touched this session (a prior buffered run caches its pages).
# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't
# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right
# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read.
# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS):
COLI_MODEL=/path/to/glm52_i4 ./coli chat
# 3) record expert usage, then pin the hottest experts in your spare RAM:
STATS=stats.txt ./coli chat
PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
# 4) quality benchmarks (MMLU/HellaSwag/ARC):
./coli bench
```
## Back-of-envelope predictions
Decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP
speculation roughly halves the effective cost *once the cache is warm*; RAM
turns cold reads into free cache hits.
| machine | expected |
|---|---|
| the dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline |
| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s |
| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s |
| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels |
| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier |
These are estimates, not measurements.
## Community benchmarks (measured)
Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active:
| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured |
|---|---|---|---|
| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB |
| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB |
| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB |
| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run |
| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output |
| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) |
| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB |
| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB |
| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk |
| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** |
| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul |
| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% |
| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** |
| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** |
| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint |
| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** |
| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · later **1.83 tok/s** on current dev with `DIRECT=1 PIPE=1 PILOT_REAL=1 PILOT_TWO=1` ([#200](https://github.com/JustVugg/colibri/issues/200)) |
| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti ([#128](https://github.com/JustVugg/colibri/issues/128), [#273](https://github.com/JustVugg/colibri/issues/273)) | — | int8 MTP head · warm cache · GPU-resident pipeline at decode | 0.03 cold → 0.5 warm CPU → **1.07 tok/s** with the pipe2 decode gate (#274) |
| Dell Pro Max GB10 (DGX Spark: Grace, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136), [#161](https://github.com/JustVugg/colibri/issues/161)) | **5.58 GB/s** O_DIRECT | int8 MTP head · warm cache | 0.50 tok/s warm · **2.4 tok/s full-k8**, **3.33 tok/s** with `CACHE_ROUTE` (#199) |
| **6 × RTX 5090 · dual Xeon Silver 4510 · 251 GB** (author's rig, [experiment log](experiments/glm52-6x5090-2026-07-12.md)) | NVMe | `CUDA_EXPERT_GB=auto PIN_GB=all` full residency · `COLI_CUDA_PIPE=2 TC_W4A16` · DRAFT=0 | **5.8–6.8 tok/s** decode · TTFT ~13 s · hit 89–100% |
### Takeaways
With 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so
decode stays cold even on a fast disk — **on small-RAM machines the RAM cap, not
the disk, is the binding constraint**; `--topp 0.7` alone bought a clean 1.6×
end-to-end speedup. The 9950X pair is the cleanest bottleneck experiment: same
machine, same history, only the disk swapped — ×5.8 disk bandwidth bought ×2.9
tokens, and the profile **flipped from 66% disk to 57% matmul**. But the
crossover depends on the CPU kernel: with OMP hot-team tuning on, an AVX-512 CPU
can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colibri/issues/101)),
so **the GPU tier earns its VRAM only when the CPU is the weak link**. On
multi-socket hosts, NUMA placement is a further lever: interleaving the resident
weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)**
([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver
4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert
bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to
9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not
blanket-interleave a GPU host: it also spreads DMA staging pages and has measured
up to a 10× regression; generated plans enable only the selective slab policy.
## Quality benchmark
**Measured** ([#108](https://github.com/JustVugg/colibri/issues/108)): the int4
container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot
log-likelihood, n=40) — but 0-shot MC scoring underserves a reasoning model, and
the OLMoE fp16-vs-int4 A/B under the same harness measured the pure quantization
cost at **-8.2pp**, concentrated on the hardest task (per-row int4 scales erode
the small logit margins hard questions depend on — grouped scales recover ~63%
of that loss, see [#225](https://github.com/JustVugg/colibri/issues/225)). The
scale-granularity/rotation/lattice ablation lives in
`tools/quant_ablation.py` ([#81](https://github.com/JustVugg/colibri/issues/81)).
```bash
cd c
pip install tokenizers datasets
./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each
./coli bench hellaswag --limit 200 # one task, more questions
./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget
```
docs/cuda.md
+104
View File
@@ -0,0 +1,104 @@
# CUDA backend (Linux)
colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming
experts deliberately remain on the original CPU path: copying an expert from
NVMe to the GPU on every use would only replace the disk bottleneck with a PCIe
bottleneck. Resident quantized tensors are uploaded lazily once and reused.
```bash
cd c
make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness
make CUDA=1
# optional dense-path experiment (hot experts are configured below)
COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4
```
Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under
`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`).
`CUDA_ARCH=native` builds for the GPU in the current machine. Requesting CUDA
with a CPU-only binary, an invalid device, or an unavailable runtime fails at
startup instead of silently falling back. For Windows, see
[windows.md](windows.md) (runtime DLL path).
## The VRAM expert tier
A measured `PIN` profile promotes its hottest experts into a persistent VRAM
tier while keeping the rest in RAM:
```bash
STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first
COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \
PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4
# multi-GPU expert tier, 150 GB total budget across six 32 GB devices
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \
CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \
SNAP=/nvme/glm52_i4 ./glm 64 4 4
# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \
CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \
SNAP=/nvme/glm52_i4 ./glm 64 4 4
```
Selected experts are uploaded during startup, so capacity failures occur before
inference. The budget is clamped against free VRAM after reserving the projected
dense resident set and 2 GB of runtime headroom per device. With `COLI_GPUS`,
`CUDA_EXPERT_GB` is a total budget across the device set; experts are assigned
whole to the least-loaded device that can hold them. Multi-GPU runs default to
`PIN_FILL=1` (measured hot set first, then unused VRAM filled with zero-heat
experts) and `CUDA_RELEASE_HOST=1` (RAM copy released after upload, reloaded
from disk only if CUDA later fails).
`CUDA_EXPERT_GB=auto` fills each device up to measured free memory minus
projected dense tensors and headroom. `PIN_GB=all` then loads the remaining
routed experts into RAM **up to the `--ram` budget** (it clamps — [#229](https://github.com/JustVugg/colibri/issues/229)),
eliminating decode-time disk misses when capacity permits. This mode is intended
for dedicated high-memory inference hosts.
### Full-residency reference result (6× RTX 5090, 251 GiB host)
`CUDA_EXPERT_GB=auto PIN_GB=all` selected a 176.7 GB VRAM tier + 191.3 GB RAM
tier (all 19,456 experts resident), adapting the VRAM tier every 16 tokens.
With the GPU-resident pipeline (`COLI_CUDA_PIPE=2`) and Tensor-Core W4A16
dispatch (`COLI_CUDA_TC_W4A16=1`), 96-token greedy decode measured
**5.8–6.8 tok/s** (TTFT ~13 s; 1571-token prefill ~122 s then 4.2 tok/s).
Full experiment log: [experiments/glm52-6x5090-2026-07-12.md](experiments/glm52-6x5090-2026-07-12.md).
These are host-specific capacity results, not portable defaults.
## The GPU-resident pipeline (`COLI_CUDA_PIPE`)
`COLI_CUDA_PIPE=2` keeps the residual stream on-device across layers: rmsnorms,
residual adds, router GEMMs and the shared expert run on the GPU while the CPU
expert loop runs uninterrupted, with batched attention and grouped expert
uploads at prefill. On a single-GPU host this also pays at decode (S=1):
**+49%** measured on a 5070 Ti ([#273](https://github.com/JustVugg/colibri/issues/273)/#274);
on multi-GPU hosts the per-layer P2P hops cancel the gain, so the decode gate is
device-count aware. `COLI_CUDA_TC_W4A16=1` enables Tensor-Core int4×fp16 mixed
dispatch for batched rows (pays at ≥16 rows).
## Notes and limitations
- Text-mode timing reports prefill separately from decode.
- MTP speculation defaults off on CUDA (cold draft routes increase expert
traffic); explicit `DRAFT=n` overrides. Since #294, `SPEC_PIN=1` keeps
draft/verify kernels consistent when speculation is on.
- Devices use independent contexts; a single expert is not sharded. Kernels are
correctness-first custom kernels.
- Profile quality matters more than raw VRAM capacity: the same 150 GB tier
measured 0.94–1.64 tok/s hot-first vs 0.29 tok/s filled without routing heat.
- The GPU tier earns its VRAM only when the CPU is the weak link — a tuned
AVX-512 CPU can match a 5090 on expert matmul
([#101](https://github.com/JustVugg/colibri/issues/101)).
## Reproducible backend A/B without the full checkpoint
```bash
cd c
python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
```
The 313M-parameter fixture has random weights and is not a language model. It
preserves the real MLA/MoE/streaming shapes to compare CPU streaming, dense-only
CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
docs/media/colibri-atlas.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 875 KiB

docs/media/colibri-brain.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 421 KiB

After

Width:  |  Height:  |  Size: 428 KiB

docs/media/colibri-dashboard.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 545 KiB

After

Width:  |  Height:  |  Size: 588 KiB

docs/media/colibri-metrics.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 117 KiB

docs/media/colibri-mobile.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 151 KiB

docs/media/ladder.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 115 KiB

docs/media/sparse.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

docs/media/tiers.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 139 KiB

docs/media/token-path.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 117 KiB

docs/metal.md
+30
View File
@@ -0,0 +1,30 @@
# Metal backend (Apple Silicon, experimental)
On Apple Silicon the decode profile is matmul-bound, and unified memory removes
the PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì
has an opt-in Metal backend that runs the **routed-expert SwiGLU (batched,
zero-copy from the RAM slabs)**, the **fused decode attention** (full MLA layer
in one command buffer, S≤4), and **prefill's large GEMMs** on the GPU.
Token-exact vs the CPU path.
```bash
cd c
make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime)
make metal-test # standalone kernel/attention correctness vs CPU reference
COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96
```
Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal
**0.42 tok/s (~1.4×)** (best config adds `DIRECT=1`; ~3× vs this machine's
first cold run). An M5 Max with a 46.9 GB learned pin reached **2.06 tok/s**
([#103](https://github.com/JustVugg/colibri/issues/103); see also the
[M5 Max performance report](METAL-M5MAX-PERF-REPORT.md)).
Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a
loss — everything is batched into few command buffers per layer, and the
resident experts' GPU work is submitted *before* the missed experts' disk reads
so I/O and compute overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row
threshold (default 16). Streaming, cache, MTP, DSA and the persistence formats
are unchanged; every GPU path falls back to the CPU per-block on any fault.
Numerics are dequant→f32-MAC (same as the CUDA tier); greedy outputs are
byte-identical to the CPU engine.
docs/serve_protocol.md
+107
View File
@@ -0,0 +1,107 @@
# The serve protocols — engine ⇄ server wire format
The engine speaks two line-oriented protocols over stdin/stdout. Both are plain text
plus byte-counted payload frames; every outbound line is written with a trailing
`fflush`, one line per write. On Windows both ends of the pipe are switched to binary
mode at startup — the CRT's CRLF translation otherwise corrupts the sentinels and
stalls byte-counted reads (#195).
| protocol | entry | selected by | used by |
|---|---|---|---|
| **mux** (continuous batching, up to 16 KV slots) | `run_serve_mux` | `SERVE_BATCH=1` | `openai_server.py`, `coli web` |
| **legacy** (single slot, interactive) | `run_serve` | `SERVE=1` (without `SERVE_BATCH`) | `coli chat` |
This document is the reference for the **mux** protocol; the legacy protocol is
summarized at the end. Line formats below are quoted from the emitting `printf`s in
`glm.c` — if this document and the code disagree, the code wins and this file needs a PR.
## Startup handshake (engine → server)
```
\x01\x01READY\x01\x01
STAT 0 0.00 0.0 <rss_gb>
HWINFO <cores> <ram_total_gb> <ram_avail_gb> <ngpu> <vram_total_gb> <cpu_name>|<gpu_name>
TIERS <vram_experts> <ram_experts> <disk_experts> <vram_gb> <ram_gb>
EMAP <rows> <cols> <hex>
```
The server must not send requests before `READY`. `HWINFO`/`TIERS`/`EMAP` are
telemetry (see below) and may grow — **servers must ignore line kinds they do not
recognize**; that is the protocol's forward-compatibility rule.
## Requests (server → engine)
```
SUBMIT <id> <slot> <bytes> <max_tokens> <temperature> <top_p>\n<payload>\n
CANCEL <id>\n
```
- `id` — non-zero u64, unique among in-flight requests.
- `slot` — KV slot index, `0 … KV_SLOTS-1` (`KV_SLOTS` env, 1–16, default 1). A slot
holds one conversation's KV; the engine matches the tokenized payload against the
slot's history and reuses the common prefix (truncate-and-extend), so stateless
HTTP turns keep their cache.
- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine
reads exactly that many bytes after the header line, then one trailing `\n`.
- `payload` — the fully rendered prompt (the server owns the chat template).
- EOF on stdin = graceful shutdown: in-flight requests finish first.
Prefill is serial; decode is continuously batched — every active slot contributes
one row per forward.
## Responses (engine → server)
Per request, in order:
```
DATA <id> <n>\n<n bytes of UTF-8>\n # a decoded token's text; repeated
TOPK <id> 5 <logprob> <hextext> ... ×5 # candidates for the sampled token (SERVE_TOPK=1)
HITS <rows> <cols> <hex> # ~every 6 tokens: routed-expert bitmap since last HITS
REPIN <layer> <eid> <old_tier> <gpu> # live re-pin swap events, as they happen
...
DONE <id> STAT <emitted> <tok_s> <hit_pct> <rss_gb> <prompt_tokens> <length_limited>
```
Errors replace the stream: `ERROR <id> <CODE>` with codes `BAD_FRAME`, `BAD_REQUEST`,
`SLOT_BUSY`, `DUPLICATE_ID`, `EMPTY_PROMPT`, `NOT_FOUND` (CANCEL of unknown id),
`CANCELLED`. A `CANCEL` is acknowledged by `ERROR <id> CANCELLED` after the slot's KV
is persisted.
Immediately before each `DONE` the engine emits a telemetry block for the finished
turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats below).
`.coli_usage` is persisted at every turn end, not only at exit.
## Telemetry lines
| line | format | meaning |
|---|---|---|
| `TIERS` | `TIERS <vram> <ram> <disk> <vram_gb> <ram_gb>` | expert count per tier + resident bytes |
| `HWINFO` | `HWINFO <cores> <ram_total> <ram_avail> <ngpu> <vram_total> <cpu>\|<gpu>` | host snapshot (GBs are floats) |
| `EMAP` | `EMAP <rows> <cols> <hex>` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat |
| `HITS` | `HITS <rows> <cols> <hex>` | 1 bit per expert, experts routed since the previous `HITS` |
| `PERF` | `PERF <id> <dt> <t_edisk> <t_ewait> <t_emm> <t_attn> <t_kvb> <t_head>` | this turn's PROFILO deltas, seconds |
| `ENTROPY` | `ENTROPY <h0> <h1> …` | per-sparse-layer routing entropy of the turn, bits |
| `GPUS` | `GPUS <n> (<used_gb> <total_gb> <experts>)×n` | per-device VRAM + resident expert count (CUDA builds) |
| `TOPK` | `TOPK <id> 5 (<logprob> <hextext>)×5` | token text hex-encoded so the line stays line-shaped |
| `REPIN` | `REPIN <layer> <eid> <old_tier> <gpu>` | one line per hot-store swap (`REPIN=n` mode) |
All telemetry is advisory: servers render what they know and skip the rest.
## HTTP surface (`openai_server.py`)
- `POST /v1/chat/completions` — OpenAI-compatible; streaming responses emit one extra
SSE frame `data: {"colibri": {stats, perf, topk, entropy, gpus, repin}}` immediately
before `data: [DONE]`; non-streaming responses attach the same object as a
`"colibri"` field.
- `GET /experts` — the latest `EMAP`/`HITS` state: `{rows, cols, map, hits, seq,
gpus, entropy, repin}`.
- `GET /*` — static hosting of `web/dist` (SPA fallback, path-traversal-safe), plus
`experts.json` if published there (the measured expert atlas, #175/#218).
## Legacy protocol (`run_serve`, `coli chat`)
Interactive lines are prompts; control frames: `\x02RESET` (clear history),
`\x02MORE` (continue an NGEN-truncated answer), and
`\x02PROMPT <bytes> <max_tokens> <temperature> <top_p> [kv_slot]\n<prompt>\n`
(the pre-mux API mode). Responses are raw text terminated by `\x01\x01END\x01\x01`
plus a `STAT` line. New integrations should use the mux protocol.
docs/tuning.md
+110
View File
@@ -0,0 +1,110 @@
# Tuning & runtime knobs
Everything here is opt-in; the defaults are chosen so a plain `./coli chat`
is safe on any machine. See also [SETTINGS.md](SETTINGS.md) and
[ENVIRONMENT.md](ENVIRONMENT.md) for the full variable inventory.
## The knobs that matter most
| knob | what it does |
|---|---|
| `--temp T` | token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy) |
| `--topp 0.7` | adaptive expert top-p (30–40% less disk; lossy — prints a warning) |
| `--ngen N` | max tokens per answer (`:more` in chat continues a truncated one) |
| `--repin N` | adapt RAM/VRAM hot experts every N emitted tokens |
| `RAM_GB=<n>` | claim more RAM for the expert cache than the conservative auto-detect |
| `PIN=stats PIN_GB=g` | pin the hottest experts from a measured usage profile |
| `DRAFT=n` | MTP draft depth (0 disables speculation) |
| `GRAMMAR=g.gbnf` | grammar-forced drafts for constrained JSON/NDJSON output ([docs](grammar-draft.md)) |
| `THINK=1` | enable GLM-5.2's reasoning block |
| `PILOT=1` | router-lookahead disk prefetch (see below) |
| `URING=1` | Linux-only batched expert I/O (implies `PIPE=1`) |
| `PIPE=0` | disable the async expert-load pool (default ON — overlaps `pread` with matmul, −18% disk service) |
| `DIRECT=1` | O_DIRECT expert reads (measured **+65%** alone on a Strix Halo, [#200](https://github.com/JustVugg/colibri/issues/200)) |
| `COLI_NUMA=1` | interleave resident weights across NUMA nodes on multi-socket hosts ([#82](https://github.com/JustVugg/colibri/issues/82)) |
| `CACHE_ROUTE=1` | cache-aware max-rank routing (opt-in, [#199](https://github.com/JustVugg/colibri/issues/199)) |
| `AUTOPIN=0` | disable the learning cache's auto-pin |
| `CAP_RAISE=0` | don't auto-grow the expert cache |
| `KVSAVE=0` | disable KV-cache persistence |
| `TF=1` | teacher-forcing validation |
## Resource policy
`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing (disk)
tiers, the reason for each placement, and the expected bottleneck. The default
`--policy quality` and `--policy balanced` modes preserve checkpoint quantization
and router decisions unless `--topk` or `--topp` is passed; those explicit lossy
overrides print a warning and proceed.
Auto-tier plans size OpenMP from physical cores and bind workers across cores.
Memory-bound quantized kernels can regress sharply when SMT siblings compete for
limited memory channels; explicit `OMP_*` settings always take precedence.
```bash
coli plan --model /models/glm52_i4 --policy quality
coli run --auto-tier --policy quality "Explain MoE offloading"
# Explicit research-only router reduction:
coli run --policy experimental-fast --topk 4 "Benchmark prompt"
```
Disk is an immutable recovery source, not a normal decode target. If the plan
leaves cold expert bytes on disk, speed depends on cache hit rate; output quality
does not.
Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute
while missing experts are loaded in a bounded background I/O pool, then the cold
results join before the layer completes. The pool engages only under `PIPE=1`;
`PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports both disk
service time and the smaller foreground-visible wait time so overlap is explicit.
`--policy balanced` enables lossless live placement (`REPIN=64`). At safe request
boundaries, a per-layer LFRU score combines decaying session frequency with recent
access and replaces at most four sufficiently colder pinned experts. `--policy
quality` leaves live replacement off by default; `REPIN=0` always disables it.
## The learning cache
The engine records which experts your usage actually routes to (`.coli_usage`
next to the model, updated every turn) and at startup automatically pins the
hottest ones in spare RAM — colibrì literally gets faster the more you use it.
`PIN=auto` seeds the pin directly from the live usage history
([#301](https://github.com/JustVugg/colibri/pull/301)).
**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine
*raises* the LRU cap to fill your `--ram` budget instead of only lowering it.
If you benchmarked colibrì before that date, rerun — your numbers were capped.
**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a
decaying session heat map replaces cold pinned experts with hotter streamed
experts. A 25% hysteresis and a four-swap limit prevent tier thrashing.
Persistent `.coli_usage` remains the long-term signal and is not decayed.
## Router-lookahead prefetch (`PILOT=1`, experimental)
GLM-5.2's expert routing is measurably predictable *ahead of time* — applying
layer L+1's router to layer L's post-attention state recalls **71.6%** of the
true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` issues
next-layer expert readahead from a dedicated I/O thread while the current layer
computes. `PILOT_REAL=1` moves the prefetched loads off the critical path
(measured +11pp hit rate on a big-cache host), and `PILOT_TWO=1` folds the
computed shared-expert into the prediction (+3% recall,
[#200](https://github.com/JustVugg/colibri/issues/200)). On disk-saturated
hosts hint-only PILOT can be net negative — measure on yours.
## Speculation and reproducibility
Speculative decoding requires that the draft and verify paths compute the same
function — `SPEC_PIN=1` (default since [#294](https://github.com/JustVugg/colibri/pull/294))
pins every forward issued while drafts are live to the platform's S=1 kernel
family. For byte-exact reproducibility across runs: `DRAFT=0`, plus `IDOT=0
COLI_CUDA=0` if you also want kernel-family/GPU independence. Acceptance
percentages are not comparable across engine versions under `--topp`
([#163](https://github.com/JustVugg/colibri/issues/163) has the full story).
## Conversations reopen warm
`coli chat` persists the compressed MLA KV-cache to disk after every turn
(`.coli_kv`, ~182 KB/token, appended incrementally, crash-safe). Close the chat,
reopen it tomorrow — the model still remembers the whole conversation and **zero
re-prefill happens**: validated byte-identical to an uninterrupted session.
`:reset` clears it, `KVSAVE=0` disables it.
docs/windows.md
+172
View File
@@ -0,0 +1,172 @@
# Windows 11 native install — a complete walkthrough (no WSL)
A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build.
## 0. What you need
| Piece | Why | Get it |
|---|---|---|
| git, Python 3 | clone + `coli` launcher | winget / python.org |
| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) |
| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` |
| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" |
| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount |
RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later.
## 1. Start the model download first (it's the long pole)
```powershell
python -m pip install -U "huggingface_hub[hf_transfer]"
$env:HF_HUB_ENABLE_HF_TRANSFER = "1"
hf download <model-repo> --local-dir D:\glm52_i4
```
Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them.
## 2. Build the engine (CPU)
From a normal PowerShell, in the repo's `c\` directory:
```powershell
make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake
make iobench.exe # disk benchmark, useful before committing to the download
```
Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`.
### ⚠️ Smart App Control will block your fresh binary
On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with:
```
Program 'glm.exe' failed to run: An Application Control policy has blocked this file
```
This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything:
```powershell
(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState
# 0 = off, 1 = enforced, 2 = evaluation
```
## 3. Build the CUDA DLL (GPU tier)
nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then:
```cmd
make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ...
make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader
```
Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts:
- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted.
- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first.
Sanity check: first GPU run should print `[CUDA] device 0: <your GPU>, ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`.
## 4. First run
```powershell
cd <repo>\c
$env:OMP_NUM_THREADS = "<physical cores>"
python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48
```
The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top:
```powershell
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4"
python coli run "..." --model D:\glm52_i4 --ngen 64
```
Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in.
## 5. Reference numbers from this walkthrough's hardware
285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute.
## Quick failure index
| Symptom | Cause | Fix |
|---|---|---|
| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** |
| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) |
| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild |
| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt |
| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ |
| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container |
| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in |
---
## Reference: build flags & warmup
# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8
# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster
# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out;
# build for THIS machine to enable it:
make glm.exe ARCH=native # banner prints "idot: avx-vnni"
# Verify (tiny model, 2.4 MB):
pip install torch transformers safetensors huggingface_hub
python tools/make_glm_oracle.py # generate tiny oracle
SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions"
# Run with real model:
SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference
python coli chat --model D:\glm52_i4 # interactive chat
python coli serve --model D:\glm52_i4 # OpenAI-compatible API
```
> Windows Store's `python` alias stub is the single most common native-Windows
> trap: install real Python (python.org or `winget install Python.Python.3.12`)
> or disable the alias under *Settings → Apps → App execution aliases*.
## Warmup (overnight cache priming)
The engine's expert cache learns from your workload. The included `warmup.ps1`
script runs `coli run` in a loop with diverse prompts to build the
`.coli_usage` histogram unattended, so the next real session starts with a
large, accurate hot-expert pin. Each run saves usage atomically on clean
completion.
```powershell
.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress
```
## NVIDIA GPU (optional, via runtime DLL)
On Windows the engine is built with MinGW gcc but CUDA kernels require MSVC +
nvcc. The split is clean: build the CUDA backend into a standalone
`coli_cuda.dll` (nvcc + MSVC), then the host `glm.exe` loads it at runtime via
`LoadLibrary` (`c/backend_loader.c`). The host never links cudart directly; if
the DLL is absent the engine falls back to CPU without error.
```powershell
# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH.
# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or
# "x64 Native Tools Command Prompt for VS"):
make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120
# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and
# links backend_loader.o instead of cudart):
make glm.exe CUDA_DLL=1 ARCH=native
# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM):
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8"
python coli chat --model D:\glm52_i4 --topp 0.7
```
The DLL exports the full `extern "C"` surface (including the #111 pipeline ABI);
`backend_loader.c` resolves symbols via `GetProcAddress` on first use.
`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the
MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match
your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series,
`sm_89` for Ada / RTX 40-series). A one-shot `build_cuda.bat` wrapper is also
available.
**Measured on a single RTX 5070 Ti + Core Ultra 9 (32 GB RAM):** CPU-only 0.63
→ CUDA attention+dense 0.72 → **1.07 tok/s** with the GPU-resident pipeline at
decode ([#273](https://github.com/JustVugg/colibri/issues/273), merged in #274).
flake.lock
Generated
+61
View File
@@ -0,0 +1,61 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1784160687,
"narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-26.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}
flake.nix
+23 -10
View File
@@ -26,7 +26,9 @@
version = "1.0";
src = ./.;
nativeBuildInputs = [ pkgs.makeWrapper ];
# python3 is needed by checkPhase: `make test-c` shells out to
# `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3).
nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ];
buildInputs = [
pkgs.gcc
@@ -44,18 +46,29 @@
installPhase = ''
runHook preInstall
mkdir -p $out/bin
cp c/glm $out/bin/glm
# Wrap coli (the Python CLI) so it finds the right python and the engine
mkdir -p $out/share/colibri
cp c/coli $out/share/colibri/coli
chmod +x $out/share/colibri/coli
cp -r c/tools $out/share/colibri/tools
# Self-contained layout under $out/lib/colibri that mirrors the
# source tree `coli` runs in (see the path-resolution logic at the
# top of c/coli): the engine, the coli CLI script, the support
# modules it imports (openai_server.py, resource_plan.py,
# doctor.py), and tools/ all sit next to each other.
mkdir -p $out/lib/colibri/tools $out/bin
cp c/glm $out/lib/colibri/glm
cp c/coli $out/lib/colibri/coli
chmod +x $out/lib/colibri/coli
cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/
cp -r c/tools/* $out/lib/colibri/tools/
# $out/bin holds the user-facing entry points.
ln -s ../lib/colibri/glm $out/bin/glm
# Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is
# found by default, and at the module dir (PYTHONPATH) so
# `import openai_server` / `resource_plan` / `doctor` resolve.
makeWrapper ${pythonEnv}/bin/python $out/bin/coli \
--add-flags "$out/share/colibri/coli" \
--set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}"
--add-flags "$out/lib/colibri/coli" \
--set-default COLI_ENGINE "$out/lib/colibri/glm" \
--set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}"
runHook postInstall
'';
web/README.md
+6 -1
View File
@@ -17,9 +17,14 @@ npm test
npm run build
```
Besides Chat and Brain, the **Profiling** tab charts where the engine spent
each turn's wall time (I/O wait, expert matmul, attention, LM head) from the
server's `/profile` endpoint — a rolling window of per-turn `PROF` snapshots
emitted by the engine.
The test suite stays browser-light: API requests use a mocked `fetch`, while
runtime capability and storage behavior are covered through pure helpers. It
checks that `/health` is resolved next to (not below) the OpenAI `/v1` prefix,
checks that `/health` and `/profile` are resolved next to (not below) the OpenAI `/v1` prefix,
supports both boolean and numeric `scheduler.active` responses, and sends the
colibrì-specific `cache_slot` field only when KV-slot support was advertised.
web/src/App.tsx
+5 -2
View File
@@ -31,6 +31,7 @@ import { Textarea } from "@/components/ui/textarea"
import { getHealth, listModels, streamChat, type ChatMessage, type HealthResponse, type StreamChatResult } from "@/lib/api"
import { activeRequests, supportsCacheSlots } from "@/lib/runtime"
import { Brain } from "./Brain"
import { Profiling } from "./Profiling"
import { persistPublicSettings, stored } from "@/lib/storage"
import { cn } from "@/lib/utils"
@@ -73,7 +74,7 @@ export default function App() {
const [totalTokens, setTotalTokens] = useState({ prompt: 0, completion: 0 })
const [connecting, setConnecting] = useState(false)
const [connected, setConnected] = useState(false)
const [view, setView] = useState<"chat" | "brain">("chat")
const [view, setView] = useState<"chat" | "brain" | "profiling">("chat")
const [error, setError] = useState("")
const autoConnected = useRef(false)
const abortRef = useRef<AbortController | null>(null)
@@ -314,6 +315,7 @@ export default function App() {
<div className="view-tabs">
<button className={view === "chat" ? "active" : ""} onClick={() => setView("chat")}><MessageSquareText className="size-3.5" /> Chat</button>
<button className={view === "brain" ? "active" : ""} onClick={() => setView("brain")}><BrainCircuit className="size-3.5" /> Brain</button>
<button className={view === "profiling" ? "active" : ""} onClick={() => setView("profiling")}><Gauge className="size-3.5" /> Profiling</button>
</div>
<div className="top-actions">
{loading && tokenCount > 0 ? <Badge className="badge-live"><Zap className="size-3 flash" /> {tokenCount} tokens</Badge> : null}
@@ -326,7 +328,8 @@ export default function App() {
</div>
</header>
{view === "brain" ? <Brain baseUrl={baseUrl} apiKey={apiKey} connected={connected} /> : <>
{view === "brain" ? <Brain baseUrl={baseUrl} apiKey={apiKey} connected={connected} />
: view === "profiling" ? <Profiling baseUrl={baseUrl} apiKey={apiKey} connected={connected} /> : <>
<div className="conversation">
{!messages.length ? (
web/src/Profiling.tsx
+165
View File
@@ -0,0 +1,165 @@
import { useEffect, useState } from "react"
import { Activity, Gauge, HardDrive, Timer } from "lucide-react"
import { getProfile, type ProfileTurn } from "@/lib/api"
/* Wall-time phases stacked per turn. The order is the palette's CVD-safe slot
* order (validated as a set on this surface) — identity never leans on colour
* alone: segments keep 2px gaps, the legend is always shown and the table
* carries the exact numbers. Disk *service* time is reported separately: it
* runs on I/O threads overlapped with compute, so only the stall the compute
* thread actually felt (I/O wait) belongs inside the wall-time stack. */
const PHASES = [
{ key: "expert_wait_s", name: "I/O wait", color: "#3987e5" },
{ key: "expert_matmul_s", name: "Expert matmul", color: "#199e70" },
{ key: "attention_s", name: "Attention", color: "#c98500" },
{ key: "lm_head_s", name: "LM head", color: "#008300" },
{ key: "other_s", name: "Other", color: "#9085e9" },
] as const
interface Turn extends ProfileTurn { other_s: number; toks: number }
const derive = (turn: ProfileTurn): Turn => ({
...turn,
other_s: Math.max(0, turn.wall_s - turn.expert_wait_s - turn.expert_matmul_s - turn.attention_s - turn.lm_head_s),
toks: turn.wall_s > 0 ? turn.completion_tokens / turn.wall_s : 0,
})
const seconds = (value: number) => (value >= 10 ? value.toFixed(1) : value.toFixed(2)) + "s"
function ShareBar({ label, turns }: { label: string; turns: Turn[] }) {
const total = turns.reduce((sum, turn) => sum + turn.wall_s, 0)
const parts = PHASES.map((phase) => ({ ...phase, value: turns.reduce((sum, turn) => sum + turn[phase.key], 0) }))
return (
<div className="prof-share">
<div className="prof-share-head"><span>{label}</span><code>{seconds(total)}</code></div>
<div className="prof-share-bar" role="img" aria-label={parts.map((part) => `${part.name} ${seconds(part.value)}`).join(", ")}>
{parts.map((part) => {
const share = total > 0 ? part.value / total : 0
return share > 0.001 ? (
<span key={part.key} style={{ width: `${100 * share}%`, background: part.color }} title={`${part.name} — ${seconds(part.value)} (${(100 * share).toFixed(1)}%)`}>
{share >= 0.09 ? `${Math.round(100 * share)}%` : ""}
</span>
) : null
})}
</div>
</div>
)
}
/* Column chart over the recent turns; oldest on the left. Stacked mode draws the
* wall-time composition, plain mode a single series (no legend — the title names it). */
function TurnColumns({ turns, stacked, height, format }: { turns: Turn[]; stacked: boolean; height: number; format: (turn: Turn) => string }) {
const [hover, setHover] = useState<number | null>(null)
const peak = Math.max(...turns.map((turn) => (stacked ? turn.wall_s : turn.toks)), 1e-9)
const gap = 2
const width = Math.max(1, (100 - gap * (turns.length - 1)) / turns.length)
return (
<div className="prof-plot" onMouseLeave={() => setHover(null)}>
<svg viewBox={`0 0 100 ${height}`} preserveAspectRatio="none" aria-hidden="true">
{[0.25, 0.5, 0.75].map((line) => <line key={line} x1="0" x2="100" y1={height * line} y2={height * line} className="prof-grid" />)}
{turns.map((turn, index) => {
const x = index * (width + gap)
if (!stacked) {
const h = (height * turn.toks) / peak
return <rect key={index} x={x} y={height - h} width={width} height={h} rx="1" fill="var(--primary)" opacity={hover === null || hover === index ? 1 : 0.45} />
}
let y = height
return PHASES.map((phase) => {
const h = (height * turn[phase.key]) / peak
y -= h
return h > 0.1 ? <rect key={`${index}-${phase.key}`} x={x} y={y + 0.35} width={width} height={Math.max(h - 0.7, 0.35)} fill={phase.color} opacity={hover === null || hover === index ? 1 : 0.45} /> : null
})
})}
{/* hit targets bigger than the marks */}
{turns.map((_, index) => <rect key={index} x={index * (width + gap) - gap / 2} y="0" width={width + gap} height={height} fill="transparent" onMouseEnter={() => setHover(index)} />)}
</svg>
<div className="prof-plot-foot">
<span>{turns.length > 1 ? `${turns.length} turns · oldest → newest` : "1 turn"}</span>
<code>{hover !== null && turns[hover] ? format(turns[hover]) : `peak ${stacked ? seconds(peak) : peak.toFixed(1) + " tok/s"}`}</code>
</div>
</div>
)
}
export function Profiling({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: string; connected: boolean }) {
const [turns, setTurns] = useState<Turn[]>([])
useEffect(() => {
if (!connected) return
let disposed = false
const poll = async () => {
if (document.visibilityState === "hidden") return
try {
const result = await getProfile(baseUrl, apiKey)
if (!disposed) setTurns(result.turns.map(derive))
} catch { /* engine busy or restarting — keep the last snapshot */ }
}
void poll()
const timer = window.setInterval(() => void poll(), 2000)
return () => { disposed = true; window.clearInterval(timer) }
}, [baseUrl, apiKey, connected])
const latest = turns[turns.length - 1]
const recent = turns.slice(-40)
const diskService = turns.reduce((sum, turn) => sum + turn.expert_disk_s, 0)
return (
<div className="prof-page">
<div className="prof-head">
<div className="section-title"><Gauge className="size-4" /> Profiling — where the engine spends each turn</div>
<div className="prof-legend">
{PHASES.map((phase) => <span key={phase.key}><i style={{ background: phase.color }} />{phase.name}</span>)}
</div>
</div>
{!latest ? (
<p className="runtime-unavailable">{connected ? "No profiled turns yet — send a chat message and the breakdown appears here." : "Connect to the engine to collect per-turn timings."}</p>
) : (
<>
<div className="prof-tiles">
<div><span><Gauge className="size-3" /> Last turn</span><strong>{latest.toks.toFixed(1)}</strong><small>tok/s</small></div>
<div><span><Timer className="size-3" /> Wall time</span><strong>{seconds(latest.wall_s)}</strong><small>{latest.prompt_tokens} → {latest.completion_tokens} tokens</small></div>
<div><span><Activity className="size-3" /> Batching</span><strong>{latest.forwards > 0 ? (latest.completion_tokens / latest.forwards).toFixed(2) : "—"}</strong><small>tokens / forward</small></div>
<div><span><HardDrive className="size-3" /> Disk service</span><strong>{seconds(latest.expert_disk_s)}</strong><small>overlapped with compute</small></div>
</div>
<div className="prof-shares">
<ShareBar label="Last turn" turns={[latest]} />
{turns.length > 1 ? <ShareBar label={`Window · last ${turns.length} turns`} turns={turns} /> : null}
</div>
<div className="prof-charts">
<div className="prof-chart">
<div className="prof-chart-title">Throughput per turn (tok/s)</div>
<TurnColumns turns={recent} stacked={false} height={36} format={(turn) => `${turn.toks.toFixed(1)} tok/s · ${turn.completion_tokens} tokens`} />
</div>
<div className="prof-chart">
<div className="prof-chart-title">Turn wall time by phase (s)</div>
<TurnColumns turns={recent} stacked height={36} format={(turn) => `${seconds(turn.wall_s)} · ${PHASES.map((phase) => `${phase.name} ${seconds(turn[phase.key])}`).join(" · ")}`} />
</div>
</div>
<div className="prof-table-wrap">
<table className="prof-table">
<thead><tr><th>Turn</th><th>Tokens</th><th>tok/s</th><th>Wall</th>{PHASES.map((phase) => <th key={phase.key}><i style={{ background: phase.color }} />{phase.name}</th>)}<th>Disk service</th></tr></thead>
<tbody>
{recent.slice().reverse().map((turn, index) => (
<tr key={turns.length - index}>
<td>{turns.length - index}</td>
<td>{turn.prompt_tokens} → {turn.completion_tokens}</td>
<td>{turn.toks.toFixed(1)}</td>
<td>{seconds(turn.wall_s)}</td>
{PHASES.map((phase) => <td key={phase.key}>{seconds(turn[phase.key])}</td>)}
<td>{seconds(turn.expert_disk_s)}</td>
</tr>
))}
</tbody>
</table>
{diskService > 0 ? <p className="prof-note">Disk service is time spent reading experts on I/O threads; it overlaps with compute, so only the <em>I/O wait</em> the compute thread felt counts inside the wall-time stack. With multiple KV sessions the shares describe the whole engine over the turn's window.</p> : null}
</div>
</>
)}
</div>
)
}
web/src/index.css
+36
View File
@@ -151,3 +151,39 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-
.brain-tip-spec { color: var(--primary); font-weight: 700; }
.brain-tip-spec small, .brain-tip-aff { color: #8b9aa3; font-weight: 400; }
.brain-tip-aff { font-size: 10px; }
/* ---- Profiling page ---- */
.prof-page { flex: 1; display: flex; flex-direction: column; gap: 16px; padding: 18px 22px; overflow: auto; }
.prof-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; }
.prof-legend { display: flex; flex-wrap: wrap; gap: 12px; font-size: 11px; color: #a5b0b4; align-items: center; }
.prof-legend i { width: 9px; height: 9px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; }
.prof-tiles { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }
.prof-tiles > div { display: grid; gap: 4px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
.prof-tiles span { display: flex; align-items: center; gap: 5px; color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }
.prof-tiles strong { font: 600 22px ui-monospace, SFMono-Regular, monospace; font-variant-numeric: tabular-nums; }
.prof-tiles small { color: var(--muted-foreground); font-size: 10px; }
.prof-shares { display: grid; gap: 10px; padding: 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
.prof-share { display: grid; gap: 6px; }
.prof-share-head { display: flex; justify-content: space-between; color: #a5b0b4; font-size: 11px; font-weight: 600; }
.prof-share-head code { color: var(--foreground); font-size: 11px; font-variant-numeric: tabular-nums; }
.prof-share-bar { display: flex; gap: 2px; height: 22px; border-radius: 6px; overflow: hidden; }
.prof-share-bar span { display: grid; place-items: center; min-width: 0; overflow: hidden; color: #06090b; font-size: 10px; font-weight: 700; font-variant-numeric: tabular-nums; transition: width .5s ease; }
.prof-charts { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
.prof-chart { display: grid; gap: 8px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
.prof-chart-title { color: #a5b0b4; font-size: 11px; font-weight: 600; }
.prof-plot svg { display: block; width: 100%; height: 120px; }
.prof-grid { stroke: var(--border); stroke-width: .3; }
.prof-plot-foot { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: #7f8d92; font-size: 10px; }
.prof-plot-foot code { color: var(--foreground); font-variant-numeric: tabular-nums; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.prof-table-wrap { overflow-x: auto; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
.prof-table { width: 100%; border-collapse: collapse; font-size: 11px; font-variant-numeric: tabular-nums; }
.prof-table th { padding: 9px 12px; border-bottom: 1px solid var(--border); color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .07em; text-transform: uppercase; text-align: left; white-space: nowrap; }
.prof-table th i { width: 8px; height: 8px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; }
.prof-table td { padding: 7px 12px; border-bottom: 1px solid var(--border); color: #c3ccd0; white-space: nowrap; }
.prof-table tbody tr:last-child td { border-bottom: 0; }
.prof-note { margin: 0; padding: 10px 12px; color: var(--muted-foreground); font-size: 10px; line-height: 1.5; border-top: 1px solid var(--border); }
@media (max-width: 900px) {
.prof-page { padding: 10px; }
.prof-tiles { grid-template-columns: 1fr 1fr; }
.prof-charts { grid-template-columns: 1fr; }
}
web/src/lib/api.test.ts
+14 -1
View File
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest"
import { extractSSE, getHealth, serverEndpoint, streamChat } from "./api"
import { extractSSE, getHealth, getProfile, serverEndpoint, streamChat } from "./api"
afterEach(() => vi.unstubAllGlobals())
@@ -36,6 +36,19 @@ describe("runtime API", () => {
headers: expect.objectContaining({ Authorization: "Bearer secret" }),
}))
})
it("requests the profiling history next to the OpenAI v1 prefix", async () => {
const turn = {
wall_s: 2.5, prompt_tokens: 7, completion_tokens: 12,
expert_disk_s: 0.4, expert_wait_s: 0.1, expert_matmul_s: 0.9,
attention_s: 0.6, lm_head_s: 0.2, forwards: 15,
}
const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ seq: 1, turns: [turn] })))
vi.stubGlobal("fetch", fetchMock)
await expect(getProfile("http://localhost:8000/v1/")).resolves.toEqual({ seq: 1, turns: [turn] })
expect(fetchMock).toHaveBeenCalledWith("http://localhost:8000/profile", expect.anything())
})
})
describe("chat request extensions", () => {
web/src/lib/api.ts
+23
View File
@@ -49,6 +49,23 @@ export interface HealthResponse {
hwinfo?: HwinfoHealth
}
export interface ProfileTurn {
wall_s: number
prompt_tokens: number
completion_tokens: number
expert_disk_s: number
expert_wait_s: number
expert_matmul_s: number
attention_s: number
lm_head_s: number
forwards: number
}
export interface ProfileResponse {
seq: number
turns: ProfileTurn[]
}
export interface TokenUsage {
prompt_tokens: number
completion_tokens: number
@@ -100,6 +117,12 @@ export async function getHealth(baseUrl: string, apiKey = "", signal?: AbortSign
return (await response.json()) as HealthResponse
}
export async function getProfile(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise<ProfileResponse> {
const response = await fetch(serverEndpoint(baseUrl, "profile"), { headers: headers(apiKey), signal })
if (!response.ok) throw new Error(await responseError(response))
return (await response.json()) as ProfileResponse
}
export function extractSSE(buffer: string) {
const frames = buffer.split(/\r?\n\r?\n/)
const rest = frames.pop() || ""