# --- Target detection --- # Ask the compiler what it actually targets (rofl0r's note on #129): a gcc/clang # toolchain reports its target triple via `-dumpmachine`, e.g. # x86_64-w64-mingw32 x86_64-pc-cygwin x86_64-unknown-linux-gnu # arm64-apple-darwin powerpc64le-unknown-linux-gnu # The triple follows the toolchain, not the host shell that `uname` reports, so # it is correct under a native-Windows shell (no `uname` on PATH) AND when # cross-compiling (e.g. make CC=x86_64-w64-mingw32-gcc on Linux). CC is chosen # per-target below, so probe with the user's CC if they set one, else gcc # (present on Linux and mingw, and a clang shim on macOS). DETECT_CC := $(if $(filter default,$(origin CC)),gcc,$(CC)) TRIPLET := $(shell $(DETECT_CC) -dumpmachine 2>/dev/null) MINGW := $(findstring mingw,$(TRIPLET)) CYGWIN := $(findstring cygwin,$(TRIPLET)) DARWIN := $(findstring darwin,$(TRIPLET)) LINUX := $(findstring linux,$(TRIPLET)) IS_WIN := $(MINGW)$(CYGWIN) # Fallbacks for the rare toolchain that does not answer -dumpmachine: keep the # #129 signal (OS=Windows_NT, set in every Windows shell) for Windows, then # `uname` for everything else, so no host regresses. ifeq ($(TRIPLET),) IS_WIN := $(if $(filter Windows_NT,$(OS)),1,) ifeq ($(IS_WIN),) UNAME_S := $(shell uname -s) UNAME_M := $(shell uname -m) DARWIN := $(findstring Darwin,$(UNAME_S)) endif endif TARGET_CPU := $(firstword $(subst -, ,$(TRIPLET))) ifeq ($(TARGET_CPU),) TARGET_CPU := $(UNAME_M) endif X86_64 := $(filter x86_64 amd64,$(TARGET_CPU)) AARCH64 := $(filter aarch64 arm64,$(TARGET_CPU)) PPC64 := $(filter powerpc64% ppc64%,$(TARGET_CPU)) ifneq (,$(DARWIN)) # --- macOS / Apple Silicon --- # Apple clang non include il runtime OpenMP: se c'e' libomp di Homebrew lo usa # (brew install libomp), altrimenti compila single-thread (i pragma omp sono ignorati). # Niente -march: su arm64 NEON e' baseline (i kernel __ARM_NEON si attivano da soli). CC = clang OMPDIR := $(shell brew --prefix libomp 2>/dev/null) # `brew --prefix libomp` can print the formula's prospective path even when it # is not installed, so verify both artifacts before adding unusable flags. ifneq ($(and $(OMPDIR),$(wildcard $(OMPDIR)/include/omp.h),$(wildcard $(OMPDIR)/lib/libomp.*)),) OMPC = -Xclang -fopenmp -I$(OMPDIR)/include OMPL = -L$(OMPDIR)/lib -lomp else $(warning libomp not found: building single-threaded. For multithreading: brew install libomp) OMPC = OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), # which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. ifneq ($(ARCH),) CFLAGS += -mcpu=$(ARCH) endif LDFLAGS = -lm $(OMPL) EXE = else ifneq ($(IS_WIN),) # --- Windows 11 x86-64 (MinGW-w64 / MSYS2) --- # GCC + libgomp + winpthreads: pthread, OpenMP, clock_gettime, opendir/readdir, # AVX2 intrinsics - tutto gratis, nessun porting. # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), # which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # -lpsapi: compat.h calls GetProcessMemoryInfo (rss_gb). It's linked via # #pragma comment(lib,"psapi.lib") for MSVC, but MinGW gcc ignores that pragma # (warns), so link psapi explicitly or the build fails undefined-reference on # modern GCC (e.g. 16.x under UCRT). Harmless where the pragma also resolves it. LDFLAGS = -lm -fopenmp -static -lpsapi EXE = .exe else ifneq (,$(PPC64)) # --- Linux PowerPC (POWER8/POWER9/POWER10) --- # PowerPC GCC uses -mcpu, not -march. ARCH=native works on gcc >= 4.7. # The AVX2/NEON kernels fall back to the portable scalar C path # (validated token-exact vs the transformers oracle on a POWER8 S824). CC = gcc ARCH ?= native CFLAGS = -O3 -mcpu=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function LDFLAGS = -lm -fopenmp -pthread EXE = else # --- Linux / *BSD x86-64 (percorso originale, invariato) --- CC = gcc # ARCH=native -> ottimizzato per QUESTA macchina (default, piu' veloce). # ARCH=x86-64-v3 -> binario PORTABILE su qualsiasi x86-64 moderno con AVX2 (per distribuire). # ARCH=x86-64 -> massima compatibilita' (niente AVX2: usa il path scalare di fallback). # -pthread: Linux lo tira dentro via -fopenmp, i *BSD no e le pthread_* non risolvono (#219). ARCH ?= native CFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function LDFLAGS = -lm -fopenmp -pthread EXE = endif endif # --- install --- PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin LIBEXECDIR ?= $(PREFIX)/libexec/colibri INSTALL ?= install # CUDA=1 adds an opt-in backend for resident tensors. The default build remains # pure C and keeps the original zero-dependency runtime. # # Two paths: # - Linux/macOS: CUDA=1 links backend_cuda.o directly (cudart via -l). # - Windows: CUDA_DLL=1 builds a standalone coli_cuda.dll (nvcc+MSVC), # then the host glm.exe loads it at runtime via backend_loader.c # (LoadLibrary/GetProcAddress). MinGW gcc cannot compile .cu # (nvcc needs cl.exe), and cross-linking MSVC objects into a # gcc binary is fragile — the DLL split keeps the toolchains # clean. See backend_loader.c and README "cuda-dll" below. CUDA ?= 0 CUDA_DLL ?= 0 ifneq ($(IS_WIN),) # the CUDA installer sets CUDA_PATH system-wide (e.g. C:\Program Files\NVIDIA # GPU Computing Toolkit\CUDA\v13.2); fall back to it before the POSIX default. # NVCC defaults to plain `nvcc` from PATH: CUDA_PATH contains spaces, which the # unquoted recipe checks cannot survive — and the cuda-dll recipe already # requires an MSVC environment (vcvars64) on PATH, so requiring the CUDA bin # directory too is symmetric. The installer adds it by default. CUDA_HOME ?= $(subst \,/,$(CUDA_PATH)) NVCC ?= nvcc # Host compiler override for nvcc (e.g. Fedora ships g++16 but CUDA needs 15): # make glm CUDA=1 NVCC_CCBIN=g++-15 (opt-in: unset = nvcc's own default) ifneq ($(NVCC_CCBIN),) NVCCFLAGS += -ccbin $(NVCC_CCBIN) endif else CUDA_HOME ?= /usr/local/cuda NVCC ?= $(CUDA_HOME)/bin/nvcc endif CUDA_ARCH ?= native ifneq ($(IS_WIN),) # nvcc's host compiler on Windows is MSVC cl.exe: the GCC-style # -Xcompiler=-Wall,-Wextra is rejected ("D8021 invalid numeric argument # '/Wextra'"), which made `make cuda-dll` unbuildable as shipped. -W3 is # the MSVC warning level (dash form, NOT /W3: MSYS make would mangle the # slash-form into a filesystem path). NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-W3 else NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra endif # PYTHON is a HOST tool (clean/test-c/test-python run it on the build machine), # so key it off the host, NOT $(IS_WIN) — which is derived from the *target* # triple ($(CC) -dumpmachine). Otherwise a cross build (make CC=x86_64-w64- # mingw32-gcc ... on Linux) sets IS_WIN and picks `python`, breaking clean/test # on hosts where only `python3` exists. $(OS)=Windows_NT in every Windows shell # (the #129 signal) and is empty on Linux/macOS. ifeq ($(OS),Windows_NT) PYTHON ?= python else PYTHON ?= python3 endif CUDA_OBJ = TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif # Windows CUDA DLL path: host links the loader, NOT cudart. ifneq ($(IS_WIN),) ifeq ($(CUDA_DLL),1) CFLAGS += -DCOLI_CUDA CUDA_OBJ = backend_loader.o endif endif # Linux CUDA direct-link path (unchanged). ifeq ($(CUDA),1) ifneq (,$(DARWIN)) $(error CUDA=1 is supported only on Linux) endif ifneq ($(IS_WIN),) # On Windows use CUDA_DLL=1 (runtime DLL), not CUDA=1 (direct link). $(error On Windows use: make CUDA_DLL=1 cuda-dll (see backend_loader.c)) endif CFLAGS += -DCOLI_CUDA LDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++ CUDA_OBJ = backend_cuda.o endif # METAL=1 adds an opt-in Apple-GPU backend (macOS only). The shader is compiled at # runtime, so no Xcode / offline metal compiler is required. Default build unchanged. METAL ?= 0 METAL_OBJ = METALXX = clang++ -x objective-c++ -std=gnu++17 -fobjc-arc -O3 ifeq ($(METAL),1) ifeq (,$(DARWIN)) $(error METAL=1 is supported only on macOS) endif CFLAGS += -DCOLI_METAL LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif all: colibri$(EXE) # phony targets — 'glm' kept for backward compatibility colibri: colibri$(EXE) glm: colibri$(EXE) # Config stamp: make only tracks file timestamps, not flag changes. Without this, # `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" # and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like # it worked but isn't. We record the build-affecting flags in .build-config and # rewrite it ONLY when they change (evaluated here at parse time, so the file's # timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)" $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) endif .build-config: ; colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config $(CC) $(CFLAGS) -c backend_loader.c -o $@ # Windows CUDA DLL: compile backend_cuda.cu with nvcc (+MSVC cl.exe as host # compiler, required by nvcc on Windows) into coli_cuda.dll. Run this from a # shell that has the MSVC environment set (e.g. after vcvars64.bat, or from a # "x64 Native Tools Command Prompt"). COLI_CUDA_BUILDING_DLL enables # __declspec(dllexport) so the 15 API symbols are exported. cuda-dll: backend_cuda.cu backend_cuda.h @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } @command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; } "$(NVCC)" $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \ -L"$(CUDA_HOME)/lib/x64" -lcudart \ backend_cuda.cu -o coli_cuda.dll backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } "$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@ backend_metal.o: backend_metal.mm backend_metal.h $(METALXX) -c backend_metal.mm -o $@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h $(METALXX) tests/test_backend_metal.mm backend_metal.mm -framework Metal -framework Foundation -o backend_metal_test ./backend_metal_test cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) ./backend_cuda_test$(EXE) cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) ./backend_cuda_bench$(EXE) olmoe$(EXE): olmoe.c st.h json.h compat.h $(CC) $(CFLAGS) olmoe.c -o olmoe$(EXE) $(LDFLAGS) # Use a baseline that matches the compiler target. macOS already targets a # portable baseline when ARCH is empty; forcing the x86 value there breaks # Apple Silicon. Unknown targets use native rather than an invalid x86 flag. ifneq (,$(DARWIN)) PORTABLE_ARCH = else ifneq (,$(AARCH64)) PORTABLE_ARCH = armv8-a else ifneq (,$(PPC64)) PORTABLE_ARCH = power8 else ifneq (,$(X86_64)) PORTABLE_ARCH = x86-64-v3 else PORTABLE_ARCH = native endif portable: $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_tier$(EXE): tests/test_tier.c tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_grammar$(EXE): tests/test_grammar.c grammar.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test # gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), # NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) test-c: $(TEST_BINS) $(PYTHON) tools/run_tests.py $(TEST_BINS) test-python: $(PYTHON) -m unittest discover -s tests -p 'test_*.py' test: test-c test-python # Local validation: one portable CPU build and dependency-free tests. check: $(MAKE) clean $(MAKE) portable $(MAKE) test install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ uninstall: rm -f $(DESTDIR)$(BINDIR)/coli rm -rf $(DESTDIR)$(LIBEXECDIR) clean: $(PYTHON) tools/clean.py bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi .PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench