diff --git a/c/Makefile b/c/Makefile index 27902cb..9887c74 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -293,6 +293,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c new file mode 100644 index 0000000..b86bda9 --- /dev/null +++ b/c/tests/test_tok_o200k.c @@ -0,0 +1,61 @@ +/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations. + * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a + * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB, + * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids + * were produced by HF `tokenizers` on the same file. Guards the case-aware + * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail, + * whitespace branches, and added-token atomicity; round-trips every case. + * The cl100k path is untouched by construction (dispatch requires \p{Lu} in + * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */ +#define _GNU_SOURCE +#include "../tok.h" + +int main(void) { + Tok T; + tok_load(&T, "tests/tok_o200k_tiny.json"); + if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } + FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); + if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } + char *line = NULL; size_t cap = 0; ssize_t nr; + int pass = 0, tot = 0, dpass = 0; + while ((nr = getline(&line, &cap, f)) >= 0) { + if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0; + if (nr == 0) continue; + char *tab = strchr(line, '\t'); if (!tab) continue; + *tab = 0; + const char *text = line, *idstr = tab + 1; + char tbuf[4096]; int tn = 0; + for (const char *q = text; *q && tn < 4095; q++) { + if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; } + else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; } + else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; } + else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; } + else tbuf[tn++] = *q; + } + tbuf[tn] = 0; + int exp[512], ne = 0; + for (const char *q = idstr; *q; ) { + while (*q == ',' || *q == ' ') q++; + if (!*q) break; + exp[ne++] = atoi(q); + while (*q && *q != ',') q++; + } + int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512); + int ok = (ng == ne); + for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]); + tot++; if (ok) pass++; + char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191); + int drt = (dn == tn) && !memcmp(dec, tbuf, tn); + if (drt) dpass++; + if (!ok || !drt) { + fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne); + for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]); + fprintf(stderr, "\n got(%d):", ng); + for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]); + fprintf(stderr, "\n decode_ok=%d\n", drt); + } + } + fclose(f); + printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot); + return (pass == tot && dpass == tot) ? 0 : 2; +} diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txt new file mode 100644 index 0000000..38e37fa --- /dev/null +++ b/c/tests/tok_o200k_cases.txt @@ -0,0 +1,40 @@ +hello world 259,32,119,111,114,263 +HelloWorld 72,101,257,111,262 +XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116 +helloWORLDhello 259,87,79,82,76,68,259 +dog's 100,111,103,270 +DON'T 68,79,78,39,84 +don't 100,111,110,39,116 +I'll've 73,39,257,39,118,101 +O'Brien 79,39,66,114,105,101,110 +the theatre 265,32,116,256,97,116,114,101 +12345 268,52,53 +a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52 +3.14 51,46,49,52 +http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98 +path/to/file 112,97,264,47,116,111,47,102,105,108,101 +a//b///c 97,47,47,98,47,47,47,99 +!!\r\n//x 33,33,13,10,47,47,120 +one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101 + \n x 32,32,10,32,32,120 + 32,32,32 +a b c 97,32,32,98,32,32,32,99 +tab\there 116,269,9,256,114,101 +Café 67,97,102,101,204,129 +naiveBayes 110,97,105,118,101,66,97,121,101,115 +Éclair 195,137,99,108,97,105,114 +北京大学 229,140,151,228,186,172,229,164,167,229,173,166 +ΑΒαβ 206,145,206,146,206,177,206,178 +Иван 208,152,208,178,208,176,208,189 +ẞßscharf 225,186,158,195,159,115,99,104,97,114,102 +i̇stanbul 105,204,135,115,116,97,110,98,117,108 +hello<|endoftext|>world 259,274,119,111,114,263 +<|message_user|>hi 275,104,105 +mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268 +'s 270 + 's 32,39,115 +A 65 +aB 97,66 +Ab 65,98 +AB 65,66 +ab 269 diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.json new file mode 100644 index 0000000..829301e --- /dev/null +++ b/c/tests/tok_o200k_tiny.json @@ -0,0 +1 @@ +{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}} \ No newline at end of file