tests: o200k tokenizer coverage, no model download
tests/tok_o200k_tiny.json is a synthetic byte-level BPE (274 vocab, a
few KB) whose Split regex is the o200k pattern; expected ids in
tok_o200k_cases.txt were generated by HF tokenizers on that same file.
test_tok_o200k (in TEST_BINS) scores 40/40 encode + 40/40 round-trip:
case-transition splits, contractions, digit groups, the [\r\n/]* tail,
whitespace branches, CJK/Greek/Cyrillic, added-token atomicity.
The cl100k path is untouched by construction — dispatch requires
\p{Lu} in the tokenizer's own Split pattern, which cl100k lacks — and
stays covered by the GLM oracle (verified on this branch: 32/32).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nicholas Beerbower
@@ -0,0 +1,61 @@
|
||||
/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations.
|
||||
* Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a
|
||||
* synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB,
|
||||
* no model download) and scores tests/tok_o200k_cases.txt, whose expected ids
|
||||
* were produced by HF `tokenizers` on the same file. Guards the case-aware
|
||||
* letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail,
|
||||
* whitespace branches, and added-token atomicity; round-trips every case.
|
||||
* The cl100k path is untouched by construction (dispatch requires \p{Lu} in
|
||||
* the tokenizer's own Split pattern) and stays covered by the GLM oracle. */
|
||||
#define _GNU_SOURCE
|
||||
#include "../tok.h"
|
||||
|
||||
int main(void) {
|
||||
Tok T;
|
||||
tok_load(&T, "tests/tok_o200k_tiny.json");
|
||||
if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; }
|
||||
FILE *f = fopen("tests/tok_o200k_cases.txt", "rb");
|
||||
if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; }
|
||||
char *line = NULL; size_t cap = 0; ssize_t nr;
|
||||
int pass = 0, tot = 0, dpass = 0;
|
||||
while ((nr = getline(&line, &cap, f)) >= 0) {
|
||||
if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0;
|
||||
if (nr == 0) continue;
|
||||
char *tab = strchr(line, '\t'); if (!tab) continue;
|
||||
*tab = 0;
|
||||
const char *text = line, *idstr = tab + 1;
|
||||
char tbuf[4096]; int tn = 0;
|
||||
for (const char *q = text; *q && tn < 4095; q++) {
|
||||
if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }
|
||||
else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }
|
||||
else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }
|
||||
else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }
|
||||
else tbuf[tn++] = *q;
|
||||
}
|
||||
tbuf[tn] = 0;
|
||||
int exp[512], ne = 0;
|
||||
for (const char *q = idstr; *q; ) {
|
||||
while (*q == ',' || *q == ' ') q++;
|
||||
if (!*q) break;
|
||||
exp[ne++] = atoi(q);
|
||||
while (*q && *q != ',') q++;
|
||||
}
|
||||
int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);
|
||||
int ok = (ng == ne);
|
||||
for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);
|
||||
tot++; if (ok) pass++;
|
||||
char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);
|
||||
int drt = (dn == tn) && !memcmp(dec, tbuf, tn);
|
||||
if (drt) dpass++;
|
||||
if (!ok || !drt) {
|
||||
fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);
|
||||
for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);
|
||||
fprintf(stderr, "\n got(%d):", ng);
|
||||
for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);
|
||||
fprintf(stderr, "\n decode_ok=%d\n", drt);
|
||||
}
|
||||
}
|
||||
fclose(f);
|
||||
printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);
|
||||
return (pass == tot && dpass == tot) ? 0 : 2;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
hello world 259,32,119,111,114,263
|
||||
HelloWorld 72,101,257,111,262
|
||||
XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116
|
||||
helloWORLDhello 259,87,79,82,76,68,259
|
||||
dog's 100,111,103,270
|
||||
DON'T 68,79,78,39,84
|
||||
don't 100,111,110,39,116
|
||||
I'll've 73,39,257,39,118,101
|
||||
O'Brien 79,39,66,114,105,101,110
|
||||
the theatre 265,32,116,256,97,116,114,101
|
||||
12345 268,52,53
|
||||
a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52
|
||||
3.14 51,46,49,52
|
||||
http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98
|
||||
path/to/file 112,97,264,47,116,111,47,102,105,108,101
|
||||
a//b///c 97,47,47,98,47,47,47,99
|
||||
!!\r\n//x 33,33,13,10,47,47,120
|
||||
one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101
|
||||
\n x 32,32,10,32,32,120
|
||||
32,32,32
|
||||
a b c 97,32,32,98,32,32,32,99
|
||||
tab\there 116,269,9,256,114,101
|
||||
Café 67,97,102,101,204,129
|
||||
naiveBayes 110,97,105,118,101,66,97,121,101,115
|
||||
Éclair 195,137,99,108,97,105,114
|
||||
北京大学 229,140,151,228,186,172,229,164,167,229,173,166
|
||||
ΑΒαβ 206,145,206,146,206,177,206,178
|
||||
Иван 208,152,208,178,208,176,208,189
|
||||
ẞßscharf 225,186,158,195,159,115,99,104,97,114,102
|
||||
i̇stanbul 105,204,135,115,116,97,110,98,117,108
|
||||
hello<|endoftext|>world 259,274,119,111,114,263
|
||||
<|message_user|>hi 275,104,105
|
||||
mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268
|
||||
's 270
|
||||
's 32,39,115
|
||||
A 65
|
||||
aB 97,66
|
||||
Ab 65,98
|
||||
AB 65,66
|
||||
ab 269
|
||||
@@ -0,0 +1 @@
|
||||
{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}}
|
||||
Reference in New Issue
Block a user