Meet Gigatoken: A Rust BPE Tokenizer that Encodes Text at 24.53 GB/s, up to 989x Faster than HuggingFace Tokenizers

‘;
}
function dash(){ return ‘—‘; }

function renderBench(key){
var d = DATA[key], max = d.rows[0][1], h=””;
h += ”;
h += ‘

CPU: ‘+d.label+’. Corpus: owt_train.txt (11.9 GB). ‘
+ ‘Best of 3 interleaved rounds, one fresh process per measurement, parallelism enabled everywhere. ‘
+ ‘gigatoken encodes the whole file un-split and finds its own split boundaries; HF tokenizers ‘
+ ‘(encode_batch_fast) gets the first 100 MB and tiktoken (encode_ordinary_batch) the first 1 GB, both presplit on <|endoftext|>. ‘
+ ‘Dimmed bars are SentencePiece-based tokenizers, which are less optimized in gigatoken. ‘
+ ‘tiktoken rows are filled only where official support exists.

‘;
return h;
}

function renderPerf(){
var max = 1049, h=”

”
+ ‘Single-threaded GPT-2 pretokenizer throughput on 100 MB of OpenWebText, Apple Silicon, cargo bench with lto = “fat”. ‘
+ ‘A fancy-regex baseline runs the same pretokenizer at ~47 MiB/s.

‘;
PERF.forEach(function(s){
h += ‘

‘;
});
h += ‘

Net result: 2.27× over the winnow + NEON baseline, 22.3× over the regex implementation.

‘;
return h;
}

function fmt(sec){
if(sec < 1) return (sec*1000).toFixed(0)+’ ms’;
if(sec < 90) return sec.toFixed(1)+’ s’;
if(sec < 5400) return (sec/60).toFixed(1)+’ min’;
if(sec < 172800) return (sec/3600).toFixed(1)+’ hours’;
return (sec/86400).toFixed(1)+’ days’;
}

function renderCalc(){
var opts=””;
Object.keys(DATA).forEach(function(k){
opts += ‘‘+DATA[k].label+’‘;
});
return ‘

‘
+ ‘

Linear extrapolation from the measured throughput above. It assumes the reported rate holds ‘
+ ‘for the whole corpus, single machine, no I/O ceiling. Real pipelines are usually storage-bound before they are tokenizer-bound. ‘
+ ‘For scale: the gigatoken README notes that at EPYC rates you could tokenize all of Common Crawl — ‘
+ ‘roughly 130 trillion tokens — in just under 6.5 hours.

‘;
}

function calcUpdate(){
var cpu = R.querySelector(‘#gt-cpu’).value;
var idx = parseInt(R.querySelector(‘#gt-tok’).value,10);
var gb = parseInt(R.querySelector(‘#gt-sz’).value,10);
R.querySelector(‘#gt-szv’).textContent = gb.toLocaleString();
var row = DATA[cpu].rows[idx];
var tg = gb / row[1];
var h=”

‘;
if(row[2] !== null){
var th = (gb*1000) / row[2];
h += ‘

‘;
h += ‘

‘;
} else {
h += ‘

‘;
}
if(row[3] !== null){
var tt = (gb*1000) / row[3];
h += ‘

‘;
}
R.querySelector(‘#gt-res’).innerHTML = h;
}

function fillTok(){
var cpu = R.querySelector(‘#gt-cpu’).value, sel = R.querySelector(‘#gt-tok’), h=””;
DATA[cpu].rows.forEach(function(r,i){ h += ‘‘+r[0]+’‘; });
sel.innerHTML = h;
}

var USAGE =
‘

Install

‘
+ ‘

pip install gigatoken

‘
+ ‘

Compatibility mode — drop-in, roughly 200–300× per the author

‘
+ ‘

import gigatoken as gtnn'
  + '# Minimum change from existing HuggingFace tokenizers usagen'
  + 'hf_tokenizer = ...ntokenizer = gt.Tokenizer(hf_tokenizer).as_hf()nn'
  + 'tokens = tokenizer.encode_batch(["This is a test string", "And here is another"])nn'
  + '# OR with tiktokenntiktokenizer = ...ntokenizer = gt.Tokenizer(tiktokenizer).as_tiktoken()

‘
+ ‘

Gigatoken API — fastest path, Rust reads the files directly

‘
+ ‘

import gigatoken as gtnn'
  + 'tokenizer = gt.Tokenizer("Qwen/Qwen3-8B")  # accepts HF model namesn'
  + 'file_source = gt.TextFileSource(["owt_train.txt"], separator=b"<|endoftext|>")n'
  + 'tokens = tokenizer.encode_files(file_source)

‘
+ ‘

Check your own tokenizer without installing anything

‘
+ ‘

uvx --with tokenizers gigatoken bench 'openai-community/gpt2' owt_train.txt \n'
  + '    --validate --doc-separator "<|endoftext|>"

‘
+ ‘

The benchmark subcommand validates output against HuggingFace tokenizers and prints a timing comparison. ‘
+ ‘On macOS, run it twice: the first run triggers a security scan that slows the Rust code.

‘;

var LIMITS =
‘

Known issues, stated by the author

‘
+ ‘
• WordPiece is not supported yet. BERT-family WordPiece models are out of scope for now.

‘
+ ‘

• SentencePiece is only partly optimized. Gemma, Mistral, CodeLlama and Llama 2 vocabularies land in the 7–22× band, not the 1,000× band.

‘
+ ‘

• File sinks are not implemented in the Gigatoken API.

‘
+ ‘

• Python iteration uses ABI3, which is slower than version-specific CPython APIs. Early experiments show a 2× gain for overhead-bound cases.

‘
+ ‘

• Windows is barely tested. The author recommends WSL for now.

‘
+ ‘

• Compatibility mode costs performance. Exact output parity with HuggingFace tokenizers is preserved, but not the full speedup.

‘
+ ‘

Stated AI use

‘
+ ‘
• The author states most of the code was written by hand, and that AI assisted with the user-facing API, compatibility widening, ‘
  + ‘porting SIMD strategies between AVX512 / AVX2 / NEON, the final profiling stages and roughly the last 4× of performance, and refactoring.

‘
+ ‘

License and packaging

‘
+ ‘
• MIT licensed. Rust 66.2% / Python 33.3% by repository language share. Published to PyPI as gigatoken.

‘
+ ‘

‘;

var CMDS = {
epyc:{line:’gigatoken bench –cpu epyc-9565’, get:function(){return renderBench(‘epyc’);}},
m4:{line:’gigatoken bench –cpu m4-max’, get:function(){return renderBench(‘m4′);}},
ryzen:{line:’gigatoken bench –cpu ryzen-9800x3d’, get:function(){return renderBench(‘ryzen’);}},
perf:{line:’gigatoken perf –log pretokenizer’, get:renderPerf},
calc:{line:’gigatoken estimate –corpus’, get:renderCalc},
usage:{line:’gigatoken install –show-usage’, get:function(){return USAGE;}},
limits:{line:’gigatoken limits’, get:function(){return LIMITS;}}
};

function show(v){
var c = CMDS[v]; if(!c) return;
LINE.textContent = c.line;
BODY.innerHTML = c.get();
R.querySelectorAll(‘.gt-cmd’).forEach(function(b){
b.setAttribute(‘aria-selected’, b.dataset.v === v ? ‘true’ : ‘false’);
});
if(v === ‘calc’){
fillTok(); calcUpdate();
R.querySelector(‘#gt-cpu’).addEventListener(‘change’, function(){ fillTok(); calcUpdate(); });
R.querySelector(‘#gt-tok’).addEventListener(‘change’, calcUpdate);
R.querySelector(‘#gt-sz’).addEventListener(‘input’, calcUpdate);
}
resize();
}

R.querySelectorAll(‘.gt-cmd’).forEach(function(b){
b.addEventListener(‘click’, function(){ show(b.dataset.v); });
});

function resize(){
try{
if(window.parent !== window){
window.parent.postMessage({gigatokenHeight: R.offsetHeight + 40}, ‘*’);
}
}catch(e){}
}
window.addEventListener(‘resize’, resize);

show(‘epyc’);
})();