htmlp/
tokenizer.rs

1use crate::TokenCounter;
2/// Offline, exact CL100K token counting. Special-token-looking strings are ordinary
3/// text. This counts content, not any provider's message framing overhead.
4pub struct Cl100k(tiktoken_rs::CoreBPE);
5impl Cl100k {
6    pub fn new() -> Result<Self, String> {
7        tiktoken_rs::cl100k_base()
8            .map(Self)
9            .map_err(|e| e.to_string())
10    }
11}
12impl TokenCounter for Cl100k {
13    fn name(&self) -> &str {
14        "cl100k_base"
15    }
16    fn count(&self, text: &str) -> u64 {
17        self.0.encode_ordinary(text).len() as u64
18    }
19}
htmlp/tokenizer.rs

htmlp/
tokenizer.rs
