Implement a utility to count tokens in text and estimate API costs.
Code Playground
JavaScript (V8)
Loading...
Console Output
// Output will appear here...
// For React: ReactDOM.createRoot(rootElement).render(<App />);
// For React: ReactDOM.createRoot(rootElement).render(<App />);
Recommended Answer
Implementation:
// First install: npm install tiktoken
import { encoding_for_model } from 'tiktoken';
class TokenCounter {
constructor(model = 'gpt-4') {
this.model = model;
this.encoding = encoding_for_model(model);
// Pricing per 1K tokens (as of 2024)
this.pricing = {
'gpt-4': { input: 0.03, output: 0.06 },
'gpt-4-turbo': { input: 0.01, output: 0.03 },
'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 }
};
}
countTokens(text) {
const tokens = this.encoding.encode(text);
return tokens.length;
}
countMessagesTokens(messages) {
let totalTokens = 0;
// Each message has some overhead tokens
const tokensPerMessage = this.model.startsWith('gpt-4') ? 3 : 4;
const tokensPerName = 1;
for (const message of messages) {
totalTokens += tokensPerMessage;
totalTokens += this.countTokens(message.content || '');
totalTokens += this.countTokens(message.role);
if (message.name) {
totalTokens += tokensPerName;
}
}
totalTokens += 3; // Every reply is primed with assistant
return totalTokens;
}
estimateCost(inputTokens, outputTokens) {
const pricing = this.pricing[this.model];
if (!pricing) return null;
const inputCost = (inputTokens / 1000) * pricing.input;
const outputCost = (outputTokens / 1000) * pricing.output;
return {
inputCost: inputCost.toFixed(6),
outputCost: outputCost.toFixed(6),
totalCost: (inputCost + outputCost).toFixed(6)
};
}
willFitInContext(text, contextWindow = 8192, reserveForOutput = 1000) {
const tokens = this.countTokens(text);
return tokens <= (contextWindow - reserveForOutput);
}
cleanup() {
this.encoding.free();
}
}
// Usage
const counter = new TokenCounter('gpt-4');
const text = "This is a sample text to count tokens.";
console.log('Tokens:', counter.countTokens(text));
const messages = [
{ role: 'system', content: 'You are a helpful assistant.' },
{ role: 'user', content: 'Hello, how are you?' }
];
console.log('Message tokens:', counter.countMessagesTokens(messages));
const cost = counter.estimateCost(1000, 500);
console.log('Estimated cost:', cost);
console.log('Fits in 8K context?', counter.willFitInContext(text));
counter.cleanup();