Build a context management system that handles long conversations by intelligently truncating or summarizing old messages.
Code Playground
JavaScript (V8)
Loading...
Console Output
// Output will appear here...
// For React: ReactDOM.createRoot(rootElement).render(<App />);
// For React: ReactDOM.createRoot(rootElement).render(<App />);
Recommended Answer
Implementation:
import OpenAI from 'openai';
import { encoding_for_model } from 'tiktoken';
class ContextManager {
constructor(maxTokens = 4000, model = 'gpt-4') {
this.maxTokens = maxTokens;
this.model = model;
this.encoding = encoding_for_model(model);
this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
}
countTokens(text) {
return this.encoding.encode(text).length;
}
countMessagesTokens(messages) {
return messages.reduce((total, msg) => {
return total + this.countTokens(msg.content) + 4; // 4 tokens overhead per message
}, 3); // 3 tokens for priming
}
async summarizeOldMessages(messages) {
const messagesToSummarize = messages.slice(1, -5); // Keep first (system) and last 5
const conversation = messagesToSummarize
.map(m => `${m.role}: ${m.content}`)
.join('\n');
const response = await this.openai.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: [
{
role: 'system',
content: 'Summarize this conversation concisely, preserving key points.'
},
{ role: 'user', content: conversation }
],
max_tokens: 200
});
return {
role: 'system',
content: `Previous conversation summary: ${response.choices[0].message.content}`
};
}
async manageContext(messages) {
const currentTokens = this.countMessagesTokens(messages);
if (currentTokens <= this.maxTokens) {
return messages; // Fits within limit
}
console.log(`Context too large (${currentTokens} tokens), managing...`);
// Strategy 1: Remove middle messages, keep recent ones
if (messages.length > 10) {
const systemMsg = messages[0];
const recentMessages = messages.slice(-8); // Keep last 8 messages
const truncated = [systemMsg, ...recentMessages];
const truncatedTokens = this.countMessagesTokens(truncated);
if (truncatedTokens <= this.maxTokens) {
console.log('Used truncation strategy');
return truncated;
}
}
// Strategy 2: Summarize old messages
if (messages.length > 6) {
const systemMsg = messages[0];
const summary = await this.summarizeOldMessages(messages);
const recentMessages = messages.slice(-5);
const summarized = [systemMsg, summary, ...recentMessages];
const summarizedTokens = this.countMessagesTokens(summarized);
if (summarizedTokens <= this.maxTokens) {
console.log('Used summarization strategy');
return summarized;
}
}
// Strategy 3: Keep only system message and last message (emergency fallback)
console.log('Using emergency fallback');
return [messages[0], messages[messages.length - 1]];
}
cleanup() {
this.encoding.free();
}
}
// Usage
const contextManager = new ContextManager(4000, 'gpt-4');
const longConversation = [
{ role: 'system', content: 'You are a helpful assistant.' },
// ... many messages ...
{ role: 'user', content: 'New question' }
];
const managedMessages = await contextManager.manageContext(longConversation);
console.log(`Reduced from ${longConversation.length} to ${managedMessages.length} messages`);
// Use managedMessages in API call
this.model = model;
this.encoding = encoding_for_model(model);
// Pricing per 1K tokens (as of 2024)
this.pricing = {
'gpt-4': { input: 0.03, output: 0.06 },
'gpt-4-turbo': { input: 0.01, output: 0.03 },
'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 }
};
}
countTokens(text) {
const tokens = this.encoding.encode(text);
return tokens.length;
}
countMessagesTokens(messages) {
let totalTokens = 0;
// Each message has some overhead tokens
const tokensPerMessage = this.model.startsWith('gpt-4') ? 3 : 4;
const tokensPerName = 1;
for (const message of messages) {
totalTokens += tokensPerMessage;
totalTokens += this.countTokens(message.content || '');
totalTokens += this.countTokens(message.role);
if (message.name) {
totalTokens += tokensPerName;
}
}
totalTokens += 3; // Every reply is primed with assistant
return totalTokens;
}
estimateCost(inputTokens, outputTokens) {
const pricing = this.pricing[this.model];
if (!pricing) return null;
const inputCost = (inputTokens / 1000) * pricing.input;
const outputCost = (outputTokens / 1000) * pricing.output;
return {
inputCost: inputCost.toFixed(6),
outputCost: outputCost.toFixed(6),
totalCost: (inputCost + outputCost).toFixed(6)
};
}
willFitInContext(text, contextWindow = 8192, reserveForOutput = 1000) {
const tokens = this.countTokens(text);
return tokens <= (contextWindow - reserveForOutput);
}
cleanup() {
this.encoding.free();
}
}
// Usage
const counter = new TokenCounter('gpt-4');
const text = "This is a sample text to count tokens.";
console.log('Tokens:', counter.countTokens(text));
const messages = [
{ role: 'system', content: 'You are a helpful assistant.' },
{ role: 'user', content: 'Hello, how are you?' }
];
console.log('Message tokens:', counter.countMessagesTokens(messages));
const cost = counter.estimateCost(1000, 500);
console.log('Estimated cost:', cost);
console.log('Fits in 8K context?', counter.willFitInContext(text));
counter.cleanup();