Build a context management system that handles long conversations by intelligently truncating or summarizing old messages.

Code Playground
JavaScript (V8)
Loading...
Console Output
// Output will appear here...
// For React: ReactDOM.createRoot(rootElement).render(<App />);
Recommended Answer

Implementation:

import OpenAI from 'openai';
import { encoding_for_model } from 'tiktoken';

class ContextManager {
  constructor(maxTokens = 4000, model = 'gpt-4') {
    this.maxTokens = maxTokens;
    this.model = model;
    this.encoding = encoding_for_model(model);
    this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
  }

  countTokens(text) {
    return this.encoding.encode(text).length;
  }

  countMessagesTokens(messages) {
    return messages.reduce((total, msg) => {
      return total + this.countTokens(msg.content) + 4; // 4 tokens overhead per message
    }, 3); // 3 tokens for priming
  }

  async summarizeOldMessages(messages) {
    const messagesToSummarize = messages.slice(1, -5); // Keep first (system) and last 5
    
    const conversation = messagesToSummarize
      .map(m => `${m.role}: ${m.content}`)
      .join('\n');

    const response = await this.openai.chat.completions.create({
      model: 'gpt-3.5-turbo',
      messages: [
        {
          role: 'system',
          content: 'Summarize this conversation concisely, preserving key points.'
        },
        { role: 'user', content: conversation }
      ],
      max_tokens: 200
    });

    return {
      role: 'system',
      content: `Previous conversation summary: ${response.choices[0].message.content}`
    };
  }

  async manageContext(messages) {
    const currentTokens = this.countMessagesTokens(messages);

    if (currentTokens <= this.maxTokens) {
      return messages; // Fits within limit
    }

    console.log(`Context too large (${currentTokens} tokens), managing...`);

    // Strategy 1: Remove middle messages, keep recent ones
    if (messages.length > 10) {
      const systemMsg = messages[0];
      const recentMessages = messages.slice(-8); // Keep last 8 messages
      
      const truncated = [systemMsg, ...recentMessages];
      const truncatedTokens = this.countMessagesTokens(truncated);
      
      if (truncatedTokens <= this.maxTokens) {
        console.log('Used truncation strategy');
        return truncated;
      }
    }

    // Strategy 2: Summarize old messages
    if (messages.length > 6) {
      const systemMsg = messages[0];
      const summary = await this.summarizeOldMessages(messages);
      const recentMessages = messages.slice(-5);
      
      const summarized = [systemMsg, summary, ...recentMessages];
      const summarizedTokens = this.countMessagesTokens(summarized);
      
      if (summarizedTokens <= this.maxTokens) {
        console.log('Used summarization strategy');
        return summarized;
      }
    }

    // Strategy 3: Keep only system message and last message (emergency fallback)
    console.log('Using emergency fallback');
    return [messages[0], messages[messages.length - 1]];
  }

  cleanup() {
    this.encoding.free();
  }
}

// Usage
const contextManager = new ContextManager(4000, 'gpt-4');

const longConversation = [
  { role: 'system', content: 'You are a helpful assistant.' },
  // ... many messages ...
  { role: 'user', content: 'New question' }
];

const managedMessages = await contextManager.manageContext(longConversation);
console.log(`Reduced from ${longConversation.length} to ${managedMessages.length} messages`);

// Use managedMessages in API call
    this.model = model;
    this.encoding = encoding_for_model(model);
    
    // Pricing per 1K tokens (as of 2024)
    this.pricing = {
      'gpt-4': { input: 0.03, output: 0.06 },
      'gpt-4-turbo': { input: 0.01, output: 0.03 },
      'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 }
    };
  }

  countTokens(text) {
    const tokens = this.encoding.encode(text);
    return tokens.length;
  }

  countMessagesTokens(messages) {
    let totalTokens = 0;
    
    // Each message has some overhead tokens
    const tokensPerMessage = this.model.startsWith('gpt-4') ? 3 : 4;
    const tokensPerName = 1;
    
    for (const message of messages) {
      totalTokens += tokensPerMessage;
      totalTokens += this.countTokens(message.content || '');
      totalTokens += this.countTokens(message.role);
      if (message.name) {
        totalTokens += tokensPerName;
      }
    }
    
    totalTokens += 3; // Every reply is primed with assistant
    return totalTokens;
  }

  estimateCost(inputTokens, outputTokens) {
    const pricing = this.pricing[this.model];
    if (!pricing) return null;
    
    const inputCost = (inputTokens / 1000) * pricing.input;
    const outputCost = (outputTokens / 1000) * pricing.output;
    
    return {
      inputCost: inputCost.toFixed(6),
      outputCost: outputCost.toFixed(6),
      totalCost: (inputCost + outputCost).toFixed(6)
    };
  }

  willFitInContext(text, contextWindow = 8192, reserveForOutput = 1000) {
    const tokens = this.countTokens(text);
    return tokens <= (contextWindow - reserveForOutput);
  }

  cleanup() {
    this.encoding.free();
  }
}

// Usage
const counter = new TokenCounter('gpt-4');

const text = "This is a sample text to count tokens.";
console.log('Tokens:', counter.countTokens(text));

const messages = [
  { role: 'system', content: 'You are a helpful assistant.' },
  { role: 'user', content: 'Hello, how are you?' }
];
console.log('Message tokens:', counter.countMessagesTokens(messages));

const cost = counter.estimateCost(1000, 500);
console.log('Estimated cost:', cost);

console.log('Fits in 8K context?', counter.willFitInContext(text));

counter.cleanup();