feat: implement token counter with real-time breakdown

Add comprehensive token counting functionality to provide visibility into
context usage:

Backend (Rust):
- Add tiktoken-rs dependency for OpenAI-compatible token counting
- Implement get_token_count command with detailed breakdown
- Count tokens for: system prompt, preset instructions, persona, world info,
  author's note, message history, and current input
- Per-section token breakdown for optimization insights

Frontend (JavaScript/HTML/CSS):
- Add token counter widget in status bar
- Real-time updates as user types (debounced 300ms)
- Expandable breakdown tooltip showing per-section counts
- Automatic update when chat history loads or changes
- Clean, minimal UI with hover interactions

Features:
- Accurate token counting using cl100k_base tokenizer
- Debounced updates for performance
- Detailed breakdown by context section
- Visual indicator with total token count
- Click to expand/collapse detailed breakdown
- Auto-hide when no character is active

This completes the "Must-Have for Basic Roleplay" features from the roadmap:
 World Info/Lorebooks
 Author's Note
 Token Counter
- Message Examples Usage (next)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-10-16 13:24:49 -07:00
parent 828475ae4f
commit 2444ca0811
5 changed files with 332 additions and 0 deletions

View File

@@ -1366,9 +1366,67 @@ function setupKeyboardShortcuts() {
messageInput.addEventListener('input', () => {
autoResize(messageInput);
updateTokenCount();
});
}
// Token Counter
let tokenUpdateTimeout = null;
async function updateTokenCount() {
// Debounce token count updates
if (tokenUpdateTimeout) {
clearTimeout(tokenUpdateTimeout);
}
tokenUpdateTimeout = setTimeout(async () => {
try {
const currentInput = messageInput.value;
const tokenData = await invoke('get_token_count', {
characterId: null, // Use active character
currentInput
});
// Update total display
const tokenCounter = document.getElementById('token-counter');
const tokenCountTotal = document.getElementById('token-count-total');
tokenCountTotal.textContent = `${tokenData.total} tokens`;
tokenCounter.style.display = 'flex';
// Update breakdown
document.getElementById('token-system').textContent = tokenData.system_prompt;
document.getElementById('token-preset').textContent = tokenData.preset_instructions;
document.getElementById('token-persona').textContent = tokenData.persona;
document.getElementById('token-worldinfo').textContent = tokenData.world_info;
document.getElementById('token-authorsnote').textContent = tokenData.authors_note;
document.getElementById('token-history').textContent = tokenData.message_history;
document.getElementById('token-input').textContent = tokenData.current_input;
document.getElementById('token-total-detail').textContent = tokenData.total;
} catch (error) {
console.error('Failed to update token count:', error);
// Hide token counter on error
document.getElementById('token-counter').style.display = 'none';
}
}, 300); // Update after 300ms of no typing
}
// Toggle token breakdown display
document.getElementById('token-details-btn').addEventListener('click', (e) => {
e.stopPropagation();
const breakdown = document.getElementById('token-breakdown');
breakdown.style.display = breakdown.style.display === 'none' ? 'block' : 'none';
});
// Close breakdown when clicking outside
document.addEventListener('click', (e) => {
const breakdown = document.getElementById('token-breakdown');
const detailsBtn = document.getElementById('token-details-btn');
if (!breakdown.contains(e.target) && !detailsBtn.contains(e.target)) {
breakdown.style.display = 'none';
}
});
// Load characters and populate dropdown
async function loadCharacters() {
console.log('Loading characters...');
@@ -1535,6 +1593,9 @@ async function loadChatHistory() {
messagesContainer.innerHTML = '';
addMessage('API configured. Ready to chat.', false, true);
}
// Update token count after loading history
updateTokenCount();
}
// Clear chat history