feat: add streaming responses with optional toggle

- Add server-sent events (SSE) streaming support for real-time token display - Implement progressive markdown rendering during streaming - Add stream toggle in API settings (defaults to disabled for compatibility) - Add visual streaming indicator with pulsing animation - Graceful fallback to non-streaming mode when disabled - Fix character saving bug (camelCase parameter naming) Backend changes: - New chat_stream command with SSE parsing - Added futures and bytes dependencies - Emit chat-token events progressively to frontend - Support for OpenAI-compatible and Anthropic streaming formats Frontend changes: - Dual code paths for streaming/non-streaming - Real-time markdown and syntax highlighting during streaming - Stream status indicator with animation
2025-10-12 11:38:40 -07:00
parent 2761fe395d
commit 6a8f1e0996
6 changed files with 310 additions and 21 deletions
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -1032,6 +1032,21 @@ dependencies = [
 "new_debug_unreachable",
 ]

+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.31"
@@ -1039,6 +1054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
 dependencies = [
 "futures-core",
+ "futures-sink",
 ]

 [[package]]
@@ -1106,6 +1122,7 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
+ "futures-channel",
 "futures-core",
 "futures-io",
 "futures-macro",
@@ -3920,6 +3937,8 @@ dependencies = [
 name = "tauri-app"
 version = "0.1.0"
 dependencies = [
+ "bytes",
+ "futures",
 "reqwest",
 "serde",
 "serde_json",
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -22,7 +22,9 @@ tauri = { version = "2", features = [] }
 tauri-plugin-opener = "2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-reqwest = { version = "0.12", features = ["json"] }
+reqwest = { version = "0.12", features = ["json", "stream"] }
 tokio = { version = "1", features = ["full"] }
 uuid = { version = "1", features = ["v4"] }
+futures = "0.3"
+bytes = "1"

--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -2,6 +2,8 @@ use serde::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use uuid::Uuid;
+use futures::StreamExt;
+use tauri::Emitter;

 #[derive(Debug, Clone, Serialize, Deserialize)]
 struct ApiConfig {
@@ -10,6 +12,8 @@ struct ApiConfig {
    model: String,
    #[serde(default)]
    active_character_id: Option<String>,
+    #[serde(default)]
+    stream: bool,
 }

 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -66,6 +70,30 @@ struct ModelsResponse {
    data: Vec<Model>,
 }

+#[derive(Debug, Serialize, Deserialize)]
+struct StreamChatRequest {
+    model: String,
+    max_tokens: u32,
+    messages: Vec<Message>,
+    stream: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct StreamChoice {
+    delta: Delta,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Delta {
+    #[serde(default)]
+    content: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct StreamResponse {
+    choices: Vec<StreamChoice>,
+}
+
 fn get_config_path() -> PathBuf {
    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
    PathBuf::from(home).join(".config/claudia/config.json")
@@ -228,7 +256,7 @@ async fn validate_api(base_url: String, api_key: String) -> Result<Vec<String>,
 }

 #[tauri::command]
-async fn save_api_config(base_url: String, api_key: String, model: String) -> Result<(), String> {
+async fn save_api_config(base_url: String, api_key: String, model: String, stream: bool) -> Result<(), String> {
    // Preserve existing active_character_id if it exists
    let active_character_id = load_config().and_then(|c| c.active_character_id);

@@ -237,6 +265,7 @@ async fn save_api_config(base_url: String, api_key: String, model: String) -> Re
        api_key,
        model,
        active_character_id,
+        stream,
    };
    save_config(&config)
 }
@@ -316,6 +345,107 @@ async fn chat(message: String) -> Result<String, String> {
    Ok(assistant_message)
 }

+#[tauri::command]
+async fn chat_stream(app_handle: tauri::AppHandle, message: String) -> Result<String, String> {
+    let config = load_config().ok_or_else(|| "API not configured".to_string())?;
+    let character = get_active_character();
+    let mut history = load_history(&character.id);
+
+    // Add user message to history
+    history.messages.push(Message {
+        role: "user".to_string(),
+        content: message.clone(),
+    });
+
+    let client = reqwest::Client::new();
+    let base = config.base_url.trim_end_matches('/');
+    let url = if base.ends_with("/v1") {
+        format!("{}/chat/completions", base)
+    } else {
+        format!("{}/v1/chat/completions", base)
+    };
+
+    // Build messages with system prompt first
+    let mut api_messages = vec![Message {
+        role: "system".to_string(),
+        content: character.system_prompt.clone(),
+    }];
+    api_messages.extend(history.messages.clone());
+
+    let request = StreamChatRequest {
+        model: config.model.clone(),
+        max_tokens: 4096,
+        messages: api_messages,
+        stream: true,
+    };
+
+    let response = client
+        .post(&url)
+        .header("authorization", format!("Bearer {}", &config.api_key))
+        .header("content-type", "application/json")
+        .json(&request)
+        .send()
+        .await
+        .map_err(|e| format!("Request failed: {}", e))?;
+
+    if !response.status().is_success() {
+        return Err(format!("API error: {}", response.status()));
+    }
+
+    // Process streaming response
+    let mut full_content = String::new();
+    let mut stream = response.bytes_stream();
+
+    let mut buffer = String::new();
+    while let Some(chunk_result) = stream.next().await {
+        let chunk = chunk_result.map_err(|e| format!("Stream error: {}", e))?;
+        let chunk_str = String::from_utf8_lossy(&chunk);
+        buffer.push_str(&chunk_str);
+
+        // Process complete lines
+        while let Some(line_end) = buffer.find('\n') {
+            let line = buffer[..line_end].trim().to_string();
+            buffer = buffer[line_end + 1..].to_string();
+
+            // Parse SSE data lines
+            if line.starts_with("data: ") {
+                let data = &line[6..];
+
+                // Check for stream end
+                if data == "[DONE]" {
+                    break;
+                }
+
+                // Parse JSON and extract content
+                if let Ok(stream_response) = serde_json::from_str::<StreamResponse>(data) {
+                    if let Some(choice) = stream_response.choices.first() {
+                        if let Some(content) = &choice.delta.content {
+                            full_content.push_str(content);
+
+                            // Emit token to frontend
+                            let _ = app_handle.emit_to("main", "chat-token", content.clone());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Add assistant message to history
+    history.messages.push(Message {
+        role: "assistant".to_string(),
+        content: full_content.clone(),
+    });
+
+    // Save history
+    save_history(&character.id, &history).ok();
+
+    // Emit completion event
+    let _ = app_handle.emit_to("main", "chat-complete", ());
+
+    Ok(full_content)
+}
+
 #[tauri::command]
 fn get_chat_history() -> Result<Vec<Message>, String> {
    let character = get_active_character();
@@ -415,6 +545,7 @@ pub fn run() {
        .plugin(tauri_plugin_opener::init())
        .invoke_handler(tauri::generate_handler![
            chat,
+            chat_stream,
            validate_api,
            save_api_config,
            get_api_config,
--- a/src/index.html
+++ b/src/index.html
@@ -104,6 +104,13 @@
              </select>
            </div>

+            <div class="form-group">
+              <label>
+                <input type="checkbox" id="stream-toggle" />
+                Enable streaming responses (real-time token display)
+              </label>
+            </div>
+
            <div id="validation-message" class="validation-message"></div>

            <button type="submit" id="save-settings-btn" class="btn-primary" disabled>
--- a/src/main.js
+++ b/src/main.js
@@ -162,26 +162,140 @@ async function handleSubmit(e) {
  messageInput.disabled = true;
  setStatus('Thinking...');

-  showTypingIndicator();
-
+  // Check if streaming is enabled
+  let streamEnabled = false;
  try {
-    const response = await invoke('chat', { message });
-    removeTypingIndicator();
-    addMessage(response, false);
-    setStatus('Ready');
+    const config = await invoke('get_api_config');
+    streamEnabled = config.stream || false;
  } catch (error) {
-    removeTypingIndicator();
-    if (error.includes('not configured')) {
-      addMessage('API not configured. Please configure your API settings.', false);
-      setTimeout(showSettings, 1000);
-    } else {
-      addMessage(`Error: ${error}`, false);
+    console.error('Failed to get config:', error);
+  }
+
+  if (streamEnabled) {
+    // Use streaming
+    setStatus('Streaming...');
+    statusText.classList.add('streaming');
+
+    let streamingMessageDiv = null;
+    let streamingContentDiv = null;
+    let fullContent = '';
+
+    // Create streaming message container
+    const messageDiv = document.createElement('div');
+    messageDiv.className = 'message assistant';
+
+    const avatar = document.createElement('div');
+    avatar.className = 'avatar-circle';
+
+    const contentDiv = document.createElement('div');
+    contentDiv.className = 'message-content';
+
+    messageDiv.appendChild(avatar);
+    messageDiv.appendChild(contentDiv);
+    messagesContainer.appendChild(messageDiv);
+
+    streamingMessageDiv = messageDiv;
+    streamingContentDiv = contentDiv;
+
+    // Set up event listeners for streaming
+    const { listen } = window.__TAURI__.event;
+
+    const tokenUnlisten = await listen('chat-token', (event) => {
+      const token = event.payload;
+      fullContent += token;
+
+      // Update content with markdown rendering
+      streamingContentDiv.innerHTML = marked.parse(fullContent);
+
+      // Apply syntax highlighting
+      streamingContentDiv.querySelectorAll('pre code').forEach((block) => {
+        hljs.highlightElement(block);
+
+        // Add copy button
+        const pre = block.parentElement;
+        if (!pre.querySelector('.copy-btn')) {
+          const copyBtn = document.createElement('button');
+          copyBtn.className = 'copy-btn';
+          copyBtn.innerHTML = `<svg width="16" height="16" viewBox="0 0 16 16" fill="none">
+            <rect x="5" y="5" width="9" height="9" stroke="currentColor" stroke-width="1.5" fill="none" rx="1"/>
+            <path d="M3 11V3a1 1 0 0 1 1-1h8" stroke="currentColor" stroke-width="1.5" fill="none"/>
+          </svg>`;
+          copyBtn.title = 'Copy code';
+          copyBtn.addEventListener('click', () => {
+            navigator.clipboard.writeText(block.textContent);
+            copyBtn.innerHTML = `<svg width="16" height="16" viewBox="0 0 16 16" fill="none">
+              <path d="M3 8l3 3 7-7" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+            </svg>`;
+            copyBtn.classList.add('copied');
+            setTimeout(() => {
+              copyBtn.innerHTML = `<svg width="16" height="16" viewBox="0 0 16 16" fill="none">
+                <rect x="5" y="5" width="9" height="9" stroke="currentColor" stroke-width="1.5" fill="none" rx="1"/>
+                <path d="M3 11V3a1 1 0 0 1 1-1h8" stroke="currentColor" stroke-width="1.5" fill="none"/>
+              </svg>`;
+              copyBtn.classList.remove('copied');
+            }, 2000);
+          });
+          pre.style.position = 'relative';
+          pre.appendChild(copyBtn);
+        }
+      });
+
+      messagesContainer.scrollTop = messagesContainer.scrollHeight;
+    });
+
+    const completeUnlisten = await listen('chat-complete', () => {
+      setStatus('Ready');
+      statusText.classList.remove('streaming');
+      sendBtn.disabled = false;
+      messageInput.disabled = false;
+      messageInput.focus();
+      tokenUnlisten();
+      completeUnlisten();
+    });
+
+    try {
+      await invoke('chat_stream', { message });
+    } catch (error) {
+      tokenUnlisten();
+      completeUnlisten();
+      statusText.classList.remove('streaming');
+      if (streamingMessageDiv) {
+        streamingMessageDiv.remove();
+      }
+      if (error.includes('not configured')) {
+        addMessage('API not configured. Please configure your API settings.', false);
+        setTimeout(showSettings, 1000);
+      } else {
+        addMessage(`Error: ${error}`, false);
+      }
+      setStatus('Error');
+      sendBtn.disabled = false;
+      messageInput.disabled = false;
+      messageInput.focus();
+    }
+  } else {
+    // Use non-streaming
+    showTypingIndicator();
+
+    try {
+      const response = await invoke('chat', { message });
+      removeTypingIndicator();
+      addMessage(response, false);
+      setStatus('Ready');
+    } catch (error) {
+      removeTypingIndicator();
+      if (error.includes('not configured')) {
+        addMessage('API not configured. Please configure your API settings.', false);
+        setTimeout(showSettings, 1000);
+      } else {
+        addMessage(`Error: ${error}`, false);
+      }
+      setStatus('Error');
+    } finally {
+      sendBtn.disabled = false;
+      messageInput.disabled = false;
+      messageInput.focus();
    }
-    setStatus('Error');
-  } finally {
-    sendBtn.disabled = false;
-    messageInput.disabled = false;
-    messageInput.focus();
  }
 }

@@ -238,6 +352,7 @@ async function handleSaveSettings(e) {
  const baseUrl = document.getElementById('api-base-url').value.trim();
  const apiKey = document.getElementById('api-key').value.trim();
  const model = document.getElementById('model-select').value;
+  const stream = document.getElementById('stream-toggle').checked;
  const saveBtn = document.getElementById('save-settings-btn');
  const validationMsg = document.getElementById('validation-message');

@@ -251,7 +366,7 @@ async function handleSaveSettings(e) {
  saveBtn.textContent = 'Saving...';

  try {
-    await invoke('save_api_config', { baseUrl, apiKey, model });
+    await invoke('save_api_config', { baseUrl, apiKey, model, stream });
    validationMsg.textContent = 'Configuration saved successfully';
    validationMsg.className = 'validation-message success';

@@ -465,7 +580,7 @@ async function handleSaveCharacter(e) {
  try {
    await invoke('update_character', {
      name,
-      system_prompt: systemPrompt,
+      systemPrompt,
      greeting,
      personality
    });
@@ -494,6 +609,7 @@ async function loadExistingConfig() {
    console.log('Loaded config:', config);
    document.getElementById('api-base-url').value = config.base_url;
    document.getElementById('api-key').value = config.api_key;
+    document.getElementById('stream-toggle').checked = config.stream || false;

    const modelSelect = document.getElementById('model-select');
    modelSelect.innerHTML = ''; // Clear existing options
--- a/src/styles.css
+++ b/src/styles.css
@@ -506,6 +506,20 @@ body {
  color: var(--text-secondary);
 }

+.status-text.streaming {
+  color: var(--accent);
+  animation: pulse 2s ease-in-out infinite;
+}
+
+@keyframes pulse {
+  0%, 100% {
+    opacity: 1;
+  }
+  50% {
+    opacity: 0.5;
+  }
+}
+
 /* Light mode support */
@media (prefers-color-scheme: light) {
  :root {