Implement comprehensive LLM provider system with global cost protection
- Add multi-provider LLM architecture supporting OpenRouter, OpenAI, Gemini, and custom providers - Implement global LLM on/off switch with default DISABLED state for cost protection - Add per-character LLM configuration with provider-specific models and settings - Create performance-optimized caching system for LLM enabled status checks - Add API key validation before enabling LLM providers to prevent broken configurations - Implement audit logging for all LLM enable/disable actions for cost accountability - Create comprehensive admin UI with prominent cost warnings and confirmation dialogs - Add visual indicators in character list for custom AI model configurations - Build character-specific LLM client system with global fallback mechanism - Add database schema support for per-character LLM settings - Implement graceful fallback responses when LLM is globally disabled - Create provider testing and validation system for reliable connections
This commit is contained in:
root
164
scripts/test_llm_providers.py
Executable file
164
scripts/test_llm_providers.py
Executable file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for multi-provider LLM system
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
from llm.multi_provider_client import MultiProviderLLMClient
|
||||
from llm.providers import LLMRequest
|
||||
from utils.config import get_settings
|
||||
|
||||
|
||||
async def test_provider_health():
|
||||
"""Test health check for all providers"""
|
||||
print("Testing provider health...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
health_status = await client.health_check()
|
||||
provider_info = client.get_provider_info()
|
||||
|
||||
print("\nProvider Health Status:")
|
||||
print("-" * 30)
|
||||
for name, healthy in health_status.items():
|
||||
status = "✅ Healthy" if healthy else "❌ Unhealthy"
|
||||
print(f"{name}: {status}")
|
||||
|
||||
print("\nProvider Information:")
|
||||
print("-" * 30)
|
||||
for name, info in provider_info.items():
|
||||
print(f"{name}:")
|
||||
print(f" Type: {info['type']}")
|
||||
print(f" Model: {info['current_model']}")
|
||||
print(f" Priority: {info['priority']}")
|
||||
print(f" Enabled: {info['enabled']}")
|
||||
print()
|
||||
|
||||
current = client.get_current_provider()
|
||||
print(f"Current primary provider: {current}")
|
||||
|
||||
return health_status, provider_info
|
||||
|
||||
|
||||
async def test_simple_request():
|
||||
"""Test a simple LLM request"""
|
||||
print("\nTesting simple LLM request...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
# Test backwards-compatible method
|
||||
response = await client.generate_response_with_fallback(
|
||||
prompt="Say hello in exactly 5 words.",
|
||||
character_name="TestCharacter",
|
||||
max_tokens=50
|
||||
)
|
||||
|
||||
if response:
|
||||
print(f"✅ Response: {response}")
|
||||
else:
|
||||
print("❌ No response received")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def test_new_request_format():
|
||||
"""Test new request/response format"""
|
||||
print("\nTesting new request format...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
request = LLMRequest(
|
||||
prompt="Respond with just the word 'working' if you understand this.",
|
||||
character_name="TestCharacter",
|
||||
max_tokens=10,
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
response = await client.generate_response(request)
|
||||
|
||||
print(f"Success: {response.success}")
|
||||
print(f"Provider: {response.provider}")
|
||||
print(f"Model: {response.model}")
|
||||
print(f"Content: {response.content}")
|
||||
print(f"Tokens used: {response.tokens_used}")
|
||||
|
||||
if response.error:
|
||||
print(f"Error: {response.error}")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def test_provider_fallback():
|
||||
"""Test provider fallback functionality"""
|
||||
print("\nTesting provider fallback...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
# Get current provider
|
||||
original_provider = client.get_current_provider()
|
||||
print(f"Original provider: {original_provider}")
|
||||
|
||||
# Try to use a non-existent provider (this should fallback)
|
||||
provider_info = client.get_provider_info()
|
||||
print(f"Available providers: {list(provider_info.keys())}")
|
||||
|
||||
# Test multiple requests to see if fallback works
|
||||
for i in range(3):
|
||||
request = LLMRequest(
|
||||
prompt=f"Test request #{i+1}: respond with 'OK'",
|
||||
max_tokens=10
|
||||
)
|
||||
|
||||
response = await client.generate_response(request)
|
||||
print(f"Request {i+1}: Provider={response.provider}, Success={response.success}")
|
||||
|
||||
if not response.success:
|
||||
print(f" Error: {response.error}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main test function"""
|
||||
print("Discord Fishbowl Multi-Provider LLM Test")
|
||||
print("=" * 50)
|
||||
|
||||
try:
|
||||
# Test 1: Provider health
|
||||
health_status, provider_info = await test_provider_health()
|
||||
|
||||
# Only continue if we have at least one healthy provider
|
||||
healthy_providers = [name for name, healthy in health_status.items() if healthy]
|
||||
if not healthy_providers:
|
||||
print("\n❌ No healthy providers found. Check your configuration.")
|
||||
return
|
||||
|
||||
# Test 2: Simple request (backwards compatibility)
|
||||
await test_simple_request()
|
||||
|
||||
# Test 3: New request format
|
||||
await test_new_request_format()
|
||||
|
||||
# Test 4: Provider fallback
|
||||
await test_provider_fallback()
|
||||
|
||||
print("\n✅ All tests completed!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
140
scripts/update_llm_config.py
Executable file
140
scripts/update_llm_config.py
Executable file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to help migrate from single LLM provider to multi-provider configuration
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def update_fishbowl_config():
|
||||
"""Update fishbowl_config.json to include multi-provider LLM configuration"""
|
||||
|
||||
config_path = Path("config/fishbowl_config.json")
|
||||
|
||||
if not config_path.exists():
|
||||
print(f"Configuration file not found: {config_path}")
|
||||
return False
|
||||
|
||||
# Read existing config
|
||||
with open(config_path, 'r') as f:
|
||||
config = json.load(f)
|
||||
|
||||
# Check if already has providers config
|
||||
if 'providers' in config.get('llm', {}):
|
||||
print("Multi-provider configuration already exists")
|
||||
return True
|
||||
|
||||
# Get current LLM config
|
||||
current_llm = config.get('llm', {})
|
||||
|
||||
# Create new multi-provider config
|
||||
providers_config = {
|
||||
"custom": {
|
||||
"type": "custom",
|
||||
"enabled": True,
|
||||
"priority": 70,
|
||||
"config": {
|
||||
"base_url": current_llm.get('base_url', 'http://localhost:11434'),
|
||||
"model": current_llm.get('model', 'llama2'),
|
||||
"api_key": os.getenv('LLM_API_KEY', 'x'),
|
||||
"timeout": current_llm.get('timeout', 300),
|
||||
"max_tokens": current_llm.get('max_tokens', 2000),
|
||||
"temperature": current_llm.get('temperature', 0.8),
|
||||
"api_format": "openai"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Add example provider configurations (disabled by default)
|
||||
providers_config.update({
|
||||
"openrouter": {
|
||||
"type": "openrouter",
|
||||
"enabled": False,
|
||||
"priority": 100,
|
||||
"config": {
|
||||
"api_key": "${OPENROUTER_API_KEY:}",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"model": "${OPENROUTER_MODEL:anthropic/claude-3-sonnet}",
|
||||
"timeout": 300,
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.8,
|
||||
"app_name": "discord-fishbowl"
|
||||
}
|
||||
},
|
||||
"openai": {
|
||||
"type": "openai",
|
||||
"enabled": False,
|
||||
"priority": 90,
|
||||
"config": {
|
||||
"api_key": "${OPENAI_API_KEY:}",
|
||||
"base_url": "https://api.openai.com/v1",
|
||||
"model": "${OPENAI_MODEL:gpt-4o-mini}",
|
||||
"timeout": 300,
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.8
|
||||
}
|
||||
},
|
||||
"gemini": {
|
||||
"type": "gemini",
|
||||
"enabled": False,
|
||||
"priority": 80,
|
||||
"config": {
|
||||
"api_key": "${GEMINI_API_KEY:}",
|
||||
"base_url": "https://generativelanguage.googleapis.com/v1beta",
|
||||
"model": "${GEMINI_MODEL:gemini-1.5-flash}",
|
||||
"timeout": 300,
|
||||
"max_tokens": 2000,
|
||||
"temperature": 0.8
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
# Update config
|
||||
config['llm']['providers'] = providers_config
|
||||
|
||||
# Create backup
|
||||
backup_path = config_path.with_suffix('.json.backup')
|
||||
with open(backup_path, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
print(f"Created backup: {backup_path}")
|
||||
|
||||
# Write updated config
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
print(f"Updated {config_path} with multi-provider configuration")
|
||||
print("\nTo enable additional providers:")
|
||||
print("1. Set environment variables for the provider you want to use")
|
||||
print("2. Change 'enabled': false to 'enabled': true in the config")
|
||||
print("3. Restart the application")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
"""Main script function"""
|
||||
print("Discord Fishbowl LLM Configuration Updater")
|
||||
print("=" * 50)
|
||||
|
||||
if update_fishbowl_config():
|
||||
print("\n✅ Configuration updated successfully!")
|
||||
print("\nAvailable providers:")
|
||||
print("- OpenRouter (supports Claude, GPT, Llama, etc.)")
|
||||
print("- OpenAI (GPT models)")
|
||||
print("- Google Gemini")
|
||||
print("- Custom/Local (current setup)")
|
||||
|
||||
print("\nNext steps:")
|
||||
print("1. Update your .env file with API keys for desired providers")
|
||||
print("2. Enable providers in config/fishbowl_config.json")
|
||||
print("3. Restart the application")
|
||||
else:
|
||||
print("\n❌ Configuration update failed!")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user