Implement comprehensive LLM provider system with global cost protection
- Add multi-provider LLM architecture supporting OpenRouter, OpenAI, Gemini, and custom providers - Implement global LLM on/off switch with default DISABLED state for cost protection - Add per-character LLM configuration with provider-specific models and settings - Create performance-optimized caching system for LLM enabled status checks - Add API key validation before enabling LLM providers to prevent broken configurations - Implement audit logging for all LLM enable/disable actions for cost accountability - Create comprehensive admin UI with prominent cost warnings and confirmation dialogs - Add visual indicators in character list for custom AI model configurations - Build character-specific LLM client system with global fallback mechanism - Add database schema support for per-character LLM settings - Implement graceful fallback responses when LLM is globally disabled - Create provider testing and validation system for reliable connections
This commit is contained in:
root
164
scripts/test_llm_providers.py
Executable file
164
scripts/test_llm_providers.py
Executable file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for multi-provider LLM system
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
from llm.multi_provider_client import MultiProviderLLMClient
|
||||
from llm.providers import LLMRequest
|
||||
from utils.config import get_settings
|
||||
|
||||
|
||||
async def test_provider_health():
|
||||
"""Test health check for all providers"""
|
||||
print("Testing provider health...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
health_status = await client.health_check()
|
||||
provider_info = client.get_provider_info()
|
||||
|
||||
print("\nProvider Health Status:")
|
||||
print("-" * 30)
|
||||
for name, healthy in health_status.items():
|
||||
status = "✅ Healthy" if healthy else "❌ Unhealthy"
|
||||
print(f"{name}: {status}")
|
||||
|
||||
print("\nProvider Information:")
|
||||
print("-" * 30)
|
||||
for name, info in provider_info.items():
|
||||
print(f"{name}:")
|
||||
print(f" Type: {info['type']}")
|
||||
print(f" Model: {info['current_model']}")
|
||||
print(f" Priority: {info['priority']}")
|
||||
print(f" Enabled: {info['enabled']}")
|
||||
print()
|
||||
|
||||
current = client.get_current_provider()
|
||||
print(f"Current primary provider: {current}")
|
||||
|
||||
return health_status, provider_info
|
||||
|
||||
|
||||
async def test_simple_request():
|
||||
"""Test a simple LLM request"""
|
||||
print("\nTesting simple LLM request...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
# Test backwards-compatible method
|
||||
response = await client.generate_response_with_fallback(
|
||||
prompt="Say hello in exactly 5 words.",
|
||||
character_name="TestCharacter",
|
||||
max_tokens=50
|
||||
)
|
||||
|
||||
if response:
|
||||
print(f"✅ Response: {response}")
|
||||
else:
|
||||
print("❌ No response received")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def test_new_request_format():
|
||||
"""Test new request/response format"""
|
||||
print("\nTesting new request format...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
request = LLMRequest(
|
||||
prompt="Respond with just the word 'working' if you understand this.",
|
||||
character_name="TestCharacter",
|
||||
max_tokens=10,
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
response = await client.generate_response(request)
|
||||
|
||||
print(f"Success: {response.success}")
|
||||
print(f"Provider: {response.provider}")
|
||||
print(f"Model: {response.model}")
|
||||
print(f"Content: {response.content}")
|
||||
print(f"Tokens used: {response.tokens_used}")
|
||||
|
||||
if response.error:
|
||||
print(f"Error: {response.error}")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def test_provider_fallback():
|
||||
"""Test provider fallback functionality"""
|
||||
print("\nTesting provider fallback...")
|
||||
|
||||
client = MultiProviderLLMClient()
|
||||
await client.initialize()
|
||||
|
||||
# Get current provider
|
||||
original_provider = client.get_current_provider()
|
||||
print(f"Original provider: {original_provider}")
|
||||
|
||||
# Try to use a non-existent provider (this should fallback)
|
||||
provider_info = client.get_provider_info()
|
||||
print(f"Available providers: {list(provider_info.keys())}")
|
||||
|
||||
# Test multiple requests to see if fallback works
|
||||
for i in range(3):
|
||||
request = LLMRequest(
|
||||
prompt=f"Test request #{i+1}: respond with 'OK'",
|
||||
max_tokens=10
|
||||
)
|
||||
|
||||
response = await client.generate_response(request)
|
||||
print(f"Request {i+1}: Provider={response.provider}, Success={response.success}")
|
||||
|
||||
if not response.success:
|
||||
print(f" Error: {response.error}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main test function"""
|
||||
print("Discord Fishbowl Multi-Provider LLM Test")
|
||||
print("=" * 50)
|
||||
|
||||
try:
|
||||
# Test 1: Provider health
|
||||
health_status, provider_info = await test_provider_health()
|
||||
|
||||
# Only continue if we have at least one healthy provider
|
||||
healthy_providers = [name for name, healthy in health_status.items() if healthy]
|
||||
if not healthy_providers:
|
||||
print("\n❌ No healthy providers found. Check your configuration.")
|
||||
return
|
||||
|
||||
# Test 2: Simple request (backwards compatibility)
|
||||
await test_simple_request()
|
||||
|
||||
# Test 3: New request format
|
||||
await test_new_request_format()
|
||||
|
||||
# Test 4: Provider fallback
|
||||
await test_provider_fallback()
|
||||
|
||||
print("\n✅ All tests completed!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user