108 lines
3.6 KiB
Plaintext
108 lines
3.6 KiB
Plaintext
# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture
|
|
# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000
|
|
# Last updated: 2026-06-16
|
|
|
|
general_settings:
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
|
# database_url: using DATABASE_URL env var instead
|
|
store_model_in_db: true
|
|
|
|
model_list:
|
|
# Content-based auto-routing (router picks GPU via 5-tier analysis)
|
|
- model_name: syslog-auto
|
|
litellm_params:
|
|
model: openai/syslog-auto
|
|
api_base: http://router:9000/v1
|
|
api_key: os.environ/ROUTER_API_KEY
|
|
rpm: 600
|
|
|
|
# Individual GPU strict passthrough (exact GPU, no silent fallback)
|
|
- model_name: qwen3.6-35B-A3B
|
|
litellm_params:
|
|
model: openai/qwen3.6-35B-A3B
|
|
api_base: http://router:9000/v1
|
|
api_key: os.environ/ROUTER_API_KEY
|
|
|
|
- model_name: qwen3.6-27B-code
|
|
litellm_params:
|
|
model: openai/qwen3.6-27B-code
|
|
api_base: http://router:9000/v1
|
|
api_key: os.environ/ROUTER_API_KEY
|
|
|
|
- model_name: gemma-4-12b
|
|
litellm_params:
|
|
model: openai/gemma-4-12b
|
|
api_base: http://router:9000/v1
|
|
api_key: os.environ/ROUTER_API_KEY
|
|
|
|
# Guardrails: Pre-call and post-call content moderation
|
|
guardrails:
|
|
- guardrail_name: "input-moderation"
|
|
litellm_params:
|
|
guardrail: openai_moderation
|
|
mode: "pre_call"
|
|
|
|
- guardrail_name: "output-moderation"
|
|
litellm_params:
|
|
guardrail: openai_moderation
|
|
mode: "post_call"
|
|
|
|
- guardrail_name: "harmful-content-filter"
|
|
litellm_params:
|
|
guardrail: litellm_content_filter
|
|
mode: "pre_call"
|
|
categories:
|
|
- category: "harmful_self_harm"
|
|
enabled: true
|
|
action: "BLOCK"
|
|
severity_threshold: "medium"
|
|
- category: "harmful_violence"
|
|
enabled: true
|
|
action: "BLOCK"
|
|
severity_threshold: "medium"
|
|
- category: "harmful_illegal_weapons"
|
|
enabled: true
|
|
action: "BLOCK"
|
|
severity_threshold: "medium"
|
|
|
|
litellm_settings:
|
|
num_retries: 0 # Disabled — our router handles retry logic
|
|
request_timeout: 600 # Match 10-min llama-server timeout
|
|
set_verbose: true
|
|
failure_callback: ["prometheus"] # Export metrics to Prometheus
|
|
|
|
router_settings:
|
|
routing_strategy: "usage-based-routing" # For external models only
|
|
enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB
|
|
allowed_fails: 100 # Router returns 503 on saturated GPUs
|
|
# Fallback chains: LiteLLM retries down the chain when router returns saturated.
|
|
# This gives accurate per-model metrics because router no longer silently reroutes.
|
|
# The router's circuit breaker prevents cascading failures to dead GPUs.
|
|
fallbacks:
|
|
- syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"]
|
|
- qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"]
|
|
- qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"]
|
|
- gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"]
|
|
|
|
# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional)
|
|
litellm_settings:
|
|
model_cost:
|
|
syslog-auto:
|
|
input_cost_per_token: 0.0
|
|
output_cost_per_token: 0.0
|
|
qwen3.6-35B-A3B:
|
|
input_cost_per_token: 0.0
|
|
output_cost_per_token: 0.0
|
|
qwen3.6-27B-code:
|
|
input_cost_per_token: 0.0
|
|
output_cost_per_token: 0.0
|
|
gemma-4-12b:
|
|
input_cost_per_token: 0.0
|
|
output_cost_per_token: 0.0
|
|
# For internal cost allocation, set symbolic rates:
|
|
# e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens
|
|
|
|
# SSO/OIDC Configuration
|
|
litellm_settings:
|
|
sso_callback: "/sso/callback"
|