fix: LiteLLM OIDC + Admin UI fixes - Authentik integration restored
- Added extra_hosts for auth.sysloggh.net to LiteLLM container - Fixed DOCS_URL=/docs (was /litellm/docs - path mismatch) - Added Authentik self-signed cert to CA bundle - Added nginx auth proxy for token/userinfo endpoints (SSL verify off) - Changed OIDC token/userinfo endpoints to use nginx internal proxy - Admin UI serving correctly on :4001/ui/ and /litellm/ui/ - Swagger API docs working at /docs and /litellm/docs - ReDoc API docs working at /redoc and /litellm/redoc - OIDC login flow verified working end-to-end
This commit is contained in:
Abiba
+76
-94
@@ -1,107 +1,89 @@
|
||||
# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture
|
||||
# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000
|
||||
# Last updated: 2026-06-16
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
# database_url: using DATABASE_URL env var instead
|
||||
store_model_in_db: true
|
||||
|
||||
model_list:
|
||||
# Content-based auto-routing (router picks GPU via 5-tier analysis)
|
||||
- model_name: syslog-auto
|
||||
litellm_params:
|
||||
model: openai/syslog-auto
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
rpm: 600
|
||||
|
||||
# Individual GPU strict passthrough (exact GPU, no silent fallback)
|
||||
- model_name: qwen3.6-35B-A3B
|
||||
litellm_params:
|
||||
model: openai/qwen3.6-35B-A3B
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
|
||||
- model_name: qwen3.6-27B-code
|
||||
litellm_params:
|
||||
model: openai/qwen3.6-27B-code
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
|
||||
- model_name: gemma-4-12b
|
||||
litellm_params:
|
||||
model: openai/gemma-4-12b
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
|
||||
# Guardrails: Pre-call and post-call content moderation
|
||||
guardrails:
|
||||
- guardrail_name: "input-moderation"
|
||||
litellm_params:
|
||||
guardrail: openai_moderation
|
||||
mode: "pre_call"
|
||||
|
||||
- guardrail_name: "output-moderation"
|
||||
litellm_params:
|
||||
guardrail: openai_moderation
|
||||
mode: "post_call"
|
||||
|
||||
- guardrail_name: "harmful-content-filter"
|
||||
litellm_params:
|
||||
guardrail: litellm_content_filter
|
||||
mode: "pre_call"
|
||||
categories:
|
||||
- category: "harmful_self_harm"
|
||||
enabled: true
|
||||
action: "BLOCK"
|
||||
severity_threshold: "medium"
|
||||
- category: "harmful_violence"
|
||||
enabled: true
|
||||
action: "BLOCK"
|
||||
severity_threshold: "medium"
|
||||
- category: "harmful_illegal_weapons"
|
||||
enabled: true
|
||||
action: "BLOCK"
|
||||
severity_threshold: "medium"
|
||||
|
||||
litellm_settings:
|
||||
num_retries: 0 # Disabled — our router handles retry logic
|
||||
request_timeout: 600 # Match 10-min llama-server timeout
|
||||
set_verbose: true
|
||||
failure_callback: ["prometheus"] # Export metrics to Prometheus
|
||||
|
||||
router_settings:
|
||||
routing_strategy: "usage-based-routing" # For external models only
|
||||
enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB
|
||||
allowed_fails: 100 # Router returns 503 on saturated GPUs
|
||||
# Fallback chains: LiteLLM retries down the chain when router returns saturated.
|
||||
# This gives accurate per-model metrics because router no longer silently reroutes.
|
||||
# The router's circuit breaker prevents cascading failures to dead GPUs.
|
||||
fallbacks:
|
||||
- syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"]
|
||||
- qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"]
|
||||
- qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"]
|
||||
- gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"]
|
||||
|
||||
# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional)
|
||||
- guardrail_name: input-moderation
|
||||
litellm_params:
|
||||
guardrail: openai_moderation
|
||||
mode: pre_call
|
||||
- guardrail_name: output-moderation
|
||||
litellm_params:
|
||||
guardrail: openai_moderation
|
||||
mode: post_call
|
||||
- guardrail_name: harmful-content-filter
|
||||
litellm_params:
|
||||
categories:
|
||||
- action: BLOCK
|
||||
category: harmful_self_harm
|
||||
enabled: true
|
||||
severity_threshold: medium
|
||||
- action: BLOCK
|
||||
category: harmful_violence
|
||||
enabled: true
|
||||
severity_threshold: medium
|
||||
- action: BLOCK
|
||||
category: harmful_illegal_weapons
|
||||
enabled: true
|
||||
severity_threshold: medium
|
||||
guardrail: litellm_content_filter
|
||||
mode: pre_call
|
||||
litellm_settings:
|
||||
failure_callback:
|
||||
- prometheus
|
||||
model_cost:
|
||||
syslog-auto:
|
||||
input_cost_per_token: 0.0
|
||||
output_cost_per_token: 0.0
|
||||
qwen3.6-35B-A3B:
|
||||
gemma-4-12b:
|
||||
input_cost_per_token: 0.0
|
||||
output_cost_per_token: 0.0
|
||||
qwen3.6-27B-code:
|
||||
input_cost_per_token: 0.0
|
||||
output_cost_per_token: 0.0
|
||||
gemma-4-12b:
|
||||
qwen3.6-35B-A3B:
|
||||
input_cost_per_token: 0.0
|
||||
output_cost_per_token: 0.0
|
||||
# For internal cost allocation, set symbolic rates:
|
||||
# e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens
|
||||
|
||||
# SSO/OIDC Configuration
|
||||
litellm_settings:
|
||||
sso_callback: "/sso/callback"
|
||||
syslog-auto:
|
||||
input_cost_per_token: 0.0
|
||||
output_cost_per_token: 0.0
|
||||
num_retries: 0
|
||||
request_timeout: 600
|
||||
set_verbose: true
|
||||
sso_callback: /sso/callback
|
||||
model_list:
|
||||
- litellm_params:
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
model: openai/syslog-auto
|
||||
rpm: 600
|
||||
model_name: syslog-auto
|
||||
- litellm_params:
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
model: openai/qwen3.6-35B-A3B
|
||||
model_name: qwen3.6-35B-A3B
|
||||
- litellm_params:
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
model: openai/qwen3.6-27B-code
|
||||
model_name: qwen3.6-27B-code
|
||||
- litellm_params:
|
||||
api_base: http://router:9000/v1
|
||||
api_key: os.environ/ROUTER_API_KEY
|
||||
model: openai/gemma-4-12b
|
||||
model_name: gemma-4-12b
|
||||
router_settings:
|
||||
allowed_fails: 100
|
||||
enable_loadbalancing_on_proxy: false
|
||||
fallbacks:
|
||||
- syslog-auto:
|
||||
- qwen3.6-35B-A3B
|
||||
- qwen3.6-27B-code
|
||||
- gemma-4-12b
|
||||
- qwen3.6-35B-A3B:
|
||||
- qwen3.6-27B-code
|
||||
- gemma-4-12b
|
||||
- qwen3.6-27B-code:
|
||||
- qwen3.6-35B-A3B
|
||||
- gemma-4-12b
|
||||
- gemma-4-12b:
|
||||
- qwen3.6-27B-code
|
||||
- qwen3.6-35B-A3B
|
||||
routing_strategy: usage-based-routing
|
||||
|
||||
Reference in New Issue
Block a user