abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: triple litellm_settings merged into one + add /ui/, /sso/, /litellm-asset-prefix/ to port 80 nginx

Browse Source
This commit is contained in:
kagentz-bot
2026-06-24 13:50:09 -04:00
parent 3d6b8173b0
commit 621fb3540a
2 changed files with 44 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm_config.yaml
+13 -21
View File
@@ -70,22 +70,7 @@ litellm_settings:
request_timeout: 600 # Match 10-min llama-server timeout
set_verbose: true
failure_callback: ["prometheus"] # Export metrics to Prometheus
router_settings:
routing_strategy: "usage-based-routing" # For external models only
enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB
allowed_fails: 100 # Router returns 503 on saturated GPUs
# Fallback chains: LiteLLM retries down the chain when router returns saturated.
# This gives accurate per-model metrics because router no longer silently reroutes.
# The router's circuit breaker prevents cascading failures to dead GPUs.
fallbacks:
- syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"]
- qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"]
- qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"]
- gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"]
# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional)
litellm_settings:
sso_callback: "/sso/callback"
model_cost:
syslog-auto:
input_cost_per_token: 0.0
@@ -99,9 +84,16 @@ litellm_settings:
gemma-4-12b:
input_cost_per_token: 0.0
output_cost_per_token: 0.0
# For internal cost allocation, set symbolic rates:
# e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens
# SSO/OIDC Configuration
litellm_settings:
sso_callback: "/sso/callback"
router_settings:
routing_strategy: "usage-based-routing" # For external models only
enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB
allowed_fails: 100 # Router returns 503 on saturated GPUs
# Fallback chains: LiteLLM retries down the chain when router returns saturated.
# This gives accurate per-model metrics because router no longer silently reroutes.
# The router's circuit breaker prevents cascading failures to dead GPUs.
fallbacks:
- syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"]
- qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"]
- qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"]
- gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"]
nginx/nginx.conf
+31
View File
@@ -118,6 +118,37 @@ http {
proxy_set_header Host $host;
}
# LiteLLM Admin UI at /ui/ (public access via Traefik → port 80)
location /ui/ {
proxy_pass $litellm_backend_url/ui/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 86400s;
proxy_buffering off;
}
# SSO callback
location /sso/ {
proxy_pass $litellm_backend_url/sso/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_read_timeout 600s;
}
# LiteLLM UI static assets on port 80
location /litellm-asset-prefix/ {
proxy_pass $litellm_backend_url/litellm-asset-prefix/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
# LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80)
location /litellm/ {
proxy_pass $litellm_backend_url/;