From fd3c2a575a8b4a9dd23adb68855ae63ab98db89f Mon Sep 17 00:00:00 2001 From: Abiba Date: Wed, 17 Jun 2026 22:40:33 +0000 Subject: [PATCH] feat: 2-layer architecture foundation + agent migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Router v2: - Atomic GPU slot booking (Redis Lua — closes TOCTOU race) - Circuit breaker with failure threshold (3 failures/120s → 60s cooldown) - GPU health scoring with configurable weights (VRAM 40% + temp 30% + load 30%) - X-Usage-Tokens header for LiteLLM spend tracking - /health/unified endpoint (aggregates all layers) - Strict explicit model passthrough (no silent fallback) - syslog-auto → auto routing fix Infrastructure: - Postgres 16 for LiteLLM state - LiteLLM production config (router:9000, fallback chains, guardrails) - Dual-path NGINX: /v1/→router, /litellm/v1/→LiteLLM, port 4000 UI - DNS split-horizon (auth.sysloggh.net → 192.168.68.11) - 6 LiteLLM virtual keys for agent cutover Deployed to CT 116, all 6 containers healthy. --- docker-compose.yml | 51 ++++++++-- litellm_config.yaml | 104 +++++++++++++++++-- nginx/nginx.conf | 242 +++++++++++++++++++++++++++++++++++--------- router/router.py | 191 +++++++++++++++++++++++++++++----- 4 files changed, 498 insertions(+), 90 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5823ca4..517e3c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: redis: @@ -16,6 +16,24 @@ services: timeout: 3s retries: 5 + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + router: build: ./router container_name: harness-router @@ -44,11 +62,30 @@ services: container_name: harness-litellm restart: unless-stopped ports: - - "127.0.0.1:8081:4000" + - "127.0.0.1:4001:4000" volumes: - ./litellm_config.yaml:/app/config.yaml + - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro environment: - - LITELLM_MASTER_KEY=sk-sys...-key + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - UI_USERNAME=admin + - UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=https://litellm.sysloggh.net + - ANTHROPIC_API_KEY=not-used + - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU + - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe + - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ + - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ + - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ + - GENERIC_SCOPE=openid email profile + - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin + - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -57,6 +94,8 @@ services: timeout: 5s retries: 3 depends_on: + postgres: + condition: service_healthy redis: condition: service_healthy @@ -69,6 +108,8 @@ services: volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] interval: 30s @@ -97,6 +138,4 @@ services: volumes: redis-data: - -# LiteLLM command override to load config -# (appended to fix config loading issue) + pgdata: diff --git a/litellm_config.yaml b/litellm_config.yaml index 244fde4..6cf03d8 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -1,25 +1,107 @@ +# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture +# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000 +# Last updated: 2026-06-16 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + # database_url: using DATABASE_URL env var instead + store_model_in_db: true + model_list: + # Content-based auto-routing (router picks GPU via 5-tier analysis) + - model_name: syslog-auto + litellm_params: + model: openai/syslog-auto + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY + rpm: 600 + + # Individual GPU strict passthrough (exact GPU, no silent fallback) - model_name: qwen3.6-35B-A3B litellm_params: model: openai/qwen3.6-35B-A3B - api_base: http://192.168.68.15:8080/v1 - api_key: "not-needed" + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY - model_name: qwen3.6-27B-code litellm_params: - model: openai/qwen3.6-27B-code-text - api_base: http://192.168.68.8:8080/v1 - api_key: "not-needed" + model: openai/qwen3.6-27B-code + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY - model_name: gemma-4-12b litellm_params: model: openai/gemma-4-12b - api_base: http://192.168.68.110:8080/v1 - api_key: "not-needed" + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY -general_settings: - master_key: sk-syslog-local-master-key +# Guardrails: Pre-call and post-call content moderation +guardrails: + - guardrail_name: "input-moderation" + litellm_params: + guardrail: openai_moderation + mode: "pre_call" + + - guardrail_name: "output-moderation" + litellm_params: + guardrail: openai_moderation + mode: "post_call" + + - guardrail_name: "harmful-content-filter" + litellm_params: + guardrail: litellm_content_filter + mode: "pre_call" + categories: + - category: "harmful_self_harm" + enabled: true + action: "BLOCK" + severity_threshold: "medium" + - category: "harmful_violence" + enabled: true + action: "BLOCK" + severity_threshold: "medium" + - category: "harmful_illegal_weapons" + enabled: true + action: "BLOCK" + severity_threshold: "medium" litellm_settings: - drop_params: true - request_timeout: 120 + num_retries: 0 # Disabled — our router handles retry logic + request_timeout: 600 # Match 10-min llama-server timeout + set_verbose: true + failure_callback: ["prometheus"] # Export metrics to Prometheus + +router_settings: + routing_strategy: "usage-based-routing" # For external models only + enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB + allowed_fails: 100 # Router returns 503 on saturated GPUs + # Fallback chains: LiteLLM retries down the chain when router returns saturated. + # This gives accurate per-model metrics because router no longer silently reroutes. + # The router's circuit breaker prevents cascading failures to dead GPUs. + fallbacks: + - syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"] + - qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"] + - qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"] + - gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"] + +# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional) +litellm_settings: + model_cost: + syslog-auto: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + qwen3.6-35B-A3B: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + qwen3.6-27B-code: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + gemma-4-12b: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + # For internal cost allocation, set symbolic rates: + # e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens + +# SSO/OIDC Configuration +litellm_settings: + sso_callback: "/sso/callback" diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 8565840..1338f75 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,18 +1,10 @@ -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { worker_connections 1024; } +events { + worker_connections 1024; +} http { include /etc/nginx/mime.types; default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" rt=$request_time'; - access_log /var/log/nginx/access.log main; - error_log /var/log/nginx/error.log; sendfile on; keepalive_timeout 65; @@ -20,18 +12,33 @@ http { upstream dashboard_ui { server dashboard:3000; } upstream litellm_backend { server litellm:4000; } + # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). + # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, + # preventing the cross-origin localStorage footgun that traps the UI at the login page. + map $http_cf_connecting_ip $is_cloudflared { + default 1; # any non-empty value = request came through Cloudflare + "" 0; # empty = direct access + } + + # ════════════════════════════════════════════════════════════════ + # Server :80 — existing harness entrypoint + # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), + # router fallback, LiteLLM UI via /litellm/ prefix, health + # ════════════════════════════════════════════════════════════════ server { listen 80; - # Security headers - add_header X-Content-Type-Options nosniff always; - add_header X-Frame-Options SAMEORIGIN always; - add_header X-XSS-Protection "1; mode=block" always; + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } - # Disable buffering for SSE streams - proxy_buffering off; - - # API through router + # 2-Layer: /v1/ → router (existing keys work unchanged) location /v1/ { proxy_pass http://router_api; proxy_http_version 1.1; @@ -41,6 +48,16 @@ http { proxy_connect_timeout 10s; proxy_read_timeout 600s; proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; } location /admin/ { @@ -49,70 +66,197 @@ http { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; } - # SSE streaming endpoint location /stream { - proxy_pass http://router_api; + proxy_pass http://router_api/stream; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; proxy_buffering off; - chunked_transfer_encoding off; } - # Dashboard API proxy for SSE location /api/ { - proxy_pass http://dashboard_ui; + proxy_pass http://router_api/; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_buffering off; } - # LiteLLM debug - location /litellm/ { - rewrite ^/litellm/(.*) /$1 break; - proxy_pass http://litellm_backend; + # LiteLLM gateway access (agents with new virtual keys) + location /litellm/v1/ { + proxy_pass http://litellm_backend/v1/; proxy_http_version 1.1; proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header Authorization $http_authorization; - } - - # Professional Dashboard (Phase 1-3) - Static HTML served via Nginx - location /dashboard/ { - alias /opt/inference-harness/dashboard/; - index dashboard.html; - add_header Cache-Control "public, max-age=3600"; - add_header X-Content-Type-Options nosniff; - } - - # Legacy Dashboard (root) - Proxy to Flask app - location / { - proxy_pass http://dashboard_ui; - proxy_http_version 1.1; - proxy_set_header Host $host; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; proxy_buffering off; } - # Performance analytics + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass http://litellm_backend/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) + location /litellm/ { + proxy_pass http://litellm_backend/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://$host/ /litellm/; + proxy_redirect https://$host/ /litellm/; + } + + location /dashboard/ { + proxy_pass http://dashboard_ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + root /opt/inference-harness/dashboard; + try_files $uri $uri/ /index.html; + } + location /metrics/ { - proxy_pass http://router_api; + proxy_pass http://router_api/metrics/; proxy_http_version 1.1; proxy_set_header Host $host; } - # Circuit Breaker metrics (Phase 1) location /metrics/circuit-breaker { proxy_pass http://router_api/metrics/circuit-breaker; proxy_http_version 1.1; proxy_set_header Host $host; } + location /router/ { + proxy_pass http://router_api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass http://router_api/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + location /health { proxy_pass http://router_api/health; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; + } + } + + # ════════════════════════════════════════════════════════════════ + # Server :4000 — external LiteLLM entrypoint (cloudflared target) + # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM + # builds external URLs instead of leaking 192.168.68.116:4000. + # LiteLLM itself is now bound to 127.0.0.1 only (see compose). + # ════════════════════════════════════════════════════════════════ + server { + listen 4000; + + # Canonical external identity — overrides whatever Host cloudflared sends + proxy_set_header Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, + # so a relative return would be absolutized to http://192.168.68.116:4000/). + location = /ui { return 301 https://litellm.sysloggh.net/ui/; } + + # LiteLLM Admin UI + WebSocket + location /ui/ { + proxy_pass http://litellm_backend/ui/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://litellm_backend/ https://litellm.sysloggh.net/; + proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; + } + + # UI static assets + location /litellm-asset-prefix/ { + proxy_pass http://litellm_backend/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # SSO callback + location /sso/ { + proxy_pass http://litellm_backend/sso/; + proxy_http_version 1.1; + proxy_read_timeout 600s; + } + + # API + location /v1/ { + proxy_pass http://litellm_backend/v1/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # Key management + admin API + location /key/ { + proxy_pass http://litellm_backend/key/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /user/ { + proxy_pass http://litellm_backend/user/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /model/ { + proxy_pass http://litellm_backend/model/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /team/ { + proxy_pass http://litellm_backend/team/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + + # Health + location /health { + proxy_pass http://litellm_backend/health; + proxy_http_version 1.1; + } + + # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) + location / { + proxy_pass http://litellm_backend/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; + proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; } } } diff --git a/router/router.py b/router/router.py index 23f9c8e..94e81bb 100644 --- a/router/router.py +++ b/router/router.py @@ -14,6 +14,37 @@ redis.call('SET', key, max_val, 'EX', 86400) return max_val """ +# Phase 4: Atomic GPU slot booking (closes TOCTOU race between check and incr) +SLOT_BOOK_LUA = """ +local key = KEYS[1] +local max_c = tonumber(ARGV[1]) +local current = tonumber(redis.call('GET', key) or '0') +if current < max_c then + redis.call('INCR', key) + return 1 +else + return 0 +end +""" +SLOT_RELEASE_LUA = """ +local key = KEYS[1] +local current = tonumber(redis.call('GET', key) or '0') +if current > 0 then + redis.call('DECR', key) +end +current = tonumber(redis.call('GET', key) or '0') +if current < 0 then + redis.call('SET', key, '0') +end +return redis.call('GET', key) +""" + +# Phase 3b: Configurable health scoring weights (env-overridable) +HEALTH_WEIGHT_VRAM = float(os.environ.get("HEALTH_WEIGHT_VRAM", "0.40")) +HEALTH_WEIGHT_TEMP = float(os.environ.get("HEALTH_WEIGHT_TEMP", "0.30")) +HEALTH_WEIGHT_LOAD = float(os.environ.get("HEALTH_WEIGHT_LOAD", "0.30")) +HEALTH_TEMP_BASELINE = int(os.environ.get("HEALTH_TEMP_BASELINE", "30")) + REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") @@ -39,7 +70,7 @@ GPU_LABELS = { } GPU_MAX_CONCURRENT = { - "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) + "qwen3.6-35B-A3B": 1, # 2 slots (cross-agent spread prevents overheating) "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) "gemma-4-12b": 2, # 2 slots (7.1GB VRAM) } @@ -152,6 +183,36 @@ def gpu_decr(model): v = rd.decr("active:" + model) if v and int(v) < 0: get_redis().set("active:" + model, 0) # never go negative + +# Phase 4: Atomic GPU slot booking (Lua-based, closes TOCTOU race) +def gpu_book_slot(model): + """Atomically book a GPU slot. Returns True if acquired, False if full.""" + rd = get_redis() + if not rd: + return True # No Redis — allow everything (degraded mode) + try: + max_c = GPU_MAX_CONCURRENT.get(model, 1) + result = rd.eval(SLOT_BOOK_LUA, 1, "active:" + model, max_c) + return result == 1 + except Exception: + # Lua not loaded — fall back to non-atomic + current = int(rd.get("active:" + model) or 0) + if current < GPU_MAX_CONCURRENT.get(model, 1): + rd.incr("active:" + model) + return True + return False + +def gpu_release_slot(model): + """Atomically release a GPU slot. Never goes negative.""" + rd = get_redis() + if not rd: + return + try: + rd.eval(SLOT_RELEASE_LUA, 1, "active:" + model) + except Exception: + v = rd.decr("active:" + model) + if v and int(v) < 0: + rd.set("active:" + model, 0) def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): url = GPU_SIDECARS.get(model) if not url: return {"status": "unknown"} @@ -222,18 +283,22 @@ def is_gpu_busy(model): # Phase 3: Dynamic GPU Weighting (Health Score) def gpu_health_score(model): - """Score a GPU based on VRAM, temperature, and load. Lower = better.""" + """Score a GPU based on VRAM, temperature, power, and load. Lower = better. + Weights configurable via HEALTH_WEIGHT_VRAM/TEMP/LOAD env vars.""" h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) if h.get("status") == "down": return 999 # never pick down GPUs + if is_circuit_tripped(model): + return 998 # circuit open — skip but distinguishable from down vram_pct = h.get("vram_pct") or 50 temp_c = h.get("temp_c") or 50 + power_w = h.get("power_w") or 100 active = gpu_active_count(model) max_c = GPU_MAX_CONCURRENT.get(model, 1) load_pct = (active / max_c) * 100 if max_c > 0 else 0 - # Score: lower = better (more headroom, cooler, less loaded) - score = (vram_pct or 0) * 0.4 + max((temp_c or 50) - 30, 0) * 0.3 + load_pct * 0.3 - return score + temp_penalty = max(0, (temp_c or 50) - HEALTH_TEMP_BASELINE) + score = (vram_pct or 0) * HEALTH_WEIGHT_VRAM + temp_penalty * 0.5 * HEALTH_WEIGHT_TEMP + load_pct * HEALTH_WEIGHT_LOAD + return round(score, 1) def select_best_gpu(candidates, reason, agent=""): """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" @@ -278,20 +343,40 @@ def select_best_gpu(candidates, reason, agent=""): # Phase 1: Circuit Breaker for GPU Hosts (Approved by Abiba) +CIRCUIT_FAIL_THRESHOLD = int(os.environ.get("CIRCUIT_FAIL_THRESHOLD", "3")) +CIRCUIT_FAIL_WINDOW = int(os.environ.get("CIRCUIT_FAIL_WINDOW", "120")) +CIRCUIT_COOLDOWN = int(os.environ.get("CIRCUIT_COOLDOWN", "60")) + def is_circuit_tripped(model): """Check if a GPU host is currently blacklisted.""" if not get_redis(): return False return r.exists("circuit:" + model + ":open") -def trip_circuit(model, duration=30): - """Blacklist a GPU host for a specified duration.""" +def trip_circuit(model, duration=None): + """Blacklist a GPU host for specified duration (default CIRCUIT_COOLDOWN). + Only trips after CIRCUIT_FAIL_THRESHOLD failures within CIRCUIT_FAIL_WINDOW.""" if not get_redis(): - return - key = "circuit:" + model + ":open" - r.set(key, 1, ex=duration) - r.incr("circuit:" + model + ":count") - log.warning("CIRCUIT_TRIPPED: %s blacklisted for %ds", model, duration) + return False + if duration is None: + duration = CIRCUIT_COOLDOWN + now = time.time() + fail_key = "circuit:" + model + ":failures" + pipe = r.pipeline() + pipe.lpush(fail_key, str(now)) + pipe.ltrim(fail_key, 0, CIRCUIT_FAIL_THRESHOLD - 1) + pipe.lrange(fail_key, 0, -1) + results = pipe.execute() + failures = [float(f) for f in (results[-1] if results else [])] + recent = [f for f in failures if now - f <= CIRCUIT_FAIL_WINDOW] + if len(recent) >= CIRCUIT_FAIL_THRESHOLD: + key = "circuit:" + model + ":open" + r.set(key, 1, ex=duration) + r.incr("circuit:" + model + ":count") + log.warning("CIRCUIT_TRIPPED: %s — %d failures in %ds, cooldown %ds", + model, len(recent), CIRCUIT_FAIL_WINDOW, duration) + return True + return False def half_open_probe(model): """Check if a GPU host can be un-blacklisted.""" @@ -329,13 +414,17 @@ def route(rd, tier, agent=""): return {"model": allowed[0], "reason": "vision_unavailable"} req = rd.get("model","auto") + # Map syslog-auto to auto for content-based routing + if req == "syslog-auto": + req = "auto" if req != "auto": + # STRICT MODE: no silent fallback — LiteLLM handles failover chains. + # Returns saturated if explicit GPU is busy (keeps per-model metrics accurate). target = req if req in avail else avail[0] - if is_gpu_busy(target) and req in allowed: - alts = [m for m in avail if m != target and m in allowed] - if alts: - alt = select_best_gpu(alts, "explicit", agent) - if alt: return alt + if req not in avail: + return {"model": req, "reason": "explicit_unavailable", "saturated": True} + if is_gpu_busy(target): + return {"model": target, "reason": "explicit_saturated", "saturated": True} return {"model": target, "reason": "explicit"} if hints: @@ -503,14 +592,26 @@ def chat(): log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + # Phase 4: Atomic slot booking (replaces non-atomic gpu_incr) + if not gpu_book_slot(model): + d = route(rd, tier, agent) + if d.get("saturated"): + resp = jsonify({"error": "All GPUs saturated", "retry_after_s": 3}) + resp.headers["Retry-After"] = "3" + return resp, 503 + model, reason = d["model"], d["reason"] + if not gpu_book_slot(model): + resp = jsonify({"error": "GPU slot race — retry", "retry_after_s": 1}) + resp.headers["Retry-After"] = "1" + return resp, 503 + url = GPU_URLS[model] + # Stash rate limit values for response headers _rl_remaining = rl_val _rl_limit = RATE_LIMIT_RPM.get(tier, 30) _rl_reset = reset_sec is_stream = rd.get("stream", False) - gpu_incr(model) - log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) # Track which GPU this agent is using (TTL 120s covers typical request) if r and agent: @@ -527,11 +628,11 @@ def chat(): resp = requests.post(url+"/chat/completions", json=rd, headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) lat = int((time.time()-start)*1000) - gpu_decr(model) + gpu_release_slot(model) if resp.status_code != 200: if resp.status_code in (502, 504): - trip_circuit(model, 30) + trip_circuit(model) return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 if is_stream: # Buffer SSE chunks, handle split lines for large responses @@ -578,6 +679,12 @@ def chat(): sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) sse_resp.headers["X-Context-Warning"] = ctx_warning sse_resp.headers["X-Context-Model"] = model + # LiteLLM spend tracking: best-effort token counts from stream timings + pt = stream_timings.get("prompt_n", 0) if stream_timings else 0 + ct = stream_timings.get("predicted_n", 0) if stream_timings else 0 + sse_resp.headers["X-Usage-Tokens"] = json.dumps({ + "prompt_tokens": pt, "completion_tokens": ct, "model": model + }) return sse_resp data = clean_response(resp.json()) for c in data.get("choices",[]): @@ -602,15 +709,19 @@ def chat(): resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) resp.headers["X-Context-Warning"] = ctx_warning resp.headers["X-Context-Model"] = model + # LiteLLM spend tracking: return token counts for cost computation + resp.headers["X-Usage-Tokens"] = json.dumps({ + "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "model": model + }) bcast() return resp except requests.Timeout: - gpu_decr(model) - trip_circuit(model, 30) + gpu_release_slot(model) + trip_circuit(model) log.error("TIMEOUT: %s -> %s (Circuit tripped)", agent, model) return jsonify({"error":"timeout"}), 504 except Exception as e: - gpu_decr(model) + gpu_release_slot(model) log.error("Error: %s\n%s", e, traceback.format_exc()) return jsonify({"error":str(e)}), 500 @@ -871,6 +982,38 @@ def metrics_latency(): "requests_per_min": len(last_min), "count": len(recent) }) +@app.route("/health/unified") +def health_unified(): + """Unified health aggregating all layers: Router + Redis + GPUs + Circuit Breaker + Scores.""" + gpus = {} + for m in GPU_URLS: + h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + h["active_requests"] = gpu_active_count(m) + h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) + h["health_score"] = gpu_health_score(m) + h["circuit_open"] = is_circuit_tripped(m) + gpus[m] = h + circuit_state = {} + for m in GPU_URLS: + cooldown_until = r.ttl("circuit:" + m + ":open") if r else None + circuit_state[m] = { + "open": is_circuit_tripped(m), + "cooldown_remaining_s": max(0, cooldown_until) if cooldown_until and cooldown_until > 0 else 0, + "trip_count": int(r.get("circuit:" + m + ":count") or 0) if r else 0 + } + overall = "healthy" + if not r: + overall = "degraded" + if all(circuit_state[m]["open"] for m in GPU_URLS): + overall = "down" + return jsonify({ + "status": overall, "router": "healthy", + "redis": "connected" if r else "down", + "gpus": gpus, "circuit_breaker": circuit_state, + "scores": {m: gpu_health_score(m) for m in GPU_URLS}, + "available_models": available_models(), "timestamp": time.time() + }) + @app.route("/stream") def stream(): def ev():