feat: 2-layer architecture foundation + agent migration
Router v2: - Atomic GPU slot booking (Redis Lua — closes TOCTOU race) - Circuit breaker with failure threshold (3 failures/120s → 60s cooldown) - GPU health scoring with configurable weights (VRAM 40% + temp 30% + load 30%) - X-Usage-Tokens header for LiteLLM spend tracking - /health/unified endpoint (aggregates all layers) - Strict explicit model passthrough (no silent fallback) - syslog-auto → auto routing fix Infrastructure: - Postgres 16 for LiteLLM state - LiteLLM production config (router:9000, fallback chains, guardrails) - Dual-path NGINX: /v1/→router, /litellm/v1/→LiteLLM, port 4000 UI - DNS split-horizon (auth.sysloggh.net → 192.168.68.11) - 6 LiteLLM virtual keys for agent cutover Deployed to CT 116, all 6 containers healthy.
This commit is contained in:
+45
-6
@@ -1,4 +1,4 @@
|
||||
version: '3.8'
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
redis:
|
||||
@@ -16,6 +16,24 @@ services:
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: harness-postgres
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:5432:5432"
|
||||
environment:
|
||||
- POSTGRES_DB=litellm
|
||||
- POSTGRES_USER=litellm
|
||||
- POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U litellm"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
router:
|
||||
build: ./router
|
||||
container_name: harness-router
|
||||
@@ -44,11 +62,30 @@ services:
|
||||
container_name: harness-litellm
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:8081:4000"
|
||||
- "127.0.0.1:4001:4000"
|
||||
volumes:
|
||||
- ./litellm_config.yaml:/app/config.yaml
|
||||
- /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
environment:
|
||||
- LITELLM_MASTER_KEY=sk-sys...-key
|
||||
- LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796
|
||||
- ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64
|
||||
- DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm
|
||||
- STORE_MODEL_IN_DB=True
|
||||
- LITELLM_UI_USERNAME=admin
|
||||
- LITELLM_UI_PASSWORD=syslog-admin-2026
|
||||
- UI_USERNAME=admin
|
||||
- UI_PASSWORD=syslog-admin-2026
|
||||
- OPENAI_API_KEY=not-used
|
||||
- PROXY_BASE_URL=https://litellm.sysloggh.net
|
||||
- ANTHROPIC_API_KEY=not-used
|
||||
- GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU
|
||||
- GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe
|
||||
- GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/
|
||||
- GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/
|
||||
- GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/
|
||||
- GENERIC_SCOPE=openid email profile
|
||||
- GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin
|
||||
- SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
healthcheck:
|
||||
@@ -57,6 +94,8 @@ services:
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -69,6 +108,8 @@ services:
|
||||
volumes:
|
||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./dashboard:/opt/inference-harness/dashboard:ro
|
||||
extra_hosts:
|
||||
- "auth.sysloggh.net:192.168.68.11"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1/health"]
|
||||
interval: 30s
|
||||
@@ -97,6 +138,4 @@ services:
|
||||
|
||||
volumes:
|
||||
redis-data:
|
||||
|
||||
# LiteLLM command override to load config
|
||||
# (appended to fix config loading issue)
|
||||
pgdata:
|
||||
|
||||
Reference in New Issue
Block a user