diff --git a/dashboard/gpu-monitor.html b/dashboard/gpu-monitor.html new file mode 100644 index 0000000..80f32fe --- /dev/null +++ b/dashboard/gpu-monitor.html @@ -0,0 +1,135 @@ + + + + + +Syslog GPU Monitor + + + +

⚡ Syslog GPU Monitor

+

Real-time GPU health — loading...

+ +
+
-
GPUs Online
+
-
Active Requests
+
-
Circuit Trips
+
-
Slots Used
+
+ +
+ + + + + + diff --git a/docker-compose.yml.bak.20260616-215242 b/docker-compose.yml.bak.20260616-215242 new file mode 100644 index 0000000..5823ca4 --- /dev/null +++ b/docker-compose.yml.bak.20260616-215242 @@ -0,0 +1,102 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: ghcr.io/berriai/litellm:main-stable + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "127.0.0.1:8081:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + environment: + - LITELLM_MASTER_KEY=sk-sys...-key + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + +# LiteLLM command override to load config +# (appended to fix config loading issue) diff --git a/docker-compose.yml.bak.20260617-104030 b/docker-compose.yml.bak.20260617-104030 new file mode 100644 index 0000000..497d407 --- /dev/null +++ b/docker-compose.yml.bak.20260617-104030 @@ -0,0 +1,131 @@ +version: "3.8" + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: ghcr.io/berriai/litellm:main-stable + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "4000:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + environment: + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=http://192.168.68.116/litellm + - ANTHROPIC_API_KEY=not-used + extra_hosts: + - "host.docker.internal:host-gateway" + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + pgdata: diff --git a/docker-compose.yml.bak.20260625-192313-fix b/docker-compose.yml.bak.20260625-192313-fix new file mode 100644 index 0000000..97d7be9 --- /dev/null +++ b/docker-compose.yml.bak.20260625-192313-fix @@ -0,0 +1,142 @@ +version: "3.8" + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "4001:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro + environment: + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - UI_USERNAME=admin + - UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=https://litellm.sysloggh.net + - DOCS_URL=/litellm/docs + - ANTHROPIC_API_KEY=not-used + - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU + - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe + - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ + - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ + - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ + - GENERIC_SCOPE=openid email profile + - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin + - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + pgdata: diff --git a/docker-compose.yml.bak.20260625-195716-remove-docs-url b/docker-compose.yml.bak.20260625-195716-remove-docs-url new file mode 100644 index 0000000..209addd --- /dev/null +++ b/docker-compose.yml.bak.20260625-195716-remove-docs-url @@ -0,0 +1,143 @@ +version: "3.8" + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "4001:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro + environment: + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - UI_USERNAME=admin + - UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=https://litellm.sysloggh.net + - DOCS_URL=/litellm/docs + - ANTHROPIC_API_KEY=not-used + - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU + - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe + - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ + - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ + - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ + - GENERIC_SCOPE=openid email profile + - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin + - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + extra_hosts: + - "host.docker.internal:host-gateway" + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + pgdata: diff --git a/docker-compose.yml.bak.20260625-195847-set-docs b/docker-compose.yml.bak.20260625-195847-set-docs new file mode 100644 index 0000000..aa7d965 --- /dev/null +++ b/docker-compose.yml.bak.20260625-195847-set-docs @@ -0,0 +1,142 @@ +version: "3.8" + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "4001:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro + environment: + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - UI_USERNAME=admin + - UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=https://litellm.sysloggh.net + - ANTHROPIC_API_KEY=not-used + - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU + - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe + - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ + - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ + - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ + - GENERIC_SCOPE=openid email profile + - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin + - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + extra_hosts: + - "host.docker.internal:host-gateway" + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + pgdata: diff --git a/docker-compose.yml.bak.20260625-202429-auth-proxy b/docker-compose.yml.bak.20260625-202429-auth-proxy new file mode 100644 index 0000000..b003a0a --- /dev/null +++ b/docker-compose.yml.bak.20260625-202429-auth-proxy @@ -0,0 +1,143 @@ +version: "3.8" + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + postgres: + image: postgres:16-alpine + container_name: harness-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "127.0.0.1:9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} + - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + redis: + condition: service_healthy + + litellm: + image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "4001:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro + environment: + - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 + - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 + - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm + - STORE_MODEL_IN_DB=True + - LITELLM_UI_USERNAME=admin + - LITELLM_UI_PASSWORD=syslog-admin-2026 + - UI_USERNAME=admin + - UI_PASSWORD=syslog-admin-2026 + - OPENAI_API_KEY=not-used + - PROXY_BASE_URL=https://litellm.sysloggh.net + - DOCS_URL=/docs + - ANTHROPIC_API_KEY=not-used + - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU + - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe + - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ + - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ + - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ + - GENERIC_SCOPE=openid email profile + - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin + - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + extra_hosts: + - "host.docker.internal:host-gateway" + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro + extra_hosts: + - "auth.sysloggh.net:192.168.68.11" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 30s + timeout: 15s + retries: 3 + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 + depends_on: + - redis + +volumes: + redis-data: + pgdata: diff --git a/litellm_config.yaml.bak.20260617-110719 b/litellm_config.yaml.bak.20260617-110719 new file mode 100644 index 0000000..6cf03d8 --- /dev/null +++ b/litellm_config.yaml.bak.20260617-110719 @@ -0,0 +1,107 @@ +# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture +# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000 +# Last updated: 2026-06-16 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + # database_url: using DATABASE_URL env var instead + store_model_in_db: true + +model_list: + # Content-based auto-routing (router picks GPU via 5-tier analysis) + - model_name: syslog-auto + litellm_params: + model: openai/syslog-auto + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY + rpm: 600 + + # Individual GPU strict passthrough (exact GPU, no silent fallback) + - model_name: qwen3.6-35B-A3B + litellm_params: + model: openai/qwen3.6-35B-A3B + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY + + - model_name: qwen3.6-27B-code + litellm_params: + model: openai/qwen3.6-27B-code + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY + + - model_name: gemma-4-12b + litellm_params: + model: openai/gemma-4-12b + api_base: http://router:9000/v1 + api_key: os.environ/ROUTER_API_KEY + +# Guardrails: Pre-call and post-call content moderation +guardrails: + - guardrail_name: "input-moderation" + litellm_params: + guardrail: openai_moderation + mode: "pre_call" + + - guardrail_name: "output-moderation" + litellm_params: + guardrail: openai_moderation + mode: "post_call" + + - guardrail_name: "harmful-content-filter" + litellm_params: + guardrail: litellm_content_filter + mode: "pre_call" + categories: + - category: "harmful_self_harm" + enabled: true + action: "BLOCK" + severity_threshold: "medium" + - category: "harmful_violence" + enabled: true + action: "BLOCK" + severity_threshold: "medium" + - category: "harmful_illegal_weapons" + enabled: true + action: "BLOCK" + severity_threshold: "medium" + +litellm_settings: + num_retries: 0 # Disabled — our router handles retry logic + request_timeout: 600 # Match 10-min llama-server timeout + set_verbose: true + failure_callback: ["prometheus"] # Export metrics to Prometheus + +router_settings: + routing_strategy: "usage-based-routing" # For external models only + enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB + allowed_fails: 100 # Router returns 503 on saturated GPUs + # Fallback chains: LiteLLM retries down the chain when router returns saturated. + # This gives accurate per-model metrics because router no longer silently reroutes. + # The router's circuit breaker prevents cascading failures to dead GPUs. + fallbacks: + - syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"] + - qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"] + - qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"] + - gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"] + +# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional) +litellm_settings: + model_cost: + syslog-auto: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + qwen3.6-35B-A3B: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + qwen3.6-27B-code: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + gemma-4-12b: + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + # For internal cost allocation, set symbolic rates: + # e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens + +# SSO/OIDC Configuration +litellm_settings: + sso_callback: "/sso/callback" diff --git a/litellm_config.yaml.bak.poc b/litellm_config.yaml.bak.poc new file mode 100644 index 0000000..244fde4 --- /dev/null +++ b/litellm_config.yaml.bak.poc @@ -0,0 +1,25 @@ +model_list: + - model_name: qwen3.6-35B-A3B + litellm_params: + model: openai/qwen3.6-35B-A3B + api_base: http://192.168.68.15:8080/v1 + api_key: "not-needed" + + - model_name: qwen3.6-27B-code + litellm_params: + model: openai/qwen3.6-27B-code-text + api_base: http://192.168.68.8:8080/v1 + api_key: "not-needed" + + - model_name: gemma-4-12b + litellm_params: + model: openai/gemma-4-12b + api_base: http://192.168.68.110:8080/v1 + api_key: "not-needed" + +general_settings: + master_key: sk-syslog-local-master-key + +litellm_settings: + drop_params: true + request_timeout: 120 diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..e69de29 diff --git a/nginx/nginx.conf.bak.20260616-220643 b/nginx/nginx.conf.bak.20260616-220643 new file mode 100644 index 0000000..295186e --- /dev/null +++ b/nginx/nginx.conf.bak.20260616-220643 @@ -0,0 +1,124 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" rt=$request_time'; + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log; + sendfile on; + keepalive_timeout 65; + + upstream router_api { server router:9000; } + upstream dashboard_ui { server dashboard:3000; } + upstream litellm_backend { server litellm:4000; } + + server { + listen 80; + + # Security headers + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options SAMEORIGIN always; + add_header X-XSS-Protection "1; mode=block" always; + + # Disable buffering for SSE streams + proxy_buffering off; + + # API through router + location /v1/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location /admin/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + } + + # SSE streaming endpoint + location /stream { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + proxy_buffering off; + chunked_transfer_encoding off; + } + + # Dashboard API proxy for SSE + location /api/ { + proxy_pass http://dashboard_ui; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + # LiteLLM debug + location /litellm/ { + rewrite ^/litellm/(.*) /$1 break; + proxy_pass http://litellm_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + } + + # Professional Dashboard (Phase 1-3) - Static HTML served via Nginx + location /dashboard/ { + alias /opt/inference-harness/dashboard/; + index dashboard.html; + add_header Cache-Control "public, max-age=3600"; + add_header X-Content-Type-Options nosniff; + } + + # Legacy Dashboard (root) - Proxy to Flask app + location / { + proxy_pass http://dashboard_ui; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + # Performance analytics + location /metrics/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # Circuit Breaker metrics (Phase 1) + location /metrics/circuit-breaker { + proxy_pass http://router_api/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass http://router_api/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health { + proxy_pass http://router_api/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + } +} diff --git a/nginx/nginx.conf.bak.20260617-104030 b/nginx/nginx.conf.bak.20260617-104030 new file mode 100644 index 0000000..7f83b04 --- /dev/null +++ b/nginx/nginx.conf.bak.20260617-104030 @@ -0,0 +1,162 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + upstream router_api { server router:9000; } + upstream dashboard_ui { server dashboard:3000; } + upstream litellm_backend { server litellm:4000; } + + server { + listen 80; + + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2-Layer: LiteLLM (Layer 1) -> Router (Layer 2) -> GPUs + # /v1/ routes to router (all existing keys work) + # Agents using new LiteLLM keys: change OPENAI_API_BASE to /litellm/v1 + location /v1/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /admin/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /stream { + proxy_pass http://router_api/stream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + location /api/ { + proxy_pass http://router_api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM gateway access (for agents with new virtual keys) + location /litellm/v1/ { + proxy_pass http://litellm_backend/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass http://litellm_backend/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI (WebSocket support for live updates) + location /litellm/ { + proxy_pass http://litellm_backend/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + # Rewrite all redirects to include /litellm/ prefix + proxy_redirect http://172.18.0.7:4000/ /litellm/; + proxy_redirect http://127.0.0.1:4000/ /litellm/; + } + + location /dashboard/ { + proxy_pass http://dashboard_ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + root /opt/inference-harness/dashboard; + try_files $uri $uri/ /index.html; + } + + location /metrics/ { + proxy_pass http://router_api/metrics/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /metrics/circuit-breaker { + proxy_pass http://router_api/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /router/ { + proxy_pass http://router_api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /ui/ { + return 302 http://192.168.68.116:4000/ui/; + } + + location /health/unified { + proxy_pass http://router_api/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /ui/ { + return 302 http://192.168.68.116:4000/ui/; + } + + location /health { + proxy_pass http://router_api/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + } +} diff --git a/nginx/nginx.conf.bak.20260624_174945 b/nginx/nginx.conf.bak.20260624_174945 new file mode 100644 index 0000000..c31e840 --- /dev/null +++ b/nginx/nginx.conf.bak.20260624_174945 @@ -0,0 +1,351 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. + # Without this, nginx resolves upstream hostnames at config load time, + # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. + resolver 127.0.0.11 valid=30s; + + # Dynamic upstream resolution via nginx variables. + # Using $var in proxy_pass forces request-time resolution through the resolver. + # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. + map $host $router_api_url { + default http://harness-router:9000; + } + map $host $dashboard_ui_url { + default http://harness-dashboard:3000; + } + map $host $litellm_backend_url { + default http://harness-litellm:4000; + } + + # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). + # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, + # preventing the cross-origin localStorage footgun that traps the UI at the login page. + map $http_cf_connecting_ip $is_cloudflared { + default 1; # any non-empty value = request came through Cloudflare + "" 0; # empty = direct access + } + + # ════════════════════════════════════════════════════════════════ + # Server :80 — existing harness entrypoint + # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), + # router fallback, LiteLLM UI via /litellm/ prefix, health + # ════════════════════════════════════════════════════════════════ + server { + listen 80; + + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2-Layer: /v1/ → router (existing keys work unchanged) + location /v1/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /admin/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /stream { + proxy_pass $router_api_url/stream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + location /api/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM gateway access (agents with new virtual keys) + location /litellm/v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) + # LiteLLM Admin UI at /ui/ (must be before catch-all /) + location /ui/ { + proxy_pass http://litellm_backend/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + error_page 301 302 = @ui_redirect; + } + + location @ui_redirect { + proxy_pass http://litellm_backend/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_read_timeout 600s; + } + + # LiteLLM UI static assets on port 80 + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) + location /litellm/ { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://$host/ /litellm/; + proxy_redirect https://$host/ /litellm/; + } + + location /dashboard/ { + proxy_pass $dashboard_ui_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # Dedicated /openapi.json block — must come before catch-all / + # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, + # but the catch-all location / strips the URI path. This block + # preserves the full path so the spec JSON is returned instead of + # the Swagger UI SPA HTML. + location /openapi.json { + proxy_pass $litellm_backend_url/openapi.json; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location /metrics/ { + proxy_pass $router_api_url/metrics/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /metrics/circuit-breaker { + proxy_pass $router_api_url/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /router/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass $router_api_url/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health { + proxy_pass $router_api_url/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + } + + # ════════════════════════════════════════════════════════════════ + # Server :4000 — external LiteLLM entrypoint (cloudflared target) + # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM + # builds external URLs instead of leaking 192.168.68.116:4000. + # LiteLLM itself is now bound to 127.0.0.1 only (see compose). + # ════════════════════════════════════════════════════════════════ + server { + listen 4000; + + # Canonical external identity — overrides whatever Host cloudflared sends + proxy_set_header Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, + # so a relative return would be absolutized to http://192.168.68.116:4000/). + location = /ui { return 301 https://litellm.sysloggh.net/ui/; } + + # LiteLLM Admin UI + WebSocket + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; + proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; + } + + # UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_read_timeout 600s; + } + + # API + location /v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # Key management + admin API + location /key/ { + proxy_pass $litellm_backend_url/key/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /user/ { + proxy_pass $litellm_backend_url/user/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /model/ { + proxy_pass $litellm_backend_url/model/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /team/ { + proxy_pass $litellm_backend_url/team/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + + # Health + location /health { + proxy_pass $litellm_backend_url/health; + proxy_http_version 1.1; + } + + # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; + proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; + } + } +} diff --git a/nginx/nginx.conf.bak.20260624_201123 b/nginx/nginx.conf.bak.20260624_201123 new file mode 100644 index 0000000..1338f75 --- /dev/null +++ b/nginx/nginx.conf.bak.20260624_201123 @@ -0,0 +1,262 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + upstream router_api { server router:9000; } + upstream dashboard_ui { server dashboard:3000; } + upstream litellm_backend { server litellm:4000; } + + # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). + # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, + # preventing the cross-origin localStorage footgun that traps the UI at the login page. + map $http_cf_connecting_ip $is_cloudflared { + default 1; # any non-empty value = request came through Cloudflare + "" 0; # empty = direct access + } + + # ════════════════════════════════════════════════════════════════ + # Server :80 — existing harness entrypoint + # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), + # router fallback, LiteLLM UI via /litellm/ prefix, health + # ════════════════════════════════════════════════════════════════ + server { + listen 80; + + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2-Layer: /v1/ → router (existing keys work unchanged) + location /v1/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /admin/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /stream { + proxy_pass http://router_api/stream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + location /api/ { + proxy_pass http://router_api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM gateway access (agents with new virtual keys) + location /litellm/v1/ { + proxy_pass http://litellm_backend/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass http://litellm_backend/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) + location /litellm/ { + proxy_pass http://litellm_backend/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://$host/ /litellm/; + proxy_redirect https://$host/ /litellm/; + } + + location /dashboard/ { + proxy_pass http://dashboard_ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location / { + root /opt/inference-harness/dashboard; + try_files $uri $uri/ /index.html; + } + + location /metrics/ { + proxy_pass http://router_api/metrics/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /metrics/circuit-breaker { + proxy_pass http://router_api/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /router/ { + proxy_pass http://router_api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass http://router_api/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health { + proxy_pass http://router_api/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + } + + # ════════════════════════════════════════════════════════════════ + # Server :4000 — external LiteLLM entrypoint (cloudflared target) + # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM + # builds external URLs instead of leaking 192.168.68.116:4000. + # LiteLLM itself is now bound to 127.0.0.1 only (see compose). + # ════════════════════════════════════════════════════════════════ + server { + listen 4000; + + # Canonical external identity — overrides whatever Host cloudflared sends + proxy_set_header Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, + # so a relative return would be absolutized to http://192.168.68.116:4000/). + location = /ui { return 301 https://litellm.sysloggh.net/ui/; } + + # LiteLLM Admin UI + WebSocket + location /ui/ { + proxy_pass http://litellm_backend/ui/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://litellm_backend/ https://litellm.sysloggh.net/; + proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; + } + + # UI static assets + location /litellm-asset-prefix/ { + proxy_pass http://litellm_backend/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # SSO callback + location /sso/ { + proxy_pass http://litellm_backend/sso/; + proxy_http_version 1.1; + proxy_read_timeout 600s; + } + + # API + location /v1/ { + proxy_pass http://litellm_backend/v1/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # Key management + admin API + location /key/ { + proxy_pass http://litellm_backend/key/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /user/ { + proxy_pass http://litellm_backend/user/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /model/ { + proxy_pass http://litellm_backend/model/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /team/ { + proxy_pass http://litellm_backend/team/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + + # Health + location /health { + proxy_pass http://litellm_backend/health; + proxy_http_version 1.1; + } + + # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) + location / { + proxy_pass http://litellm_backend/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; + proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; + } + } +} diff --git a/nginx/nginx.conf.bak.predup2 b/nginx/nginx.conf.bak.predup2 new file mode 100644 index 0000000..c331911 --- /dev/null +++ b/nginx/nginx.conf.bak.predup2 @@ -0,0 +1,351 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. + # Without this, nginx resolves upstream hostnames at config load time, + # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. + resolver 127.0.0.11 valid=30s; + + # Dynamic upstream resolution via nginx variables. + # Using $var in proxy_pass forces request-time resolution through the resolver. + # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. + map $host $router_api_url { + default http://harness-router:9000; + } + map $host $dashboard_ui_url { + default http://harness-dashboard:3000; + } + map $host $litellm_backend_url { + default http://harness-litellm:4000; + } + + # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). + # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, + # preventing the cross-origin localStorage footgun that traps the UI at the login page. + map $http_cf_connecting_ip $is_cloudflared { + default 1; # any non-empty value = request came through Cloudflare + "" 0; # empty = direct access + } + + # ════════════════════════════════════════════════════════════════ + # Server :80 — existing harness entrypoint + # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), + # router fallback, LiteLLM UI via /litellm/ prefix, health + # ════════════════════════════════════════════════════════════════ + server { + listen 80; + + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2-Layer: /v1/ → router (existing keys work unchanged) + location /v1/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /admin/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /stream { + proxy_pass $router_api_url/stream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + location /api/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM gateway access (agents with new virtual keys) + location /litellm/v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) + # LiteLLM Admin UI at /ui/ (must be before catch-all /) + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + error_page 301 302 = @ui_redirect; + } + + location @ui_redirect { + proxy_pass $litellm_backend_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_read_timeout 600s; + } + + # LiteLLM UI static assets on port 80 + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) + location /litellm/ { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://$host/ /litellm/; + proxy_redirect https://$host/ /litellm/; + } + + location /dashboard/ { + proxy_pass $dashboard_ui_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # Dedicated /openapi.json block — must come before catch-all / + # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, + # but the catch-all location / strips the URI path. This block + # preserves the full path so the spec JSON is returned instead of + # the Swagger UI SPA HTML. + location /openapi.json { + proxy_pass $litellm_backend_url/openapi.json; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location /metrics/ { + proxy_pass $router_api_url/metrics/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /metrics/circuit-breaker { + proxy_pass $router_api_url/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /router/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass $router_api_url/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health { + proxy_pass $router_api_url/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + } + + # ════════════════════════════════════════════════════════════════ + # Server :4000 — external LiteLLM entrypoint (cloudflared target) + # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM + # builds external URLs instead of leaking 192.168.68.116:4000. + # LiteLLM itself is now bound to 127.0.0.1 only (see compose). + # ════════════════════════════════════════════════════════════════ + server { + listen 4000; + + # Canonical external identity — overrides whatever Host cloudflared sends + proxy_set_header Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, + # so a relative return would be absolutized to http://192.168.68.116:4000/). + location = /ui { return 301 https://litellm.sysloggh.net/ui/; } + + # LiteLLM Admin UI + WebSocket + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; + proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; + } + + # UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_read_timeout 600s; + } + + # API + location /v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # Key management + admin API + location /key/ { + proxy_pass $litellm_backend_url/key/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /user/ { + proxy_pass $litellm_backend_url/user/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /model/ { + proxy_pass $litellm_backend_url/model/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /team/ { + proxy_pass $litellm_backend_url/team/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + + # Health + location /health { + proxy_pass $litellm_backend_url/health; + proxy_http_version 1.1; + } + + # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; + proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; + } + } +} diff --git a/nginx/nginx.conf.bak.preui b/nginx/nginx.conf.bak.preui new file mode 100644 index 0000000..470844f --- /dev/null +++ b/nginx/nginx.conf.bak.preui @@ -0,0 +1,343 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. + # Without this, nginx resolves upstream hostnames at config load time, + # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. + resolver 127.0.0.11 valid=30s; + + # Dynamic upstream resolution via nginx variables. + # Using $var in proxy_pass forces request-time resolution through the resolver. + # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. + map $host $router_api_url { + default http://harness-router:9000; + } + map $host $dashboard_ui_url { + default http://harness-dashboard:3000; + } + map $host $litellm_backend_url { + default http://harness-litellm:4000; + } + + # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). + # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, + # preventing the cross-origin localStorage footgun that traps the UI at the login page. + map $http_cf_connecting_ip $is_cloudflared { + default 1; # any non-empty value = request came through Cloudflare + "" 0; # empty = direct access + } + + # ════════════════════════════════════════════════════════════════ + # Server :80 — existing harness entrypoint + # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), + # router fallback, LiteLLM UI via /litellm/ prefix, health + # ════════════════════════════════════════════════════════════════ + server { + listen 80; + + # Authentik OIDC subrequest + location /authentik/auth { + internal; + proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # 2-Layer: /v1/ → router (existing keys work unchanged) + location /v1/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + error_page 502 503 = @router_fallback; + } + + location @router_fallback { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /admin/ { + proxy_pass $router_api_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + } + + location /stream { + proxy_pass $router_api_url/stream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + location /api/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM gateway access (agents with new virtual keys) + location /litellm/v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # LiteLLM UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) + # LiteLLM Admin UI at /ui/ (must be before catch-all /) + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + error_page 301 302 = @ui_redirect; + } + + location @ui_redirect { + proxy_pass $litellm_backend_url; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_read_timeout 600s; + } + + + # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) + location /litellm/ { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://$host/ /litellm/; + proxy_redirect https://$host/ /litellm/; + } + + location /dashboard/ { + proxy_pass $dashboard_ui_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # Dedicated /openapi.json block — must come before catch-all / + # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, + # but the catch-all location / strips the URI path. This block + # preserves the full path so the spec JSON is returned instead of + # the Swagger UI SPA HTML. + location /openapi.json { + proxy_pass $litellm_backend_url/openapi.json; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 600s; + proxy_buffering off; + } + + location /metrics/ { + proxy_pass $router_api_url/metrics/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /metrics/circuit-breaker { + proxy_pass $router_api_url/metrics/circuit-breaker; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /router/ { + proxy_pass $router_api_url/; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health/unified { + proxy_pass $router_api_url/health/unified; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + location /health { + proxy_pass $router_api_url/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + } + + # ════════════════════════════════════════════════════════════════ + # Server :4000 — external LiteLLM entrypoint (cloudflared target) + # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM + # builds external URLs instead of leaking 192.168.68.116:4000. + # LiteLLM itself is now bound to 127.0.0.1 only (see compose). + # ════════════════════════════════════════════════════════════════ + server { + listen 4000; + + # Canonical external identity — overrides whatever Host cloudflared sends + proxy_set_header Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Host litellm.sysloggh.net; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, + # so a relative return would be absolutized to http://192.168.68.116:4000/). + location = /ui { return 301 https://litellm.sysloggh.net/ui/; } + + # LiteLLM Admin UI + WebSocket + location /ui/ { + proxy_pass $litellm_backend_url/ui/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; + proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; + } + + # UI static assets + location /litellm-asset-prefix/ { + proxy_pass $litellm_backend_url/litellm-asset-prefix/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # SSO callback + location /sso/ { + proxy_pass $litellm_backend_url/sso/; + proxy_http_version 1.1; + proxy_read_timeout 600s; + } + + # API + location /v1/ { + proxy_pass $litellm_backend_url/v1/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + proxy_read_timeout 600s; + proxy_buffering off; + } + + # Key management + admin API + location /key/ { + proxy_pass $litellm_backend_url/key/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /user/ { + proxy_pass $litellm_backend_url/user/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /model/ { + proxy_pass $litellm_backend_url/model/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + location /team/ { + proxy_pass $litellm_backend_url/team/; + proxy_http_version 1.1; + proxy_set_header Authorization $http_authorization; + } + + # Health + location /health { + proxy_pass $litellm_backend_url/health; + proxy_http_version 1.1; + } + + # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) + location / { + proxy_pass $litellm_backend_url/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_buffering off; + proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; + proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; + } + } +} diff --git a/router/router-fixed.py b/router/router-fixed.py new file mode 100644 index 0000000..97d0f0e --- /dev/null +++ b/router/router-fixed.py @@ -0,0 +1,831 @@ +import os, json, time, logging, traceback, threading, queue, statistics, math +import requests, redis +from flask import Flask, request, jsonify, Response, stream_with_context + +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") +GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") +GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") +GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") + +GPU_SIDECARS = { + "qwen3.6-35B-A3B": "http://192.168.68.15:8090", + "qwen3.6-27B-code": "http://192.168.68.8:8090", + "gemma-4-12b": "http://192.168.68.110:8090", +} +GPU_URLS = { + "qwen3.6-35B-A3B": GPU_MOE_URL, + "qwen3.6-27B-code": GPU_DENSE_URL, + "gemma-4-12b": GPU_LIGHT_URL, +} +# Max concurrent requests per GPU (based on llama.cpp --parallel) +GPU_MAX_CONCURRENT = { + "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) + "qwen3.6-27B-code": 2, # 2 slots + "gemma-4-12b": 2, # 2 slots (12GB VRAM, 4GB headroom) +} + +# Context window sizes (tokens) — used for compaction signals +GPU_CONTEXT = { + "qwen3.6-35B-A3B": 262144, + "qwen3.6-27B-code": 262144, + "gemma-4-12b": 262144, +} + +TIER_MODELS = { + "starter": ["gemma-4-12b"], + "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], + "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], +} +# ── PHASE 0: Dual-Key API Key System ── +# API_KEYS env var is REQUIRED (JSON string). No hardcoded fallback. +# Format: {"sk-xxx": {"tier": "enterprise", "agent": "Name"}} +# Deprecated keys get {"deprecated": true} — still accepted, logged with warning. +_raw_keys = os.environ.get("API_KEYS") +if not _raw_keys: + raise RuntimeError("FATAL: API_KEYS environment variable is required. " + "Set it in docker-compose.yml or .env file. " + "No hardcoded keys fallback — this is a security feature.") +API_KEYS = json.loads(_raw_keys) +log.info("Loaded %d API keys from env var (%d deprecated)", + len(API_KEYS), sum(1 for v in API_KEYS.values() if v.get("deprecated"))) +# Rate limits: requests per minute per API key tier +RATE_LIMIT_RPM = { + "enterprise": 120, + "professional": 60, + "starter": 20, +} + +def check_rate_limit(api_key, tier): + """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" + if not r: + return True, 999, 60 + limit = RATE_LIMIT_RPM.get(tier, 30) + key = f"ratelimit:{api_key}" + current = int(r.get(key) or 0) + if current >= limit: + ttl = r.ttl(key) + retry = max(ttl, 1) if ttl and ttl > 0 else 60 + return False, retry, 0 + pipe = r.pipeline() + pipe.incr(key) + pipe.expire(key, 60) # 1-minute sliding window + pipe.execute() + remaining = limit - (current + 1) + reset_seconds = r.ttl(key) or 60 + return True, remaining, reset_seconds + + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") +log = logging.getLogger("router") +try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() +except Exception: r = None + + +def counter_audit_loop(): + """Every 30s, check GPU slots and reset counters if all slots idle.""" + while True: + time.sleep(30) + if not r: continue + for model, url in GPU_URLS.items(): + try: + resp = requests.get(url.replace("/v1","") + "/slots", + headers={"Authorization": "Bearer not-needed"}, timeout=5) + if resp.status_code == 200: + slots = resp.json() + all_idle = all(not s.get("is_processing", False) for s in slots) + if all_idle: + current = int(r.get("active:" + model) or 0) + if current > 0: + r.set("active:" + model, 0) + log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) + except Exception: + pass + +threading.Thread(target=counter_audit_loop, daemon=True).start() + +app = Flask(__name__) +sse_subscribers = []; sse_lock = threading.Lock() + +def gpu_active_count(model): + """Get number of in-flight requests for a GPU.""" + if r: + return int(r.get("active:" + model) or 0) + return 0 + +def gpu_incr(model): + if r: r.incr("active:" + model) + +def gpu_decr(model): + if r: + v = r.decr("active:" + model) + if v and int(v) < 0: + r.set("active:" + model, 0) # never go negative + +def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): + url = GPU_SIDECARS.get(model) + if not url: return {"status": "unknown"} + try: + resp = requests.get(url, timeout=sidecar_timeout) + if resp.status_code == 200: + d = resp.json() + pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 + status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy() + vram_warning = pct >= 95 + # Also check if llama.cpp endpoint is actually responding + gpu_url = GPU_URLS.get(model, "") + try: + hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) + if hr.status_code != 200: + status = "down" + except Exception: + status = "down" + return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} + except Exception: pass + return {"status": "down"} + +def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] + +def estimate_tokens(msgs): + """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" + return len(json.dumps(msgs, default=str)) // 3.5 + +def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): + """Store detailed performance record in Redis for analytics.""" + if not r: return + try: + total_ms = queue_ms + inference_ms + tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 + rec = json.dumps({ + "ts": time.time(), + "model": model, "agent": agent, "tier": tier, "reason": reason, + "queue_ms": round(queue_ms, 1), + "inference_ms": round(inference_ms, 1), + "total_ms": round(total_ms, 1), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "tokens_per_sec": round(tps, 1), + "stream": stream + }) + # Global recent list (last 500) + r.lpush("perf:recent", rec) + r.ltrim("perf:recent", 0, 499) + # Per-model list (last 200) + r.lpush("perf:model:" + model, rec) + r.ltrim("perf:model:" + model, 0, 199) + # Per-reason list (last 200) + r.lpush("perf:reason:" + reason, rec) + r.ltrim("perf:reason:" + reason, 0, 199) + # Per-agent list (last 200) + r.lpush("perf:agent:" + agent, rec) + r.ltrim("perf:agent:" + agent, 0, 199) + except Exception: + pass + +def is_gpu_busy(model): + """Check if GPU is at or near max concurrent capacity.""" + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + return active >= max_c + +def select_best_gpu(candidates, reason, agent=""): + """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" + # Count how many distinct agents are on each GPU + gpu_agent_counts = {} + if r: + for m in GPU_URLS: + count = 0 + for ak in API_KEYS.values(): + if r.get("agent_gpu:" + ak["agent"] + ":" + m): + count += 1 + gpu_agent_counts[m] = count + # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) + for m in candidates: + if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: + return {"model": m, "reason": reason} + # Second pass: prefer GPU this agent is NOT already on (skip own GPU) + if agent: + for m in candidates: + if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): + return {"model": m, "reason": reason} + # Third pass: any non-busy GPU + for m in candidates: + if not is_gpu_busy(m): + return {"model": m, "reason": reason} + # All busy — pick least loaded + best = None + best_load = 999 + for m in candidates: + load = gpu_active_count(m) + if load < best_load: + best_load = load + best = m + if best: + return {"model": best, "reason": "load_balanced_" + reason} + return None + +def route(rd, tier, agent=""): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) + avail = [m for m in available_models() if m in allowed] + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + # Check if all available GPUs are at max capacity + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} + + req = rd.get("model","auto") + if req != "auto": + # STRICT MODE: no silent fallback — LiteLLM handles failover chains. + # This keeps per-model metrics accurate. Returns saturated if busy. + target = req if req in avail else avail[0] + if req not in avail: + return {"model": req, "reason": "explicit_unavailable", "saturated": True} + if is_gpu_busy(target): + return {"model": target, "reason": "explicit_saturated", "saturated": True} + return {"model": target, "reason": "explicit"} + + if hints: + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} + + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + + # TIER 1: Lightweight — single-turn short queries → VLM (fastest) + if not sys and turns <= 1 and t <= 500 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"lightweight"} + # VLM busy — Dense is faster for short queries than MoE + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "lightweight_fallback", agent) + if result: return result + + # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat + if t <= 15000 and turns <= 12 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"simple_conv"} + # VLM busy — fall back to Dense, then MoE + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "simple_conv_fallback", agent) + if result: return result + + # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance) + if t <= 25000: + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback + if t > 25000: + candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "heavy_reasoning", agent) + if result: return result + + # TIER 5: Default — Dense primary, MoE fallback + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "default", agent) + if result: return result + return {"model":avail[0],"reason":"last_resort"} + +def clean_unicode(text): + if not isinstance(text, str): return text + text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") + text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'") + text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"') + text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ") + return text.encode("ascii", "ignore").decode("ascii") + +def clean_response(d): + if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} + if isinstance(d, list): return [clean_response(v) for v in d] + if isinstance(d, str): return clean_unicode(d) + return d + +def get_metrics(): + d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}} + for m in GPU_URLS: + h = check_gpu_health(m) + d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)}) + d["active_requests"][m] = gpu_active_count(m) + if r: + try: + for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) + for k,v in API_KEYS.items(): + c = int(r.get("routes:agent:"+v["agent"]) or 0) + if c>0: d["agent_counts"][v["agent"]] = c + for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) + raw = r.lrange("routes:recent",0,49) + d["recent"] = [json.loads(x) for x in raw] if raw else [] + except Exception: pass + return d + +def bcast(): + data = get_metrics(); payload = json.dumps(data) + with sse_lock: + dead = [] + for q in sse_subscribers: + try: q.put(payload) + except Exception: dead.append(q) + for q in dead: sse_subscribers.remove(q) + +QUEUE_TIMEOUT = int(os.environ.get("QUEUE_TIMEOUT", "30")) # max seconds to queue before 503 + +@app.route("/v1/chat/completions", methods=["POST"]) +def chat(): + try: + rd = request.get_json(force=True) + ak = request.headers.get("Authorization","").replace("Bearer ","") + if not ak or ak not in API_KEYS: + log.warning("AUTH_REJECTED: no/invalid API key from %s", request.remote_addr) + return jsonify({"error": "Unauthorized — valid API key required"}), 401 + ki = API_KEYS[ak] + tier, agent = ki["tier"], ki["agent"] + # Phase 0: dual-key transition — log deprecated key usage + if ki.get("deprecated"): + new_key = next((k for k, v in API_KEYS.items() + if v.get("agent") == agent and not v.get("deprecated")), None) + log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s", + agent, ak[:12], ak[-8:], + new_key[:12] if new_key else "N/A", + new_key[-8:] if new_key else "N/A") + if r: + r.incr("deprecated_usage:" + agent) + + # Rate limit check + allowed, rl_val, reset_sec = check_rate_limit(ak, tier) + if not allowed: + resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val}) + resp.headers["Retry-After"] = str(rl_val) + resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30)) + resp.headers["X-RateLimit-Remaining"] = "0" + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val)) + log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:]) + return resp, 429 + + # Allow agent to override queue timeout via header + q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) + + # Cross-turn context tracking: accumulate tokens per session + session_id = request.headers.get("X-Session-Id", "") + session_tokens = 0 + if session_id and r: + try: + prev = int(r.get("session:" + session_id) or 0) + current = estimate_tokens(rd.get("messages",[])) + session_tokens = max(prev, current) # context only grows + r.set("session:" + session_id, session_tokens, ex=86400) # TTL 24h + except Exception: pass + + d = route(rd, tier, agent) + queue_start = time.time() + + # Queue loop: wait for a GPU slot instead of immediate 503 + while d.get("saturated"): + elapsed = time.time() - queue_start + if elapsed > q_timeout: + resp = jsonify({"error": "All GPUs saturated", "queued_s": round(elapsed,1), "retry_after_s": 5}) + resp.headers["Retry-After"] = "5" + log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed) + return resp, 503 + time.sleep(0.5) # poll every 500ms + d = route(rd, tier, agent) + + queue_ms = (time.time() - queue_start) * 1000 + if queue_ms > 500: + log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) + model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + + # Stash rate limit values for response headers + _rl_remaining = rl_val + _rl_limit = RATE_LIMIT_RPM.get(tier, 30) + _rl_reset = reset_sec + is_stream = rd.get("stream", False) + + gpu_incr(model) + + log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) + # Track which GPU this agent is using (TTL 120s covers typical request) + if r and agent: + try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1") + except: pass + if r: + try: + r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) + r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) + r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)})) + r.ltrim("routes:recent",0,999) + except Exception: pass + start = time.time() + resp = requests.post(url+"/chat/completions", json=rd, + headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) + lat = int((time.time()-start)*1000) + gpu_decr(model) + + if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 + if is_stream: + # Buffer SSE chunks, handle split lines for large responses + chunks = [] + stream_timings = {} + buf = "" # accumulate partial lines + for raw in resp.iter_content(chunk_size=None, decode_unicode=True): + if raw: + cleaned = clean_unicode(raw) + chunks.append(cleaned) + buf += cleaned + # Process complete lines from buffer + while "\n" in buf: + line, buf = buf.split("\n", 1) + line = line.strip() + if line.startswith("data: ") and not stream_timings: + js = line[6:].strip() + if js.startswith("{") and "timings" in js and "predicted_n" in js: + try: + tj = json.loads(js).get("timings", {}) + if tj: + stream_timings = tj + except: pass + # Store perf record with real token counts from stream + if stream_timings: + pt = stream_timings.get("prompt_n", 0) + ct = stream_timings.get("predicted_n", 0) + tps = stream_timings.get("predicted_per_second", 0) + gen_ms = stream_timings.get("predicted_ms", lat) + store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True) + else: + store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True) + # Yield all chunks to client + def gen(): + for c in chunks: yield c + bcast() + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) + sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") + sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) + sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + sse_resp.headers["X-Context-Warning"] = ctx_warning + sse_resp.headers["X-Context-Model"] = model + return sse_resp + data = clean_response(resp.json()) + for c in data.get("choices",[]): + msg = c.get("message",{}) + if not msg.get("content") and msg.get("reasoning_content"): + msg["content"] = msg["reasoning_content"] + # Extract performance data from llama.cpp response + usage = data.get("usage", {}) + timings = data.get("timings", {}) + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + inference_ms = lat # total GPU round-trip + store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False) + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) + data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} + resp = jsonify(data) + resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) + resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + resp.headers["X-Context-Warning"] = ctx_warning + resp.headers["X-Context-Model"] = model + bcast() + return resp + except requests.Timeout: + gpu_decr(model) + log.error("TIMEOUT: %s -> %s", agent, model) + return jsonify({"error":"timeout"}), 504 + except Exception as e: + gpu_decr(model) + log.error("Error: %s\n%s", e, traceback.format_exc()) + return jsonify({"error":str(e)}), 500 + +@app.route("/metrics/performance") +def performance(): + """Per-request performance analytics with percentiles per model/reason/agent.""" + if not r: return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24")) + model_filter = request.args.get("model", "all") + + # Load recent records + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + records = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + records.append(rec) + except: pass + + # Filter by model if specified + if model_filter != "all": + records = [r for r in records if r["model"] == model_filter] + + if not records: + return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}) + + def pct(values, p): + if len(values) < 2: return round(values[0], 1) if values else 0 + return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1) + + # Per-model stats + model_groups = {} + for rec in records: + m = rec["model"] + if m not in model_groups: model_groups[m] = [] + model_groups[m].append(rec) + + models = [] + for m, recs in sorted(model_groups.items()): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + non_stream = [r for r in recs if not r["stream"]] + queue_times = [r["queue_ms"] for r in non_stream] + models.append({ + "model": m, + "count": len(recs), + "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1), + "latency": { + "avg": round(statistics.mean(latencies), 1), + "p50": pct(latencies, 50), + "p95": pct(latencies, 95), + "p99": pct(latencies, 99) + }, + "throughput": { + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0, + "p50": pct(tps_vals, 50) if tps_vals else 0, + "p95": pct(tps_vals, 95) if tps_vals else 0, + }, + "queue": { + "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0, + "p95_ms": pct(queue_times, 95) if queue_times else 0, + } if queue_times else None + }) + + # Per-reason stats + reason_groups = {} + for rec in records: + rsn = rec["reason"] + if rsn not in reason_groups: reason_groups[rsn] = [] + reason_groups[rsn].append(rec) + + reasons = [] + for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + reasons.append({ + "reason": rsn, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "p95_total_ms": pct(latencies, 95) + }) + + # Per-agent stats + agent_groups = {} + for rec in records: + ag = rec["agent"] + if ag not in agent_groups: agent_groups[ag] = [] + agent_groups[ag].append(rec) + + agents = [] + for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + agents.append({ + "agent": ag, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0 + }) + + all_lat = [r["total_ms"] for r in records] + all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0] + summary = { + "total_requests": len(records), + "window_hours": window_hours, + "latency": { + "avg_ms": round(statistics.mean(all_lat), 1), + "p50_ms": pct(all_lat, 50), + "p95_ms": pct(all_lat, 95), + "p99_ms": pct(all_lat, 99) + }, + "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0 + } + + return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/metrics/scatter") +def scatter(): + """Return individual data points for scatter plots (prompt_tokens vs latency).""" + if not r: return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24")) + model_filter = request.args.get("model", "all") + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + points = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + if model_filter == "all" or rec["model"] == model_filter: + points.append({ + "model": rec["model"], + "agent": rec["agent"], + "reason": rec["reason"], + "prompt_tokens": int(rec.get("prompt_tokens", 0)), + "completion_tokens": rec.get("completion_tokens", 0), + "inference_ms": round(rec["inference_ms"], 1), + "tokens_per_sec": rec.get("tokens_per_sec", 0), + "stream": rec.get("stream", False) + }) + except: pass + return jsonify({"points": points, "count": len(points)}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/v1/models") +def models(): + def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) + +@app.route("/health") +def health(): + gpus = {} + for m in GPU_URLS: + h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + h["active_requests"] = gpu_active_count(m) + h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) + gpus[m] = h + return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()}) + +@app.route("/metrics") +def metrics(): return jsonify(get_metrics()) + +@app.route("/metrics/timeseries") +def metrics_timeseries(): + period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) + data = {"models": {}, "labels": []} + if period == "day": + buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + elif period == "week": + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + else: + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + if r: + for model in models_list: + counts = [] + for bucket in buckets: + total = 0 + if period in ("week","month"): + for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) + else: total = int(r.get("ts:"+model+":"+bucket) or 0) + counts.append(total) + data["models"][model] = counts + return jsonify(data) + +@app.route("/stream") +def stream(): + def ev(): + q = queue.Queue() + with sse_lock: sse_subscribers.append(q) + try: + yield "data: "+json.dumps(get_metrics())+"\n\n" + while True: + try: yield "data: "+q.get(timeout=3)+"\n\n" + except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" + except GeneratorExit: pass + finally: + with sse_lock: + if q in sse_subscribers: sse_subscribers.remove(q) + return Response(stream_with_context(ev()), mimetype="text/event-stream", + headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) + +# ── Phase 0: Admin Key Management ── +ADMIN_KEY = os.environ.get("ADMIN_KEY", "") + +def _admin_auth(): + """Require admin key for management endpoints.""" + if not ADMIN_KEY: + return False, "ADMIN_KEY not configured on server" + ak = request.headers.get("Authorization","").replace("Bearer ","") + if ak != ADMIN_KEY: + return False, "Admin key required" + return True, None + +@app.route("/admin/keys") +def admin_keys(): + """List all API keys (masked) with agent, tier, and deprecation status.""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + keys = [] + for key, info in API_KEYS.items(): + masked = key[:8] + "..." + key[-8:] + keys.append({ + "masked": masked, + "prefix": key[:8], + "agent": info["agent"], + "tier": info["tier"], + "deprecated": info.get("deprecated", False), + "length": len(key) + }) + return jsonify({ + "total": len(keys), + "active": sum(1 for k in keys if not k["deprecated"]), + "deprecated": sum(1 for k in keys if k["deprecated"]), + "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"])) + }) + +@app.route("/admin/keys/deprecation-summary") +def admin_deprecation_summary(): + """Summary of deprecated key usage (from Redis logs, if available).""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + deprecated_agents = [] + for key, info in API_KEYS.items(): + if info.get("deprecated"): + # Check Redis for usage count + count = 0 + if r: + count = int(r.get("deprecated_usage:" + info["agent"]) or 0) + deprecated_agents.append({ + "agent": info["agent"], + "deprecated_uses": count, + "needs_migration": count > 0 + }) + return jsonify({ + "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]), + "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage" + }) + +@app.route("/admin/keys/generate", methods=["POST"]) +def admin_generate_key(): + """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "").strip() + tier = body.get("tier", "enterprise") + if not agent: + return jsonify({"error": "agent field required"}), 400 + if tier not in ("starter", "professional", "enterprise"): + return jsonify({"error": "tier must be starter/professional/enterprise"}), 400 + # Generate secure key + import secrets, hashlib + prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8] + suffix = secrets.token_hex(20) + new_key = f"sk-{prefix}-{suffix}" + # Update in-memory dict (note: not persisted across restarts without env var update) + API_KEYS[new_key] = {"tier": tier, "agent": agent} + log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:]) + return jsonify({ + "agent": agent, + "tier": tier, + "key": new_key, + "masked": new_key[:8] + "..." + new_key[-8:], + "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist." + }), 201 + +@app.route("/admin/keys/revoke", methods=["POST"]) +def admin_revoke_key(): + """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "") + key_prefix = body.get("key_prefix", "") + revoked = [] + keys_to_remove = [] + for key, info in API_KEYS.items(): + if not info.get("deprecated"): + continue + if agent and info["agent"] == agent: + keys_to_remove.append(key) + elif key_prefix and key.startswith(key_prefix): + keys_to_remove.append(key) + for key in keys_to_remove: + info = API_KEYS.pop(key) + revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]}) + log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:]) + return jsonify({ + "revoked": len(revoked), + "keys": revoked, + "remaining_total": len(API_KEYS), + "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist." + }) + +if __name__ == "__main__": + log.info("Router on :9000 (load-aware)") + app.run(host="0.0.0.0", port=9000, debug=False) diff --git a/router/router.py.bak.20260616-214735 b/router/router.py.bak.20260616-214735 new file mode 100644 index 0000000..23f9c8e --- /dev/null +++ b/router/router.py.bak.20260616-214735 @@ -0,0 +1,1006 @@ +import os, json, time, logging, traceback, threading, queue, statistics, math +import requests, redis +from flask import Flask, request, jsonify, Response, stream_with_context + + + +# Phase 2: Atomic session token update Redis Lua script +SESSION_LUA_SCRIPT = """ +local key = KEYS[1] +local new_val = tonumber(ARGV[1]) +local current = tonumber(redis.call('GET', key) or 0) +local max_val = math.max(current, new_val) +redis.call('SET', key, max_val, 'EX', 86400) +return max_val +""" + + +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") +GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") +GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") +GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") + +GPU_SIDECARS = { + "qwen3.6-35B-A3B": "http://192.168.68.15:8090", + "qwen3.6-27B-code": "http://192.168.68.8:8090", + "gemma-4-12b": "http://192.168.68.110:8090", +} +GPU_URLS = { + "qwen3.6-35B-A3B": GPU_MOE_URL, + "qwen3.6-27B-code": GPU_DENSE_URL, + "gemma-4-12b": GPU_LIGHT_URL, +} +# Max concurrent requests per GPU (based on llama.cpp --parallel) + +GPU_LABELS = { + "qwen3.6-35B-A3B": "Qwen3.6 35B (Strix Halo)", + "qwen3.6-27B-code": "Qwen3.6 27B Code (RTX 3090)", + "gemma-4-12b": "Gemma-4 12B (RTX 5070)", +} + +GPU_MAX_CONCURRENT = { + "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) + "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) + "gemma-4-12b": 2, # 2 slots (7.1GB VRAM) +} + +# Context window sizes (tokens) — used for compaction signals +GPU_CONTEXT = { + "qwen3.6-35B-A3B": 262144, + "qwen3.6-27B-code": 262144, + "gemma-4-12b": 262144, +} + +TIER_MODELS = { + "starter": ["gemma-4-12b"], + "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], + "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], +} +# API keys loaded from API_KEYS env var (set in docker-compose.yml) +# Fallback is dev-only — production MUST set API_KEYS env var +API_KEYS = json.loads(os.environ.get("API_KEYS", json.dumps({ + "sk-dev-local-only": {"tier": "enterprise", "agent": "dev"}, +}))) +# Rate limits: requests per minute per API key tier +RATE_LIMIT_RPM = { + "enterprise": 120, + "professional": 60, + "starter": 20, +} + +def check_rate_limit(api_key, tier): + """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" + if not get_redis(): + return True, 999, 60 + limit = RATE_LIMIT_RPM.get(tier, 30) + key = f"ratelimit:{api_key}" + current = int(get_redis().get(key) or 0) + if current >= limit: + ttl = r.ttl(key) + retry = max(ttl, 1) if ttl and ttl > 0 else 60 + return False, retry, 0 + pipe = r.pipeline() + pipe.incr(key) + pipe.expire(key, 60) # 1-minute sliding window + pipe.execute() + remaining = limit - (current + 1) + reset_seconds = r.ttl(key) or 60 + return True, remaining, reset_seconds + + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") +log = logging.getLogger("router") +# Redis connection — initialized lazily, retries on first use +def get_redis(): + global r + if r is not None: + try: + r.ping() + return r + except Exception: + r = None + try: + r = redis.from_url(REDIS_URL, decode_responses=True) + r.ping() + return r + except Exception: + return None + +r = None +try: get_redis() +except Exception: pass +except Exception: r = None + + +def counter_audit_loop(): + """Every 30s, check GPU slots and reset counters if all slots idle.""" + while True: + time.sleep(30) + if not get_redis(): continue + for model, url in GPU_URLS.items(): + try: + resp = requests.get(url.replace("/v1","") + "/slots", + headers={"Authorization": "Bearer not-needed"}, timeout=5) + if resp.status_code == 200: + slots = resp.json() + all_idle = all(not s.get("is_processing", False) for s in slots) + if all_idle: + current = int(get_redis().get("active:" + model) or 0) + if current > 0: + get_redis().set("active:" + model, 0) + log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) + except Exception: + pass + +threading.Thread(target=counter_audit_loop, daemon=True).start() + +app = Flask(__name__) +sse_subscribers = []; sse_lock = threading.Lock() + +def gpu_active_count(model): + """Get number of in-flight requests for a GPU.""" + if r: + return int(r.get("active:" + model) or 0) + return 0 + +def gpu_incr(model): + if get_redis(): get_redis().incr("active:" + model) + +def gpu_decr(model): + rd = get_redis() + if rd: + v = rd.decr("active:" + model) + if v and int(v) < 0: + get_redis().set("active:" + model, 0) # never go negative +def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): + url = GPU_SIDECARS.get(model) + if not url: return {"status": "unknown"} + try: + resp = requests.get(url, timeout=sidecar_timeout) + if resp.status_code == 200: + d = resp.json() + pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 + status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy() + vram_warning = pct >= 95 + # Also check if llama.cpp endpoint is actually responding + gpu_url = GPU_URLS.get(model, "") + try: + hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) + if hr.status_code != 200: + status = "down" + except Exception: + status = "down" + return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} + except Exception: pass + return {"status": "down"} + +def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] + +def estimate_tokens(msgs): + """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" + return len(json.dumps(msgs, default=str)) // 3.5 + +def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): + """Store detailed performance record in Redis for analytics.""" + if not get_redis(): return + try: + total_ms = queue_ms + inference_ms + tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 + rec = json.dumps({ + "ts": time.time(), + "model": model, "agent": agent, "tier": tier, "reason": reason, + "queue_ms": round(queue_ms, 1), + "inference_ms": round(inference_ms, 1), + "total_ms": round(total_ms, 1), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "tokens_per_sec": round(tps, 1), + "stream": stream + }) + # Global recent list (last 500) + r.lpush("perf:recent", rec) + r.ltrim("perf:recent", 0, 499) + # Per-model list (last 200) + r.lpush("perf:model:" + model, rec) + r.ltrim("perf:model:" + model, 0, 199) + # Per-reason list (last 200) + r.lpush("perf:reason:" + reason, rec) + r.ltrim("perf:reason:" + reason, 0, 199) + # Per-agent list (last 200) + r.lpush("perf:agent:" + agent, rec) + r.ltrim("perf:agent:" + agent, 0, 199) + except Exception: + pass + +def is_gpu_busy(model): + """Check if GPU is at or near max concurrent capacity.""" + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + return active >= max_c + + + +# Phase 3: Dynamic GPU Weighting (Health Score) +def gpu_health_score(model): + """Score a GPU based on VRAM, temperature, and load. Lower = better.""" + h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) + if h.get("status") == "down": + return 999 # never pick down GPUs + vram_pct = h.get("vram_pct") or 50 + temp_c = h.get("temp_c") or 50 + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + load_pct = (active / max_c) * 100 if max_c > 0 else 0 + # Score: lower = better (more headroom, cooler, less loaded) + score = (vram_pct or 0) * 0.4 + max((temp_c or 50) - 30, 0) * 0.3 + load_pct * 0.3 + return score + +def select_best_gpu(candidates, reason, agent=""): + """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" + # Count how many distinct agents are on each GPU + gpu_agent_counts = {} + if r: + for m in GPU_URLS: + count = 0 + for ak in API_KEYS.values(): + if r.get("agent_gpu:" + ak["agent"] + ":" + m): + count += 1 + gpu_agent_counts[m] = count + # Phase 3: Sort candidates by health score before selection + sorted_candidates = sorted(candidates, key=gpu_health_score) + # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) + for m in sorted_candidates: + if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: + return {"model": m, "reason": reason} + # Second pass: prefer GPU this agent is NOT already on (skip own GPU) + if agent: + for m in sorted_candidates: + if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): + return {"model": m, "reason": reason} + # Third pass: any non-busy GPU + for m in sorted_candidates: + if not is_gpu_busy(m): + return {"model": m, "reason": reason} + # All busy — pick least loaded + best = None + best_load = 999 + for m in candidates: + load = gpu_active_count(m) + if load < best_load: + best_load = load + best = m + if best: + return {"model": best, "reason": "load_balanced_" + reason} + return None + + + + + +# Phase 1: Circuit Breaker for GPU Hosts (Approved by Abiba) +def is_circuit_tripped(model): + """Check if a GPU host is currently blacklisted.""" + if not get_redis(): + return False + return r.exists("circuit:" + model + ":open") + +def trip_circuit(model, duration=30): + """Blacklist a GPU host for a specified duration.""" + if not get_redis(): + return + key = "circuit:" + model + ":open" + r.set(key, 1, ex=duration) + r.incr("circuit:" + model + ":count") + log.warning("CIRCUIT_TRIPPED: %s blacklisted for %ds", model, duration) + +def half_open_probe(model): + """Check if a GPU host can be un-blacklisted.""" + if not get_redis(): + return True + key = "circuit:" + model + ":open" + if not r.exists(key): + return True # no circuit + return False # still open + +def route(rd, tier, agent=""): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) + # Phase 1: Filter out models with tripped circuit breakers + avail = [m for m in available_models() if m in allowed and not is_circuit_tripped(m)] + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} + + # GUARD: multimodal -> VLM only (sole vision model) + has_image = any( + isinstance(m.get("content"), list) and + any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) + for m in msgs + ) + if has_image: + if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): + return {"model": "gemma-4-12b", "reason": "vision"} + elif "gemma-4-12b" in avail: + return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} + else: + return {"model": allowed[0], "reason": "vision_unavailable"} + + req = rd.get("model","auto") + if req != "auto": + target = req if req in avail else avail[0] + if is_gpu_busy(target) and req in allowed: + alts = [m for m in avail if m != target and m in allowed] + if alts: + alt = select_best_gpu(alts, "explicit", agent) + if alt: return alt + return {"model": target, "reason": "explicit"} + + if hints: + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} + if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: + return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} + + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + + # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) + if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"tiny"} + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "tiny_fallback", agent) + if result: return result + + # TIER 2: Light - moderate chat -> VLM first (fastest), Dense fallback + if t <= 5000 and turns <= 4: + candidates = [m for m in ["gemma-4-12b","qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "light", agent) + if result: return result + + # TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%) + if t <= 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy - quality first -> Dense primary, MoE fallback + if t > 30000: + candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "heavy", agent) + if result: return result + + # TIER 5: Default - MoE primary (60%), Dense spillover (40%) + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "default", agent) + if result: return result + return {"model":avail[0],"reason":"last_resort"} + + +def moe_spillover(avail, default_order): + """Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating. + Only applies when MoE is first candidate, available, and not busy.""" + import random + if (default_order[0] == "qwen3.6-35B-A3B" + and "qwen3.6-35B-A3B" in avail + and not is_gpu_busy("qwen3.6-35B-A3B") + and "qwen3.6-27B-code" in avail + and not is_gpu_busy("qwen3.6-27B-code") + and random.random() < 0.4): + # Swap: Dense first, MoE second + return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")] + return [m for m in default_order if m in avail] +def clean_unicode(text): + if not isinstance(text, str): return text + text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") + text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'") + text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"') + text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ") + return text.encode("ascii", "ignore").decode("ascii") + +def clean_response(d): + if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} + if isinstance(d, list): return [clean_response(v) for v in d] + if isinstance(d, str): return clean_unicode(d) + return d + +def get_metrics(): + d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}} + for m in GPU_URLS: + h = check_gpu_health(m) + d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)}) + d["active_requests"][m] = gpu_active_count(m) + if r: + try: + for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) + for k,v in API_KEYS.items(): + c = int(r.get("routes:agent:"+v["agent"]) or 0) + if c>0: d["agent_counts"][v["agent"]] = c + for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) + raw = r.lrange("routes:recent",0,49) + d["recent"] = [json.loads(x) for x in raw] if raw else [] + except Exception: pass + return d + +def bcast(): + data = get_metrics(); payload = json.dumps(data) + with sse_lock: + dead = [] + for q in sse_subscribers: + try: q.put(payload) + except Exception: dead.append(q) + for q in dead: sse_subscribers.remove(q) + +QUEUE_TIMEOUT = int(os.environ.get("QUEUE_TIMEOUT", "30")) # max seconds to queue before 503 + +@app.route("/v1/chat/completions", methods=["POST"]) +def chat(): + try: + rd = request.get_json(force=True) + ak = request.headers.get("Authorization","").replace("Bearer ","") + if not ak or ak not in API_KEYS: + log.warning("AUTH_REJECTED: no/invalid API key from %s", request.remote_addr) + return jsonify({"error": "Unauthorized — valid API key required"}), 401 + ki = API_KEYS[ak] + tier, agent = ki["tier"], ki["agent"] + + # Phase 0: dual-key transition — log deprecated key usage + if ki.get("deprecated"): + new_key = next((k for k, v in API_KEYS.items() + if v.get("agent") == agent and not v.get("deprecated")), None) + log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s", + agent, ak[:12], ak[-8:], + new_key[:12] if new_key else "N/A", + new_key[-8:] if new_key else "N/A") + if r: + r.incr("deprecated_usage:" + agent) + + # Rate limit check + allowed, rl_val, reset_sec = check_rate_limit(ak, tier) + if not allowed: + resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val}) + resp.headers["Retry-After"] = str(rl_val) + resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30)) + resp.headers["X-RateLimit-Remaining"] = "0" + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val)) + log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:]) + return resp, 429 + + # Allow agent to override queue timeout via header + q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) + + # Cross-turn context tracking: accumulate tokens per session (Phase 2: atomic Lua) + session_id = request.headers.get("X-Session-Id", "") + session_tokens = 0 + if session_id and r: + try: + current = estimate_tokens(rd.get("messages",[])) + # Atomic GET/MAX/SET via Lua script prevents race conditions + session_tokens = r.eval(SESSION_LUA_SCRIPT, 1, "session:" + session_id, current) + except Exception: pass + + d = route(rd, tier, agent) + queue_start = time.time() + + # Queue loop: wait for a GPU slot instead of immediate 503 + while d.get("saturated"): + elapsed = time.time() - queue_start + if elapsed > q_timeout: + resp = jsonify({"error": "All GPUs saturated", "queued_s": round(elapsed,1), "retry_after_s": 5}) + resp.headers["Retry-After"] = "5" + log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed) + return resp, 503 + time.sleep(0.5) # poll every 500ms + d = route(rd, tier, agent) + + queue_ms = (time.time() - queue_start) * 1000 + if queue_ms > 500: + log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) + model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + + # Stash rate limit values for response headers + _rl_remaining = rl_val + _rl_limit = RATE_LIMIT_RPM.get(tier, 30) + _rl_reset = reset_sec + is_stream = rd.get("stream", False) + + gpu_incr(model) + + log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) + # Track which GPU this agent is using (TTL 120s covers typical request) + if r and agent: + try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1") + except: pass + if r: + try: + r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) + r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) + r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)})) + r.ltrim("routes:recent",0,999) + except Exception: pass + start = time.time() + resp = requests.post(url+"/chat/completions", json=rd, + headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) + lat = int((time.time()-start)*1000) + gpu_decr(model) + + if resp.status_code != 200: + if resp.status_code in (502, 504): + trip_circuit(model, 30) + return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 + if is_stream: + # Buffer SSE chunks, handle split lines for large responses + chunks = [] + stream_timings = {} + buf = "" # accumulate partial lines + for raw in resp.iter_content(chunk_size=None, decode_unicode=True): + if raw: + cleaned = clean_unicode(raw) + chunks.append(cleaned) + buf += cleaned + # Process complete lines from buffer + while "\n" in buf: + line, buf = buf.split("\n", 1) + line = line.strip() + if line.startswith("data: ") and not stream_timings: + js = line[6:].strip() + if js.startswith("{") and "timings" in js and "predicted_n" in js: + try: + tj = json.loads(js).get("timings", {}) + if tj: + stream_timings = tj + except: pass + # Store perf record with real token counts from stream + if stream_timings: + pt = stream_timings.get("prompt_n", 0) + ct = stream_timings.get("predicted_n", 0) + tps = stream_timings.get("predicted_per_second", 0) + gen_ms = stream_timings.get("predicted_ms", lat) + store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True) + else: + store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True) + # Yield all chunks to client + def gen(): + for c in chunks: yield c + bcast() + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) + sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") + sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) + sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + sse_resp.headers["X-Context-Warning"] = ctx_warning + sse_resp.headers["X-Context-Model"] = model + return sse_resp + data = clean_response(resp.json()) + for c in data.get("choices",[]): + msg = c.get("message",{}) + if not msg.get("content") and msg.get("reasoning_content"): + msg["content"] = msg["reasoning_content"] + # Extract performance data from llama.cpp response + usage = data.get("usage", {}) + timings = data.get("timings", {}) + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + inference_ms = lat # total GPU round-trip + store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False) + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) + data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} + resp = jsonify(data) + resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) + resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + resp.headers["X-Context-Warning"] = ctx_warning + resp.headers["X-Context-Model"] = model + bcast() + return resp + except requests.Timeout: + gpu_decr(model) + trip_circuit(model, 30) + log.error("TIMEOUT: %s -> %s (Circuit tripped)", agent, model) + return jsonify({"error":"timeout"}), 504 + except Exception as e: + gpu_decr(model) + log.error("Error: %s\n%s", e, traceback.format_exc()) + return jsonify({"error":str(e)}), 500 + +@app.route("/metrics/performance") +def performance(): + """Per-request performance analytics with percentiles per model/reason/agent.""" + if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24").replace("h","")) + model_filter = request.args.get("model", "all") + + # Load recent records + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + records = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + records.append(rec) + except: pass + + # Filter by model if specified + if model_filter != "all": + records = [r for r in records if r["model"] == model_filter] + + if not records: + return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}) + + def pct(values, p): + if len(values) < 2: return round(values[0], 1) if values else 0 + return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1) + + # Per-model stats + model_groups = {} + for rec in records: + m = rec["model"] + if m not in model_groups: model_groups[m] = [] + model_groups[m].append(rec) + + models = [] + for m, recs in sorted(model_groups.items()): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + non_stream = [r for r in recs if not r["stream"]] + queue_times = [r["queue_ms"] for r in non_stream] + models.append({ + "model": m, + "count": len(recs), + "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1), + "latency": { + "avg": round(statistics.mean(latencies), 1), + "p50": pct(latencies, 50), + "p95": pct(latencies, 95), + "p99": pct(latencies, 99) + }, + "throughput": { + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0, + "p50": pct(tps_vals, 50) if tps_vals else 0, + "p95": pct(tps_vals, 95) if tps_vals else 0, + }, + "queue": { + "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0, + "p95_ms": pct(queue_times, 95) if queue_times else 0, + } if queue_times else None + }) + + # Per-reason stats + reason_groups = {} + for rec in records: + rsn = rec["reason"] + if rsn not in reason_groups: reason_groups[rsn] = [] + reason_groups[rsn].append(rec) + + reasons = [] + for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + reasons.append({ + "reason": rsn, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "p95_total_ms": pct(latencies, 95) + }) + + # Per-agent stats + agent_groups = {} + for rec in records: + ag = rec["agent"] + if ag not in agent_groups: agent_groups[ag] = [] + agent_groups[ag].append(rec) + + agents = [] + for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + agents.append({ + "agent": ag, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0 + }) + + all_lat = [r["total_ms"] for r in records] + all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0] + summary = { + "total_requests": len(records), + "window_hours": window_hours, + "latency": { + "avg_ms": round(statistics.mean(all_lat), 1), + "p50_ms": pct(all_lat, 50), + "p95_ms": pct(all_lat, 95), + "p99_ms": pct(all_lat, 99) + }, + "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0 + } + + return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/metrics/scatter") +def scatter(): + """Return individual data points for scatter plots (prompt_tokens vs latency).""" + if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24").replace("h","")) + model_filter = request.args.get("model", "all") + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + points = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + if model_filter == "all" or rec["model"] == model_filter: + points.append({ + "model": rec["model"], + "agent": rec["agent"], + "reason": rec["reason"], + "prompt_tokens": int(rec.get("prompt_tokens", 0)), + "completion_tokens": rec.get("completion_tokens", 0), + "inference_ms": round(rec["inference_ms"], 1), + "tokens_per_sec": rec.get("tokens_per_sec", 0), + "stream": rec.get("stream", False) + }) + except: pass + return jsonify({"points": points, "count": len(points)}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/v1/models") +def models(): + def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) + +@app.route("/health") +def health(): + gpus = {} + for m in GPU_URLS: + h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + h["active_requests"] = gpu_active_count(m) + h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) + gpus[m] = h + return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()}) + +@app.route("/metrics") +def metrics(): return jsonify(get_metrics()) + +@app.route("/metrics/timeseries") +def metrics_timeseries(): + period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) + data = {"models": {}, "labels": []} + if period == "day": + buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + elif period == "week": + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + else: + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + if r: + for model in models_list: + counts = [] + for bucket in buckets: + total = 0 + if period in ("week","month"): + for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) + else: total = int(r.get("ts:"+model+":"+bucket) or 0) + counts.append(total) + data["models"][model] = counts + return jsonify(data) + + + +@app.route("/metrics/circuit-breaker") +def metrics_circuit_breaker(): + """Expose circuit breaker status per model. Phase 1.""" + result = {} + if r: + for model in GPU_URLS: + key = "circuit:" + model + ":open" + duration = r.ttl(key) + trip_count = int(r.get("circuit:" + model + ":count") or 0) + result[model] = { + "tripped": r.exists(key), + "remaining_ttl": duration, + "trip_count": trip_count + } + return jsonify(result) + + +@app.route("/metrics/gpu-health") +def metrics_gpu_health(): + """Live GPU health scores + circuit breaker + KPIs.""" + result = {"gpus": [], "ts": time.time()} + for model in GPU_URLS: + h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) + score = gpu_health_score(model) + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + cb_tripped = bool(r and r.exists("circuit:" + model + ":open")) + cb_count = int(r.get("circuit:" + model + ":count") or 0) if r else 0 + result["gpus"].append({ + "id": model, + "label": GPU_LABELS.get(model, model), + "status": h.get("status", "unknown"), + "vram_pct": h.get("vram_pct", 0), + "temp_c": h.get("temp_c", 0), + "vram_used_mb": h.get("vram_used_mb", 0), + "vram_total_mb": h.get("vram_total_mb", 0), + "gpu_name": h.get("gpu_name", model), + "health_score": round(score, 1), + "active_requests": active, + "max_concurrent": max_c, + "circuit_tripped": cb_tripped, + "circuit_trip_count": cb_count + }) + online = sum(1 for g in result["gpus"] if g["status"] in ("healthy", "saturated")) + trips = sum(g["circuit_trip_count"] for g in result["gpus"]) + result["kpi"] = {"gpus_online": online, "total_trips": trips, "total_gpus": len(GPU_URLS)} + return jsonify(result) + +@app.route("/metrics/latency") +def metrics_latency(): + """Lightweight latency summary for dashboard KPIs.""" + if not r: return jsonify({"avg_ms": 0, "requests_per_min": 0}) + recent = [] + for x in (r.lrange("routes:recent", 0, 49) or []): + try: recent.append(json.loads(x)) + except: pass + if not recent: return jsonify({"avg_ms": 0, "requests_per_min": 0, "count": 0}) + now = time.time() + last_min = [x for x in recent if now - x.get("ts", 0) < 60] + latencies = [x.get("queue_ms", 0) + x.get("inference_ms", 0) for x in last_min if "inference_ms" in x] + return jsonify({ + "avg_ms": round(sum(latencies) / len(latencies), 1) if latencies else 0, + "requests_per_min": len(last_min), + "count": len(recent) + }) +@app.route("/stream") +def stream(): + def ev(): + q = queue.Queue() + with sse_lock: sse_subscribers.append(q) + try: + yield "data: "+json.dumps(get_metrics())+"\n\n" + while True: + try: yield "data: "+q.get(timeout=3)+"\n\n" + except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" + except GeneratorExit: pass + finally: + with sse_lock: + if q in sse_subscribers: sse_subscribers.remove(q) + return Response(stream_with_context(ev()), mimetype="text/event-stream", + headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) + +# ── Phase 0: Admin Key Management ── +ADMIN_KEY = os.environ.get("ADMIN_KEY", "") + +def _admin_auth(): + """Require admin key for management endpoints.""" + if not ADMIN_KEY: + return False, "ADMIN_KEY not configured on server" + ak = request.headers.get("Authorization","").replace("Bearer ","") + if ak != ADMIN_KEY: + return False, "Admin key required" + return True, None + +@app.route("/admin/keys") +def admin_keys(): + """List all API keys (masked) with agent, tier, and deprecation status.""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + keys = [] + for key, info in API_KEYS.items(): + masked = key[:8] + "..." + key[-8:] + keys.append({ + "masked": masked, + "prefix": key[:8], + "agent": info["agent"], + "tier": info["tier"], + "deprecated": info.get("deprecated", False), + "length": len(key) + }) + return jsonify({ + "total": len(keys), + "active": sum(1 for k in keys if not k["deprecated"]), + "deprecated": sum(1 for k in keys if k["deprecated"]), + "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"])) + }) + +@app.route("/admin/keys/deprecation-summary") +def admin_deprecation_summary(): + """Summary of deprecated key usage (from Redis logs, if available).""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + deprecated_agents = [] + for key, info in API_KEYS.items(): + if info.get("deprecated"): + # Check Redis for usage count + count = 0 + if r: + count = int(r.get("deprecated_usage:" + info["agent"]) or 0) + deprecated_agents.append({ + "agent": info["agent"], + "deprecated_uses": count, + "needs_migration": count > 0 + }) + return jsonify({ + "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]), + "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage" + }) + +@app.route("/admin/keys/generate", methods=["POST"]) +def admin_generate_key(): + """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "").strip() + tier = body.get("tier", "enterprise") + if not agent: + return jsonify({"error": "agent field required"}), 400 + if tier not in ("starter", "professional", "enterprise"): + return jsonify({"error": "tier must be starter/professional/enterprise"}), 400 + # Generate secure key + import secrets, hashlib + prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8] + suffix = secrets.token_hex(20) + new_key = f"sk-{prefix}-{suffix}" + # Update in-memory dict (note: not persisted across restarts without env var update) + API_KEYS[new_key] = {"tier": tier, "agent": agent} + log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:]) + return jsonify({ + "agent": agent, + "tier": tier, + "key": new_key, + "masked": new_key[:8] + "..." + new_key[-8:], + "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist." + }), 201 + +@app.route("/admin/keys/revoke", methods=["POST"]) +def admin_revoke_key(): + """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "") + key_prefix = body.get("key_prefix", "") + revoked = [] + keys_to_remove = [] + for key, info in API_KEYS.items(): + if not info.get("deprecated"): + continue + if agent and info["agent"] == agent: + keys_to_remove.append(key) + elif key_prefix and key.startswith(key_prefix): + keys_to_remove.append(key) + for key in keys_to_remove: + info = API_KEYS.pop(key) + revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]}) + log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:]) + return jsonify({ + "revoked": len(revoked), + "keys": revoked, + "remaining_total": len(API_KEYS), + "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist." + }) + +if __name__ == "__main__": + log.info("Router on :9000 (load-aware)") + app.run(host="0.0.0.0", port=9000, debug=False)