diff --git a/dashboard/gpu-monitor.html b/dashboard/gpu-monitor.html deleted file mode 100644 index 80f32fe..0000000 --- a/dashboard/gpu-monitor.html +++ /dev/null @@ -1,135 +0,0 @@ - - - - - -Syslog GPU Monitor - - - -

⚡ Syslog GPU Monitor

-

Real-time GPU health — loading...

- -
-
-
GPUs Online
-
-
Active Requests
-
-
Circuit Trips
-
-
Slots Used
-
- -
- - - - - - diff --git a/docker-compose.yml.bak.20260616-215242 b/docker-compose.yml.bak.20260616-215242 deleted file mode 100644 index 5823ca4..0000000 --- a/docker-compose.yml.bak.20260616-215242 +++ /dev/null @@ -1,102 +0,0 @@ -version: '3.8' - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: ghcr.io/berriai/litellm:main-stable - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "127.0.0.1:8081:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - environment: - - LITELLM_MASTER_KEY=sk-sys...-key - extra_hosts: - - "host.docker.internal:host-gateway" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - -# LiteLLM command override to load config -# (appended to fix config loading issue) diff --git a/docker-compose.yml.bak.20260617-104030 b/docker-compose.yml.bak.20260617-104030 deleted file mode 100644 index 497d407..0000000 --- a/docker-compose.yml.bak.20260617-104030 +++ /dev/null @@ -1,131 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - container_name: harness-postgres - restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_DB=litellm - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm"] - interval: 5s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: ghcr.io/berriai/litellm:main-stable - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "4000:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - environment: - - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 - - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 - - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm - - STORE_MODEL_IN_DB=True - - LITELLM_UI_USERNAME=admin - - LITELLM_UI_PASSWORD=syslog-admin-2026 - - OPENAI_API_KEY=not-used - - PROXY_BASE_URL=http://192.168.68.116/litellm - - ANTHROPIC_API_KEY=not-used - extra_hosts: - - "host.docker.internal:host-gateway" - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - extra_hosts: - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - pgdata: diff --git a/docker-compose.yml.bak.20260625-192313-fix b/docker-compose.yml.bak.20260625-192313-fix deleted file mode 100644 index 97d7be9..0000000 --- a/docker-compose.yml.bak.20260625-192313-fix +++ /dev/null @@ -1,142 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - container_name: harness-postgres - restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_DB=litellm - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm"] - interval: 5s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "4001:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro - environment: - - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 - - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 - - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm - - STORE_MODEL_IN_DB=True - - LITELLM_UI_USERNAME=admin - - LITELLM_UI_PASSWORD=syslog-admin-2026 - - UI_USERNAME=admin - - UI_PASSWORD=syslog-admin-2026 - - OPENAI_API_KEY=not-used - - PROXY_BASE_URL=https://litellm.sysloggh.net - - DOCS_URL=/litellm/docs - - ANTHROPIC_API_KEY=not-used - - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU - - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe - - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ - - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ - - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ - - GENERIC_SCOPE=openid email profile - - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin - - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - extra_hosts: - - "host.docker.internal:host-gateway" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - extra_hosts: - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - pgdata: diff --git a/docker-compose.yml.bak.20260625-195716-remove-docs-url b/docker-compose.yml.bak.20260625-195716-remove-docs-url deleted file mode 100644 index 209addd..0000000 --- a/docker-compose.yml.bak.20260625-195716-remove-docs-url +++ /dev/null @@ -1,143 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - container_name: harness-postgres - restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_DB=litellm - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm"] - interval: 5s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "4001:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro - environment: - - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 - - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 - - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm - - STORE_MODEL_IN_DB=True - - LITELLM_UI_USERNAME=admin - - LITELLM_UI_PASSWORD=syslog-admin-2026 - - UI_USERNAME=admin - - UI_PASSWORD=syslog-admin-2026 - - OPENAI_API_KEY=not-used - - PROXY_BASE_URL=https://litellm.sysloggh.net - - DOCS_URL=/litellm/docs - - ANTHROPIC_API_KEY=not-used - - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU - - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe - - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ - - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ - - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ - - GENERIC_SCOPE=openid email profile - - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin - - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - extra_hosts: - - "host.docker.internal:host-gateway" - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - extra_hosts: - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - pgdata: diff --git a/docker-compose.yml.bak.20260625-195847-set-docs b/docker-compose.yml.bak.20260625-195847-set-docs deleted file mode 100644 index aa7d965..0000000 --- a/docker-compose.yml.bak.20260625-195847-set-docs +++ /dev/null @@ -1,142 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - container_name: harness-postgres - restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_DB=litellm - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm"] - interval: 5s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "4001:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro - environment: - - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 - - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 - - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm - - STORE_MODEL_IN_DB=True - - LITELLM_UI_USERNAME=admin - - LITELLM_UI_PASSWORD=syslog-admin-2026 - - UI_USERNAME=admin - - UI_PASSWORD=syslog-admin-2026 - - OPENAI_API_KEY=not-used - - PROXY_BASE_URL=https://litellm.sysloggh.net - - ANTHROPIC_API_KEY=not-used - - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU - - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe - - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ - - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ - - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ - - GENERIC_SCOPE=openid email profile - - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin - - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - extra_hosts: - - "host.docker.internal:host-gateway" - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - extra_hosts: - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - pgdata: diff --git a/docker-compose.yml.bak.20260625-202429-auth-proxy b/docker-compose.yml.bak.20260625-202429-auth-proxy deleted file mode 100644 index b003a0a..0000000 --- a/docker-compose.yml.bak.20260625-202429-auth-proxy +++ /dev/null @@ -1,143 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis:7-alpine - container_name: harness-redis - restart: unless-stopped - ports: - - "127.0.0.1:6379:6379" - volumes: - - redis-data:/data - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - container_name: harness-postgres - restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" - environment: - - POSTGRES_DB=litellm - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=d9fc143e3dc1a7a8e672c359fea95c5e - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm"] - interval: 5s - timeout: 3s - retries: 5 - - router: - build: ./router - container_name: harness-router - restart: unless-stopped - ports: - - "127.0.0.1:9000:9000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_MOE_URL=http://192.168.68.15:8080/v1 - - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}} - - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - redis: - condition: service_healthy - - litellm: - image: docker.litellm.ai/berriai/litellm:1.90.0-rc.1 - command: ["--config", "/app/config.yaml", "--port", "4000"] - container_name: harness-litellm - restart: unless-stopped - ports: - - "4001:4000" - volumes: - - ./litellm_config.yaml:/app/config.yaml - - /opt/combined-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro - environment: - - LITELLM_MASTER_KEY=sk-litellm-7f96080dd99b15c36bd4b333b58a6796 - - ROUTER_API_KEY=sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64 - - DATABASE_URL=postgresql://litellm:d9fc143e3dc1a7a8e672c359fea95c5e@postgres:5432/litellm - - STORE_MODEL_IN_DB=True - - LITELLM_UI_USERNAME=admin - - LITELLM_UI_PASSWORD=syslog-admin-2026 - - UI_USERNAME=admin - - UI_PASSWORD=syslog-admin-2026 - - OPENAI_API_KEY=not-used - - PROXY_BASE_URL=https://litellm.sysloggh.net - - DOCS_URL=/docs - - ANTHROPIC_API_KEY=not-used - - GENERIC_CLIENT_ID=FHd7bs9dP5gHad2Ki23iUL5kQvFa0GRaj3nlLnNU - - GENERIC_CLIENT_SECRET=aDkXQx82duqpxc98xxqp0quzzUf1mawsnOTqj7sx1acaS7rWSt02N5ksBCi92n8ZilRavigoYME6fLakP20Ixc9H2pxnSZFiOqQLb7BPi8UtsvfxmzXklD0HJIdbKFxe - - GENERIC_AUTHORIZATION_ENDPOINT=https://auth.sysloggh.net/application/o/authorize/ - - GENERIC_TOKEN_ENDPOINT=https://auth.sysloggh.net/application/o/token/ - - GENERIC_USERINFO_ENDPOINT=https://auth.sysloggh.net/application/o/userinfo/ - - GENERIC_SCOPE=openid email profile - - GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE=proxy_admin - - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - extra_hosts: - - "host.docker.internal:host-gateway" - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - - nginx: - image: nginx:alpine - container_name: harness-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./dashboard:/opt/inference-harness/dashboard:ro - extra_hosts: - - "auth.sysloggh.net:192.168.68.11" - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 30s - timeout: 15s - retries: 3 - depends_on: - - litellm - - dashboard - - dashboard: - build: ./dashboard - container_name: harness-dashboard - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - - REDIS_URL=redis://redis:6379 - - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] - interval: 15s - timeout: 5s - retries: 3 - depends_on: - - redis - -volumes: - redis-data: - pgdata: diff --git a/litellm_config.yaml.bak.20260617-110719 b/litellm_config.yaml.bak.20260617-110719 deleted file mode 100644 index 6cf03d8..0000000 --- a/litellm_config.yaml.bak.20260617-110719 +++ /dev/null @@ -1,107 +0,0 @@ -# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture -# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000 -# Last updated: 2026-06-16 - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - # database_url: using DATABASE_URL env var instead - store_model_in_db: true - -model_list: - # Content-based auto-routing (router picks GPU via 5-tier analysis) - - model_name: syslog-auto - litellm_params: - model: openai/syslog-auto - api_base: http://router:9000/v1 - api_key: os.environ/ROUTER_API_KEY - rpm: 600 - - # Individual GPU strict passthrough (exact GPU, no silent fallback) - - model_name: qwen3.6-35B-A3B - litellm_params: - model: openai/qwen3.6-35B-A3B - api_base: http://router:9000/v1 - api_key: os.environ/ROUTER_API_KEY - - - model_name: qwen3.6-27B-code - litellm_params: - model: openai/qwen3.6-27B-code - api_base: http://router:9000/v1 - api_key: os.environ/ROUTER_API_KEY - - - model_name: gemma-4-12b - litellm_params: - model: openai/gemma-4-12b - api_base: http://router:9000/v1 - api_key: os.environ/ROUTER_API_KEY - -# Guardrails: Pre-call and post-call content moderation -guardrails: - - guardrail_name: "input-moderation" - litellm_params: - guardrail: openai_moderation - mode: "pre_call" - - - guardrail_name: "output-moderation" - litellm_params: - guardrail: openai_moderation - mode: "post_call" - - - guardrail_name: "harmful-content-filter" - litellm_params: - guardrail: litellm_content_filter - mode: "pre_call" - categories: - - category: "harmful_self_harm" - enabled: true - action: "BLOCK" - severity_threshold: "medium" - - category: "harmful_violence" - enabled: true - action: "BLOCK" - severity_threshold: "medium" - - category: "harmful_illegal_weapons" - enabled: true - action: "BLOCK" - severity_threshold: "medium" - -litellm_settings: - num_retries: 0 # Disabled — our router handles retry logic - request_timeout: 600 # Match 10-min llama-server timeout - set_verbose: true - failure_callback: ["prometheus"] # Export metrics to Prometheus - -router_settings: - routing_strategy: "usage-based-routing" # For external models only - enable_loadbalancing_on_proxy: false # Disable LiteLLM internal LB - allowed_fails: 100 # Router returns 503 on saturated GPUs - # Fallback chains: LiteLLM retries down the chain when router returns saturated. - # This gives accurate per-model metrics because router no longer silently reroutes. - # The router's circuit breaker prevents cascading failures to dead GPUs. - fallbacks: - - syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"] - - qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"] - - qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"] - - gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"] - -# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional) -litellm_settings: - model_cost: - syslog-auto: - input_cost_per_token: 0.0 - output_cost_per_token: 0.0 - qwen3.6-35B-A3B: - input_cost_per_token: 0.0 - output_cost_per_token: 0.0 - qwen3.6-27B-code: - input_cost_per_token: 0.0 - output_cost_per_token: 0.0 - gemma-4-12b: - input_cost_per_token: 0.0 - output_cost_per_token: 0.0 - # For internal cost allocation, set symbolic rates: - # e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens - -# SSO/OIDC Configuration -litellm_settings: - sso_callback: "/sso/callback" diff --git a/litellm_config.yaml.bak.poc b/litellm_config.yaml.bak.poc deleted file mode 100644 index 244fde4..0000000 --- a/litellm_config.yaml.bak.poc +++ /dev/null @@ -1,25 +0,0 @@ -model_list: - - model_name: qwen3.6-35B-A3B - litellm_params: - model: openai/qwen3.6-35B-A3B - api_base: http://192.168.68.15:8080/v1 - api_key: "not-needed" - - - model_name: qwen3.6-27B-code - litellm_params: - model: openai/qwen3.6-27B-code-text - api_base: http://192.168.68.8:8080/v1 - api_key: "not-needed" - - - model_name: gemma-4-12b - litellm_params: - model: openai/gemma-4-12b - api_base: http://192.168.68.110:8080/v1 - api_key: "not-needed" - -general_settings: - master_key: sk-syslog-local-master-key - -litellm_settings: - drop_params: true - request_timeout: 120 diff --git a/nginx/default.conf b/nginx/default.conf deleted file mode 100644 index e69de29..0000000 diff --git a/nginx/nginx.conf.bak.20260616-220643 b/nginx/nginx.conf.bak.20260616-220643 deleted file mode 100644 index 295186e..0000000 --- a/nginx/nginx.conf.bak.20260616-220643 +++ /dev/null @@ -1,124 +0,0 @@ -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { worker_connections 1024; } - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" rt=$request_time'; - access_log /var/log/nginx/access.log main; - error_log /var/log/nginx/error.log; - sendfile on; - keepalive_timeout 65; - - upstream router_api { server router:9000; } - upstream dashboard_ui { server dashboard:3000; } - upstream litellm_backend { server litellm:4000; } - - server { - listen 80; - - # Security headers - add_header X-Content-Type-Options nosniff always; - add_header X-Frame-Options SAMEORIGIN always; - add_header X-XSS-Protection "1; mode=block" always; - - # Disable buffering for SSE streams - proxy_buffering off; - - # API through router - location /v1/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location /admin/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - } - - # SSE streaming endpoint - location /stream { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header Connection ""; - proxy_buffering off; - chunked_transfer_encoding off; - } - - # Dashboard API proxy for SSE - location /api/ { - proxy_pass http://dashboard_ui; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_buffering off; - } - - # LiteLLM debug - location /litellm/ { - rewrite ^/litellm/(.*) /$1 break; - proxy_pass http://litellm_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header Authorization $http_authorization; - } - - # Professional Dashboard (Phase 1-3) - Static HTML served via Nginx - location /dashboard/ { - alias /opt/inference-harness/dashboard/; - index dashboard.html; - add_header Cache-Control "public, max-age=3600"; - add_header X-Content-Type-Options nosniff; - } - - # Legacy Dashboard (root) - Proxy to Flask app - location / { - proxy_pass http://dashboard_ui; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_buffering off; - } - - # Performance analytics - location /metrics/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # Circuit Breaker metrics (Phase 1) - location /metrics/circuit-breaker { - proxy_pass http://router_api/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health/unified { - proxy_pass http://router_api/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health { - proxy_pass http://router_api/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - } -} diff --git a/nginx/nginx.conf.bak.20260617-104030 b/nginx/nginx.conf.bak.20260617-104030 deleted file mode 100644 index 7f83b04..0000000 --- a/nginx/nginx.conf.bak.20260617-104030 +++ /dev/null @@ -1,162 +0,0 @@ -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - - upstream router_api { server router:9000; } - upstream dashboard_ui { server dashboard:3000; } - upstream litellm_backend { server litellm:4000; } - - server { - listen 80; - - # Authentik OIDC subrequest - location /authentik/auth { - internal; - proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # 2-Layer: LiteLLM (Layer 1) -> Router (Layer 2) -> GPUs - # /v1/ routes to router (all existing keys work) - # Agents using new LiteLLM keys: change OPENAI_API_BASE to /litellm/v1 - location /v1/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - error_page 502 503 = @router_fallback; - } - - location @router_fallback { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /admin/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /stream { - proxy_pass http://router_api/stream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_buffering off; - } - - location /api/ { - proxy_pass http://router_api/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM gateway access (for agents with new virtual keys) - location /litellm/v1/ { - proxy_pass http://litellm_backend/v1/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # LiteLLM UI static assets - location /litellm-asset-prefix/ { - proxy_pass http://litellm_backend/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM Admin UI (WebSocket support for live updates) - location /litellm/ { - proxy_pass http://litellm_backend/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - # Rewrite all redirects to include /litellm/ prefix - proxy_redirect http://172.18.0.7:4000/ /litellm/; - proxy_redirect http://127.0.0.1:4000/ /litellm/; - } - - location /dashboard/ { - proxy_pass http://dashboard_ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location / { - root /opt/inference-harness/dashboard; - try_files $uri $uri/ /index.html; - } - - location /metrics/ { - proxy_pass http://router_api/metrics/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /metrics/circuit-breaker { - proxy_pass http://router_api/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /router/ { - proxy_pass http://router_api/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /ui/ { - return 302 http://192.168.68.116:4000/ui/; - } - - location /health/unified { - proxy_pass http://router_api/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /ui/ { - return 302 http://192.168.68.116:4000/ui/; - } - - location /health { - proxy_pass http://router_api/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - } -} diff --git a/nginx/nginx.conf.bak.20260624_174945 b/nginx/nginx.conf.bak.20260624_174945 deleted file mode 100644 index c31e840..0000000 --- a/nginx/nginx.conf.bak.20260624_174945 +++ /dev/null @@ -1,351 +0,0 @@ -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - - # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. - # Without this, nginx resolves upstream hostnames at config load time, - # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. - resolver 127.0.0.11 valid=30s; - - # Dynamic upstream resolution via nginx variables. - # Using $var in proxy_pass forces request-time resolution through the resolver. - # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. - map $host $router_api_url { - default http://harness-router:9000; - } - map $host $dashboard_ui_url { - default http://harness-dashboard:3000; - } - map $host $litellm_backend_url { - default http://harness-litellm:4000; - } - - # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). - # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, - # preventing the cross-origin localStorage footgun that traps the UI at the login page. - map $http_cf_connecting_ip $is_cloudflared { - default 1; # any non-empty value = request came through Cloudflare - "" 0; # empty = direct access - } - - # ════════════════════════════════════════════════════════════════ - # Server :80 — existing harness entrypoint - # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), - # router fallback, LiteLLM UI via /litellm/ prefix, health - # ════════════════════════════════════════════════════════════════ - server { - listen 80; - - # Authentik OIDC subrequest - location /authentik/auth { - internal; - proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # 2-Layer: /v1/ → router (existing keys work unchanged) - location /v1/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - error_page 502 503 = @router_fallback; - } - - location @router_fallback { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /admin/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /stream { - proxy_pass $router_api_url/stream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_buffering off; - } - - location /api/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM gateway access (agents with new virtual keys) - location /litellm/v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # LiteLLM UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) - # LiteLLM Admin UI at /ui/ (must be before catch-all /) - location /ui/ { - proxy_pass http://litellm_backend/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - error_page 301 302 = @ui_redirect; - } - - location @ui_redirect { - proxy_pass http://litellm_backend/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_buffering off; - } - - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_read_timeout 600s; - } - - # LiteLLM UI static assets on port 80 - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) - location /litellm/ { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://$host/ /litellm/; - proxy_redirect https://$host/ /litellm/; - } - - location /dashboard/ { - proxy_pass $dashboard_ui_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # Dedicated /openapi.json block — must come before catch-all / - # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, - # but the catch-all location / strips the URI path. This block - # preserves the full path so the spec JSON is returned instead of - # the Swagger UI SPA HTML. - location /openapi.json { - proxy_pass $litellm_backend_url/openapi.json; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location /metrics/ { - proxy_pass $router_api_url/metrics/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /metrics/circuit-breaker { - proxy_pass $router_api_url/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /router/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health/unified { - proxy_pass $router_api_url/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health { - proxy_pass $router_api_url/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - } - - # ════════════════════════════════════════════════════════════════ - # Server :4000 — external LiteLLM entrypoint (cloudflared target) - # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM - # builds external URLs instead of leaking 192.168.68.116:4000. - # LiteLLM itself is now bound to 127.0.0.1 only (see compose). - # ════════════════════════════════════════════════════════════════ - server { - listen 4000; - - # Canonical external identity — overrides whatever Host cloudflared sends - proxy_set_header Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port 443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - - # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, - # so a relative return would be absolutized to http://192.168.68.116:4000/). - location = /ui { return 301 https://litellm.sysloggh.net/ui/; } - - # LiteLLM Admin UI + WebSocket - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; - proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; - } - - # UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_read_timeout 600s; - } - - # API - location /v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # Key management + admin API - location /key/ { - proxy_pass $litellm_backend_url/key/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /user/ { - proxy_pass $litellm_backend_url/user/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /model/ { - proxy_pass $litellm_backend_url/model/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /team/ { - proxy_pass $litellm_backend_url/team/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - - # Health - location /health { - proxy_pass $litellm_backend_url/health; - proxy_http_version 1.1; - } - - # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; - proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; - } - } -} diff --git a/nginx/nginx.conf.bak.20260624_201123 b/nginx/nginx.conf.bak.20260624_201123 deleted file mode 100644 index 1338f75..0000000 --- a/nginx/nginx.conf.bak.20260624_201123 +++ /dev/null @@ -1,262 +0,0 @@ -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - - upstream router_api { server router:9000; } - upstream dashboard_ui { server dashboard:3000; } - upstream litellm_backend { server litellm:4000; } - - # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). - # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, - # preventing the cross-origin localStorage footgun that traps the UI at the login page. - map $http_cf_connecting_ip $is_cloudflared { - default 1; # any non-empty value = request came through Cloudflare - "" 0; # empty = direct access - } - - # ════════════════════════════════════════════════════════════════ - # Server :80 — existing harness entrypoint - # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), - # router fallback, LiteLLM UI via /litellm/ prefix, health - # ════════════════════════════════════════════════════════════════ - server { - listen 80; - - # Authentik OIDC subrequest - location /authentik/auth { - internal; - proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # 2-Layer: /v1/ → router (existing keys work unchanged) - location /v1/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - error_page 502 503 = @router_fallback; - } - - location @router_fallback { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /admin/ { - proxy_pass http://router_api; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /stream { - proxy_pass http://router_api/stream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_buffering off; - } - - location /api/ { - proxy_pass http://router_api/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM gateway access (agents with new virtual keys) - location /litellm/v1/ { - proxy_pass http://litellm_backend/v1/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # LiteLLM UI static assets - location /litellm-asset-prefix/ { - proxy_pass http://litellm_backend/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) - location /litellm/ { - proxy_pass http://litellm_backend/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://$host/ /litellm/; - proxy_redirect https://$host/ /litellm/; - } - - location /dashboard/ { - proxy_pass http://dashboard_ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location / { - root /opt/inference-harness/dashboard; - try_files $uri $uri/ /index.html; - } - - location /metrics/ { - proxy_pass http://router_api/metrics/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /metrics/circuit-breaker { - proxy_pass http://router_api/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /router/ { - proxy_pass http://router_api/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health/unified { - proxy_pass http://router_api/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health { - proxy_pass http://router_api/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - } - - # ════════════════════════════════════════════════════════════════ - # Server :4000 — external LiteLLM entrypoint (cloudflared target) - # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM - # builds external URLs instead of leaking 192.168.68.116:4000. - # LiteLLM itself is now bound to 127.0.0.1 only (see compose). - # ════════════════════════════════════════════════════════════════ - server { - listen 4000; - - # Canonical external identity — overrides whatever Host cloudflared sends - proxy_set_header Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port 443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - - # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, - # so a relative return would be absolutized to http://192.168.68.116:4000/). - location = /ui { return 301 https://litellm.sysloggh.net/ui/; } - - # LiteLLM Admin UI + WebSocket - location /ui/ { - proxy_pass http://litellm_backend/ui/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://litellm_backend/ https://litellm.sysloggh.net/; - proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; - } - - # UI static assets - location /litellm-asset-prefix/ { - proxy_pass http://litellm_backend/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # SSO callback - location /sso/ { - proxy_pass http://litellm_backend/sso/; - proxy_http_version 1.1; - proxy_read_timeout 600s; - } - - # API - location /v1/ { - proxy_pass http://litellm_backend/v1/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # Key management + admin API - location /key/ { - proxy_pass http://litellm_backend/key/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /user/ { - proxy_pass http://litellm_backend/user/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /model/ { - proxy_pass http://litellm_backend/model/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /team/ { - proxy_pass http://litellm_backend/team/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - - # Health - location /health { - proxy_pass http://litellm_backend/health; - proxy_http_version 1.1; - } - - # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) - location / { - proxy_pass http://litellm_backend/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; - proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; - } - } -} diff --git a/nginx/nginx.conf.bak.predup2 b/nginx/nginx.conf.bak.predup2 deleted file mode 100644 index c331911..0000000 --- a/nginx/nginx.conf.bak.predup2 +++ /dev/null @@ -1,351 +0,0 @@ -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - - # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. - # Without this, nginx resolves upstream hostnames at config load time, - # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. - resolver 127.0.0.11 valid=30s; - - # Dynamic upstream resolution via nginx variables. - # Using $var in proxy_pass forces request-time resolution through the resolver. - # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. - map $host $router_api_url { - default http://harness-router:9000; - } - map $host $dashboard_ui_url { - default http://harness-dashboard:3000; - } - map $host $litellm_backend_url { - default http://harness-litellm:4000; - } - - # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). - # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, - # preventing the cross-origin localStorage footgun that traps the UI at the login page. - map $http_cf_connecting_ip $is_cloudflared { - default 1; # any non-empty value = request came through Cloudflare - "" 0; # empty = direct access - } - - # ════════════════════════════════════════════════════════════════ - # Server :80 — existing harness entrypoint - # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), - # router fallback, LiteLLM UI via /litellm/ prefix, health - # ════════════════════════════════════════════════════════════════ - server { - listen 80; - - # Authentik OIDC subrequest - location /authentik/auth { - internal; - proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # 2-Layer: /v1/ → router (existing keys work unchanged) - location /v1/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - error_page 502 503 = @router_fallback; - } - - location @router_fallback { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /admin/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /stream { - proxy_pass $router_api_url/stream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_buffering off; - } - - location /api/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM gateway access (agents with new virtual keys) - location /litellm/v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # LiteLLM UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) - # LiteLLM Admin UI at /ui/ (must be before catch-all /) - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - error_page 301 302 = @ui_redirect; - } - - location @ui_redirect { - proxy_pass $litellm_backend_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_buffering off; - } - - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_read_timeout 600s; - } - - # LiteLLM UI static assets on port 80 - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) - location /litellm/ { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://$host/ /litellm/; - proxy_redirect https://$host/ /litellm/; - } - - location /dashboard/ { - proxy_pass $dashboard_ui_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # Dedicated /openapi.json block — must come before catch-all / - # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, - # but the catch-all location / strips the URI path. This block - # preserves the full path so the spec JSON is returned instead of - # the Swagger UI SPA HTML. - location /openapi.json { - proxy_pass $litellm_backend_url/openapi.json; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location /metrics/ { - proxy_pass $router_api_url/metrics/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /metrics/circuit-breaker { - proxy_pass $router_api_url/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /router/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health/unified { - proxy_pass $router_api_url/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health { - proxy_pass $router_api_url/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - } - - # ════════════════════════════════════════════════════════════════ - # Server :4000 — external LiteLLM entrypoint (cloudflared target) - # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM - # builds external URLs instead of leaking 192.168.68.116:4000. - # LiteLLM itself is now bound to 127.0.0.1 only (see compose). - # ════════════════════════════════════════════════════════════════ - server { - listen 4000; - - # Canonical external identity — overrides whatever Host cloudflared sends - proxy_set_header Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port 443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - - # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, - # so a relative return would be absolutized to http://192.168.68.116:4000/). - location = /ui { return 301 https://litellm.sysloggh.net/ui/; } - - # LiteLLM Admin UI + WebSocket - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; - proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; - } - - # UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_read_timeout 600s; - } - - # API - location /v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # Key management + admin API - location /key/ { - proxy_pass $litellm_backend_url/key/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /user/ { - proxy_pass $litellm_backend_url/user/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /model/ { - proxy_pass $litellm_backend_url/model/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /team/ { - proxy_pass $litellm_backend_url/team/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - - # Health - location /health { - proxy_pass $litellm_backend_url/health; - proxy_http_version 1.1; - } - - # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; - proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; - } - } -} diff --git a/nginx/nginx.conf.bak.preui b/nginx/nginx.conf.bak.preui deleted file mode 100644 index 470844f..0000000 --- a/nginx/nginx.conf.bak.preui +++ /dev/null @@ -1,343 +0,0 @@ -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - - # Docker DNS resolver — forces request-time resolution for variable-based proxy_pass. - # Without this, nginx resolves upstream hostnames at config load time, - # which fails when Docker DNS (127.0.0.11) isn't ready yet on container start. - resolver 127.0.0.11 valid=30s; - - # Dynamic upstream resolution via nginx variables. - # Using $var in proxy_pass forces request-time resolution through the resolver. - # Without this, 'host not found in upstream' crashes nginx when Docker DNS is slow. - map $host $router_api_url { - default http://harness-router:9000; - } - map $host $dashboard_ui_url { - default http://harness-dashboard:3000; - } - map $host $litellm_backend_url { - default http://harness-litellm:4000; - } - - # Detect Cloudflare Tunnel requests (cloudflared always sets CF-Connecting-IP). - # Direct LAN browser access to :4000 has no such header -> redirect to canonical https, - # preventing the cross-origin localStorage footgun that traps the UI at the login page. - map $http_cf_connecting_ip $is_cloudflared { - default 1; # any non-empty value = request came through Cloudflare - "" 0; # empty = direct access - } - - # ════════════════════════════════════════════════════════════════ - # Server :80 — existing harness entrypoint - # dashboard (/), router API (/v1/, /admin/, /stream, /api/, /metrics), - # router fallback, LiteLLM UI via /litellm/ prefix, health - # ════════════════════════════════════════════════════════════════ - server { - listen 80; - - # Authentik OIDC subrequest - location /authentik/auth { - internal; - proxy_pass https://auth.sysloggh.net/outpost.goauthentik.io/auth/nginx; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # 2-Layer: /v1/ → router (existing keys work unchanged) - location /v1/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - error_page 502 503 = @router_fallback; - } - - location @router_fallback { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /admin/ { - proxy_pass $router_api_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - } - - location /stream { - proxy_pass $router_api_url/stream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_buffering off; - } - - location /api/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM gateway access (agents with new virtual keys) - location /litellm/v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # LiteLLM UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # LiteLLM Admin UI at /ui/ (public access via Traefik → port 80) - # LiteLLM Admin UI at /ui/ (must be before catch-all /) - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - error_page 301 302 = @ui_redirect; - } - - location @ui_redirect { - proxy_pass $litellm_backend_url; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_buffering off; - } - - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_read_timeout 600s; - } - - - # LiteLLM Admin UI via /litellm/ prefix (LAN/internal access on :80) - location /litellm/ { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://$host/ /litellm/; - proxy_redirect https://$host/ /litellm/; - } - - location /dashboard/ { - proxy_pass $dashboard_ui_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - # Dedicated /openapi.json block — must come before catch-all / - # LiteLLM serves valid openapi: 3.1.0 JSON at this path internally, - # but the catch-all location / strips the URI path. This block - # preserves the full path so the spec JSON is returned instead of - # the Swagger UI SPA HTML. - location /openapi.json { - proxy_pass $litellm_backend_url/openapi.json; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Authorization $http_authorization; - proxy_connect_timeout 10s; - proxy_read_timeout 600s; - proxy_buffering off; - } - - location /metrics/ { - proxy_pass $router_api_url/metrics/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /metrics/circuit-breaker { - proxy_pass $router_api_url/metrics/circuit-breaker; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /router/ { - proxy_pass $router_api_url/; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health/unified { - proxy_pass $router_api_url/health/unified; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - - location /health { - proxy_pass $router_api_url/health; - proxy_http_version 1.1; - proxy_set_header Host $host; - } - } - - # ════════════════════════════════════════════════════════════════ - # Server :4000 — external LiteLLM entrypoint (cloudflared target) - # Fixes the /ui redirect: forces correct Host + scheme so LiteLLM - # builds external URLs instead of leaking 192.168.68.116:4000. - # LiteLLM itself is now bound to 127.0.0.1 only (see compose). - # ════════════════════════════════════════════════════════════════ - server { - listen 4000; - - # Canonical external identity — overrides whatever Host cloudflared sends - proxy_set_header Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Host litellm.sysloggh.net; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port 443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - - # /ui → /ui/ — absolute redirect (cloudflared rewrites Host to origin IP, - # so a relative return would be absolutized to http://192.168.68.116:4000/). - location = /ui { return 301 https://litellm.sysloggh.net/ui/; } - - # LiteLLM Admin UI + WebSocket - location /ui/ { - proxy_pass $litellm_backend_url/ui/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect $litellm_backend_url/ https://litellm.sysloggh.net/; - proxy_redirect http://litellm.sysloggh.net:4000/ https://litellm.sysloggh.net/; - } - - # UI static assets - location /litellm-asset-prefix/ { - proxy_pass $litellm_backend_url/litellm-asset-prefix/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - - # SSO callback - location /sso/ { - proxy_pass $litellm_backend_url/sso/; - proxy_http_version 1.1; - proxy_read_timeout 600s; - } - - # API - location /v1/ { - proxy_pass $litellm_backend_url/v1/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - proxy_read_timeout 600s; - proxy_buffering off; - } - - # Key management + admin API - location /key/ { - proxy_pass $litellm_backend_url/key/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /user/ { - proxy_pass $litellm_backend_url/user/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /model/ { - proxy_pass $litellm_backend_url/model/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - location /team/ { - proxy_pass $litellm_backend_url/team/; - proxy_http_version 1.1; - proxy_set_header Authorization $http_authorization; - } - - # Health - location /health { - proxy_pass $litellm_backend_url/health; - proxy_http_version 1.1; - } - - # Root + everything else → LiteLLM (UI index, /configs, /spend, etc.) - location / { - proxy_pass $litellm_backend_url/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 86400s; - proxy_buffering off; - proxy_redirect http://192.168.68.116:4000/ https://litellm.sysloggh.net/; - proxy_redirect https://192.168.68.116:4000/ https://litellm.sysloggh.net/; - } - } -} diff --git a/router/router-fixed.py b/router/router-fixed.py deleted file mode 100644 index 97d0f0e..0000000 --- a/router/router-fixed.py +++ /dev/null @@ -1,831 +0,0 @@ -import os, json, time, logging, traceback, threading, queue, statistics, math -import requests, redis -from flask import Flask, request, jsonify, Response, stream_with_context - -REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") -GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") -GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") -GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") - -GPU_SIDECARS = { - "qwen3.6-35B-A3B": "http://192.168.68.15:8090", - "qwen3.6-27B-code": "http://192.168.68.8:8090", - "gemma-4-12b": "http://192.168.68.110:8090", -} -GPU_URLS = { - "qwen3.6-35B-A3B": GPU_MOE_URL, - "qwen3.6-27B-code": GPU_DENSE_URL, - "gemma-4-12b": GPU_LIGHT_URL, -} -# Max concurrent requests per GPU (based on llama.cpp --parallel) -GPU_MAX_CONCURRENT = { - "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) - "qwen3.6-27B-code": 2, # 2 slots - "gemma-4-12b": 2, # 2 slots (12GB VRAM, 4GB headroom) -} - -# Context window sizes (tokens) — used for compaction signals -GPU_CONTEXT = { - "qwen3.6-35B-A3B": 262144, - "qwen3.6-27B-code": 262144, - "gemma-4-12b": 262144, -} - -TIER_MODELS = { - "starter": ["gemma-4-12b"], - "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], - "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], -} -# ── PHASE 0: Dual-Key API Key System ── -# API_KEYS env var is REQUIRED (JSON string). No hardcoded fallback. -# Format: {"sk-xxx": {"tier": "enterprise", "agent": "Name"}} -# Deprecated keys get {"deprecated": true} — still accepted, logged with warning. -_raw_keys = os.environ.get("API_KEYS") -if not _raw_keys: - raise RuntimeError("FATAL: API_KEYS environment variable is required. " - "Set it in docker-compose.yml or .env file. " - "No hardcoded keys fallback — this is a security feature.") -API_KEYS = json.loads(_raw_keys) -log.info("Loaded %d API keys from env var (%d deprecated)", - len(API_KEYS), sum(1 for v in API_KEYS.values() if v.get("deprecated"))) -# Rate limits: requests per minute per API key tier -RATE_LIMIT_RPM = { - "enterprise": 120, - "professional": 60, - "starter": 20, -} - -def check_rate_limit(api_key, tier): - """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" - if not r: - return True, 999, 60 - limit = RATE_LIMIT_RPM.get(tier, 30) - key = f"ratelimit:{api_key}" - current = int(r.get(key) or 0) - if current >= limit: - ttl = r.ttl(key) - retry = max(ttl, 1) if ttl and ttl > 0 else 60 - return False, retry, 0 - pipe = r.pipeline() - pipe.incr(key) - pipe.expire(key, 60) # 1-minute sliding window - pipe.execute() - remaining = limit - (current + 1) - reset_seconds = r.ttl(key) or 60 - return True, remaining, reset_seconds - - -logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") -log = logging.getLogger("router") -try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() -except Exception: r = None - - -def counter_audit_loop(): - """Every 30s, check GPU slots and reset counters if all slots idle.""" - while True: - time.sleep(30) - if not r: continue - for model, url in GPU_URLS.items(): - try: - resp = requests.get(url.replace("/v1","") + "/slots", - headers={"Authorization": "Bearer not-needed"}, timeout=5) - if resp.status_code == 200: - slots = resp.json() - all_idle = all(not s.get("is_processing", False) for s in slots) - if all_idle: - current = int(r.get("active:" + model) or 0) - if current > 0: - r.set("active:" + model, 0) - log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) - except Exception: - pass - -threading.Thread(target=counter_audit_loop, daemon=True).start() - -app = Flask(__name__) -sse_subscribers = []; sse_lock = threading.Lock() - -def gpu_active_count(model): - """Get number of in-flight requests for a GPU.""" - if r: - return int(r.get("active:" + model) or 0) - return 0 - -def gpu_incr(model): - if r: r.incr("active:" + model) - -def gpu_decr(model): - if r: - v = r.decr("active:" + model) - if v and int(v) < 0: - r.set("active:" + model, 0) # never go negative - -def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): - url = GPU_SIDECARS.get(model) - if not url: return {"status": "unknown"} - try: - resp = requests.get(url, timeout=sidecar_timeout) - if resp.status_code == 200: - d = resp.json() - pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 - status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy() - vram_warning = pct >= 95 - # Also check if llama.cpp endpoint is actually responding - gpu_url = GPU_URLS.get(model, "") - try: - hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) - if hr.status_code != 200: - status = "down" - except Exception: - status = "down" - return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} - except Exception: pass - return {"status": "down"} - -def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] - -def estimate_tokens(msgs): - """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" - return len(json.dumps(msgs, default=str)) // 3.5 - -def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): - """Store detailed performance record in Redis for analytics.""" - if not r: return - try: - total_ms = queue_ms + inference_ms - tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 - rec = json.dumps({ - "ts": time.time(), - "model": model, "agent": agent, "tier": tier, "reason": reason, - "queue_ms": round(queue_ms, 1), - "inference_ms": round(inference_ms, 1), - "total_ms": round(total_ms, 1), - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "tokens_per_sec": round(tps, 1), - "stream": stream - }) - # Global recent list (last 500) - r.lpush("perf:recent", rec) - r.ltrim("perf:recent", 0, 499) - # Per-model list (last 200) - r.lpush("perf:model:" + model, rec) - r.ltrim("perf:model:" + model, 0, 199) - # Per-reason list (last 200) - r.lpush("perf:reason:" + reason, rec) - r.ltrim("perf:reason:" + reason, 0, 199) - # Per-agent list (last 200) - r.lpush("perf:agent:" + agent, rec) - r.ltrim("perf:agent:" + agent, 0, 199) - except Exception: - pass - -def is_gpu_busy(model): - """Check if GPU is at or near max concurrent capacity.""" - active = gpu_active_count(model) - max_c = GPU_MAX_CONCURRENT.get(model, 1) - return active >= max_c - -def select_best_gpu(candidates, reason, agent=""): - """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" - # Count how many distinct agents are on each GPU - gpu_agent_counts = {} - if r: - for m in GPU_URLS: - count = 0 - for ak in API_KEYS.values(): - if r.get("agent_gpu:" + ak["agent"] + ":" + m): - count += 1 - gpu_agent_counts[m] = count - # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) - for m in candidates: - if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: - return {"model": m, "reason": reason} - # Second pass: prefer GPU this agent is NOT already on (skip own GPU) - if agent: - for m in candidates: - if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): - return {"model": m, "reason": reason} - # Third pass: any non-busy GPU - for m in candidates: - if not is_gpu_busy(m): - return {"model": m, "reason": reason} - # All busy — pick least loaded - best = None - best_load = 999 - for m in candidates: - load = gpu_active_count(m) - if load < best_load: - best_load = load - best = m - if best: - return {"model": best, "reason": "load_balanced_" + reason} - return None - -def route(rd, tier, agent=""): - msgs = rd.get("messages",[]); t = estimate_tokens(msgs) - sys = any(m.get("role")=="system" for m in msgs) - turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) - hints = rd.get("routing_hints",{}) - allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) - avail = [m for m in available_models() if m in allowed] - if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} - # Check if all available GPUs are at max capacity - if all(is_gpu_busy(m) for m in avail): - return {"model": avail[0], "reason": "all_saturated", "saturated": True} - - req = rd.get("model","auto") - if req != "auto": - # STRICT MODE: no silent fallback — LiteLLM handles failover chains. - # This keeps per-model metrics accurate. Returns saturated if busy. - target = req if req in avail else avail[0] - if req not in avail: - return {"model": req, "reason": "explicit_unavailable", "saturated": True} - if is_gpu_busy(target): - return {"model": target, "reason": "explicit_saturated", "saturated": True} - return {"model": target, "reason": "explicit"} - - if hints: - if hints.get("priority")=="speed" and "gemma-4-12b" in avail: - return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} - if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: - return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} - - first_msg = msgs[0].get("content","") if msgs else "" - words = len(first_msg.split()) if isinstance(first_msg, str) else 99 - - # TIER 1: Lightweight — single-turn short queries → VLM (fastest) - if not sys and turns <= 1 and t <= 500 and words <= 100 and "gemma-4-12b" in avail: - if not is_gpu_busy("gemma-4-12b"): - return {"model":"gemma-4-12b","reason":"lightweight"} - # VLM busy — Dense is faster for short queries than MoE - fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(fallback, "lightweight_fallback", agent) - if result: return result - - # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat - if t <= 15000 and turns <= 12 and "gemma-4-12b" in avail: - if not is_gpu_busy("gemma-4-12b"): - return {"model":"gemma-4-12b","reason":"simple_conv"} - # VLM busy — fall back to Dense, then MoE - fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(fallback, "simple_conv_fallback", agent) - if result: return result - - # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance) - if t <= 25000: - candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(candidates, "medium", agent) - if result: return result - - # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback - if t > 25000: - candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] - result = select_best_gpu(candidates, "heavy_reasoning", agent) - if result: return result - - # TIER 5: Default — Dense primary, MoE fallback - candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(candidates, "default", agent) - if result: return result - return {"model":avail[0],"reason":"last_resort"} - -def clean_unicode(text): - if not isinstance(text, str): return text - text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") - text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'") - text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"') - text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ") - return text.encode("ascii", "ignore").decode("ascii") - -def clean_response(d): - if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} - if isinstance(d, list): return [clean_response(v) for v in d] - if isinstance(d, str): return clean_unicode(d) - return d - -def get_metrics(): - d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}} - for m in GPU_URLS: - h = check_gpu_health(m) - d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)}) - d["active_requests"][m] = gpu_active_count(m) - if r: - try: - for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) - for k,v in API_KEYS.items(): - c = int(r.get("routes:agent:"+v["agent"]) or 0) - if c>0: d["agent_counts"][v["agent"]] = c - for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) - raw = r.lrange("routes:recent",0,49) - d["recent"] = [json.loads(x) for x in raw] if raw else [] - except Exception: pass - return d - -def bcast(): - data = get_metrics(); payload = json.dumps(data) - with sse_lock: - dead = [] - for q in sse_subscribers: - try: q.put(payload) - except Exception: dead.append(q) - for q in dead: sse_subscribers.remove(q) - -QUEUE_TIMEOUT = int(os.environ.get("QUEUE_TIMEOUT", "30")) # max seconds to queue before 503 - -@app.route("/v1/chat/completions", methods=["POST"]) -def chat(): - try: - rd = request.get_json(force=True) - ak = request.headers.get("Authorization","").replace("Bearer ","") - if not ak or ak not in API_KEYS: - log.warning("AUTH_REJECTED: no/invalid API key from %s", request.remote_addr) - return jsonify({"error": "Unauthorized — valid API key required"}), 401 - ki = API_KEYS[ak] - tier, agent = ki["tier"], ki["agent"] - # Phase 0: dual-key transition — log deprecated key usage - if ki.get("deprecated"): - new_key = next((k for k, v in API_KEYS.items() - if v.get("agent") == agent and not v.get("deprecated")), None) - log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s", - agent, ak[:12], ak[-8:], - new_key[:12] if new_key else "N/A", - new_key[-8:] if new_key else "N/A") - if r: - r.incr("deprecated_usage:" + agent) - - # Rate limit check - allowed, rl_val, reset_sec = check_rate_limit(ak, tier) - if not allowed: - resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val}) - resp.headers["Retry-After"] = str(rl_val) - resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30)) - resp.headers["X-RateLimit-Remaining"] = "0" - resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val)) - log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:]) - return resp, 429 - - # Allow agent to override queue timeout via header - q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) - - # Cross-turn context tracking: accumulate tokens per session - session_id = request.headers.get("X-Session-Id", "") - session_tokens = 0 - if session_id and r: - try: - prev = int(r.get("session:" + session_id) or 0) - current = estimate_tokens(rd.get("messages",[])) - session_tokens = max(prev, current) # context only grows - r.set("session:" + session_id, session_tokens, ex=86400) # TTL 24h - except Exception: pass - - d = route(rd, tier, agent) - queue_start = time.time() - - # Queue loop: wait for a GPU slot instead of immediate 503 - while d.get("saturated"): - elapsed = time.time() - queue_start - if elapsed > q_timeout: - resp = jsonify({"error": "All GPUs saturated", "queued_s": round(elapsed,1), "retry_after_s": 5}) - resp.headers["Retry-After"] = "5" - log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed) - return resp, 503 - time.sleep(0.5) # poll every 500ms - d = route(rd, tier, agent) - - queue_ms = (time.time() - queue_start) * 1000 - if queue_ms > 500: - log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) - model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] - - # Stash rate limit values for response headers - _rl_remaining = rl_val - _rl_limit = RATE_LIMIT_RPM.get(tier, 30) - _rl_reset = reset_sec - is_stream = rd.get("stream", False) - - gpu_incr(model) - - log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) - # Track which GPU this agent is using (TTL 120s covers typical request) - if r and agent: - try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1") - except: pass - if r: - try: - r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) - r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) - r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)})) - r.ltrim("routes:recent",0,999) - except Exception: pass - start = time.time() - resp = requests.post(url+"/chat/completions", json=rd, - headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) - lat = int((time.time()-start)*1000) - gpu_decr(model) - - if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 - if is_stream: - # Buffer SSE chunks, handle split lines for large responses - chunks = [] - stream_timings = {} - buf = "" # accumulate partial lines - for raw in resp.iter_content(chunk_size=None, decode_unicode=True): - if raw: - cleaned = clean_unicode(raw) - chunks.append(cleaned) - buf += cleaned - # Process complete lines from buffer - while "\n" in buf: - line, buf = buf.split("\n", 1) - line = line.strip() - if line.startswith("data: ") and not stream_timings: - js = line[6:].strip() - if js.startswith("{") and "timings" in js and "predicted_n" in js: - try: - tj = json.loads(js).get("timings", {}) - if tj: - stream_timings = tj - except: pass - # Store perf record with real token counts from stream - if stream_timings: - pt = stream_timings.get("prompt_n", 0) - ct = stream_timings.get("predicted_n", 0) - tps = stream_timings.get("predicted_per_second", 0) - gen_ms = stream_timings.get("predicted_ms", lat) - store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True) - else: - store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True) - # Yield all chunks to client - def gen(): - for c in chunks: yield c - bcast() - ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) - ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 - ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) - sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") - sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit) - sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) - sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) - sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) - sse_resp.headers["X-Context-Warning"] = ctx_warning - sse_resp.headers["X-Context-Model"] = model - return sse_resp - data = clean_response(resp.json()) - for c in data.get("choices",[]): - msg = c.get("message",{}) - if not msg.get("content") and msg.get("reasoning_content"): - msg["content"] = msg["reasoning_content"] - # Extract performance data from llama.cpp response - usage = data.get("usage", {}) - timings = data.get("timings", {}) - prompt_tokens = usage.get("prompt_tokens", 0) - completion_tokens = usage.get("completion_tokens", 0) - inference_ms = lat # total GPU round-trip - store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False) - ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) - ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 - ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) - data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} - resp = jsonify(data) - resp.headers["X-RateLimit-Limit"] = str(_rl_limit) - resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) - resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) - resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) - resp.headers["X-Context-Warning"] = ctx_warning - resp.headers["X-Context-Model"] = model - bcast() - return resp - except requests.Timeout: - gpu_decr(model) - log.error("TIMEOUT: %s -> %s", agent, model) - return jsonify({"error":"timeout"}), 504 - except Exception as e: - gpu_decr(model) - log.error("Error: %s\n%s", e, traceback.format_exc()) - return jsonify({"error":str(e)}), 500 - -@app.route("/metrics/performance") -def performance(): - """Per-request performance analytics with percentiles per model/reason/agent.""" - if not r: return jsonify({"error": "Redis unavailable"}), 503 - try: - window_hours = int(request.args.get("window", "24")) - model_filter = request.args.get("model", "all") - - # Load recent records - cutoff = time.time() - (window_hours * 3600) - raw = r.lrange("perf:recent", 0, -1) - records = [] - for x in raw: - try: - rec = json.loads(x) - if rec["ts"] >= cutoff: - records.append(rec) - except: pass - - # Filter by model if specified - if model_filter != "all": - records = [r for r in records if r["model"] == model_filter] - - if not records: - return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}) - - def pct(values, p): - if len(values) < 2: return round(values[0], 1) if values else 0 - return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1) - - # Per-model stats - model_groups = {} - for rec in records: - m = rec["model"] - if m not in model_groups: model_groups[m] = [] - model_groups[m].append(rec) - - models = [] - for m, recs in sorted(model_groups.items()): - latencies = [r["total_ms"] for r in recs] - tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] - non_stream = [r for r in recs if not r["stream"]] - queue_times = [r["queue_ms"] for r in non_stream] - models.append({ - "model": m, - "count": len(recs), - "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1), - "latency": { - "avg": round(statistics.mean(latencies), 1), - "p50": pct(latencies, 50), - "p95": pct(latencies, 95), - "p99": pct(latencies, 99) - }, - "throughput": { - "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0, - "p50": pct(tps_vals, 50) if tps_vals else 0, - "p95": pct(tps_vals, 95) if tps_vals else 0, - }, - "queue": { - "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0, - "p95_ms": pct(queue_times, 95) if queue_times else 0, - } if queue_times else None - }) - - # Per-reason stats - reason_groups = {} - for rec in records: - rsn = rec["reason"] - if rsn not in reason_groups: reason_groups[rsn] = [] - reason_groups[rsn].append(rec) - - reasons = [] - for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])): - latencies = [r["total_ms"] for r in recs] - reasons.append({ - "reason": rsn, - "count": len(recs), - "avg_total_ms": round(statistics.mean(latencies), 1), - "p95_total_ms": pct(latencies, 95) - }) - - # Per-agent stats - agent_groups = {} - for rec in records: - ag = rec["agent"] - if ag not in agent_groups: agent_groups[ag] = [] - agent_groups[ag].append(rec) - - agents = [] - for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])): - latencies = [r["total_ms"] for r in recs] - tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] - agents.append({ - "agent": ag, - "count": len(recs), - "avg_total_ms": round(statistics.mean(latencies), 1), - "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0 - }) - - all_lat = [r["total_ms"] for r in records] - all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0] - summary = { - "total_requests": len(records), - "window_hours": window_hours, - "latency": { - "avg_ms": round(statistics.mean(all_lat), 1), - "p50_ms": pct(all_lat, 50), - "p95_ms": pct(all_lat, 95), - "p99_ms": pct(all_lat, 99) - }, - "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0 - } - - return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - -@app.route("/metrics/scatter") -def scatter(): - """Return individual data points for scatter plots (prompt_tokens vs latency).""" - if not r: return jsonify({"error": "Redis unavailable"}), 503 - try: - window_hours = int(request.args.get("window", "24")) - model_filter = request.args.get("model", "all") - cutoff = time.time() - (window_hours * 3600) - raw = r.lrange("perf:recent", 0, -1) - points = [] - for x in raw: - try: - rec = json.loads(x) - if rec["ts"] >= cutoff: - if model_filter == "all" or rec["model"] == model_filter: - points.append({ - "model": rec["model"], - "agent": rec["agent"], - "reason": rec["reason"], - "prompt_tokens": int(rec.get("prompt_tokens", 0)), - "completion_tokens": rec.get("completion_tokens", 0), - "inference_ms": round(rec["inference_ms"], 1), - "tokens_per_sec": rec.get("tokens_per_sec", 0), - "stream": rec.get("stream", False) - }) - except: pass - return jsonify({"points": points, "count": len(points)}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - -@app.route("/v1/models") -def models(): - def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) - return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) - -@app.route("/health") -def health(): - gpus = {} - for m in GPU_URLS: - h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) - h["active_requests"] = gpu_active_count(m) - h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) - gpus[m] = h - return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()}) - -@app.route("/metrics") -def metrics(): return jsonify(get_metrics()) - -@app.route("/metrics/timeseries") -def metrics_timeseries(): - period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) - data = {"models": {}, "labels": []} - if period == "day": - buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - elif period == "week": - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - else: - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - if r: - for model in models_list: - counts = [] - for bucket in buckets: - total = 0 - if period in ("week","month"): - for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) - else: total = int(r.get("ts:"+model+":"+bucket) or 0) - counts.append(total) - data["models"][model] = counts - return jsonify(data) - -@app.route("/stream") -def stream(): - def ev(): - q = queue.Queue() - with sse_lock: sse_subscribers.append(q) - try: - yield "data: "+json.dumps(get_metrics())+"\n\n" - while True: - try: yield "data: "+q.get(timeout=3)+"\n\n" - except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" - except GeneratorExit: pass - finally: - with sse_lock: - if q in sse_subscribers: sse_subscribers.remove(q) - return Response(stream_with_context(ev()), mimetype="text/event-stream", - headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) - -# ── Phase 0: Admin Key Management ── -ADMIN_KEY = os.environ.get("ADMIN_KEY", "") - -def _admin_auth(): - """Require admin key for management endpoints.""" - if not ADMIN_KEY: - return False, "ADMIN_KEY not configured on server" - ak = request.headers.get("Authorization","").replace("Bearer ","") - if ak != ADMIN_KEY: - return False, "Admin key required" - return True, None - -@app.route("/admin/keys") -def admin_keys(): - """List all API keys (masked) with agent, tier, and deprecation status.""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - keys = [] - for key, info in API_KEYS.items(): - masked = key[:8] + "..." + key[-8:] - keys.append({ - "masked": masked, - "prefix": key[:8], - "agent": info["agent"], - "tier": info["tier"], - "deprecated": info.get("deprecated", False), - "length": len(key) - }) - return jsonify({ - "total": len(keys), - "active": sum(1 for k in keys if not k["deprecated"]), - "deprecated": sum(1 for k in keys if k["deprecated"]), - "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"])) - }) - -@app.route("/admin/keys/deprecation-summary") -def admin_deprecation_summary(): - """Summary of deprecated key usage (from Redis logs, if available).""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - deprecated_agents = [] - for key, info in API_KEYS.items(): - if info.get("deprecated"): - # Check Redis for usage count - count = 0 - if r: - count = int(r.get("deprecated_usage:" + info["agent"]) or 0) - deprecated_agents.append({ - "agent": info["agent"], - "deprecated_uses": count, - "needs_migration": count > 0 - }) - return jsonify({ - "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]), - "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage" - }) - -@app.route("/admin/keys/generate", methods=["POST"]) -def admin_generate_key(): - """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - body = request.get_json(force=True) - agent = body.get("agent", "").strip() - tier = body.get("tier", "enterprise") - if not agent: - return jsonify({"error": "agent field required"}), 400 - if tier not in ("starter", "professional", "enterprise"): - return jsonify({"error": "tier must be starter/professional/enterprise"}), 400 - # Generate secure key - import secrets, hashlib - prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8] - suffix = secrets.token_hex(20) - new_key = f"sk-{prefix}-{suffix}" - # Update in-memory dict (note: not persisted across restarts without env var update) - API_KEYS[new_key] = {"tier": tier, "agent": agent} - log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:]) - return jsonify({ - "agent": agent, - "tier": tier, - "key": new_key, - "masked": new_key[:8] + "..." + new_key[-8:], - "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist." - }), 201 - -@app.route("/admin/keys/revoke", methods=["POST"]) -def admin_revoke_key(): - """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - body = request.get_json(force=True) - agent = body.get("agent", "") - key_prefix = body.get("key_prefix", "") - revoked = [] - keys_to_remove = [] - for key, info in API_KEYS.items(): - if not info.get("deprecated"): - continue - if agent and info["agent"] == agent: - keys_to_remove.append(key) - elif key_prefix and key.startswith(key_prefix): - keys_to_remove.append(key) - for key in keys_to_remove: - info = API_KEYS.pop(key) - revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]}) - log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:]) - return jsonify({ - "revoked": len(revoked), - "keys": revoked, - "remaining_total": len(API_KEYS), - "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist." - }) - -if __name__ == "__main__": - log.info("Router on :9000 (load-aware)") - app.run(host="0.0.0.0", port=9000, debug=False) diff --git a/router/router.py.bak.20260518074236 b/router/router.py.bak.20260518074236 deleted file mode 100644 index 46d38d9..0000000 --- a/router/router.py.bak.20260518074236 +++ /dev/null @@ -1,342 +0,0 @@ -import os, json, time, logging, traceback, threading, queue -import requests, redis -from flask import Flask, request, jsonify, Response, stream_with_context - -REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") -GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") -GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") -GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") - -GPU_SIDECARS = { - "qwen3.6-35B-A3B": "http://192.168.68.15:8090", - "qwen3.6-27B-code": "http://192.168.68.8:8090", - "qwen3.5-9b-vlm": "http://192.168.68.110:8090", -} -GPU_URLS = { - "qwen3.6-35B-A3B": GPU_MOE_URL, - "qwen3.6-27B-code": GPU_DENSE_URL, - "qwen3.5-9b-vlm": GPU_LIGHT_URL, -} -# Max concurrent requests per GPU (based on llama.cpp --parallel) -GPU_MAX_CONCURRENT = { - "qwen3.6-35B-A3B": 2, # 2 slots - "qwen3.6-27B-code": 2, # 2 slots - "qwen3.5-9b-vlm": 1, # 1 slot -} - -TIER_MODELS = { - "starter": ["qwen3.5-9b-vlm"], - "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"], - "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"], -} -API_KEYS = { - "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin"}, - "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba"}, - "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni"}, - "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko"}, - "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby"}, - "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0"}, - "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo"}, - "sk-starter-abc123": {"tier": "starter", "agent": "test-starter"}, - "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro"}, -} - -logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") -log = logging.getLogger("router") -try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() -except Exception: r = None - - -def counter_audit_loop(): - """Every 30s, check GPU slots and reset counters if all slots idle.""" - while True: - time.sleep(30) - if not r: continue - for model, url in GPU_URLS.items(): - try: - resp = requests.get(url.replace("/v1","") + "/slots", - headers={"Authorization": "Bearer not-needed"}, timeout=5) - if resp.status_code == 200: - slots = resp.json() - all_idle = all(not s.get("is_processing", False) for s in slots) - if all_idle: - current = int(r.get("active:" + model) or 0) - if current > 0: - r.set("active:" + model, 0) - log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) - except Exception: - pass - -threading.Thread(target=counter_audit_loop, daemon=True).start() - -app = Flask(__name__) -sse_subscribers = []; sse_lock = threading.Lock() - -def gpu_active_count(model): - """Get number of in-flight requests for a GPU.""" - if r: - return int(r.get("active:" + model) or 0) - return 0 - -def gpu_incr(model): - if r: r.incr("active:" + model) - -def gpu_decr(model): - if r: - v = r.decr("active:" + model) - if v and int(v) < 0: - r.set("active:" + model, 0) # never go negative - -def check_gpu_health(model): - url = GPU_SIDECARS.get(model) - if not url: return {"status": "unknown"} - try: - resp = requests.get(url, timeout=5) - if resp.status_code == 200: - d = resp.json() - pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 - status = "healthy" if pct < 90 else "saturated" - # Also check if llama.cpp endpoint is actually responding - gpu_url = GPU_URLS.get(model, "") - try: - hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3) - if hr.status_code != 200: - status = "down" - except Exception: - status = "down" - return {"status": status, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} - except Exception: pass - return {"status": "down"} - -def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] - -def estimate_tokens(msgs): return sum(len(str(m.get("content",""))) for m in msgs) // 4 - -def is_gpu_busy(model): - """Check if GPU is at or near max concurrent capacity.""" - active = gpu_active_count(model) - max_c = GPU_MAX_CONCURRENT.get(model, 1) - return active >= max_c - -def select_best_gpu(candidates, reason): - """Pick the best GPU from candidates, preferring least-loaded.""" - best = None - best_load = 999 - for m in candidates: - load = gpu_active_count(m) - if load < best_load: - best_load = load - best = m - if best: - actual_reason = reason - if is_gpu_busy(best): - actual_reason = "load_balanced_" + reason - return {"model": best, "reason": actual_reason} - return None - -def route(rd, tier): - msgs = rd.get("messages",[]); t = estimate_tokens(msgs) - sys = any(m.get("role")=="system" for m in msgs) - turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) - hints = rd.get("routing_hints",{}) - allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"]) - avail = [m for m in available_models() if m in allowed] - if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} - - req = rd.get("model","auto") - if req != "auto": - target = req if req in avail else avail[0] - # If explicit model is busy, check if another can take it - if is_gpu_busy(target) and req in allowed: - alts = [m for m in avail if m != target and m in allowed] - if alts: - alt = select_best_gpu(alts, "explicit") - if alt: return alt - return {"model": target, "reason": "explicit"} - - if hints: - if hints.get("priority")=="speed" and "qwen3.5-9b-vlm" in avail: - return select_best_gpu(["qwen3.5-9b-vlm"], "hint_speed") or {"model":"qwen3.5-9b-vlm","reason":"hint_speed"} - if hints.get("priority")=="quality" and "qwen3.6-27B-code" in avail: - return select_best_gpu(["qwen3.6-27B-code"], "hint_quality") or {"model":"qwen3.6-27B-code","reason":"hint_quality"} - - # Heavy -> dense (but fall back to MoE if dense is busy) - if t > 4000 or sys or turns > 6: - candidates = ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] - candidates = [m for m in candidates if m in avail] - result = select_best_gpu(candidates, "heavy_reasoning") - if result: return result - - # Ultra-light -> VLM - first_msg = msgs[0].get("content","") if msgs else "" - words = len(first_msg.split()) if isinstance(first_msg, str) else 99 - if words <= 3 and turns <= 1 and not sys and "qwen3.5-9b-vlm" in avail: - if not is_gpu_busy("qwen3.5-9b-vlm"): - return {"model":"qwen3.5-9b-vlm","reason":"ultra_light"} - - # Default: MoE, fall back to dense if MoE is busy - if "qwen3.6-35B-A3B" in avail: - if is_gpu_busy("qwen3.6-35B-A3B") and "qwen3.6-27B-code" in avail: - return {"model": "qwen3.6-27B-code", "reason": "load_balanced_default"} - return {"model":"qwen3.6-35B-A3B","reason":"default_moe"} - - return {"model":avail[0],"reason":"fallback"} - -def clean_unicode(text): - if not isinstance(text, str): return text - text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") - text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'") - text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"') - text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ") - return text.encode("ascii", "ignore").decode("ascii") - -def clean_response(d): - if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} - if isinstance(d, list): return [clean_response(v) for v in d] - if isinstance(d, str): return clean_unicode(d) - return d - -def get_metrics(): - d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}} - for m in GPU_URLS: - h = check_gpu_health(m) - d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)}) - d["active_requests"][m] = gpu_active_count(m) - if r: - try: - for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) - for k,v in API_KEYS.items(): - c = int(r.get("routes:agent:"+v["agent"]) or 0) - if c>0: d["agent_counts"][v["agent"]] = c - for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) - raw = r.lrange("routes:recent",0,49) - d["recent"] = [json.loads(x) for x in raw] if raw else [] - except Exception: pass - return d - -def bcast(): - data = get_metrics(); payload = json.dumps(data) - with sse_lock: - dead = [] - for q in sse_subscribers: - try: q.put(payload) - except Exception: dead.append(q) - for q in dead: sse_subscribers.remove(q) - -@app.route("/v1/chat/completions", methods=["POST"]) -def chat(): - try: - rd = request.get_json(force=True) - ak = request.headers.get("Authorization","").replace("Bearer ","") - ki = API_KEYS.get(ak, {"tier":"starter","agent":"unknown"}) - tier, agent = ki["tier"], ki["agent"] - d = route(rd, tier) - if d.get("saturated"): - resp = jsonify({"error": "All GPUs saturated", "retry_after_s": 5}) - resp.headers["Retry-After"] = "5" - return resp, 503 - model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] - is_stream = rd.get("stream", False) - - gpu_incr(model) - decremented = False - - log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) - if r: - try: - r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) - r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) - r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent})) - r.ltrim("routes:recent",0,999) - except Exception: pass - start = time.time() - resp = requests.post(url+"/chat/completions", json=rd, - headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) - lat = int((time.time()-start)*1000) - gpu_decr(model) - decremented = True # Release slot - - if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 - if is_stream: - def gen(): - for raw in resp.iter_content(chunk_size=None, decode_unicode=True): - if raw: yield clean_unicode(raw) - bcast() - return Response(stream_with_context(gen()), mimetype="text/event-stream") - data = clean_response(resp.json()) - for c in data.get("choices",[]): - msg = c.get("message",{}) - if not msg.get("content") and msg.get("reasoning_content"): - msg["content"] = msg["reasoning_content"] - data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model)} - bcast() - return jsonify(data) - if not decremented: - try: gpu_decr(model) - except: pass - except requests.Timeout: - return jsonify({"error":"timeout"}), 504 - log.error("Error: %s\n%s", e, traceback.format_exc()) - return jsonify({"error":str(e)}), 500 - -@app.route("/v1/models") -def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]}) - -@app.route("/health") -def health(): - gpus = {} - for m in GPU_URLS: - h = check_gpu_health(m) - h["active_requests"] = gpu_active_count(m) - h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) - gpus[m] = h - return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()}) - -@app.route("/metrics") -def metrics(): return jsonify(get_metrics()) - -@app.route("/metrics/timeseries") -def metrics_timeseries(): - period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) - data = {"models": {}, "labels": []} - if period == "day": - buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - elif period == "week": - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - else: - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - if r: - for model in models_list: - counts = [] - for bucket in buckets: - total = 0 - if period in ("week","month"): - for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) - else: total = int(r.get("ts:"+model+":"+bucket) or 0) - counts.append(total) - data["models"][model] = counts - return jsonify(data) - -@app.route("/stream") -def stream(): - def ev(): - q = queue.Queue() - with sse_lock: sse_subscribers.append(q) - try: - yield "data: "+json.dumps(get_metrics())+"\n\n" - while True: - try: yield "data: "+q.get(timeout=3)+"\n\n" - except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" - except GeneratorExit: pass - finally: - with sse_lock: - if q in sse_subscribers: sse_subscribers.remove(q) - return Response(stream_with_context(ev()), mimetype="text/event-stream", - headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) - -if __name__ == "__main__": - log.info("Router on :9000 (load-aware)") - app.run(host="0.0.0.0", port=9000, debug=False) diff --git a/router/router.py.bak.20260616-214735 b/router/router.py.bak.20260616-214735 deleted file mode 100644 index 23f9c8e..0000000 --- a/router/router.py.bak.20260616-214735 +++ /dev/null @@ -1,1006 +0,0 @@ -import os, json, time, logging, traceback, threading, queue, statistics, math -import requests, redis -from flask import Flask, request, jsonify, Response, stream_with_context - - - -# Phase 2: Atomic session token update Redis Lua script -SESSION_LUA_SCRIPT = """ -local key = KEYS[1] -local new_val = tonumber(ARGV[1]) -local current = tonumber(redis.call('GET', key) or 0) -local max_val = math.max(current, new_val) -redis.call('SET', key, max_val, 'EX', 86400) -return max_val -""" - - -REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") -GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") -GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") -GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") - -GPU_SIDECARS = { - "qwen3.6-35B-A3B": "http://192.168.68.15:8090", - "qwen3.6-27B-code": "http://192.168.68.8:8090", - "gemma-4-12b": "http://192.168.68.110:8090", -} -GPU_URLS = { - "qwen3.6-35B-A3B": GPU_MOE_URL, - "qwen3.6-27B-code": GPU_DENSE_URL, - "gemma-4-12b": GPU_LIGHT_URL, -} -# Max concurrent requests per GPU (based on llama.cpp --parallel) - -GPU_LABELS = { - "qwen3.6-35B-A3B": "Qwen3.6 35B (Strix Halo)", - "qwen3.6-27B-code": "Qwen3.6 27B Code (RTX 3090)", - "gemma-4-12b": "Gemma-4 12B (RTX 5070)", -} - -GPU_MAX_CONCURRENT = { - "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) - "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) - "gemma-4-12b": 2, # 2 slots (7.1GB VRAM) -} - -# Context window sizes (tokens) — used for compaction signals -GPU_CONTEXT = { - "qwen3.6-35B-A3B": 262144, - "qwen3.6-27B-code": 262144, - "gemma-4-12b": 262144, -} - -TIER_MODELS = { - "starter": ["gemma-4-12b"], - "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], - "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], -} -# API keys loaded from API_KEYS env var (set in docker-compose.yml) -# Fallback is dev-only — production MUST set API_KEYS env var -API_KEYS = json.loads(os.environ.get("API_KEYS", json.dumps({ - "sk-dev-local-only": {"tier": "enterprise", "agent": "dev"}, -}))) -# Rate limits: requests per minute per API key tier -RATE_LIMIT_RPM = { - "enterprise": 120, - "professional": 60, - "starter": 20, -} - -def check_rate_limit(api_key, tier): - """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" - if not get_redis(): - return True, 999, 60 - limit = RATE_LIMIT_RPM.get(tier, 30) - key = f"ratelimit:{api_key}" - current = int(get_redis().get(key) or 0) - if current >= limit: - ttl = r.ttl(key) - retry = max(ttl, 1) if ttl and ttl > 0 else 60 - return False, retry, 0 - pipe = r.pipeline() - pipe.incr(key) - pipe.expire(key, 60) # 1-minute sliding window - pipe.execute() - remaining = limit - (current + 1) - reset_seconds = r.ttl(key) or 60 - return True, remaining, reset_seconds - - -logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") -log = logging.getLogger("router") -# Redis connection — initialized lazily, retries on first use -def get_redis(): - global r - if r is not None: - try: - r.ping() - return r - except Exception: - r = None - try: - r = redis.from_url(REDIS_URL, decode_responses=True) - r.ping() - return r - except Exception: - return None - -r = None -try: get_redis() -except Exception: pass -except Exception: r = None - - -def counter_audit_loop(): - """Every 30s, check GPU slots and reset counters if all slots idle.""" - while True: - time.sleep(30) - if not get_redis(): continue - for model, url in GPU_URLS.items(): - try: - resp = requests.get(url.replace("/v1","") + "/slots", - headers={"Authorization": "Bearer not-needed"}, timeout=5) - if resp.status_code == 200: - slots = resp.json() - all_idle = all(not s.get("is_processing", False) for s in slots) - if all_idle: - current = int(get_redis().get("active:" + model) or 0) - if current > 0: - get_redis().set("active:" + model, 0) - log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) - except Exception: - pass - -threading.Thread(target=counter_audit_loop, daemon=True).start() - -app = Flask(__name__) -sse_subscribers = []; sse_lock = threading.Lock() - -def gpu_active_count(model): - """Get number of in-flight requests for a GPU.""" - if r: - return int(r.get("active:" + model) or 0) - return 0 - -def gpu_incr(model): - if get_redis(): get_redis().incr("active:" + model) - -def gpu_decr(model): - rd = get_redis() - if rd: - v = rd.decr("active:" + model) - if v and int(v) < 0: - get_redis().set("active:" + model, 0) # never go negative -def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): - url = GPU_SIDECARS.get(model) - if not url: return {"status": "unknown"} - try: - resp = requests.get(url, timeout=sidecar_timeout) - if resp.status_code == 200: - d = resp.json() - pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 - status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy() - vram_warning = pct >= 95 - # Also check if llama.cpp endpoint is actually responding - gpu_url = GPU_URLS.get(model, "") - try: - hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) - if hr.status_code != 200: - status = "down" - except Exception: - status = "down" - return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} - except Exception: pass - return {"status": "down"} - -def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] - -def estimate_tokens(msgs): - """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" - return len(json.dumps(msgs, default=str)) // 3.5 - -def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): - """Store detailed performance record in Redis for analytics.""" - if not get_redis(): return - try: - total_ms = queue_ms + inference_ms - tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 - rec = json.dumps({ - "ts": time.time(), - "model": model, "agent": agent, "tier": tier, "reason": reason, - "queue_ms": round(queue_ms, 1), - "inference_ms": round(inference_ms, 1), - "total_ms": round(total_ms, 1), - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "tokens_per_sec": round(tps, 1), - "stream": stream - }) - # Global recent list (last 500) - r.lpush("perf:recent", rec) - r.ltrim("perf:recent", 0, 499) - # Per-model list (last 200) - r.lpush("perf:model:" + model, rec) - r.ltrim("perf:model:" + model, 0, 199) - # Per-reason list (last 200) - r.lpush("perf:reason:" + reason, rec) - r.ltrim("perf:reason:" + reason, 0, 199) - # Per-agent list (last 200) - r.lpush("perf:agent:" + agent, rec) - r.ltrim("perf:agent:" + agent, 0, 199) - except Exception: - pass - -def is_gpu_busy(model): - """Check if GPU is at or near max concurrent capacity.""" - active = gpu_active_count(model) - max_c = GPU_MAX_CONCURRENT.get(model, 1) - return active >= max_c - - - -# Phase 3: Dynamic GPU Weighting (Health Score) -def gpu_health_score(model): - """Score a GPU based on VRAM, temperature, and load. Lower = better.""" - h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) - if h.get("status") == "down": - return 999 # never pick down GPUs - vram_pct = h.get("vram_pct") or 50 - temp_c = h.get("temp_c") or 50 - active = gpu_active_count(model) - max_c = GPU_MAX_CONCURRENT.get(model, 1) - load_pct = (active / max_c) * 100 if max_c > 0 else 0 - # Score: lower = better (more headroom, cooler, less loaded) - score = (vram_pct or 0) * 0.4 + max((temp_c or 50) - 30, 0) * 0.3 + load_pct * 0.3 - return score - -def select_best_gpu(candidates, reason, agent=""): - """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" - # Count how many distinct agents are on each GPU - gpu_agent_counts = {} - if r: - for m in GPU_URLS: - count = 0 - for ak in API_KEYS.values(): - if r.get("agent_gpu:" + ak["agent"] + ":" + m): - count += 1 - gpu_agent_counts[m] = count - # Phase 3: Sort candidates by health score before selection - sorted_candidates = sorted(candidates, key=gpu_health_score) - # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) - for m in sorted_candidates: - if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: - return {"model": m, "reason": reason} - # Second pass: prefer GPU this agent is NOT already on (skip own GPU) - if agent: - for m in sorted_candidates: - if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): - return {"model": m, "reason": reason} - # Third pass: any non-busy GPU - for m in sorted_candidates: - if not is_gpu_busy(m): - return {"model": m, "reason": reason} - # All busy — pick least loaded - best = None - best_load = 999 - for m in candidates: - load = gpu_active_count(m) - if load < best_load: - best_load = load - best = m - if best: - return {"model": best, "reason": "load_balanced_" + reason} - return None - - - - - -# Phase 1: Circuit Breaker for GPU Hosts (Approved by Abiba) -def is_circuit_tripped(model): - """Check if a GPU host is currently blacklisted.""" - if not get_redis(): - return False - return r.exists("circuit:" + model + ":open") - -def trip_circuit(model, duration=30): - """Blacklist a GPU host for a specified duration.""" - if not get_redis(): - return - key = "circuit:" + model + ":open" - r.set(key, 1, ex=duration) - r.incr("circuit:" + model + ":count") - log.warning("CIRCUIT_TRIPPED: %s blacklisted for %ds", model, duration) - -def half_open_probe(model): - """Check if a GPU host can be un-blacklisted.""" - if not get_redis(): - return True - key = "circuit:" + model + ":open" - if not r.exists(key): - return True # no circuit - return False # still open - -def route(rd, tier, agent=""): - msgs = rd.get("messages",[]); t = estimate_tokens(msgs) - sys = any(m.get("role")=="system" for m in msgs) - turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) - hints = rd.get("routing_hints",{}) - allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) - # Phase 1: Filter out models with tripped circuit breakers - avail = [m for m in available_models() if m in allowed and not is_circuit_tripped(m)] - if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} - if all(is_gpu_busy(m) for m in avail): - return {"model": avail[0], "reason": "all_saturated", "saturated": True} - - # GUARD: multimodal -> VLM only (sole vision model) - has_image = any( - isinstance(m.get("content"), list) and - any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) - for m in msgs - ) - if has_image: - if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): - return {"model": "gemma-4-12b", "reason": "vision"} - elif "gemma-4-12b" in avail: - return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} - else: - return {"model": allowed[0], "reason": "vision_unavailable"} - - req = rd.get("model","auto") - if req != "auto": - target = req if req in avail else avail[0] - if is_gpu_busy(target) and req in allowed: - alts = [m for m in avail if m != target and m in allowed] - if alts: - alt = select_best_gpu(alts, "explicit", agent) - if alt: return alt - return {"model": target, "reason": "explicit"} - - if hints: - if hints.get("priority")=="speed" and "gemma-4-12b" in avail: - return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} - if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: - return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} - if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: - return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} - - first_msg = msgs[0].get("content","") if msgs else "" - words = len(first_msg.split()) if isinstance(first_msg, str) else 99 - - # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) - if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: - if not is_gpu_busy("gemma-4-12b"): - return {"model":"gemma-4-12b","reason":"tiny"} - fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(fallback, "tiny_fallback", agent) - if result: return result - - # TIER 2: Light - moderate chat -> VLM first (fastest), Dense fallback - if t <= 5000 and turns <= 4: - candidates = [m for m in ["gemma-4-12b","qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(candidates, "light", agent) - if result: return result - - # TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%) - if t <= 30000: - candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) - result = select_best_gpu(candidates, "medium", agent) - if result: return result - - # TIER 4: Heavy - quality first -> Dense primary, MoE fallback - if t > 30000: - candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","gemma-4-12b"] if m in avail] - result = select_best_gpu(candidates, "heavy", agent) - if result: return result - - # TIER 5: Default - MoE primary (60%), Dense spillover (40%) - candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) - result = select_best_gpu(candidates, "default", agent) - if result: return result - return {"model":avail[0],"reason":"last_resort"} - - -def moe_spillover(avail, default_order): - """Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating. - Only applies when MoE is first candidate, available, and not busy.""" - import random - if (default_order[0] == "qwen3.6-35B-A3B" - and "qwen3.6-35B-A3B" in avail - and not is_gpu_busy("qwen3.6-35B-A3B") - and "qwen3.6-27B-code" in avail - and not is_gpu_busy("qwen3.6-27B-code") - and random.random() < 0.4): - # Swap: Dense first, MoE second - return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")] - return [m for m in default_order if m in avail] -def clean_unicode(text): - if not isinstance(text, str): return text - text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") - text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'") - text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"') - text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ") - return text.encode("ascii", "ignore").decode("ascii") - -def clean_response(d): - if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} - if isinstance(d, list): return [clean_response(v) for v in d] - if isinstance(d, str): return clean_unicode(d) - return d - -def get_metrics(): - d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}} - for m in GPU_URLS: - h = check_gpu_health(m) - d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)}) - d["active_requests"][m] = gpu_active_count(m) - if r: - try: - for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) - for k,v in API_KEYS.items(): - c = int(r.get("routes:agent:"+v["agent"]) or 0) - if c>0: d["agent_counts"][v["agent"]] = c - for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) - raw = r.lrange("routes:recent",0,49) - d["recent"] = [json.loads(x) for x in raw] if raw else [] - except Exception: pass - return d - -def bcast(): - data = get_metrics(); payload = json.dumps(data) - with sse_lock: - dead = [] - for q in sse_subscribers: - try: q.put(payload) - except Exception: dead.append(q) - for q in dead: sse_subscribers.remove(q) - -QUEUE_TIMEOUT = int(os.environ.get("QUEUE_TIMEOUT", "30")) # max seconds to queue before 503 - -@app.route("/v1/chat/completions", methods=["POST"]) -def chat(): - try: - rd = request.get_json(force=True) - ak = request.headers.get("Authorization","").replace("Bearer ","") - if not ak or ak not in API_KEYS: - log.warning("AUTH_REJECTED: no/invalid API key from %s", request.remote_addr) - return jsonify({"error": "Unauthorized — valid API key required"}), 401 - ki = API_KEYS[ak] - tier, agent = ki["tier"], ki["agent"] - - # Phase 0: dual-key transition — log deprecated key usage - if ki.get("deprecated"): - new_key = next((k for k, v in API_KEYS.items() - if v.get("agent") == agent and not v.get("deprecated")), None) - log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s", - agent, ak[:12], ak[-8:], - new_key[:12] if new_key else "N/A", - new_key[-8:] if new_key else "N/A") - if r: - r.incr("deprecated_usage:" + agent) - - # Rate limit check - allowed, rl_val, reset_sec = check_rate_limit(ak, tier) - if not allowed: - resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val}) - resp.headers["Retry-After"] = str(rl_val) - resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30)) - resp.headers["X-RateLimit-Remaining"] = "0" - resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val)) - log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:]) - return resp, 429 - - # Allow agent to override queue timeout via header - q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) - - # Cross-turn context tracking: accumulate tokens per session (Phase 2: atomic Lua) - session_id = request.headers.get("X-Session-Id", "") - session_tokens = 0 - if session_id and r: - try: - current = estimate_tokens(rd.get("messages",[])) - # Atomic GET/MAX/SET via Lua script prevents race conditions - session_tokens = r.eval(SESSION_LUA_SCRIPT, 1, "session:" + session_id, current) - except Exception: pass - - d = route(rd, tier, agent) - queue_start = time.time() - - # Queue loop: wait for a GPU slot instead of immediate 503 - while d.get("saturated"): - elapsed = time.time() - queue_start - if elapsed > q_timeout: - resp = jsonify({"error": "All GPUs saturated", "queued_s": round(elapsed,1), "retry_after_s": 5}) - resp.headers["Retry-After"] = "5" - log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed) - return resp, 503 - time.sleep(0.5) # poll every 500ms - d = route(rd, tier, agent) - - queue_ms = (time.time() - queue_start) * 1000 - if queue_ms > 500: - log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) - model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] - - # Stash rate limit values for response headers - _rl_remaining = rl_val - _rl_limit = RATE_LIMIT_RPM.get(tier, 30) - _rl_reset = reset_sec - is_stream = rd.get("stream", False) - - gpu_incr(model) - - log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) - # Track which GPU this agent is using (TTL 120s covers typical request) - if r and agent: - try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1") - except: pass - if r: - try: - r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) - r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) - r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)})) - r.ltrim("routes:recent",0,999) - except Exception: pass - start = time.time() - resp = requests.post(url+"/chat/completions", json=rd, - headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) - lat = int((time.time()-start)*1000) - gpu_decr(model) - - if resp.status_code != 200: - if resp.status_code in (502, 504): - trip_circuit(model, 30) - return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 - if is_stream: - # Buffer SSE chunks, handle split lines for large responses - chunks = [] - stream_timings = {} - buf = "" # accumulate partial lines - for raw in resp.iter_content(chunk_size=None, decode_unicode=True): - if raw: - cleaned = clean_unicode(raw) - chunks.append(cleaned) - buf += cleaned - # Process complete lines from buffer - while "\n" in buf: - line, buf = buf.split("\n", 1) - line = line.strip() - if line.startswith("data: ") and not stream_timings: - js = line[6:].strip() - if js.startswith("{") and "timings" in js and "predicted_n" in js: - try: - tj = json.loads(js).get("timings", {}) - if tj: - stream_timings = tj - except: pass - # Store perf record with real token counts from stream - if stream_timings: - pt = stream_timings.get("prompt_n", 0) - ct = stream_timings.get("predicted_n", 0) - tps = stream_timings.get("predicted_per_second", 0) - gen_ms = stream_timings.get("predicted_ms", lat) - store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True) - else: - store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True) - # Yield all chunks to client - def gen(): - for c in chunks: yield c - bcast() - ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) - ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 - ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) - sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") - sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit) - sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) - sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) - sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) - sse_resp.headers["X-Context-Warning"] = ctx_warning - sse_resp.headers["X-Context-Model"] = model - return sse_resp - data = clean_response(resp.json()) - for c in data.get("choices",[]): - msg = c.get("message",{}) - if not msg.get("content") and msg.get("reasoning_content"): - msg["content"] = msg["reasoning_content"] - # Extract performance data from llama.cpp response - usage = data.get("usage", {}) - timings = data.get("timings", {}) - prompt_tokens = usage.get("prompt_tokens", 0) - completion_tokens = usage.get("completion_tokens", 0) - inference_ms = lat # total GPU round-trip - store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False) - ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) - ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 - ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) - data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} - resp = jsonify(data) - resp.headers["X-RateLimit-Limit"] = str(_rl_limit) - resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) - resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) - resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) - resp.headers["X-Context-Warning"] = ctx_warning - resp.headers["X-Context-Model"] = model - bcast() - return resp - except requests.Timeout: - gpu_decr(model) - trip_circuit(model, 30) - log.error("TIMEOUT: %s -> %s (Circuit tripped)", agent, model) - return jsonify({"error":"timeout"}), 504 - except Exception as e: - gpu_decr(model) - log.error("Error: %s\n%s", e, traceback.format_exc()) - return jsonify({"error":str(e)}), 500 - -@app.route("/metrics/performance") -def performance(): - """Per-request performance analytics with percentiles per model/reason/agent.""" - if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 - try: - window_hours = int(request.args.get("window", "24").replace("h","")) - model_filter = request.args.get("model", "all") - - # Load recent records - cutoff = time.time() - (window_hours * 3600) - raw = r.lrange("perf:recent", 0, -1) - records = [] - for x in raw: - try: - rec = json.loads(x) - if rec["ts"] >= cutoff: - records.append(rec) - except: pass - - # Filter by model if specified - if model_filter != "all": - records = [r for r in records if r["model"] == model_filter] - - if not records: - return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}) - - def pct(values, p): - if len(values) < 2: return round(values[0], 1) if values else 0 - return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1) - - # Per-model stats - model_groups = {} - for rec in records: - m = rec["model"] - if m not in model_groups: model_groups[m] = [] - model_groups[m].append(rec) - - models = [] - for m, recs in sorted(model_groups.items()): - latencies = [r["total_ms"] for r in recs] - tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] - non_stream = [r for r in recs if not r["stream"]] - queue_times = [r["queue_ms"] for r in non_stream] - models.append({ - "model": m, - "count": len(recs), - "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1), - "latency": { - "avg": round(statistics.mean(latencies), 1), - "p50": pct(latencies, 50), - "p95": pct(latencies, 95), - "p99": pct(latencies, 99) - }, - "throughput": { - "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0, - "p50": pct(tps_vals, 50) if tps_vals else 0, - "p95": pct(tps_vals, 95) if tps_vals else 0, - }, - "queue": { - "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0, - "p95_ms": pct(queue_times, 95) if queue_times else 0, - } if queue_times else None - }) - - # Per-reason stats - reason_groups = {} - for rec in records: - rsn = rec["reason"] - if rsn not in reason_groups: reason_groups[rsn] = [] - reason_groups[rsn].append(rec) - - reasons = [] - for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])): - latencies = [r["total_ms"] for r in recs] - reasons.append({ - "reason": rsn, - "count": len(recs), - "avg_total_ms": round(statistics.mean(latencies), 1), - "p95_total_ms": pct(latencies, 95) - }) - - # Per-agent stats - agent_groups = {} - for rec in records: - ag = rec["agent"] - if ag not in agent_groups: agent_groups[ag] = [] - agent_groups[ag].append(rec) - - agents = [] - for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])): - latencies = [r["total_ms"] for r in recs] - tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] - agents.append({ - "agent": ag, - "count": len(recs), - "avg_total_ms": round(statistics.mean(latencies), 1), - "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0 - }) - - all_lat = [r["total_ms"] for r in records] - all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0] - summary = { - "total_requests": len(records), - "window_hours": window_hours, - "latency": { - "avg_ms": round(statistics.mean(all_lat), 1), - "p50_ms": pct(all_lat, 50), - "p95_ms": pct(all_lat, 95), - "p99_ms": pct(all_lat, 99) - }, - "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0 - } - - return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - -@app.route("/metrics/scatter") -def scatter(): - """Return individual data points for scatter plots (prompt_tokens vs latency).""" - if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 - try: - window_hours = int(request.args.get("window", "24").replace("h","")) - model_filter = request.args.get("model", "all") - cutoff = time.time() - (window_hours * 3600) - raw = r.lrange("perf:recent", 0, -1) - points = [] - for x in raw: - try: - rec = json.loads(x) - if rec["ts"] >= cutoff: - if model_filter == "all" or rec["model"] == model_filter: - points.append({ - "model": rec["model"], - "agent": rec["agent"], - "reason": rec["reason"], - "prompt_tokens": int(rec.get("prompt_tokens", 0)), - "completion_tokens": rec.get("completion_tokens", 0), - "inference_ms": round(rec["inference_ms"], 1), - "tokens_per_sec": rec.get("tokens_per_sec", 0), - "stream": rec.get("stream", False) - }) - except: pass - return jsonify({"points": points, "count": len(points)}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - -@app.route("/v1/models") -def models(): - def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) - return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) - -@app.route("/health") -def health(): - gpus = {} - for m in GPU_URLS: - h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) - h["active_requests"] = gpu_active_count(m) - h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) - gpus[m] = h - return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()}) - -@app.route("/metrics") -def metrics(): return jsonify(get_metrics()) - -@app.route("/metrics/timeseries") -def metrics_timeseries(): - period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) - data = {"models": {}, "labels": []} - if period == "day": - buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] - elif period == "week": - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] - else: - buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] - if r: - for model in models_list: - counts = [] - for bucket in buckets: - total = 0 - if period in ("week","month"): - for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) - else: total = int(r.get("ts:"+model+":"+bucket) or 0) - counts.append(total) - data["models"][model] = counts - return jsonify(data) - - - -@app.route("/metrics/circuit-breaker") -def metrics_circuit_breaker(): - """Expose circuit breaker status per model. Phase 1.""" - result = {} - if r: - for model in GPU_URLS: - key = "circuit:" + model + ":open" - duration = r.ttl(key) - trip_count = int(r.get("circuit:" + model + ":count") or 0) - result[model] = { - "tripped": r.exists(key), - "remaining_ttl": duration, - "trip_count": trip_count - } - return jsonify(result) - - -@app.route("/metrics/gpu-health") -def metrics_gpu_health(): - """Live GPU health scores + circuit breaker + KPIs.""" - result = {"gpus": [], "ts": time.time()} - for model in GPU_URLS: - h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) - score = gpu_health_score(model) - active = gpu_active_count(model) - max_c = GPU_MAX_CONCURRENT.get(model, 1) - cb_tripped = bool(r and r.exists("circuit:" + model + ":open")) - cb_count = int(r.get("circuit:" + model + ":count") or 0) if r else 0 - result["gpus"].append({ - "id": model, - "label": GPU_LABELS.get(model, model), - "status": h.get("status", "unknown"), - "vram_pct": h.get("vram_pct", 0), - "temp_c": h.get("temp_c", 0), - "vram_used_mb": h.get("vram_used_mb", 0), - "vram_total_mb": h.get("vram_total_mb", 0), - "gpu_name": h.get("gpu_name", model), - "health_score": round(score, 1), - "active_requests": active, - "max_concurrent": max_c, - "circuit_tripped": cb_tripped, - "circuit_trip_count": cb_count - }) - online = sum(1 for g in result["gpus"] if g["status"] in ("healthy", "saturated")) - trips = sum(g["circuit_trip_count"] for g in result["gpus"]) - result["kpi"] = {"gpus_online": online, "total_trips": trips, "total_gpus": len(GPU_URLS)} - return jsonify(result) - -@app.route("/metrics/latency") -def metrics_latency(): - """Lightweight latency summary for dashboard KPIs.""" - if not r: return jsonify({"avg_ms": 0, "requests_per_min": 0}) - recent = [] - for x in (r.lrange("routes:recent", 0, 49) or []): - try: recent.append(json.loads(x)) - except: pass - if not recent: return jsonify({"avg_ms": 0, "requests_per_min": 0, "count": 0}) - now = time.time() - last_min = [x for x in recent if now - x.get("ts", 0) < 60] - latencies = [x.get("queue_ms", 0) + x.get("inference_ms", 0) for x in last_min if "inference_ms" in x] - return jsonify({ - "avg_ms": round(sum(latencies) / len(latencies), 1) if latencies else 0, - "requests_per_min": len(last_min), - "count": len(recent) - }) -@app.route("/stream") -def stream(): - def ev(): - q = queue.Queue() - with sse_lock: sse_subscribers.append(q) - try: - yield "data: "+json.dumps(get_metrics())+"\n\n" - while True: - try: yield "data: "+q.get(timeout=3)+"\n\n" - except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" - except GeneratorExit: pass - finally: - with sse_lock: - if q in sse_subscribers: sse_subscribers.remove(q) - return Response(stream_with_context(ev()), mimetype="text/event-stream", - headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) - -# ── Phase 0: Admin Key Management ── -ADMIN_KEY = os.environ.get("ADMIN_KEY", "") - -def _admin_auth(): - """Require admin key for management endpoints.""" - if not ADMIN_KEY: - return False, "ADMIN_KEY not configured on server" - ak = request.headers.get("Authorization","").replace("Bearer ","") - if ak != ADMIN_KEY: - return False, "Admin key required" - return True, None - -@app.route("/admin/keys") -def admin_keys(): - """List all API keys (masked) with agent, tier, and deprecation status.""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - keys = [] - for key, info in API_KEYS.items(): - masked = key[:8] + "..." + key[-8:] - keys.append({ - "masked": masked, - "prefix": key[:8], - "agent": info["agent"], - "tier": info["tier"], - "deprecated": info.get("deprecated", False), - "length": len(key) - }) - return jsonify({ - "total": len(keys), - "active": sum(1 for k in keys if not k["deprecated"]), - "deprecated": sum(1 for k in keys if k["deprecated"]), - "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"])) - }) - -@app.route("/admin/keys/deprecation-summary") -def admin_deprecation_summary(): - """Summary of deprecated key usage (from Redis logs, if available).""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - deprecated_agents = [] - for key, info in API_KEYS.items(): - if info.get("deprecated"): - # Check Redis for usage count - count = 0 - if r: - count = int(r.get("deprecated_usage:" + info["agent"]) or 0) - deprecated_agents.append({ - "agent": info["agent"], - "deprecated_uses": count, - "needs_migration": count > 0 - }) - return jsonify({ - "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]), - "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage" - }) - -@app.route("/admin/keys/generate", methods=["POST"]) -def admin_generate_key(): - """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - body = request.get_json(force=True) - agent = body.get("agent", "").strip() - tier = body.get("tier", "enterprise") - if not agent: - return jsonify({"error": "agent field required"}), 400 - if tier not in ("starter", "professional", "enterprise"): - return jsonify({"error": "tier must be starter/professional/enterprise"}), 400 - # Generate secure key - import secrets, hashlib - prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8] - suffix = secrets.token_hex(20) - new_key = f"sk-{prefix}-{suffix}" - # Update in-memory dict (note: not persisted across restarts without env var update) - API_KEYS[new_key] = {"tier": tier, "agent": agent} - log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:]) - return jsonify({ - "agent": agent, - "tier": tier, - "key": new_key, - "masked": new_key[:8] + "..." + new_key[-8:], - "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist." - }), 201 - -@app.route("/admin/keys/revoke", methods=["POST"]) -def admin_revoke_key(): - """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}""" - ok, err = _admin_auth() - if not ok: return jsonify({"error": err}), 401 - body = request.get_json(force=True) - agent = body.get("agent", "") - key_prefix = body.get("key_prefix", "") - revoked = [] - keys_to_remove = [] - for key, info in API_KEYS.items(): - if not info.get("deprecated"): - continue - if agent and info["agent"] == agent: - keys_to_remove.append(key) - elif key_prefix and key.startswith(key_prefix): - keys_to_remove.append(key) - for key in keys_to_remove: - info = API_KEYS.pop(key) - revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]}) - log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:]) - return jsonify({ - "revoked": len(revoked), - "keys": revoked, - "remaining_total": len(API_KEYS), - "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist." - }) - -if __name__ == "__main__": - log.info("Router on :9000 (load-aware)") - app.run(host="0.0.0.0", port=9000, debug=False)