abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(router): Phase 1 - Actual Circuit Breaker for GPU hosts

Browse Source 
- Added is_circuit_tripped(), trip_circuit(), half_open_probe() functions
- Filters out models with tripped circuits in route() function
- Trips circuit on 502/504 errors and timeouts in chat() function
- Prevents hung GPU cascades (Node #480 scenario)

Approved by Abiba via relay #635

Signed-off-by: Mumuni <mumuni@sysloggh.com>
This commit is contained in:
Abiba
2026-06-11 00:29:53 +00:00
parent f1d095e411
commit b79af634d7
2 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+1 -1
View File
@@ -27,7 +27,7 @@ services:
- GPU_MOE_URL=http://192.168.68.15:8080/v1 - GPU_MOE_URL=http://192.168.68.15:8080/v1
- GPU_DENSE_URL=http://192.168.68.8:8080/v1 - GPU_DENSE_URL=http://192.168.68.8:8080/v1
- GPU_LIGHT_URL=http://192.168.68.110:8080/v1 - GPU_LIGHT_URL=http://192.168.68.110:8080/v1
- API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro"}} - API_KEYS={"sk-syslog-local-master-key":{"tier":"enterprise","agent":"admin","deprecated":true},"sk-9e65b69a67-e54af421c1b09fb8bd4f75dacb38cb64":{"tier":"enterprise","agent":"admin"},"sk-syslog-abiba":{"tier":"enterprise","agent":"Abiba","deprecated":true},"sk-856ffb0bbb-e5aaf78b10054eca608f8fbcbd73a889":{"tier":"enterprise","agent":"Abiba"},"sk-syslog-mumuni":{"tier":"enterprise","agent":"Mumuni","deprecated":true},"sk-b57e6e042e-47573660114f3138c852c47f62da807e":{"tier":"enterprise","agent":"Mumuni"},"sk-syslog-tanko":{"tier":"enterprise","agent":"Tanko","deprecated":true},"sk-620a05e95a-e93d875476b650a4d1137249ead8eaa7":{"tier":"enterprise","agent":"Tanko"},"sk-syslog-koby":{"tier":"enterprise","agent":"Koby","deprecated":true},"sk-eb3e6fc1c0-de1bf2edf35a53cb3749a2400483fdee":{"tier":"enterprise","agent":"Koby"},"sk-syslog-kagenz0":{"tier":"enterprise","agent":"Kagenz0","deprecated":true},"sk-12b66b3392-b548aed9138aeb6f698e8e521650ed9b":{"tier":"enterprise","agent":"Kagenz0"},"sk-syslog-koonimo":{"tier":"enterprise","agent":"Koonimo","deprecated":true},"sk-680d06686c-00ee8bf9dc3c93b276af122d49a14dfe":{"tier":"enterprise","agent":"Koonimo"},"sk-starter-abc123":{"tier":"starter","agent":"test-starter","deprecated":true},"sk-55da55907a-1bd7ff344e26feda50e9ac2219697860":{"tier":"starter","agent":"test-starter"},"sk-professional-xyz789":{"tier":"professional","agent":"test-pro","deprecated":true},"sk-b5159863e6-8df3ae52fb958cfe76cc2888c8c8e676":{"tier":"professional","agent":"test-pro"}}
- ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814 - ADMIN_KEY=sk-admin-ee09fffd04978b61a1569ac670c68814
healthcheck: healthcheck:
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"]
router/router.py
+8 -3
View File
@@ -242,7 +242,8 @@ def route(rd, tier, agent=""):
turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) turns = len([m for m in msgs if m.get("role") in ("user","assistant")])
hints = rd.get("routing_hints",{}) hints = rd.get("routing_hints",{})
allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) allowed = TIER_MODELS.get(tier, ["gemma-4-12b"])
avail = [m for m in available_models() if m in allowed] # Phase 1: Filter out models with tripped circuit breakers
avail = [m for m in available_models() if m in allowed and not is_circuit_tripped(m)]
if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
if all(is_gpu_busy(m) for m in avail): if all(is_gpu_busy(m) for m in avail):
return {"model": avail[0], "reason": "all_saturated", "saturated": True} return {"model": avail[0], "reason": "all_saturated", "saturated": True}
@@ -463,7 +464,10 @@ def chat():
lat = int((time.time()-start)*1000) lat = int((time.time()-start)*1000)
gpu_decr(model) gpu_decr(model)
if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 if resp.status_code != 200:
if resp.status_code in (502, 504):
trip_circuit(model, 30)
return jsonify({"error":"GPU error "+str(resp.status_code)}), 502
if is_stream: if is_stream:
# Buffer SSE chunks, handle split lines for large responses # Buffer SSE chunks, handle split lines for large responses
chunks = [] chunks = []
@@ -537,7 +541,8 @@ def chat():
return resp return resp
except requests.Timeout: except requests.Timeout:
gpu_decr(model) gpu_decr(model)
log.error("TIMEOUT: %s -> %s", agent, model) trip_circuit(model, 30)
log.error("TIMEOUT: %s -> %s (Circuit tripped)", agent, model)
return jsonify({"error":"timeout"}), 504 return jsonify({"error":"timeout"}), 504
except Exception as e: except Exception as e:
gpu_decr(model) gpu_decr(model)