feat(router): Phase 1 - Circuit Breaker + /metrics/circuit-breaker endpoint

This commit is contained in:
Abiba
2026-06-11 00:47:15 +00:00
parent 19f7d90cc1
commit 2e24ee5598
+45
View File
@@ -236,6 +236,33 @@ def select_best_gpu(candidates, reason, agent=""):
# Phase 1: Circuit Breaker for GPU Hosts (Approved by Abiba)
def is_circuit_tripped(model):
"""Check if a GPU host is currently blacklisted."""
if not get_redis():
return False
return r.exists("circuit:" + model + ":open")
def trip_circuit(model, duration=30):
"""Blacklist a GPU host for a specified duration."""
if not get_redis():
return
key = "circuit:" + model + ":open"
r.set(key, 1, ex=duration)
r.incr("circuit:" + model + ":count")
log.warning("CIRCUIT_TRIPPED: %s blacklisted for %ds", model, duration)
def half_open_probe(model):
"""Check if a GPU host can be un-blacklisted."""
if not get_redis():
return True
key = "circuit:" + model + ":open"
if not r.exists(key):
return True # no circuit
return False # still open
def route(rd, tier, agent=""): def route(rd, tier, agent=""):
msgs = rd.get("messages",[]); t = estimate_tokens(msgs) msgs = rd.get("messages",[]); t = estimate_tokens(msgs)
sys = any(m.get("role")=="system" for m in msgs) sys = any(m.get("role")=="system" for m in msgs)
@@ -739,6 +766,24 @@ def metrics_timeseries():
data["models"][model] = counts data["models"][model] = counts
return jsonify(data) return jsonify(data)
@app.route("/metrics/circuit-breaker")
def metrics_circuit_breaker():
"""Expose circuit breaker status per model. Phase 1."""
result = {}
if r:
for model in GPU_URLS:
key = "circuit:" + model + ":open"
duration = r.ttl(key)
trip_count = int(r.get("circuit:" + model + ":count") or 0)
result[model] = {
"tripped": r.exists(key),
"remaining_ttl": duration,
"trip_count": trip_count
}
return jsonify(result)
@app.route("/stream") @app.route("/stream")
def stream(): def stream():
def ev(): def ev():