diff --git a/router/router.py b/router/router.py index fd34c01..efb8760 100644 --- a/router/router.py +++ b/router/router.py @@ -50,7 +50,7 @@ RATE_LIMIT_RPM = { def check_rate_limit(api_key, tier): """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" - if not r: + if not get_redis(): return True, 999, 60 limit = RATE_LIMIT_RPM.get(tier, 30) key = f"ratelimit:{api_key}" @@ -70,7 +70,25 @@ def check_rate_limit(api_key, tier): logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") log = logging.getLogger("router") -try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() +# Redis connection — initialized lazily, retries on first use +def get_redis(): + global r + if r is not None: + try: + r.ping() + return r + except Exception: + r = None + try: + r = redis.from_url(REDIS_URL, decode_responses=True) + r.ping() + return r + except Exception: + return None + +r = None +try: get_redis() +except Exception: pass except Exception: r = None @@ -78,7 +96,7 @@ def counter_audit_loop(): """Every 30s, check GPU slots and reset counters if all slots idle.""" while True: time.sleep(30) - if not r: continue + if not get_redis(): continue for model, url in GPU_URLS.items(): try: resp = requests.get(url.replace("/v1","") + "/slots", @@ -89,7 +107,7 @@ def counter_audit_loop(): if all_idle: current = int(r.get("active:" + model) or 0) if current > 0: - r.set("active:" + model, 0) + rd.set("active:" + model, 0) log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) except Exception: pass @@ -106,14 +124,14 @@ def gpu_active_count(model): return 0 def gpu_incr(model): - if r: r.incr("active:" + model) + if get_redis(): get_redis().incr("active:" + model) def gpu_decr(model): - if r: - v = r.decr("active:" + model) + rd = get_redis() + if rd: + v = rd.decr("active:" + model) if v and int(v) < 0: - r.set("active:" + model, 0) # never go negative - + rd.set("active:" + model, 0) # never go negative def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): url = GPU_SIDECARS.get(model) if not url: return {"status": "unknown"} @@ -144,7 +162,7 @@ def estimate_tokens(msgs): def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): """Store detailed performance record in Redis for analytics.""" - if not r: return + if not get_redis(): return try: total_ms = queue_ms + inference_ms tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 @@ -518,7 +536,7 @@ def chat(): @app.route("/metrics/performance") def performance(): """Per-request performance analytics with percentiles per model/reason/agent.""" - if not r: return jsonify({"error": "Redis unavailable"}), 503 + if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 try: window_hours = int(request.args.get("window", "24").replace("h","")) model_filter = request.args.get("model", "all") @@ -635,7 +653,7 @@ def performance(): @app.route("/metrics/scatter") def scatter(): """Return individual data points for scatter plots (prompt_tokens vs latency).""" - if not r: return jsonify({"error": "Redis unavailable"}), 503 + if not get_redis(): return jsonify({"error": "Redis unavailable"}), 503 try: window_hours = int(request.args.get("window", "24").replace("h","")) model_filter = request.args.get("model", "all")