diff --git a/scripts/litellm-health-check.py b/scripts/litellm-health-check.py index 25ec8f2..50ec076 100755 --- a/scripts/litellm-health-check.py +++ b/scripts/litellm-health-check.py @@ -124,8 +124,10 @@ def check_model_probes(): data='{"model":"' + model + '","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}', timeout=30) + first_kind = None # Track first attempt's failure kind if code == 000 and failure_kind: # Retry once with longer timeout (45s) before declaring failure + first_kind = failure_kind time.sleep(1) code, failure_kind = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions", method="POST", @@ -134,8 +136,11 @@ def check_model_probes(): timeout=45) if code == 000 and failure_kind: - # Both attempts failed - report with duration - results.append((model, False, "probe-failed: " + model + " " + failure_kind + " (timeout after retry, 45s)")) + # Both attempts failed - report both kinds + if first_kind: + results.append((model, False, "probe-failed: " + model + " " + first_kind + " then " + failure_kind + " (2 attempts)")) + else: + results.append((model, False, "probe-failed: " + model + " " + failure_kind)) elif code == 200: results.append((model, True, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=" + model + ")")) elif code in (401, 403):