From a12abbeb14c7f8265875ed503222743e7365a626 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 19 Sep 2026 11:43:39 +0000 Subject: [PATCH] fix(litellm): Implement busy vs down with proper degraded state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three states: - healthy: passed, exit 0 (unchanged) - busy (completion timed out after retry AND host /health answered): ⚠️ DEGRADED line, does NOT fail the run, exit 0 - host unreachable or real fault: ❌, exit 1 (unchanged) Summary now reports degraded count: - All pass, no degraded: '✅ All checks passed' - All pass, 1+ degraded: '✅ All checks passed (1 degraded: gpu-dense)' - Some failed: '❌ Some checks failed' or '❌ Some checks failed (1 degraded: ...)' Host health mapping verified: - gpu-dense -> 192.168.68.8:8080/health - gpu-vision -> 192.168.68.110:8080/health - strix-moe -> 192.168.68.15:8080/health --- scripts/litellm-health-check.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/litellm-health-check.py b/scripts/litellm-health-check.py index 89a9785..19170b9 100755 --- a/scripts/litellm-health-check.py +++ b/scripts/litellm-health-check.py @@ -288,6 +288,7 @@ def main(): print("") all_pass = True + degraded = [] # Track degraded (busy) checks # Run all checks checks = [ @@ -307,9 +308,15 @@ def main(): # Model probes model_results = check_model_probes() for name, passed, detail in model_results: - status = "✅" if passed else "❌" + # Check if this is a busy (degraded) verdict + if not passed and detail.startswith("busy "): + status = "⚠️" + degraded.append(name) + else: + status = "✅" if passed else "❌" print(" " + status + " " + name + ": " + detail) - if not passed: + # Only set all_pass=False for real failures (not busy) + if not passed and not detail.startswith("busy "): all_pass = False # Admin key list @@ -335,10 +342,16 @@ def main(): print("") if all_pass: - print("✅ All checks passed") + if degraded: + print("✅ All checks passed (" + str(len(degraded)) + " degraded: " + ", ".join(degraded) + ")") + else: + print("✅ All checks passed") return 0 else: - print("❌ Some checks failed") + if degraded: + print("❌ Some checks failed (" + str(len(degraded)) + " degraded: " + ", ".join(degraded) + ")") + else: + print("❌ Some checks failed") return 1 if __name__ == "__main__":