audit-hermes-config.py

@@ -94,7 +94,16 @@ def audit(path):
         cfg = yaml.safe_load(f)
 
     model = cfg.get("model", {})
-    fb = cfg.get("fallback_providers", {})
+    fb_raw = cfg.get("fallback_providers", {})
+    # Normalize: fallback_providers may be a dict (single provider) or a list of dicts
+    # (one entry per fallback). Both shapes are valid; we must handle both without crashing.
+    if isinstance(fb_raw, dict):
+        fb_entries = [fb_raw]
+    elif isinstance(fb_raw, list):
+        fb_entries = fb_raw
+    else:
+        fb_entries = [fb_raw]  # Let it fail the check below as malformed
+    fb = fb_entries[0] if fb_entries else {}
     comp = cfg.get("compression", {})
     aux = cfg.get("auxiliary", {})
     deleg = cfg.get("delegation", {})
@@ -207,22 +216,32 @@ def audit(path):
         "Rule 14",
         f"delegation.provider must be 'harness' (got {deleg.get('provider')!r})",
     )
-    check(
-        fb.get("provider") == "deepseek",
-        "Rule 14",
-        f"fallback_providers.provider must be 'deepseek' (got {fb.get('provider')!r}) — "
-        f"true fallback diversity, not same endpoint as primary",
-    )
-    check(
-        fb.get("model") == "deepseek-v4-flash",
-        "Rule 14",
-        f"fallback_providers.model must be 'deepseek-v4-flash' (got {fb.get('model')!r})",
-    )
-    check(
-        fb.get("api_key_env") == "DEEPSEEK_API_KEY",
-        "Rule 14",
-        f"fallback_providers.api_key_env must be DEEPSEEK_API_KEY (got {fb.get('api_key_env')!r})",
-    )
+    # Check each fallback entry. A malformed entry (not a mapping) is a VIOLATION, not a crash.
+    for idx, entry in enumerate(fb_entries):
+        prefix = f"fallback_providers[{idx}]"
+        if not isinstance(entry, dict):
+            check(
+                False,
+                "Rule 14",
+                f"{prefix} must be a mapping (got {type(entry).__name__})",
+            )
+            continue
+        check(
+            entry.get("provider") == "deepseek",
+            "Rule 14",
+            f"{prefix}.provider must be 'deepseek' (got {entry.get('provider')!r}) — "
+            f"true fallback diversity, not same endpoint as primary",
+        )
+        check(
+            entry.get("model") == "deepseek-v4-flash",
+            "Rule 14",
+            f"{prefix}.model must be 'deepseek-v4-flash' (got {entry.get('model')!r})",
+        )
+        check(
+            entry.get("api_key_env") == "DEEPSEEK_API_KEY",
+            "Rule 14",
+            f"{prefix}.api_key_env must be DEEPSEEK_API_KEY (got {entry.get('api_key_env')!r})",
+        )
 
     # --- custom_providers sanity ---
     check(

tests/test_audit_hermes_fallback_list_shape.py

@@ -0,0 +1,236 @@
+"""Regression test for the fallback_providers list-shape crash in audit-hermes-config.py.
+
+WHY THIS FILE EXISTS: audit-hermes-config.py assumed `fallback_providers` was always a dict
+(single provider). Two live agents (koby, koonimo) carry it as a LIST of dicts (one entry per
+fallback), so the script crashed with:
+
+    File "audit-hermes-config.py", line 211, in audit
+        fb.get("provider") == "deepseek",
+    AttributeError: 'list' object has no attribute 'get'
+
+Both are REAL agent configs, so this is not a malformed-input case — the script simply could not
+audit two of the four agents it exists to audit. Until fixed, the key-hygiene check had no
+coverage for half the fleet while appearing to run.
+
+These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert:
+1. A config whose `fallback_providers` is a LIST of valid dicts does NOT crash (exit code is 0 or 1,
+   never a traceback/AttributeError).
+2. A config whose `fallback_providers` contains a MALFORMED entry (a list element that is not a
+   mapping) reports a VIOLATION naming the offending entry, NOT an uncaught exception.
+3. The dict shape still works (existing tests must stay green).
+
+No network, vault, or SSH access is required.
+"""
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+AUDIT = ROOT / "audit-hermes-config.py"
+
+# A valid config where fallback_providers is a LIST of dicts (the real koby/koonimo shape).
+# One entry, well-formed: provider=deepseek, model=deepseek-v4-flash, api_key_env=DEEPSEEK_API_KEY.
+# This must produce a real verdict (PASS or FAIL) without crashing.
+LIST_SHAPE_VALID = """
+model:
+  api_key: ""
+  api_key_env: LITELLM_API_KEY
+  base_url: http://192.168.68.116/litellm/v1
+  max_tokens: 4096
+  default: syslog-auto
+  provider: harness
+fallback_providers:
+  - provider: deepseek
+    model: deepseek-v4-flash
+    api_key_env: DEEPSEEK_API_KEY
+compression:
+  model: syslog-auto
+  provider: harness
+  threshold: 0.65
+  max_context_window: 131072
+auxiliary:
+  vision:
+    model: gpu-vision
+    provider: harness
+  web_extract:
+    model: gpu-vision
+    provider: harness
+  compression:
+    model: syslog-auto
+    provider: harness
+delegation:
+  provider: harness
+custom_providers:
+  - name: harness
+    key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/litellm/v1
+"""
+
+# A valid config where fallback_providers is a LIST with TWO entries (multiple fallbacks).
+# Both entries well-formed. Must not crash and should produce a real verdict.
+LIST_SHAPE_MULTI = """
+model:
+  api_key: ""
+  api_key_env: LITELLM_API_KEY
+  base_url: http://192.168.68.116/litellm/v1
+  max_tokens: 4096
+  default: syslog-auto
+  provider: harness
+fallback_providers:
+  - provider: deepseek
+    model: deepseek-v4-flash
+    api_key_env: DEEPSEEK_API_KEY
+  - provider: deepseek
+    model: deepseek-v4-flash
+    api_key_env: DEEPSEEK_API_KEY
+compression:
+  model: syslog-auto
+  provider: harness
+  threshold: 0.65
+  max_context_window: 131072
+auxiliary:
+  vision:
+    model: gpu-vision
+    provider: harness
+  web_extract:
+    model: gpu-vision
+    provider: harness
+  compression:
+    model: syslog-auto
+    provider: harness
+delegation:
+  provider: harness
+custom_providers:
+  - name: harness
+    key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/litellm/v1
+"""
+
+# A config where fallback_providers is a LIST containing a MALFORMED entry:
+# one element is a plain string, not a mapping. The checker must report a VIOLATION
+# naming the offending entry (fallback_providers[1]) and NOT crash.
+LIST_SHAPE_MALFORMED = """
+model:
+  api_key: ""
+  api_key_env: LITELLM_API_KEY
+  base_url: http://192.168.68.116/litellm/v1
+  max_tokens: 4096
+  default: syslog-auto
+  provider: harness
+fallback_providers:
+  - provider: deepseek
+    model: deepseek-v4-flash
+    api_key_env: DEEPSEEK_API_KEY
+  - "not-a-mapping"
+compression:
+  model: syslog-auto
+  provider: harness
+  threshold: 0.65
+  max_context_window: 131072
+auxiliary:
+  vision:
+    model: gpu-vision
+    provider: harness
+  web_extract:
+    model: gpu-vision
+    provider: harness
+  compression:
+    model: syslog-auto
+    provider: harness
+delegation:
+  provider: harness
+custom_providers:
+  - name: harness
+    key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/litellm/v1
+"""
+
+# The original DICT shape (single provider) must still work — existing behaviour preserved.
+DICT_SHAPE_VALID = """
+model:
+  api_key: ""
+  api_key_env: LITELLM_API_KEY
+  base_url: http://192.168.68.116/litellm/v1
+  max_tokens: 4096
+  default: syslog-auto
+  provider: harness
+fallback_providers:
+  provider: deepseek
+  model: deepseek-v4-flash
+  api_key_env: DEEPSEEK_API_KEY
+compression:
+  model: syslog-auto
+  provider: harness
+  threshold: 0.65
+  max_context_window: 131072
+auxiliary:
+  vision:
+    model: gpu-vision
+    provider: harness
+  web_extract:
+    model: gpu-vision
+    provider: harness
+  compression:
+    model: syslog-auto
+    provider: harness
+delegation:
+  provider: harness
+custom_providers:
+  - name: harness
+    key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/litellm/v1
+"""
+
+
+def _run_config(tmp_path, name, text):
+    cfg = tmp_path / name
+    cfg.write_text(text)
+    proc = subprocess.run(
+        [sys.executable, str(AUDIT), str(cfg)],
+        capture_output=True, text=True,
+    )
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+def test_list_shape_single_entry_does_not_crash(tmp_path):
+    """A LIST with one valid dict must not raise AttributeError; exit 0 (PASS)."""
+    code, out, err = _run_config(tmp_path, "list-single.yaml", LIST_SHAPE_VALID)
+    # Must NOT be a crash (traceback). A clean run exits 0 (PASS) or 1 (FAIL), never 2+ (exception).
+    assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}"
+    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
+    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
+    # The valid single-entry list should PASS (all rules satisfied).
+    assert code == 0, f"Expected PASS but got {code}\n{out}"
+    assert "RESULT: PASS" in out
+
+
+def test_list_shape_multiple_entries_does_not_crash(tmp_path):
+    """A LIST with two valid dicts must not raise AttributeError; exit 0 (PASS)."""
+    code, out, err = _run_config(tmp_path, "list-multi.yaml", LIST_SHAPE_MULTI)
+    assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}"
+    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
+    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
+    assert code == 0, f"Expected PASS but got {code}\n{out}"
+    assert "RESULT: PASS" in out
+
+
+def test_list_shape_malformed_entry_reports_violation_not_crash(tmp_path):
+    """A LIST containing a non-mapping element must be a reported VIOLATION, not a crash."""
+    code, out, err = _run_config(tmp_path, "list-malformed.yaml", LIST_SHAPE_MALFORMED)
+    # Must NOT be a crash.
+    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
+    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
+    # Should be a FAIL (exit 1) because the malformed entry is a violation.
+    assert code == 1, f"Expected FAIL (exit 1) but got {code}\n{out}"
+    assert "RESULT: FAIL" in out
+    # The violation must name the offending entry (fallback_providers[1]).
+    assert "fallback_providers[1]" in out, f"Violation did not name the offending entry:\n{out}"
+
+
+def test_dict_shape_still_passes(tmp_path):
+    """The original DICT shape (single provider) must still PASS — existing behaviour preserved."""
+    code, out, err = _run_config(tmp_path, "dict-valid.yaml", DICT_SHAPE_VALID)
+    assert code == 0, f"Expected PASS but got {code}\n{out}\nSTDERR:\n{err}"
+    assert "RESULT: PASS" in out