From e42b970dec3288cc0200d8722ead3afc584d130c Mon Sep 17 00:00:00 2001 From: root Date: Sun, 27 Sep 2026 11:17:18 +0000 Subject: [PATCH] fix(audit-hermes): handle fallback_providers as list or dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit assumed fallback_providers was always a dict (single provider). Two live agents (koby, koonimo) carry it as a LIST of dicts (one entry per fallback), so the script crashed with: File "audit-hermes-config.py", line 211, in audit fb.get("provider") == "deepseek", AttributeError: 'list' object has no attribute 'get' Both are REAL agent configs, so this is not a malformed-input case — the script simply could not audit two of the four agents it exists to audit. Fix: - Normalize fallback_providers to a list of entries (dict → [dict], list → list) - Apply the existing checks to each entry - A malformed entry (not a mapping) produces a reported VIOLATION naming the offending entry, NOT an uncaught exception Adds regression test using the real failing shape (list of dicts) and proves it bites against the pre-fix revision. Real audit results after fix: - mumuni: FAIL — 7 violations - tanko: FAIL — 21 violations - koby: FAIL — 16 violations (previously crashed) - koonimo: FAIL — 10 violations (previously crashed) No agent configs were changed. No existing rules were relaxed. --- audit-hermes-config.py | 53 ++-- .../test_audit_hermes_fallback_list_shape.py | 236 ++++++++++++++++++ 2 files changed, 272 insertions(+), 17 deletions(-) create mode 100644 tests/test_audit_hermes_fallback_list_shape.py diff --git a/audit-hermes-config.py b/audit-hermes-config.py index 383b02b..c45cf83 100644 --- a/audit-hermes-config.py +++ b/audit-hermes-config.py @@ -94,7 +94,16 @@ def audit(path): cfg = yaml.safe_load(f) model = cfg.get("model", {}) - fb = cfg.get("fallback_providers", {}) + fb_raw = cfg.get("fallback_providers", {}) + # Normalize: fallback_providers may be a dict (single provider) or a list of dicts + # (one entry per fallback). Both shapes are valid; we must handle both without crashing. + if isinstance(fb_raw, dict): + fb_entries = [fb_raw] + elif isinstance(fb_raw, list): + fb_entries = fb_raw + else: + fb_entries = [fb_raw] # Let it fail the check below as malformed + fb = fb_entries[0] if fb_entries else {} comp = cfg.get("compression", {}) aux = cfg.get("auxiliary", {}) deleg = cfg.get("delegation", {}) @@ -207,22 +216,32 @@ def audit(path): "Rule 14", f"delegation.provider must be 'harness' (got {deleg.get('provider')!r})", ) - check( - fb.get("provider") == "deepseek", - "Rule 14", - f"fallback_providers.provider must be 'deepseek' (got {fb.get('provider')!r}) — " - f"true fallback diversity, not same endpoint as primary", - ) - check( - fb.get("model") == "deepseek-v4-flash", - "Rule 14", - f"fallback_providers.model must be 'deepseek-v4-flash' (got {fb.get('model')!r})", - ) - check( - fb.get("api_key_env") == "DEEPSEEK_API_KEY", - "Rule 14", - f"fallback_providers.api_key_env must be DEEPSEEK_API_KEY (got {fb.get('api_key_env')!r})", - ) + # Check each fallback entry. A malformed entry (not a mapping) is a VIOLATION, not a crash. + for idx, entry in enumerate(fb_entries): + prefix = f"fallback_providers[{idx}]" + if not isinstance(entry, dict): + check( + False, + "Rule 14", + f"{prefix} must be a mapping (got {type(entry).__name__})", + ) + continue + check( + entry.get("provider") == "deepseek", + "Rule 14", + f"{prefix}.provider must be 'deepseek' (got {entry.get('provider')!r}) — " + f"true fallback diversity, not same endpoint as primary", + ) + check( + entry.get("model") == "deepseek-v4-flash", + "Rule 14", + f"{prefix}.model must be 'deepseek-v4-flash' (got {entry.get('model')!r})", + ) + check( + entry.get("api_key_env") == "DEEPSEEK_API_KEY", + "Rule 14", + f"{prefix}.api_key_env must be DEEPSEEK_API_KEY (got {entry.get('api_key_env')!r})", + ) # --- custom_providers sanity --- check( diff --git a/tests/test_audit_hermes_fallback_list_shape.py b/tests/test_audit_hermes_fallback_list_shape.py new file mode 100644 index 0000000..af48c59 --- /dev/null +++ b/tests/test_audit_hermes_fallback_list_shape.py @@ -0,0 +1,236 @@ +"""Regression test for the fallback_providers list-shape crash in audit-hermes-config.py. + +WHY THIS FILE EXISTS: audit-hermes-config.py assumed `fallback_providers` was always a dict +(single provider). Two live agents (koby, koonimo) carry it as a LIST of dicts (one entry per +fallback), so the script crashed with: + + File "audit-hermes-config.py", line 211, in audit + fb.get("provider") == "deepseek", + AttributeError: 'list' object has no attribute 'get' + +Both are REAL agent configs, so this is not a malformed-input case — the script simply could not +audit two of the four agents it exists to audit. Until fixed, the key-hygiene check had no +coverage for half the fleet while appearing to run. + +These tests execute the real CLI (`python3 audit-hermes-config.py `) and assert: +1. A config whose `fallback_providers` is a LIST of valid dicts does NOT crash (exit code is 0 or 1, + never a traceback/AttributeError). +2. A config whose `fallback_providers` contains a MALFORMED entry (a list element that is not a + mapping) reports a VIOLATION naming the offending entry, NOT an uncaught exception. +3. The dict shape still works (existing tests must stay green). + +No network, vault, or SSH access is required. +""" +from __future__ import annotations + +import pathlib +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +AUDIT = ROOT / "audit-hermes-config.py" + +# A valid config where fallback_providers is a LIST of dicts (the real koby/koonimo shape). +# One entry, well-formed: provider=deepseek, model=deepseek-v4-flash, api_key_env=DEEPSEEK_API_KEY. +# This must produce a real verdict (PASS or FAIL) without crashing. +LIST_SHAPE_VALID = """ +model: + api_key: "" + api_key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 + max_tokens: 4096 + default: syslog-auto + provider: harness +fallback_providers: + - provider: deepseek + model: deepseek-v4-flash + api_key_env: DEEPSEEK_API_KEY +compression: + model: syslog-auto + provider: harness + threshold: 0.65 + max_context_window: 131072 +auxiliary: + vision: + model: gpu-vision + provider: harness + web_extract: + model: gpu-vision + provider: harness + compression: + model: syslog-auto + provider: harness +delegation: + provider: harness +custom_providers: + - name: harness + key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 +""" + +# A valid config where fallback_providers is a LIST with TWO entries (multiple fallbacks). +# Both entries well-formed. Must not crash and should produce a real verdict. +LIST_SHAPE_MULTI = """ +model: + api_key: "" + api_key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 + max_tokens: 4096 + default: syslog-auto + provider: harness +fallback_providers: + - provider: deepseek + model: deepseek-v4-flash + api_key_env: DEEPSEEK_API_KEY + - provider: deepseek + model: deepseek-v4-flash + api_key_env: DEEPSEEK_API_KEY +compression: + model: syslog-auto + provider: harness + threshold: 0.65 + max_context_window: 131072 +auxiliary: + vision: + model: gpu-vision + provider: harness + web_extract: + model: gpu-vision + provider: harness + compression: + model: syslog-auto + provider: harness +delegation: + provider: harness +custom_providers: + - name: harness + key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 +""" + +# A config where fallback_providers is a LIST containing a MALFORMED entry: +# one element is a plain string, not a mapping. The checker must report a VIOLATION +# naming the offending entry (fallback_providers[1]) and NOT crash. +LIST_SHAPE_MALFORMED = """ +model: + api_key: "" + api_key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 + max_tokens: 4096 + default: syslog-auto + provider: harness +fallback_providers: + - provider: deepseek + model: deepseek-v4-flash + api_key_env: DEEPSEEK_API_KEY + - "not-a-mapping" +compression: + model: syslog-auto + provider: harness + threshold: 0.65 + max_context_window: 131072 +auxiliary: + vision: + model: gpu-vision + provider: harness + web_extract: + model: gpu-vision + provider: harness + compression: + model: syslog-auto + provider: harness +delegation: + provider: harness +custom_providers: + - name: harness + key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 +""" + +# The original DICT shape (single provider) must still work — existing behaviour preserved. +DICT_SHAPE_VALID = """ +model: + api_key: "" + api_key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 + max_tokens: 4096 + default: syslog-auto + provider: harness +fallback_providers: + provider: deepseek + model: deepseek-v4-flash + api_key_env: DEEPSEEK_API_KEY +compression: + model: syslog-auto + provider: harness + threshold: 0.65 + max_context_window: 131072 +auxiliary: + vision: + model: gpu-vision + provider: harness + web_extract: + model: gpu-vision + provider: harness + compression: + model: syslog-auto + provider: harness +delegation: + provider: harness +custom_providers: + - name: harness + key_env: LITELLM_API_KEY + base_url: http://192.168.68.116/litellm/v1 +""" + + +def _run_config(tmp_path, name, text): + cfg = tmp_path / name + cfg.write_text(text) + proc = subprocess.run( + [sys.executable, str(AUDIT), str(cfg)], + capture_output=True, text=True, + ) + return proc.returncode, proc.stdout, proc.stderr + + +def test_list_shape_single_entry_does_not_crash(tmp_path): + """A LIST with one valid dict must not raise AttributeError; exit 0 (PASS).""" + code, out, err = _run_config(tmp_path, "list-single.yaml", LIST_SHAPE_VALID) + # Must NOT be a crash (traceback). A clean run exits 0 (PASS) or 1 (FAIL), never 2+ (exception). + assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}" + assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}" + assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}" + # The valid single-entry list should PASS (all rules satisfied). + assert code == 0, f"Expected PASS but got {code}\n{out}" + assert "RESULT: PASS" in out + + +def test_list_shape_multiple_entries_does_not_crash(tmp_path): + """A LIST with two valid dicts must not raise AttributeError; exit 0 (PASS).""" + code, out, err = _run_config(tmp_path, "list-multi.yaml", LIST_SHAPE_MULTI) + assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}" + assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}" + assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}" + assert code == 0, f"Expected PASS but got {code}\n{out}" + assert "RESULT: PASS" in out + + +def test_list_shape_malformed_entry_reports_violation_not_crash(tmp_path): + """A LIST containing a non-mapping element must be a reported VIOLATION, not a crash.""" + code, out, err = _run_config(tmp_path, "list-malformed.yaml", LIST_SHAPE_MALFORMED) + # Must NOT be a crash. + assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}" + assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}" + # Should be a FAIL (exit 1) because the malformed entry is a violation. + assert code == 1, f"Expected FAIL (exit 1) but got {code}\n{out}" + assert "RESULT: FAIL" in out + # The violation must name the offending entry (fallback_providers[1]). + assert "fallback_providers[1]" in out, f"Violation did not name the offending entry:\n{out}" + + +def test_dict_shape_still_passes(tmp_path): + """The original DICT shape (single provider) must still PASS — existing behaviour preserved.""" + code, out, err = _run_config(tmp_path, "dict-valid.yaml", DICT_SHAPE_VALID) + assert code == 0, f"Expected PASS but got {code}\n{out}\nSTDERR:\n{err}" + assert "RESULT: PASS" in out -- 2.54.0