prose-contracts/tests/test_audit_hermes_fallback_list_shape.py

"""Regression test for the fallback_providers list-shape crash in audit-hermes-config.py.

WHY THIS FILE EXISTS: audit-hermes-config.py assumed `fallback_providers` was always a dict
(single provider). Two live agents (koby, koonimo) carry it as a LIST of dicts (one entry per
fallback), so the script crashed with:

    File "audit-hermes-config.py", line 211, in audit
        fb.get("provider") == "deepseek",
    AttributeError: 'list' object has no attribute 'get'

Both are REAL agent configs, so this is not a malformed-input case — the script simply could not
audit two of the four agents it exists to audit. Until fixed, the key-hygiene check had no
coverage for half the fleet while appearing to run.

These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert:
1. A config whose `fallback_providers` is a LIST of valid dicts does NOT crash (exit code is 0 or 1,
   never a traceback/AttributeError).
2. A config whose `fallback_providers` contains a MALFORMED entry (a list element that is not a
   mapping) reports a VIOLATION naming the offending entry, NOT an uncaught exception.
3. The dict shape still works (existing tests must stay green).

No network, vault, or SSH access is required.
"""
from __future__ import annotations

import pathlib
import subprocess
import sys

ROOT = pathlib.Path(__file__).resolve().parent.parent
AUDIT = ROOT / "audit-hermes-config.py"

# A valid config where fallback_providers is a LIST of dicts (the real koby/koonimo shape).
# One entry, well-formed: provider=deepseek, model=deepseek-v4-flash, api_key_env=DEEPSEEK_API_KEY.
# This must produce a real verdict (PASS or FAIL) without crashing.
LIST_SHAPE_VALID = """
model:
  api_key: ""
  api_key_env: LITELLM_API_KEY
  base_url: http://192.168.68.116/litellm/v1
  max_tokens: 4096
  default: syslog-auto
  provider: harness
fallback_providers:
  - provider: deepseek
    model: deepseek-v4-flash
    api_key_env: DEEPSEEK_API_KEY
compression:
  model: syslog-auto
  provider: harness
  threshold: 0.65
  max_context_window: 131072
auxiliary:
  vision:
    model: gpu-vision
    provider: harness
  web_extract:
    model: gpu-vision
    provider: harness
  compression:
    model: syslog-auto
    provider: harness
delegation:
  provider: harness
custom_providers:
  - name: harness
    key_env: LITELLM_API_KEY
    base_url: http://192.168.68.116/litellm/v1
"""

# A valid config where fallback_providers is a LIST with TWO entries (multiple fallbacks).
# Both entries well-formed. Must not crash and should produce a real verdict.
LIST_SHAPE_MULTI = """
model:
  api_key: ""
  api_key_env: LITELLM_API_KEY
  base_url: http://192.168.68.116/litellm/v1
  max_tokens: 4096
  default: syslog-auto
  provider: harness
fallback_providers:
  - provider: deepseek
    model: deepseek-v4-flash
    api_key_env: DEEPSEEK_API_KEY
  - provider: deepseek
    model: deepseek-v4-flash
    api_key_env: DEEPSEEK_API_KEY
compression:
  model: syslog-auto
  provider: harness
  threshold: 0.65
  max_context_window: 131072
auxiliary:
  vision:
    model: gpu-vision
    provider: harness
  web_extract:
    model: gpu-vision
    provider: harness
  compression:
    model: syslog-auto
    provider: harness
delegation:
  provider: harness
custom_providers:
  - name: harness
    key_env: LITELLM_API_KEY
    base_url: http://192.168.68.116/litellm/v1
"""

# A config where fallback_providers is a LIST containing a MALFORMED entry:
# one element is a plain string, not a mapping. The checker must report a VIOLATION
# naming the offending entry (fallback_providers[1]) and NOT crash.
LIST_SHAPE_MALFORMED = """
model:
  api_key: ""
  api_key_env: LITELLM_API_KEY
  base_url: http://192.168.68.116/litellm/v1
  max_tokens: 4096
  default: syslog-auto
  provider: harness
fallback_providers:
  - provider: deepseek
    model: deepseek-v4-flash
    api_key_env: DEEPSEEK_API_KEY
  - "not-a-mapping"
compression:
  model: syslog-auto
  provider: harness
  threshold: 0.65
  max_context_window: 131072
auxiliary:
  vision:
    model: gpu-vision
    provider: harness
  web_extract:
    model: gpu-vision
    provider: harness
  compression:
    model: syslog-auto
    provider: harness
delegation:
  provider: harness
custom_providers:
  - name: harness
    key_env: LITELLM_API_KEY
    base_url: http://192.168.68.116/litellm/v1
"""

# The original DICT shape (single provider) must still work — existing behaviour preserved.
DICT_SHAPE_VALID = """
model:
  api_key: ""
  api_key_env: LITELLM_API_KEY
  base_url: http://192.168.68.116/litellm/v1
  max_tokens: 4096
  default: syslog-auto
  provider: harness
fallback_providers:
  provider: deepseek
  model: deepseek-v4-flash
  api_key_env: DEEPSEEK_API_KEY
compression:
  model: syslog-auto
  provider: harness
  threshold: 0.65
  max_context_window: 131072
auxiliary:
  vision:
    model: gpu-vision
    provider: harness
  web_extract:
    model: gpu-vision
    provider: harness
  compression:
    model: syslog-auto
    provider: harness
delegation:
  provider: harness
custom_providers:
  - name: harness
    key_env: LITELLM_API_KEY
    base_url: http://192.168.68.116/litellm/v1
"""


def _run_config(tmp_path, name, text):
    cfg = tmp_path / name
    cfg.write_text(text)
    proc = subprocess.run(
        [sys.executable, str(AUDIT), str(cfg)],
        capture_output=True, text=True,
    )
    return proc.returncode, proc.stdout, proc.stderr


def test_list_shape_single_entry_does_not_crash(tmp_path):
    """A LIST with one valid dict must not raise AttributeError; exit 0 (PASS)."""
    code, out, err = _run_config(tmp_path, "list-single.yaml", LIST_SHAPE_VALID)
    # Must NOT be a crash (traceback). A clean run exits 0 (PASS) or 1 (FAIL), never 2+ (exception).
    assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}"
    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
    # The valid single-entry list should PASS (all rules satisfied).
    assert code == 0, f"Expected PASS but got {code}\n{out}"
    assert "RESULT: PASS" in out


def test_list_shape_multiple_entries_does_not_crash(tmp_path):
    """A LIST with two valid dicts must not raise AttributeError; exit 0 (PASS)."""
    code, out, err = _run_config(tmp_path, "list-multi.yaml", LIST_SHAPE_MULTI)
    assert code in (0, 1), f"Expected clean exit 0 or 1, got {code}\nSTDOUT:\n{out}\nSTDERR:\n{err}"
    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
    assert code == 0, f"Expected PASS but got {code}\n{out}"
    assert "RESULT: PASS" in out


def test_list_shape_malformed_entry_reports_violation_not_crash(tmp_path):
    """A LIST containing a non-mapping element must be a reported VIOLATION, not a crash."""
    code, out, err = _run_config(tmp_path, "list-malformed.yaml", LIST_SHAPE_MALFORMED)
    # Must NOT be a crash.
    assert "AttributeError" not in err, f"Crashed with AttributeError:\n{err}"
    assert "Traceback" not in err, f"Crashed with uncaught exception:\n{err}"
    # Should be a FAIL (exit 1) because the malformed entry is a violation.
    assert code == 1, f"Expected FAIL (exit 1) but got {code}\n{out}"
    assert "RESULT: FAIL" in out
    # The violation must name the offending entry (fallback_providers[1]).
    assert "fallback_providers[1]" in out, f"Violation did not name the offending entry:\n{out}"


def test_dict_shape_still_passes(tmp_path):
    """The original DICT shape (single provider) must still PASS — existing behaviour preserved."""
    code, out, err = _run_config(tmp_path, "dict-valid.yaml", DICT_SHAPE_VALID)
    assert code == 0, f"Expected PASS but got {code}\n{out}\nSTDERR:\n{err}"
    assert "RESULT: PASS" in out