prose-contracts/audit-hermes-config.py

#!/usr/bin/env python3
"""
Hermes Config Audit — validates a live config.yaml against the prose contract rules.

Usage:
    python3 audit-hermes-config.py <config.yaml>
    python3 audit-hermes-config.py /root/.hermes/config.yaml

Exit codes:
    0 = all checks pass
    1 = one or more contract violations found

This script encodes every rule from hermes-config-template.prose.md so config
changes can be verified before and after application. It is the single automated
enforcement layer for the prose contract.

Contract: /root/prose-contracts/hermes-config-template.prose.md
"""

import sys
import yaml

VIOLATIONS = []
WARNINGS = []
PASSES = []


def check(condition, rule, message):
    if condition:
        PASSES.append(f"[{rule}] {message}")
    else:
        VIOLATIONS.append(f"[{rule}] {message}")


def warn(rule, message):
    WARNINGS.append(f"[{rule}] {message}")


def audit(path):
    with open(path) as f:
        cfg = yaml.safe_load(f)

    model = cfg.get("model", {})
    fb = cfg.get("fallback_providers", {})
    comp = cfg.get("compression", {})
    aux = cfg.get("auxiliary", {})
    deleg = cfg.get("delegation", {})
    cps = cfg.get("custom_providers", [])
    cp = cps[0] if cps else {}

    # --- Rule 3: API Keys via Environment ---
    check(
        model.get("api_key") in ("", None),
        "Rule 3",
        f"model.api_key must be empty (got {model.get('api_key')!r}) — keys via env var, not hardcoded",
    )
    check(
        model.get("api_key_env") == "LITELLM_API_KEY",
        "Rule 3",
        f"model.api_key_env must be LITELLM_API_KEY (got {model.get('api_key_env')!r})",
    )

    # --- Rule 5: Main Config Base URL ---
    expected_base = "http://192.168.68.116/v1"
    check(
        model.get("base_url") == expected_base,
        "Rule 5",
        f"model.base_url must be {expected_base} (got {model.get('base_url')!r}) — /v1 not /litellm/v1",
    )

    # --- Rule 6: max_tokens Is Required ---
    check(
        isinstance(model.get("max_tokens"), int) and model.get("max_tokens") <= 8192,
        "Rule 6",
        f"model.max_tokens must be set and <= 8192 (got {model.get('max_tokens')!r}) — thermal safety",
    )

    # --- Rule 7: Auxiliary Model Consistency ---
    check(
        comp.get("model") == "syslog-auto",
        "Rule 7",
        f"compression.model must be syslog-auto (got {comp.get('model')!r}) — auto-routing to prevent Strix Halo overload",
    )
    aux_comp = aux.get("compression", {})
    check(
        aux_comp.get("model") == "syslog-auto",
        "Rule 7",
        f"auxiliary.compression.model must be syslog-auto (got {aux_comp.get('model')!r}) — must match compression.model",
    )

    # --- Rule 8: GPU Workload Distribution ---
    check(
        aux.get("vision", {}).get("model") == "gpu-light",
        "Rule 8",
        f"auxiliary.vision.model must be gpu-light (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
    )
    check(
        aux.get("web_extract", {}).get("model") == "gpu-light",
        "Rule 8",
        f"auxiliary.web_extract.model must be gpu-light (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
    )

    # --- Rule 9: Compression Threshold ---
    check(
        comp.get("threshold") == 0.65,
        "Rule 9",
        f"compression.threshold must be 0.65 for 128K models (got {comp.get('threshold')!r})",
    )
    check(
        comp.get("max_context_window") == 131072,
        "Rule 9",
        f"compression.max_context_window must be 131072 (got {comp.get('max_context_window')!r}) — matches 128K GPU capacity",
    )

    # --- Rule 10: Default Model Must Be syslog-auto ---
    check(
        model.get("default") == "syslog-auto",
        "Rule 10",
        f"model.default must be syslog-auto (got {model.get('default')!r}) — auto-routing default",
    )

    # --- Rule 14: Provider Name Must Match custom_providers Name ---
    check(
        model.get("provider") == "harness",
        "Rule 14",
        f"model.provider must be 'harness' (got {model.get('provider')!r}) — NOT 'custom'. "
        f"provider: custom causes generic resolution path that ignores key_env → 'no-key-required' → 401",
    )
    check(
        comp.get("provider") == "harness",
        "Rule 14",
        f"compression.provider must be 'harness' (got {comp.get('provider')!r})",
    )
    for aux_name in ("vision", "web_extract", "compression"):
        aux_provider = aux.get(aux_name, {}).get("provider")
        check(
            aux_provider == "harness",
            "Rule 14",
            f"auxiliary.{aux_name}.provider must be 'harness' (got {aux_provider!r})",
        )
    check(
        deleg.get("provider") == "harness",
        "Rule 14",
        f"delegation.provider must be 'harness' (got {deleg.get('provider')!r})",
    )
    check(
        fb.get("provider") == "deepseek",
        "Rule 14",
        f"fallback_providers.provider must be 'deepseek' (got {fb.get('provider')!r}) — "
        f"true fallback diversity, not same endpoint as primary",
    )
    check(
        fb.get("model") == "deepseek-v4-flash",
        "Rule 14",
        f"fallback_providers.model must be 'deepseek-v4-flash' (got {fb.get('model')!r})",
    )
    check(
        fb.get("api_key_env") == "DEEPSEEK_API_KEY",
        "Rule 14",
        f"fallback_providers.api_key_env must be DEEPSEEK_API_KEY (got {fb.get('api_key_env')!r})",
    )

    # --- custom_providers sanity ---
    check(
        cp.get("name") == "harness",
        "custom_providers",
        f"custom_providers[0].name must be 'harness' (got {cp.get('name')!r})",
    )
    check(
        cp.get("key_env") == "LITELLM_API_KEY" or cp.get("api_key_env") == "LITELLM_API_KEY",
        "custom_providers",
        f"custom_providers[0] must have key_env or api_key_env = LITELLM_API_KEY "
        f"(got key_env={cp.get('key_env')!r}, api_key_env={cp.get('api_key_env')!r})",
    )
    check(
        cp.get("base_url", "").endswith("/v1"),
        "custom_providers",
        f"custom_providers[0].base_url must end with /v1 (got {cp.get('base_url')!r})",
    )

    # --- No raw model names (Rule 7/8 spirit) ---
    raw_names = {"gemma-4-12b", "qwen3.6-27B-code", "qwen3.6-35B-udq4", "ornith-1.0-35b"}
    for section_path, section_dict in [
        ("model", model), ("compression", comp),
        ("auxiliary.vision", aux.get("vision", {})),
        ("auxiliary.web_extract", aux.get("web_extract", {})),
        ("auxiliary.compression", aux.get("compression", {})),
        ("delegation", deleg),
    ]:
        m = section_dict.get("model", "")
        if m in raw_names:
            warn(
                "Rule 7/8",
                f"{section_path}.model = {m!r} — raw model name, use stable alias instead "
                f"(gpu-light, gpu-dense, strix-moe, syslog-auto)",
            )

    # --- Report ---
    print(f"{'=' * 60}")
    print(f"Hermes Config Audit: {path}")
    print(f"{'=' * 60}")
    print(f"\n✅ PASSED ({len(PASSES)}):")
    for p in PASSES:
        print(f"  ✅ {p}")

    if WARNINGS:
        print(f"\n⚠️  WARNINGS ({len(WARNINGS)}):")
        for w in WARNINGS:
            print(f"  ⚠️  {w}")

    if VIOLATIONS:
        print(f"\n❌ VIOLATIONS ({len(VIOLATIONS)}):")
        for v in VIOLATIONS:
            print(f"  ❌ {v}")
        print(f"\n{'=' * 60}")
        print(f"RESULT: FAIL — {len(VIOLATIONS)} violation(s) must be fixed")
        print(f"{'=' * 60}")
        return 1
    else:
        print(f"\n{'=' * 60}")
        print(f"RESULT: PASS — all contract rules satisfied")
        print(f"{'=' * 60}")
        return 0


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python3 audit-hermes-config.py <config.yaml>")
        sys.exit(2)
    sys.exit(audit(sys.argv[1]))