abiba
221f9f79f3
audit-hermes-config.py Rule 8 required auxiliary.vision.model and auxiliary.web_extract.model to equal the retired 'gpu-light', so a config adopting the live canonical 'gpu-vision' FAILED our own audit - the audit was enforcing a dead alias (400 Invalid model name). Rule 8 now requires gpu-vision; retired names gpu-light/crew-auto join the raw-name rejection set; the guidance message names the live aliases. Sweep of the remaining references: gpu-self-heal stops canonicalizing gpu-light; hermes-config-template, hermes-agent-baseline, hermes-key-enforcement, inference-optimization, litellm-client-timeouts and gpu-fleet now use the live gpu-vision alias. Where a file restated model/rpm/weight/fallback state it now points at CT 116 /opt/inference-harness/litellm_config.yaml instead of duplicating it. koby's .129 config is report-only and recorded, not edited. Adds tests/test_audit_hermes_config_alias.py: executes the audit CLI and asserts gpu-vision passes while gpu-light and gemma-4-12b fail.
90 lines
2.7 KiB
Python
90 lines
2.7 KiB
Python
"""Regression tests for the 2026-09-12 retired-alias sweep in audit-hermes-config.py.
|
|
|
|
WHY THIS FILE EXISTS: the executable audit pushed agent configs toward a DEAD alias.
|
|
Rule 8 required `auxiliary.vision.model == "gpu-light"` and
|
|
`auxiliary.web_extract.model == "gpu-light"`, but `gpu-light` (and its raw predecessor
|
|
`gemma-4-12b`) were retired on 2026-09-12 and now return 400 `Invalid model name`; the
|
|
live RTX 5070 alias is `gpu-vision`. A config that adopted the correct canonical alias
|
|
therefore FAILED our own audit, so the audit was actively enforcing a broken config.
|
|
|
|
These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert
|
|
observable behaviour — exit code and the emitted rule message — for the live alias and
|
|
for both retired names. No network, vault, or SSH access is required.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
AUDIT = ROOT / "audit-hermes-config.py"
|
|
|
|
BASE = """
|
|
model:
|
|
api_key: ""
|
|
api_key_env: LITELLM_API_KEY
|
|
base_url: http://192.168.68.116/v1
|
|
max_tokens: 4096
|
|
default: syslog-auto
|
|
provider: harness
|
|
fallback_providers:
|
|
provider: deepseek
|
|
model: deepseek-v4-flash
|
|
api_key_env: DEEPSEEK_API_KEY
|
|
compression:
|
|
model: syslog-auto
|
|
provider: harness
|
|
threshold: 0.65
|
|
max_context_window: 131072
|
|
auxiliary:
|
|
vision:
|
|
model: {alias}
|
|
provider: harness
|
|
web_extract:
|
|
model: {alias}
|
|
provider: harness
|
|
compression:
|
|
model: syslog-auto
|
|
provider: harness
|
|
delegation:
|
|
provider: harness
|
|
custom_providers:
|
|
- name: harness
|
|
key_env: LITELLM_API_KEY
|
|
base_url: http://192.168.68.116/v1
|
|
"""
|
|
|
|
|
|
def _run(tmp_path, alias):
|
|
cfg = tmp_path / f"{alias}.yaml"
|
|
cfg.write_text(BASE.format(alias=alias))
|
|
proc = subprocess.run(
|
|
[sys.executable, str(AUDIT), str(cfg)],
|
|
capture_output=True, text=True,
|
|
)
|
|
return proc.returncode, proc.stdout
|
|
|
|
|
|
def test_live_canonical_alias_passes(tmp_path):
|
|
"""The RTX 5070 alias that actually resolves must satisfy Rule 8."""
|
|
code, out = _run(tmp_path, "gpu-vision")
|
|
assert code == 0, out
|
|
assert "RESULT: PASS" in out
|
|
|
|
|
|
def test_retired_gpu_light_is_rejected(tmp_path):
|
|
"""A config pinned to the retired alias must fail, not pass."""
|
|
code, out = _run(tmp_path, "gpu-light")
|
|
assert code == 1, out
|
|
assert "auxiliary.vision.model must be gpu-vision" in out
|
|
assert "RESULT: FAIL" in out
|
|
|
|
|
|
def test_retired_gemma_is_rejected(tmp_path):
|
|
"""The retired raw model name must fail Rule 8 as well."""
|
|
code, out = _run(tmp_path, "gemma-4-12b")
|
|
assert code == 1, out
|
|
assert "auxiliary.vision.model must be gpu-vision" in out
|
|
assert "RESULT: FAIL" in out
|