fix(audit): reject retired aliases; sweep gpu-light/gemma-4-12b to gpu-vision
audit-hermes-config.py Rule 8 required auxiliary.vision.model and auxiliary.web_extract.model to equal the retired 'gpu-light', so a config adopting the live canonical 'gpu-vision' FAILED our own audit - the audit was enforcing a dead alias (400 Invalid model name). Rule 8 now requires gpu-vision; retired names gpu-light/crew-auto join the raw-name rejection set; the guidance message names the live aliases. Sweep of the remaining references: gpu-self-heal stops canonicalizing gpu-light; hermes-config-template, hermes-agent-baseline, hermes-key-enforcement, inference-optimization, litellm-client-timeouts and gpu-fleet now use the live gpu-vision alias. Where a file restated model/rpm/weight/fallback state it now points at CT 116 /opt/inference-harness/litellm_config.yaml instead of duplicating it. koby's .129 config is report-only and recorded, not edited. Adds tests/test_audit_hermes_config_alias.py: executes the audit CLI and asserts gpu-vision passes while gpu-light and gemma-4-12b fail.
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
"""Regression tests for the 2026-09-12 retired-alias sweep in audit-hermes-config.py.
|
||||
|
||||
WHY THIS FILE EXISTS: the executable audit pushed agent configs toward a DEAD alias.
|
||||
Rule 8 required `auxiliary.vision.model == "gpu-light"` and
|
||||
`auxiliary.web_extract.model == "gpu-light"`, but `gpu-light` (and its raw predecessor
|
||||
`gemma-4-12b`) were retired on 2026-09-12 and now return 400 `Invalid model name`; the
|
||||
live RTX 5070 alias is `gpu-vision`. A config that adopted the correct canonical alias
|
||||
therefore FAILED our own audit, so the audit was actively enforcing a broken config.
|
||||
|
||||
These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert
|
||||
observable behaviour — exit code and the emitted rule message — for the live alias and
|
||||
for both retired names. No network, vault, or SSH access is required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
AUDIT = ROOT / "audit-hermes-config.py"
|
||||
|
||||
BASE = """
|
||||
model:
|
||||
api_key: ""
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
max_tokens: 4096
|
||||
default: syslog-auto
|
||||
provider: harness
|
||||
fallback_providers:
|
||||
provider: deepseek
|
||||
model: deepseek-v4-flash
|
||||
api_key_env: DEEPSEEK_API_KEY
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
threshold: 0.65
|
||||
max_context_window: 131072
|
||||
auxiliary:
|
||||
vision:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
web_extract:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
delegation:
|
||||
provider: harness
|
||||
custom_providers:
|
||||
- name: harness
|
||||
key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
"""
|
||||
|
||||
|
||||
def _run(tmp_path, alias):
|
||||
cfg = tmp_path / f"{alias}.yaml"
|
||||
cfg.write_text(BASE.format(alias=alias))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(AUDIT), str(cfg)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return proc.returncode, proc.stdout
|
||||
|
||||
|
||||
def test_live_canonical_alias_passes(tmp_path):
|
||||
"""The RTX 5070 alias that actually resolves must satisfy Rule 8."""
|
||||
code, out = _run(tmp_path, "gpu-vision")
|
||||
assert code == 0, out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_gpu_light_is_rejected(tmp_path):
|
||||
"""A config pinned to the retired alias must fail, not pass."""
|
||||
code, out = _run(tmp_path, "gpu-light")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_gemma_is_rejected(tmp_path):
|
||||
"""The retired raw model name must fail Rule 8 as well."""
|
||||
code, out = _run(tmp_path, "gemma-4-12b")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
Reference in New Issue
Block a user