Compare commits
2
Commits
a74229ee74
...
cd479caeec
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd479caeec | ||
|
|
1b1de8b0fc |
@@ -0,0 +1 @@
|
|||||||
|
__pycache__/
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Hermes Config Audit — validates a live config.yaml against the prose contract rules.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 audit-hermes-config.py <config.yaml>
|
||||||
|
python3 audit-hermes-config.py /root/.hermes/config.yaml
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 = all checks pass
|
||||||
|
1 = one or more contract violations found
|
||||||
|
|
||||||
|
This script encodes every rule from hermes-config-template.prose.md so config
|
||||||
|
changes can be verified before and after application. It is the single automated
|
||||||
|
enforcement layer for the prose contract.
|
||||||
|
|
||||||
|
Contract: /root/prose-contracts/hermes-config-template.prose.md
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
VIOLATIONS = []
|
||||||
|
WARNINGS = []
|
||||||
|
PASSES = []
|
||||||
|
|
||||||
|
|
||||||
|
def check(condition, rule, message):
|
||||||
|
if condition:
|
||||||
|
PASSES.append(f"[{rule}] {message}")
|
||||||
|
else:
|
||||||
|
VIOLATIONS.append(f"[{rule}] {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def warn(rule, message):
|
||||||
|
WARNINGS.append(f"[{rule}] {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def audit(path):
|
||||||
|
with open(path) as f:
|
||||||
|
cfg = yaml.safe_load(f)
|
||||||
|
|
||||||
|
model = cfg.get("model", {})
|
||||||
|
fb = cfg.get("fallback_providers", {})
|
||||||
|
comp = cfg.get("compression", {})
|
||||||
|
aux = cfg.get("auxiliary", {})
|
||||||
|
deleg = cfg.get("delegation", {})
|
||||||
|
cps = cfg.get("custom_providers", [])
|
||||||
|
cp = cps[0] if cps else {}
|
||||||
|
|
||||||
|
# --- Rule 3: API Keys via Environment ---
|
||||||
|
check(
|
||||||
|
model.get("api_key") in ("", None),
|
||||||
|
"Rule 3",
|
||||||
|
f"model.api_key must be empty (got {model.get('api_key')!r}) — keys via env var, not hardcoded",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
model.get("api_key_env") == "LITELLM_API_KEY",
|
||||||
|
"Rule 3",
|
||||||
|
f"model.api_key_env must be LITELLM_API_KEY (got {model.get('api_key_env')!r})",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 5: Main Config Base URL ---
|
||||||
|
expected_base = "http://192.168.68.116/v1"
|
||||||
|
check(
|
||||||
|
model.get("base_url") == expected_base,
|
||||||
|
"Rule 5",
|
||||||
|
f"model.base_url must be {expected_base} (got {model.get('base_url')!r}) — /v1 not /litellm/v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 6: max_tokens Is Required ---
|
||||||
|
check(
|
||||||
|
isinstance(model.get("max_tokens"), int) and model.get("max_tokens") <= 8192,
|
||||||
|
"Rule 6",
|
||||||
|
f"model.max_tokens must be set and <= 8192 (got {model.get('max_tokens')!r}) — thermal safety",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 7: Auxiliary Model Consistency ---
|
||||||
|
check(
|
||||||
|
comp.get("model") == "syslog-auto",
|
||||||
|
"Rule 7",
|
||||||
|
f"compression.model must be syslog-auto (got {comp.get('model')!r}) — auto-routing to prevent Strix Halo overload",
|
||||||
|
)
|
||||||
|
aux_comp = aux.get("compression", {})
|
||||||
|
check(
|
||||||
|
aux_comp.get("model") == "syslog-auto",
|
||||||
|
"Rule 7",
|
||||||
|
f"auxiliary.compression.model must be syslog-auto (got {aux_comp.get('model')!r}) — must match compression.model",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 8: GPU Workload Distribution ---
|
||||||
|
check(
|
||||||
|
aux.get("vision", {}).get("model") == "gpu-light",
|
||||||
|
"Rule 8",
|
||||||
|
f"auxiliary.vision.model must be gpu-light (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
aux.get("web_extract", {}).get("model") == "gpu-light",
|
||||||
|
"Rule 8",
|
||||||
|
f"auxiliary.web_extract.model must be gpu-light (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 9: Compression Threshold ---
|
||||||
|
check(
|
||||||
|
comp.get("threshold") == 0.65,
|
||||||
|
"Rule 9",
|
||||||
|
f"compression.threshold must be 0.65 for 128K models (got {comp.get('threshold')!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
comp.get("max_context_window") == 131072,
|
||||||
|
"Rule 9",
|
||||||
|
f"compression.max_context_window must be 131072 (got {comp.get('max_context_window')!r}) — matches 128K GPU capacity",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 10: Default Model Must Be syslog-auto ---
|
||||||
|
check(
|
||||||
|
model.get("default") == "syslog-auto",
|
||||||
|
"Rule 10",
|
||||||
|
f"model.default must be syslog-auto (got {model.get('default')!r}) — auto-routing default",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Rule 14: Provider Name Must Match custom_providers Name ---
|
||||||
|
check(
|
||||||
|
model.get("provider") == "harness",
|
||||||
|
"Rule 14",
|
||||||
|
f"model.provider must be 'harness' (got {model.get('provider')!r}) — NOT 'custom'. "
|
||||||
|
f"provider: custom causes generic resolution path that ignores key_env → 'no-key-required' → 401",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
comp.get("provider") == "harness",
|
||||||
|
"Rule 14",
|
||||||
|
f"compression.provider must be 'harness' (got {comp.get('provider')!r})",
|
||||||
|
)
|
||||||
|
for aux_name in ("vision", "web_extract", "compression"):
|
||||||
|
aux_provider = aux.get(aux_name, {}).get("provider")
|
||||||
|
check(
|
||||||
|
aux_provider == "harness",
|
||||||
|
"Rule 14",
|
||||||
|
f"auxiliary.{aux_name}.provider must be 'harness' (got {aux_provider!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
deleg.get("provider") == "harness",
|
||||||
|
"Rule 14",
|
||||||
|
f"delegation.provider must be 'harness' (got {deleg.get('provider')!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
fb.get("provider") == "deepseek",
|
||||||
|
"Rule 14",
|
||||||
|
f"fallback_providers.provider must be 'deepseek' (got {fb.get('provider')!r}) — "
|
||||||
|
f"true fallback diversity, not same endpoint as primary",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
fb.get("model") == "deepseek-v4-flash",
|
||||||
|
"Rule 14",
|
||||||
|
f"fallback_providers.model must be 'deepseek-v4-flash' (got {fb.get('model')!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
fb.get("api_key_env") == "DEEPSEEK_API_KEY",
|
||||||
|
"Rule 14",
|
||||||
|
f"fallback_providers.api_key_env must be DEEPSEEK_API_KEY (got {fb.get('api_key_env')!r})",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- custom_providers sanity ---
|
||||||
|
check(
|
||||||
|
cp.get("name") == "harness",
|
||||||
|
"custom_providers",
|
||||||
|
f"custom_providers[0].name must be 'harness' (got {cp.get('name')!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
cp.get("key_env") == "LITELLM_API_KEY" or cp.get("api_key_env") == "LITELLM_API_KEY",
|
||||||
|
"custom_providers",
|
||||||
|
f"custom_providers[0] must have key_env or api_key_env = LITELLM_API_KEY "
|
||||||
|
f"(got key_env={cp.get('key_env')!r}, api_key_env={cp.get('api_key_env')!r})",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
cp.get("base_url", "").endswith("/v1"),
|
||||||
|
"custom_providers",
|
||||||
|
f"custom_providers[0].base_url must end with /v1 (got {cp.get('base_url')!r})",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- No raw model names (Rule 7/8 spirit) ---
|
||||||
|
raw_names = {"gemma-4-12b", "qwen3.6-27B-code", "qwen3.6-35B-udq4", "ornith-1.0-35b"}
|
||||||
|
for section_path, section_dict in [
|
||||||
|
("model", model), ("compression", comp),
|
||||||
|
("auxiliary.vision", aux.get("vision", {})),
|
||||||
|
("auxiliary.web_extract", aux.get("web_extract", {})),
|
||||||
|
("auxiliary.compression", aux.get("compression", {})),
|
||||||
|
("delegation", deleg),
|
||||||
|
]:
|
||||||
|
m = section_dict.get("model", "")
|
||||||
|
if m in raw_names:
|
||||||
|
warn(
|
||||||
|
"Rule 7/8",
|
||||||
|
f"{section_path}.model = {m!r} — raw model name, use stable alias instead "
|
||||||
|
f"(gpu-light, gpu-dense, strix-moe, syslog-auto)",
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Report ---
|
||||||
|
print(f"{'=' * 60}")
|
||||||
|
print(f"Hermes Config Audit: {path}")
|
||||||
|
print(f"{'=' * 60}")
|
||||||
|
print(f"\n✅ PASSED ({len(PASSES)}):")
|
||||||
|
for p in PASSES:
|
||||||
|
print(f" ✅ {p}")
|
||||||
|
|
||||||
|
if WARNINGS:
|
||||||
|
print(f"\n⚠️ WARNINGS ({len(WARNINGS)}):")
|
||||||
|
for w in WARNINGS:
|
||||||
|
print(f" ⚠️ {w}")
|
||||||
|
|
||||||
|
if VIOLATIONS:
|
||||||
|
print(f"\n❌ VIOLATIONS ({len(VIOLATIONS)}):")
|
||||||
|
for v in VIOLATIONS:
|
||||||
|
print(f" ❌ {v}")
|
||||||
|
print(f"\n{'=' * 60}")
|
||||||
|
print(f"RESULT: FAIL — {len(VIOLATIONS)} violation(s) must be fixed")
|
||||||
|
print(f"{'=' * 60}")
|
||||||
|
return 1
|
||||||
|
else:
|
||||||
|
print(f"\n{'=' * 60}")
|
||||||
|
print(f"RESULT: PASS — all contract rules satisfied")
|
||||||
|
print(f"{'=' * 60}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: python3 audit-hermes-config.py <config.yaml>")
|
||||||
|
sys.exit(2)
|
||||||
|
sys.exit(audit(sys.argv[1]))
|
||||||
@@ -130,7 +130,7 @@ mcp_servers:
|
|||||||
# ─── Compression ───
|
# ─── Compression ───
|
||||||
compression:
|
compression:
|
||||||
enabled: true
|
enabled: true
|
||||||
model: strix-moe # ⚠️ Must match auxiliary.compression.model. Stable alias (gpu-fleet § Stable Role-Based Aliases). NOT ornith-1.0-35b (LiteLLM does not serve that name).
|
model: syslog-auto # ⚠️ Must match auxiliary.compression.model. Stable alias (gpu-fleet § Stable Role-Based Aliases). NOT ornith-1.0-35b (LiteLLM does not serve that name).
|
||||||
provider: harness
|
provider: harness
|
||||||
max_context_window: 131072 # MUST match actual GPU capacity. All 3 GPUs are 128K (Jul 17).
|
max_context_window: 131072 # MUST match actual GPU capacity. All 3 GPUs are 128K (Jul 17).
|
||||||
threshold: 0.65 # Fires at ~170K for 262K window, ~85K for 128K
|
threshold: 0.65 # Fires at ~170K for 262K window, ~85K for 128K
|
||||||
@@ -166,7 +166,7 @@ auxiliary:
|
|||||||
timeout: 30
|
timeout: 30
|
||||||
compression:
|
compression:
|
||||||
provider: harness
|
provider: harness
|
||||||
model: strix-moe # MUST match compression.model above. Stable alias for Strix Halo.
|
model: syslog-auto # MUST match compression.model above. Stable alias for Strix Halo (weighted pool).
|
||||||
base_url: http://192.168.68.116/v1 # Rule 5: /v1 NOT /litellm/v1
|
base_url: http://192.168.68.116/v1 # Rule 5: /v1 NOT /litellm/v1
|
||||||
api_key_env: LITELLM_API_KEY
|
api_key_env: LITELLM_API_KEY
|
||||||
timeout: 300 # gpu-fleet: 300s for large-history summarization (was 60)
|
timeout: 300 # gpu-fleet: 300s for large-history summarization (was 60)
|
||||||
@@ -252,6 +252,11 @@ The following MUST be identical across ALL profiles:
|
|||||||
- Compression uses `strix-moe` (stable alias for Strix Halo — 64GB, 128K ctx, compression-optimized)
|
- Compression uses `strix-moe` (stable alias for Strix Halo — 64GB, 128K ctx, compression-optimized)
|
||||||
- **`strix-moe` is the only valid compression model name** — LiteLLM does NOT serve `ornith-1.0-35b`
|
- **`strix-moe` is the only valid compression model name** — LiteLLM does NOT serve `ornith-1.0-35b`
|
||||||
(it serves `strix-moe`, `qwen3.6-35B-udq4`, `gpu-dense`, `gpu-light`, `syslog-auto`, `gemma-4-12b`, `qwen3.6-27B-code`). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
(it serves `strix-moe`, `qwen3.6-35B-udq4`, `gpu-dense`, `gpu-light`, `syslog-auto`, `gemma-4-12b`, `qwen3.6-27B-code`). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
||||||
|
- **OPERATIONAL DECISION (2026-07-23): Use `syslog-auto` for compression across all agents.**
|
||||||
|
The `syslog-auto` alias routes to the Strix Halo, but uses the weighted pool instead of pinning
|
||||||
|
to `strix-moe` directly. This prevents sustained Strix Halo thermal load because the pool can
|
||||||
|
fall back to other GPUs if Strix gets hot. Both `compression.model` and `auxiliary.compression.model`
|
||||||
|
MUST be `syslog-auto`.
|
||||||
- All auxiliary services MUST use identical routing:
|
- All auxiliary services MUST use identical routing:
|
||||||
- `base_url: http://192.168.68.116/v1` (Rule 5: `/v1`, NOT `/litellm/v1`)
|
- `base_url: http://192.168.68.116/v1` (Rule 5: `/v1`, NOT `/litellm/v1`)
|
||||||
- `api_key_env: LITELLM_API_KEY`
|
- `api_key_env: LITELLM_API_KEY`
|
||||||
@@ -265,11 +270,11 @@ The following MUST be identical across ALL profiles:
|
|||||||
### Rule 8: GPU Workload Distribution (UPDATED 2026-07-16)
|
### Rule 8: GPU Workload Distribution (UPDATED 2026-07-16)
|
||||||
- **RTX 3090 (24GB, 128K ctx, qwen3.6-27B-code)**: Heavy reasoning, code gen, long conversations
|
- **RTX 3090 (24GB, 128K ctx, qwen3.6-27B-code)**: Heavy reasoning, code gen, long conversations
|
||||||
- **RTX 5070 (12GB, 128K ctx, gemma-4-12b)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
- **RTX 5070 (12GB, 128K ctx, gemma-4-12b)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
||||||
- **Strix Halo (64GB, 128K ctx, strix-moe)**: Context compression, summarization, long docs
|
- **Strix Halo (64GB, 128K ctx, syslog-auto)**: Context compression, summarization, long docs
|
||||||
- Agent profiles MUST route auxiliary tasks to the correct GPU:
|
- Agent profiles MUST route auxiliary tasks to the correct GPU:
|
||||||
- `auxiliary.vision.model: gemma-4-12b` (RTX 5070)
|
- `auxiliary.vision.model: gemma-4-12b` (RTX 5070)
|
||||||
- `auxiliary.web_extract.model: gemma-4-12b` (RTX 5070)
|
- `auxiliary.web_extract.model: gemma-4-12b` (RTX 5070)
|
||||||
- `auxiliary.compression.model: strix-moe` (Strix Halo)
|
- `auxiliary.compression.model: syslog-auto` (Strix Halo)
|
||||||
- Default model (`model.default`) and custom_provider remain `syslog-auto` for auto-routing
|
- Default model (`model.default`) and custom_provider remain `syslog-auto` for auto-routing
|
||||||
- For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
|
- For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
|
||||||
- Do NOT use `threshold: 0.25` — this fires at 65K, causing premature context loss
|
- Do NOT use `threshold: 0.25` — this fires at 65K, causing premature context loss
|
||||||
@@ -371,6 +376,26 @@ curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $K" http://192.
|
|||||||
- `/etc/environment` is NO LONGER the canonical key source (stale values there caused 401s).
|
- `/etc/environment` is NO LONGER the canonical key source (stale values there caused 401s).
|
||||||
- Do NOT leave a hardcoded stale key in `/etc/environment` — it shadows the drop-in/wrapper.
|
- Do NOT leave a hardcoded stale key in `/etc/environment` — it shadows the drop-in/wrapper.
|
||||||
|
|
||||||
|
### Rule 14: Provider Name Must Match custom_providers Name (ADDED 2026-07-19, WAL #1471)
|
||||||
|
|
||||||
|
- `model.provider` MUST be `harness` (the `custom_providers[0].name`), NOT the literal string `custom`
|
||||||
|
- When `provider: custom`, Hermes' `_get_named_custom_provider("custom")` returns None (no provider is
|
||||||
|
named "custom" — it is named "harness"), causing a fall-through to the generic resolution path
|
||||||
|
(`source: env/config`) at `runtime_provider.py:1156`
|
||||||
|
- The generic path builds `api_key_candidates` from `model.api_key` (empty), host-gated
|
||||||
|
OLLAMA/OPENAI/OPENROUTER keys, and `_host_derived_api_key` (returns "" for IP addresses)
|
||||||
|
- **The generic path does NOT resolve `model.api_key_env` or `custom_providers.key_env`** —
|
||||||
|
`LITELLM_API_KEY` is never read, producing `api_key = "no-key-required"` → HTTP 401
|
||||||
|
- The named custom provider path (`source: custom_provider:harness`) DOES read `key_env` —
|
||||||
|
but only triggers when `provider` matches the `custom_providers[0].name`
|
||||||
|
- All sections MUST use `provider: harness`: `model`, `compression`, `auxiliary.vision`,
|
||||||
|
`auxiliary.web_extract`, `auxiliary.compression`, `delegation`
|
||||||
|
- Only `fallback_providers` uses a different provider (`deepseek`) for true fallback diversity
|
||||||
|
- **Diagnostic**: If you see `source: env/config` in a request dump or log, the provider name
|
||||||
|
is wrong. It should be `source: custom_provider:harness`.
|
||||||
|
- **Audit script**: Run `python3 /root/prose-contracts/audit-hermes-config.py <config.yaml>`
|
||||||
|
before and after any config change to catch this and all other rule violations.
|
||||||
|
|
||||||
## Execution
|
## Execution
|
||||||
|
|
||||||
1. **Check current config** — Read the target agent's config.yaml
|
1. **Check current config** — Read the target agent's config.yaml
|
||||||
|
|||||||
Reference in New Issue
Block a user