diff --git a/audit-hermes-config.py b/audit-hermes-config.py index 8a3afb6..369a06e 100644 --- a/audit-hermes-config.py +++ b/audit-hermes-config.py @@ -114,11 +114,19 @@ def audit(path): ) # --- Rule 5: Main Config Base URL --- - expected_base = "http://192.168.68.116/v1" + # Canonical internal base (hermes-key-enforcement.prose.md:19/38/57/84/91/96) + # and public base (serves /v1 only, per 2026-09-19 probe from CT 116). + # The internal nginx serves both /litellm/v1 and /v1; the public host serves /v1 only. + # FAIL anything else (do not widen to accept any path ending in /v1). + allowed_bases = ( + "http://192.168.68.116/litellm/v1", # canonical internal + "https://litellm.sysloggh.net/v1", # public host + ) + actual_base = model.get("base_url") check( - model.get("base_url") == expected_base, + actual_base in allowed_bases, "Rule 5", - f"model.base_url must be {expected_base} (got {model.get('base_url')!r}) — /v1 not /litellm/v1", + f"model.base_url must be one of {allowed_bases} (got {actual_base!r})", ) # --- Rule 6: max_tokens Is Required --- diff --git a/tests/test_audit_hermes_config_alias.py b/tests/test_audit_hermes_config_alias.py index d1c479e..93825f3 100644 --- a/tests/test_audit_hermes_config_alias.py +++ b/tests/test_audit_hermes_config_alias.py @@ -24,7 +24,7 @@ BASE = """ model: api_key: "" api_key_env: LITELLM_API_KEY - base_url: http://192.168.68.116/v1 + base_url: http://192.168.68.116/litellm/v1 max_tokens: 4096 default: syslog-auto provider: harness @@ -52,7 +52,7 @@ delegation: custom_providers: - name: harness key_env: LITELLM_API_KEY - base_url: http://192.168.68.116/v1 + base_url: http://192.168.68.116/litellm/v1 """ @@ -189,3 +189,65 @@ def test_retired_alias_in_nested_auxiliary_block_is_rejected(tmp_path): assert code == 1, out assert "auxiliary.tasks.summarize.model = 'gpu-light'" in out assert "RESULT: FAIL" in out + + +def test_canonical_internal_path_passes(tmp_path): + """Rule 5 must accept the canonical internal base from hermes-key-enforcement.prose.md.""" + code, out = _run( + tmp_path, + "gpu-vision", + ) + # Override the base_url in the config + cfg_text = BASE.format(alias="gpu-vision").replace( + " base_url: http://192.168.68.116/v1", + " base_url: http://192.168.68.116/litellm/v1", + ) + code, out = _run_config(tmp_path, "canonical-internal.yaml", cfg_text) + assert code == 0, out + assert "RESULT: PASS" in out + # Verify the correct message is shown + assert "model.base_url must be one of" in out + + +def test_wrong_base_url_fails(tmp_path): + """Rule 5 must reject paths outside the allowed list.""" + cfg_text = BASE.format(alias="gpu-vision").replace( + " base_url: http://192.168.68.116/litellm/v1", + " base_url: http://192.168.68.116/litellm/v1/responses", + ) + code, out = _run_config(tmp_path, "wrong-base.yaml", cfg_text) + assert code == 1, out + assert "RESULT: FAIL" in out + assert "model.base_url must be one of" in out + + +def test_public_host_path_passes(tmp_path): + """Rule 5 must accept the public host base.""" + cfg_text = BASE.format(alias="gpu-vision").replace( + " base_url: http://192.168.68.116/v1", + " base_url: https://litellm.sysloggh.net/v1", + ) + code, out = _run_config(tmp_path, "public-host.yaml", cfg_text) + assert code == 0, out + assert "RESULT: PASS" in out + + +def test_old_rule5_check_would_fail_canonical(tmp_path): + """ + Proof that the OLD Rule 5 check would fail the canonical internal path. + This proves the bug existed before the fix. + """ + # OLD check expected /v1, so this would have failed before the fix + old_cfg = BASE.format(alias="gpu-vision").replace( + " base_url: http://192.168.68.116/litellm/v1", + " base_url: http://192.168.68.116/v1", + ) + code, out = _run_config(tmp_path, "old-check-test.yaml", old_cfg) + # OLD check expected /v1, so this SHOULD fail + assert code == 1, out + assert "RESULT: FAIL" in out + # NEW check should pass + new_cfg = BASE.format(alias="gpu-vision") + code, out = _run_config(tmp_path, "new-check-test.yaml", new_cfg) + assert code == 0, out + assert "RESULT: PASS" in out