diff --git a/audit-hermes-config.py b/audit-hermes-config.py index 2c3d109..f2e6d8c 100644 --- a/audit-hermes-config.py +++ b/audit-hermes-config.py @@ -262,6 +262,41 @@ def audit(path): f"{field_path} = {value!r} is a raw-but-live model name — prefer the stable alias {raw_but_live[value]}", ) + # --- MCP Server Checks (Rule 15) --- + # Valid MCP server endpoints + VALID_MCP_ENDPOINTS = { + 'ra-h-os': 'http://192.168.68.65:3100/mcp', + 'litellm': 'https://litellm.sysloggh.net/mcp', + } + + # Check MCP servers if they exist + mcp_servers = cfg.get('mcp_servers', {}) + if mcp_servers: + for server_name, server_config in mcp_servers.items(): + url = server_config.get('url', '') + + # Check endpoint validity + if server_name in VALID_MCP_ENDPOINTS: + expected = VALID_MCP_ENDPOINTS[server_name] + check(url == expected, 'Rule 15', f'MCP server "{server_name}" URL is correct: {url}') + else: + warn('Rule 15', f'MCP server "{server_name}" URL may need validation (not in known list): {url}') + + # Check for proper authentication + headers = server_config.get('headers', {}) + has_auth = False + for key, value in headers.items(): + if 'key' in key.lower() or 'auth' in key.lower(): + has_auth = True + # Check if the value looks like a literal key vs env-var reference + if value.startswith('Bearer ') and value[7:].startswith('sk-'): + check(True, 'Rule 15', f'MCP server "{server_name}" has valid auth header: {key}') + else: + warn('Rule 15', f'MCP server "{server_name}" header may use env-var instead of literal key: {key} = {value}') + break + if not has_auth: + warn('Rule 15', f'MCP server "{server_name}" has no authentication header') + # --- Report --- print(f"{'=' * 60}") print(f"Hermes Config Audit: {path}") diff --git a/hermes-config-template.prose.md b/hermes-config-template.prose.md index 0a1a6ef..9c393df 100644 --- a/hermes-config-template.prose.md +++ b/hermes-config-template.prose.md @@ -5,7 +5,8 @@ description: > Standard Hermes configuration template for Syslog Solution LLC agents. Enforces shared infrastructure setup (Firecrawl, SearXNG, local models, RA-H OS MCP) while keeping agent-specific API keys and model choices. - UPDATED 2026-08-07: Added Rule 15 (MCP Validation) from the 2026-08-07 keyless-MCP incident. + UPDATED 2026-08-07: Added litellm MCP server entry; updated Rule 15 (MCP Validation) + to enforce REAL key headers (not env-vars) from the 2026-08-07 keyless-MCP incident. Added Rule 12 (Context-Issue Diagnostic) + Rule 13 (.env fallback enforcement) from the 2026-07-16 Mumuni root-cause investigation (WAL #1300). UPDATED 2026-07-12: GPU workload redistributed. Compression → Strix Halo. RTX 3090 context verified at 128K. Infisical .env fallback required (Rule 3/13). @@ -145,6 +146,12 @@ mcp_servers: url: http://192.168.68.65:3100/mcp timeout: 120 connect_timeout: 60 + litellm: + url: https://litellm.sysloggh.net/mcp + headers: + x-litellm-api-key: "Bearer " # Rule 15: must be a REAL key (sk-...), not an env-var name + # Note: MCP endpoint requires Accept: application/json, text/event-stream header + # This is handled by the MCP client library; don't add to config # ─── Compression ─── compression: @@ -217,6 +224,40 @@ When LiteLLM keys are regenerated (e.g., after infrastructure changes): 3. **After update**: Restart Hermes on the agent host 4. **Verify**: `curl -H "Authorization: Bearer sk-" http://192.168.68.116/v1/models` +## MCP Server Configuration + +MCP server entries in `mcp_servers:` must follow the format shown in the Template section. + +**Header requirements (Rule 15):** +- Use `headers:` field with a `x-litellm-api-key` entry +- The value must be `"Bearer "` where `` is a literal LiteLLM virtual key +- Do NOT use env-var references like `$LITELLM_API_KEY` — they resolve to empty strings in + the static config and cause "Malformed API Key" errors (2026-08-07 Tanko incident) + +**Key source:** +- Keys are stored in the Infisical vault (project=agents, env=production) +- For template-based config generation: substitute the agent's key from the agent_keys table +- For manual config updates: retrieve the key from the vault and insert the literal value + +**Verification (2026-08-07):** +- Tested MCP initialize handshake against litellm.sysloggh.net/mcp with agent virtual key +- Confirmed: 200 response with `serverInfo.name: "litellm-mcp-server"` +- Confirmed: tools/list returns 200 (MCP endpoint accessible with virtual keys) +- Note: This contradicts infrastructure-update.prose.md:214 ("only master key has access") — + the LiteLLM version may have been upgraded since that contract was written +- Key requirement: must be a valid LiteLLM virtual key (HTTP 200 on /v1/models) + +**Key rotation note:** +- MCP headers use literal keys (not env-vars), so they do NOT auto-rotate with the vault +- After key rotation, MCP server headers must be regenerated with the new key value +- This is a manual step: update the `x-litellm-api-key` header in each config file +- TODO: Consider adding MCP header regeneration to the Key Update Procedure or a generation hook + +**NetBird dependency:** +- `litellm.sysloggh.net` is a NetBird endpoint (see Rule 5 and Infrastructure Stack table) +- NetBird outages cause 502 errors on MCP requests, not auth failures +- Diagnose: if MCP requests fail with 502, check NetBird status before investigating keys + ## Violation Classification When reporting findings, separate POLICY observations from FAULT findings: @@ -455,14 +496,28 @@ curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $K" http://192. - **Audit script**: Run `python3 /root/prose-contracts/audit-hermes-config.py ` before and after any config change to catch this and all other rule violations. -### Rule 15: MCP Endpoint and Header Validation (ADDED 2026-08-07) -- Every MCP server entry must point at the correct endpoint: - - ra-h-os = http://192.168.68.65:3100/mcp - - litellm = https://litellm.sysloggh.net/mcp -- MCP entries must carry a REAL key value in the header. - - Avoid using env-var names like LITELLM_API_KEY in the header; they do not resolve for MCP - endpoints and result in "Malformed API Key" floods. - - Ensure the header value is the actual key (e.g., `sk-...`). +### Rule 15: MCP Endpoint and Header Validation (UPDATED 2026-08-07) + +**Endpoint validation:** +- ra-h-os must point to `http://192.168.68.65:3100/mcp` +- litellm must point to `https://litellm.sysloggh.net/mcp` +- Mismatched endpoints cause silent failures (e.g., 2026-08-07 incident: Tanko's config had + ra-h-os pointing to litellm's endpoint) + +**Header validation:** +- Every MCP entry with authentication must carry a `headers:` field +- The header value must be a REAL key (e.g., `Bearer sk-abc123...`), NOT an env-var name +- Env-var names like `LITELLM_API_KEY` do NOT resolve in static MCP configs and cause + "Malformed API Key" floods (401 errors in agent gateway logs) +- Verify: header value should match a valid LiteLLM key (test with `curl` against /v1/models) +- Verify MCP access: test the MCP initialize handshake against the MCP endpoint (not just /v1/models) + ```bash + curl -s -X POST -H "x-litellm-api-key: Bearer " -H "Accept: application/json, text/event-stream" \ + https://litellm.sysloggh.net/mcp -d '{"jsonrpc":"2.0","id":1,"method":"initialize",...}' \ + | jq '.data.result.serverInfo' # should show serverInfo.name and version + ``` + +**See:** § MCP Server Configuration for implementation details and key source. ## Execution