|
|
|
@@ -193,38 +193,107 @@ Run on any Hermes host to detect violations.
|
|
|
|
|
**Bounded scan (2026-10-02):** Do NOT recurse the entire `/root/.hermes/` tree. Use `--exclude-dir=state-snapshots` to skip dated snapshot directories. Rationale: a superseded config will always carry a superseded key and will report forever with zero signal content (the koby state-snapshot line has repeated on consecutive days). If you deliberately want to include snapshots, say so in the contract and the report.
|
|
|
|
|
|
|
|
|
|
```bash
|
|
|
|
|
# 1. Check config.yaml for hardcoded harness keys (bounded scan — excludes state-snapshots)
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"grep -rn 'api_key: sk-' /root/.hermes/ --exclude-dir=state-snapshots --include='config.yaml' | grep -v 'deepseek\|openai\|anthropic\|DEEPSEEK'" \
|
|
|
|
|
2>/dev/null
|
|
|
|
|
# Per-agent path resolution (2026-10-03): mumuni /home/hermes/.hermes, koonimo/koby /root/.hermes, tanko /home/jerome/.hermes (down)
|
|
|
|
|
# HERMES_HOME is ALWAYS resolvable per agent from an explicit map, independent of whether anything is running
|
|
|
|
|
# SYSTEMD_USER_DIR is the per-user systemd dir for the agent (from the same map)
|
|
|
|
|
# LIVE_GW_PID is needed ONLY by step 3; steps 1/1b use HERMES_HOME and run even when the gateway is down
|
|
|
|
|
# HARD GUARD: an empty/unresolved HERMES_HOME must never expand into `/` or an empty glob
|
|
|
|
|
|
|
|
|
|
# Interpret exit status:
|
|
|
|
|
# 0 = match found (violation)
|
|
|
|
|
# 1 = no match (pass)
|
|
|
|
|
# 124 = timeout (probe-failed, not unreachable)
|
|
|
|
|
# 255 = ssh connect failed (unreachable)
|
|
|
|
|
# other = probe-failed (record the actual code)
|
|
|
|
|
# Step A: Resolve HERMES_HOME and SYSTEMD_USER_DIR from the static per-agent map (no SSH needed)
|
|
|
|
|
# mumuni: /home/hermes/.hermes + /home/hermes/.config/systemd/user
|
|
|
|
|
# koby/koonimo: /root/.hermes + /root/.config/systemd/user
|
|
|
|
|
# tanko: /home/jerome/.hermes + /home/jerome/.config/systemd/user
|
|
|
|
|
HERMES_HOME="$(case "<agent>" in
|
|
|
|
|
mumuni) echo "/home/hermes/.hermes" ;;
|
|
|
|
|
koby|koonimo) echo "/root/.hermes" ;;
|
|
|
|
|
tanko) echo "/home/jerome/.hermes" ;;
|
|
|
|
|
*) echo "" ;;
|
|
|
|
|
esac)"
|
|
|
|
|
SYSTEMD_USER_DIR="$(case "<agent>" in
|
|
|
|
|
mumuni) echo "/home/hermes/.config/systemd/user" ;;
|
|
|
|
|
koby|koonimo) echo "/root/.config/systemd/user" ;;
|
|
|
|
|
tanko) echo "/home/jerome/.config/systemd/user" ;;
|
|
|
|
|
*) echo "" ;;
|
|
|
|
|
esac)"
|
|
|
|
|
|
|
|
|
|
# Step B: Resolve LIVE_GW_PID (only for step 3) - check if gateway.pid exists and process is alive
|
|
|
|
|
# NOTE: Simpler approach - read gateway.pid locally, parse locally, then test liveness with a separate SSH
|
|
|
|
|
LIVE_GW_PID="$(ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"if [ -n '${HERMES_HOME}' ] && [ -f '${HERMES_HOME}/gateway.pid' ]; then \
|
|
|
|
|
cat '${HERMES_HOME}/gateway.pid'; \
|
|
|
|
|
fi" 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin).get("pid",""))' 2>/dev/null)"
|
|
|
|
|
# Test liveness with a second short SSH
|
|
|
|
|
if [ -n "$LIVE_GW_PID" ]; then
|
|
|
|
|
if ! ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"[ -d /proc/${LIVE_GW_PID} ]" 2>/dev/null; then
|
|
|
|
|
LIVE_GW_PID="" # Gateway is down
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 1. Check config.yaml for hardcoded harness keys (bounded scan — excludes state-snapshots)
|
|
|
|
|
# Uses HERMES_HOME (not LIVE_GW_PID) so it runs even when the gateway is down
|
|
|
|
|
if [ -n "${HERMES_HOME}" ]; then
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"grep -rn 'api_key: sk-' ${HERMES_HOME}/ --exclude-dir=state-snapshots --include='config.yaml' | grep -v 'deepseek\|openai\|anthropic\|DEEPSEEK'" \
|
|
|
|
|
2>/dev/null
|
|
|
|
|
# Interpret exit status:
|
|
|
|
|
# 0 = match found (violation)
|
|
|
|
|
# 1 = no match (pass)
|
|
|
|
|
# 124 = timeout (probe-failed, not unreachable)
|
|
|
|
|
# 255 = ssh connect failed (unreachable)
|
|
|
|
|
# other = probe-failed (record the actual code)
|
|
|
|
|
else
|
|
|
|
|
echo "probe-failed: could not resolve the Hermes home for <agent>"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 1b. Check for double-path bug: base_url ending with /responses
|
|
|
|
|
# (Hermes appends /v1/responses when api_mode=responses, so base_url must end at /v1)
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"grep -rn 'litellm/v1/responses' /root/.hermes/config.yaml" 2>/dev/null
|
|
|
|
|
# ANY output here = WRONG. Must be 'litellm/v1' without /responses suffix.
|
|
|
|
|
# Uses HERMES_HOME (not LIVE_GW_PID) so it runs even when the gateway is down
|
|
|
|
|
if [ -n "${HERMES_HOME}" ]; then
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"grep -rn 'litellm/v1/responses' ${HERMES_HOME}/config.yaml" 2>/dev/null
|
|
|
|
|
# ANY output here = WRONG. Must be 'litellm/v1' without /responses suffix.
|
|
|
|
|
else
|
|
|
|
|
echo "probe-failed: could not resolve the Hermes home for <agent>"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 2. Check systemd drop-ins for master key leaks (2026-07-05: Tanko had this)
|
|
|
|
|
# NOTE: Both greps are inside ONE quoted remote command, separated by ; (not two separate ssh arguments)
|
|
|
|
|
# NOTE: Scan system-level /etc/systemd/system/*hermes* AND the per-user systemd dir (from the static map)
|
|
|
|
|
# Per-agent systemd user dirs (from the map, measured 2026-10-03):
|
|
|
|
|
# koby/koonimo: /root/.config/systemd/user
|
|
|
|
|
# tanko: /home/jerome/.config/systemd/user
|
|
|
|
|
# mumuni: /home/hermes/.config/systemd/user
|
|
|
|
|
# All agents: /etc/systemd/system/ (system-level units)
|
|
|
|
|
# A missing/empty scan location must render as probe-failed, never as compliant
|
|
|
|
|
# NOTE: Double-quoted ssh command so ${SYSTEMD_USER_DIR} (local var) is interpolated on the runner
|
|
|
|
|
# Each grep runs independently; a missing dir prints probe-failed, never silent empty
|
|
|
|
|
# NOTE: Credential values are masked to a 4-character prefix + "...MASKED" to avoid printing live keys
|
|
|
|
|
# The detection is identical — every hit is still found and reported; only the printed value changes
|
|
|
|
|
MASK='sk-[A-Za-z0-9_.-]{4}'
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"grep -rn 'LITELLM_API_KEY' /root/.config/systemd/user/ 2>/dev/null; grep -rn 'LITELLM_API_KEY=sk-synthetic-litellm-' /root/.config/systemd/ 2>/dev/null" ; true
|
|
|
|
|
"grep -rn 'LITELLM_API_KEY' /etc/systemd/system/*hermes* 2>/dev/null | sed -E 's/(${MASK})[A-Za-z0-9_.-]+/\1...MASKED/g'; \
|
|
|
|
|
if [ -d '${SYSTEMD_USER_DIR}' ]; then \
|
|
|
|
|
grep -rn 'LITELLM_API_KEY' '${SYSTEMD_USER_DIR}/' 2>/dev/null | sed -E 's/(${MASK})[A-Za-z0-9_.-]+/\1...MASKED/g'; \
|
|
|
|
|
else \
|
|
|
|
|
echo 'probe-failed: per-user systemd dir ${SYSTEMD_USER_DIR} not found'; \
|
|
|
|
|
fi" ; true
|
|
|
|
|
|
|
|
|
|
# 3. Verify running process env matches dedicated key
|
|
|
|
|
# NOTE: Single-quoted remote command so $(...) expands on the REMOTE host, not the runner
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
'cat /proc/$(cat /home/jerome/.hermes/gateway.pid | python3 -c "import sys,json; print(json.load(sys.stdin)['pid'])")/environ | tr "\0" "\n" | grep LITELLM_API_KEY' \
|
|
|
|
|
&& echo "step3: PASS" || echo "step3: probe-failed (exit $?; see stderr above)"
|
|
|
|
|
# NOTE: Uses LIVE_GW_PID (not HERMES_HOME) - only runs when gateway is up
|
|
|
|
|
# NOTE: Credential value is masked to a 4-character prefix + "...MASKED" to avoid printing live keys
|
|
|
|
|
if [ -n "$LIVE_GW_PID" ]; then
|
|
|
|
|
timeout 15 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@<ip> \
|
|
|
|
|
"cat /proc/${LIVE_GW_PID}/environ | tr '\0' '\n' | grep LITELLM_API_KEY | sed -E 's/(sk-[A-Za-z0-9_.-]{4})[A-Za-z0-9_.-]+/\1...MASKED/g'" \
|
|
|
|
|
&& echo "step3: PASS" || echo "step3: probe-failed (exit $?; see stderr above)"
|
|
|
|
|
else
|
|
|
|
|
echo "step3: probe-failed (no live gateway.pid for this agent)"
|
|
|
|
|
fi
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
If any output from step 2 — **critical violation** (master key leaked). Fix immediately.
|
|
|
|
|
|
|
|
|
|
> **Masking note (2026-10-03):** Credential values in steps 2 and 3 are deliberately masked to a 4-character prefix + `...MASKED` (e.g. `sk-Cggi...MASKED`) to avoid printing live keys into transcripts and logs. The detection is identical — every hit is still found and reported; only the printed value changes. Do not mistake a masked value for truncated output.
|
|
|
|
|
|
|
|
|
|
### Negative control (probe-failed vs unreachable) — deterministic
|
|
|
|
|
|
|
|
|
|
To prove the distinction between a scan timeout and a connection failure, run a command that CANNOT finish in time (sleep 5s) with a 1-second timeout:
|
|
|
|
|
abiba-bot
root