Revert "infra-update contract: v1.1.0 — fix CT→VM refs, add missing stacks, correct paths"

This reverts commit 09ffbf10aa.

.gitea/workflows/pr-pipeline.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout repository
         run: |
-          git clone --depth=50 "http://git.sysloggh.net/${{ gitea.repository }}.git" .
+          git clone --depth=50 "http://192.168.68.17:3000/${{ gitea.repository }}.git" .
           git fetch origin "${{ gitea.ref }}" --depth=50
           git checkout "${{ gitea.sha }}"
 
@@ -34,7 +34,7 @@ jobs:
     steps:
       - name: Checkout repository
         run: |
-          git clone --depth=50 "http://git.sysloggh.net/${{ gitea.repository }}.git" .
+          git clone --depth=50 "http://192.168.68.17:3000/${{ gitea.repository }}.git" .
           git fetch origin "${{ gitea.ref }}" --depth=50
           git checkout "${{ gitea.sha }}"
 
@@ -64,7 +64,7 @@ jobs:
     steps:
       - name: Checkout repository
         run: |
-          git clone --depth=50 "http://git.sysloggh.net/${{ gitea.repository }}.git" .
+          git clone --depth=50 "http://192.168.68.17:3000/${{ gitea.repository }}.git" .
           git fetch origin "${{ gitea.ref }}" --depth=50
           git checkout "${{ gitea.sha }}"
 
@@ -77,7 +77,7 @@ jobs:
     steps:
       - name: Checkout repository
         run: |
-          git clone --depth=50 "http://git.sysloggh.net/${{ gitea.repository }}.git" .
+          git clone --depth=50 "http://192.168.68.17:3000/${{ gitea.repository }}.git" .
           git fetch origin "${{ gitea.ref }}" --depth=50
           git checkout "${{ gitea.sha }}"
 

gpu-fleet.prose.md

@@ -27,6 +27,7 @@ triggers:
 - benchmarks: { tok_per_sec: map, baseline: map, history: array } — Inference speed benchmarks tracked over time
 - grafana: { status: "running", dashboards: ["gpu-fleet"] } — Grafana on CT 116 (:3001)
 - prometheus: { status: "running", targets: 5 } — Scrapes GPU :9400 exporters + LiteLLM
+- port_conflict_detection: { status: "active" } — All 3 GPU wrappers detect ghost processes before binding
 
 ## Fleet Topology (Current — June 2026)
 
@@ -112,7 +113,7 @@ triggers:
 
 ### sync-keys
 1. List all agent keys in LiteLLM DB via `GET /key/list`
-2. Compare against expected agent list: [tanko, mumuni, abiba, koby, koonimo, kagenz0]
+2. Compare against expected agent list: [tanko, mumuni, abiba, tdunna, baggy, kagenz0]
 3. Generate missing keys via `POST /key/generate` with unlimited budget
 4. Update agent configs — `/etc/environment` LITELLM_API_KEY
 5. Send Zulip DM to agents that can't be reached via SSH
@@ -124,14 +125,14 @@ Show full fleet status: GPUs, models, VRAM, active requests, circuit breakers, k
 
 ## Agent Keys (LiteLLM DB — Current 2026-06-30)
 
-| Agent | Key | Host | Access |
-|-------|-----|------|--------|
-| Tanko | `sk-3ZsdWJbbNSo9zSnJN2OsJw` | .122 | SSH jerome |
-| Mumuni | `sk-XY2aUfvy2BIs6kp1ZPh6VA` | .123 | SSH root |
-| Abiba | `sk-kxbPgbnV2Zkn6TKQbAEcEg` | .24 | local |
-| Koby | `sk-lDTsWy7H3T19UwUFEN6JSg` | ? | Zulip DM (regenerated 2026-07-01 — old `sk-eb3e6...` was router key, not LiteLLM) |
-| Koonimo | `sk-Prx_vRXYb2VEzDmhPCbNeg` | .114 (baggy) | no SSH |
-| Kagenz0 | `sk-Dh4CDkaHebMLEp8qqq20qA` | ? | no SSH |
+| Agent | CT | IP | Key | Access |
+|-------|-----|-----|-----|--------|
+| Tanko | 112 | .122 | `sk-CggiHWlamQyShxWC3Hx6uw` | SSH jerome |
+| Mumuni | 114 | .123 | `sk-VrqCNlwUgzoNGOpikJ7nwQ` | SSH root |
+| Abiba | 100 | .65 | `sk-Qvzi4uYQBhlSK_XstEhcyQ` | local (pi agent) |
+| Tdunna | 111 | ? | `sk-6sbCNjz2T6lTVDBdlNHXsA` | Zulip DM |
+| Baggy | 113 | ? | `sk-krnw_zGBwvvL5b7l2t-s-A` | no SSH |
+| Kagenz0 | 105 | ? | `sk-Dh4CDkaHebMLEp8qqq20qA` | no SSH |
 
 **Key update procedure**: Update `/etc/environment` → `LITELLM_API_KEY=sk-...` → restart Hermes.
 If no SSH access, send Zulip DM via abiba-bot.
@@ -145,10 +146,10 @@ If no SSH access, send Zulip DM via abiba-bot.
 | `/opt/inference-harness/router/router.py` | CT 116 | Router source (builds via compose) |
 | `/etc/nginx/nginx.conf` | CT 116 (nginx container) | Routes /v1→LiteLLM, /dashboard/, /litellm/, /health |
 | `/opt/monitoring/prometheus.yml` | CT 116 | Prometheus scrape config (5 targets) |
-| `/root/scripts/gpu-monitor-server.py` | pi (.24) | GPU fleet monitor v2.1.0 (with benchmarks) |
-| `/root/scripts/gpu_benchmark.py` | pi (.24) | GPU inference benchmark module (tok/s tracking) |
-| `/root/scripts/gpu-saturation-watchdog.py` | pi (.24) | Auto-restart stuck llama-server |
-| `/root/dashboard/gpu-fleet.html` | pi (.24) | Live HTML dashboard |
+| `/root/scripts/gpu-monitor-server.py` | pi (.65) | GPU fleet monitor v2.1.0 (with benchmarks) |
+| `/root/scripts/gpu_benchmark.py` | pi (.65) | GPU inference benchmark module (tok/s tracking) |
+| `/root/scripts/gpu-saturation-watchdog.py` | pi (.65) | Auto-restart stuck llama-server |
+| `/root/dashboard/gpu-fleet.html` | pi (.65) | Live HTML dashboard |
 | `/etc/systemd/system/llama-server.service` | .8, .110 | llama-server daemons (Nvidia GPUs) |
 | `/etc/systemd/system/ornith-server.service` | .15 (amdpve) | llama-server daemon (Vulkan, Strix Halo). Note: `llama-server.service` and `llama-server@.service` are **masked** on .15 to prevent port 8080 collisions. |
 
@@ -171,6 +172,7 @@ If no SSH access, send Zulip DM via abiba-bot.
 - **RTX 3090 optimization**: `--parallel 2 --batch-size 4096` — doubled concurrent throughput.
 - **RTX 5070 optimization**: Removed draft model freed ~2GB VRAM for vision tasks. `--parallel 2`.
 - **Strix Halo GPU**: Vulkan is the working backend (ROCm/HIP path abandoned — HSA runtime blocked on Debian 13). Build at `/root/llama.cpp/build-vk/`, commit `4fc4ec5` (2026-07-01), ggml 0.15.3 shared-lib arch. Mesa RADV 25.0.7, KHR_coopmat fast path active. ~70 tok/s gen, 532 tok/s prompt. Service: `ornith-server.service` on port 8080, 256K context, flash-attn + q8 KV.
+- **Port conflict detection (2026-07-05)**: All 3 GPU wrappers now detect ghost processes squatting port 8080 before starting. `.8` and `.110` use inline pre-start check in `llama-wrapper.sh`; `.15` uses `/usr/local/bin/port-cleanup.sh` ExecStartPre. Replaces the blanket `pkill -9 -x llama-server` on .15 which would kill ALL llama-server instances regardless of port. Ghost detection was the root cause of .8 crash-looping for 27+ restarts (stale pid 25836 squatting 8080 after OOM kill).
 - **Strix Halo thermal safeguard (2026-07-02)**: `ornith-server.service` has `-n 8192` (hard generation cap per request). Without it, `--predict` defaults to -1 (infinity) — a runaway request from .123 (Mumuni) decoded 39,868 tokens over 24 min, pushing Tctl to 98°C (crit 89.8°C) and throttling 70→29 t/s. The cap bounds worst-case generation to ~5 min. Do NOT remove `-n` without a replacement ceiling. Sustained load hits ~84°C even at 92s; the APU is fanless/low-flow. Clients MUST also set `max_tokens`.
 - **Port 8080 firewall**: amdpve iptables restricts 8080 to 192.168.68.116 (LiteLLM/router host) only. All inbound connections are from .116 (LiteLLM proxied via nginx). Localhost curls hang (SYN dropped). Always test from .116.
 - **Router sidecar fallback**: `router.py` `check_gpu_health()` now probes GPU `/health` directly when sidecar at :8090 is absent. Sidecar JSON exporters not deployed on any GPU host — router relies on GPU-direct fallback.

hermes-agent-baseline.prose.md

@@ -0,0 +1,214 @@
+---
+kind: reference
+name: hermes-agent-baseline
+version: 1.0.0
+description: >
+  Canonical known-good baseline for all Syslog Hermes agents. Captures the exact
+  configuration state, keys, workarounds, and audit procedure. When an agent's
+  configuration goes sideways, restore from this baseline. Last verified 2026-07-05.
+author: Abiba (pi agent)
+---
+
+# Hermes Agent Baseline — Canonical Good State
+
+## Quick Restore
+
+```bash
+# Verify all agents against baseline in one command:
+for ct in 112 114 111 113; do
+    echo "CT $ct: $(pct-run $ct grep api_key: /root/.hermes/config.yaml | grep -c sk-) api_keys found"
+done
+```
+
+## Agent Map
+
+| Agent | CT | Node | IP | LiteLLM Key | LiteLLM Alias |
+|-------|-----|------|-----|-------------|---------------|
+| Tanko | 112 | amdpve | .122 | `sk-CggiHWlamQyShxWC3Hx6uw` | `tanko` |
+| Mumuni | 114 | minipve | .123 | `sk-VrqCNlwUgzoNGOpikJ7nwQ` | `mumuni` |
+| Tdunna | 111 | amdpve | ? | `sk-6sbCNjz2T6lTVDBdlNHXsA` | `tdunna` |
+| Baggy | 113 | amdpve | ? | `sk-krnw_zGBwvvL5b7l2t-s-A` | `baggy` |
+
+Access: `pct-run <CT_ID> <command>` — no IPs needed. GPU hosts (.8, .110, .15) use SSH.
+
+## Key Architecture
+
+```
+Agent (systemd) → LITELLM_API_KEY → LiteLLM (:116/v1) → Router (:9000) → GPU (llama-server)
+                                                     └── Key DB (Postgres)
+```
+
+- **Master key**: `sk-litellm-7f96080dd99b15c36bd4b333b58a6796` — ADMIN ONLY, never in agent configs
+- **Agent keys**: Each agent has a dedicated key in LiteLLM's database with alias matching the agent name
+- **Key source**: `/etc/environment` → `LITELLM_API_KEY=sk-...` (systemd service sources this)
+- **Override**: `/home/jerome/.config/systemd/user/hermes-gateway.service.d/env.conf` (if present, must match)
+
+## Config Pattern — Mandatory Fields
+
+Every agent's `/root/.hermes/config.yaml` (or `/home/jerome/.hermes/config.yaml`) MUST have:
+
+### 1. Main Model
+```yaml
+model:
+  default: syslog-auto
+  provider: custom:harness       # or: harness
+  api_key_env: LITELLM_API_KEY
+  max_tokens: 4096
+```
+
+### 2. Custom Provider
+```yaml
+custom_providers:
+  - name: harness
+    model: syslog-auto
+    base_url: http://192.168.68.116/v1
+    api_key_env: LITELLM_API_KEY
+    api_mode: chat_completions
+```
+
+### 3. Vision (CRITICAL — must have api_key directly!)
+```yaml
+auxiliary:
+  vision:
+    provider: harness
+    model: gemma-4-12b              # or syslog-auto
+    base_url: http://192.168.68.116/v1
+    api_key_env: LITELLM_API_KEY
+    api_key: <ACTUAL_KEY_FROM_/etc/environment>   # ← MANDATORY workaround
+    timeout: 60
+    download_timeout: 30
+```
+
+### 4. Compression (must have api_key!)
+```yaml
+  compression:
+    enabled: true
+    threshold: 0.65
+    target_ratio: 0.3
+    provider: harness
+    model: syslog-auto               # or gemma-4-12b
+    base_url: http://192.168.68.116/v1
+    api_key_env: LITELLM_API_KEY
+    api_key: <ACTUAL_KEY_FROM_/etc/environment>   # ← MANDATORY workaround
+    timeout: 120
+```
+
+## Known Bug: `api_key_env` Ignored by Auxiliary Client
+
+**Bug location**: `agent/auxiliary_client.py` → `_resolve_task_provider_model()` (line ~5478)
+
+**What happens**: The function reads `api_key` from auxiliary task configs but does NOT
+resolve `api_key_env`. If only `api_key_env` is set (no `api_key`), the key resolves
+to `None`, and the explicit_base_url branch in `resolve_provider_client` falls through
+to `"no-key-required"` → 401 from LiteLLM.
+
+**Impact**: Vision analysis, compression, and any other auxiliary task calling
+LiteLLM/harness will fail with:
+```
+401: LiteLLM Virtual Key expected. Received=no-k****ired, expected to start with 'sk-'
+```
+
+**Workaround**: Set `api_key` directly (copy the value from `/etc/environment`) alongside
+`api_key_env` in every auxiliary task config that uses the harness provider.
+
+**Permanent fix**: Patch `_resolve_task_provider_model()` to resolve `api_key_env` when
+`api_key` is empty:
+```python
+cfg_api_key = str(task_config.get("api_key", "")).strip() or None
+if not cfg_api_key:
+    key_env = str(task_config.get("api_key_env", "")).strip()
+    if key_env:
+        cfg_api_key = os.getenv(key_env, "").strip() or None
+```
+
+## Audit Procedure
+
+### Full Audit (all agents)
+```bash
+for ct in 112 114 111 113; do
+    echo "=== CT $ct ==="
+    pct-run $ct grep LITELLM_API_KEY /etc/environment
+    pct-run $ct grep "api_key: sk-" /root/.hermes/config.yaml | grep -v api_key_env
+    echo ""
+done
+```
+
+### Master Key Leak Check
+```bash
+# On every agent:
+pct-run <CT> grep -rl "sk-litellm-7f96080dd" /root/ /etc/ 2>/dev/null
+# Must return empty
+```
+
+### Verify Key Works
+```bash
+curl -s http://192.168.68.116:80/v1/models \
+  -H "Authorization: Bearer <AGENT_KEY>" | grep syslog-auto
+# Must return model list
+```
+
+### Verify Vision/Compression
+```bash
+# Check both api_key and api_key_env are present:
+pct-run <CT> grep -A8 "vision:" /root/.hermes/config.yaml | grep api_key
+# Must show both api_key: sk-... and api_key_env: LITELLM_API_KEY
+```
+
+## Systemd Pattern
+
+For agents where the gateway runs as a user service:
+```ini
+# /home/jerome/.config/systemd/user/hermes-gateway.service.d/env.conf
+[Service]
+Environment="LITELLM_API_KEY=sk-..."    # must match /etc/environment
+```
+
+For agents where the gateway runs as root:
+```ini
+# /root/.config/systemd/user/hermes-gateway.service
+EnvironmentFile=/etc/environment        # sources LITELLM_API_KEY
+```
+
+**No drop-in that hardcodes the master key. Ever.**
+
+## Attachment Cache Locations
+
+| Type | Path |
+|------|------|
+| Images | `~/.hermes/cache/images/` |
+| Documents | `~/.hermes/cache/documents/` |
+| Audio | `~/.hermes/cache/audio/` |
+
+## Health Verification
+
+Run the consolidated health check:
+```bash
+python3 /root/scripts/agent-health-check.py
+```
+This validates all 4 LiteLLM keys, detects GPU port conflicts (ghost processes),
+verifies gateway liveness, confirms Zulip streaming (`edit_message` present),
+and counts recent errors. Non-disruptive — never restarts anything.
+
+## GPU Port Conflict Detection
+
+All 3 GPU hosts have pre-start ghost detection in their launch wrappers:
+- `.8` and `.110`: inline check in `llama-wrapper.sh`
+- `.15`: `/usr/local/bin/port-cleanup.sh` (ExecStartPre, replaces blanket `pkill`)
+
+Detection pattern: `ss -tlnp` on port 8080 → compare pid against `systemctl MainPID`.
+If they differ → ghost detected → kill ghost → start fresh.
+
+## Related Contracts
+
+- `hermes-key-enforcement.prose.md` — key policy, rotation, detection query
+- `hermes-config-template.prose.md` — full configuration template
+- `gpu-fleet.prose.md` — GPU fleet and agent key table
+- `infrastructure-control.prose.md` — CT inventory with pct-run access
+- `litellm-health.prose.md` — LiteLLM stack health verification
+
+## Change Log
+
+| Date | Change |
+|------|--------|
+| 2026-07-06 | Port conflict detection added to all 3 GPU wrappers. Consolidated health check script deployed. Zulip streaming edit_message enabled for Tanko/Mumuni. |
+| 2026-07-05 | Baseline created. All 4 agents audited, master key removed, api_key workaround applied |

hermes-config-template.prose.md

@@ -12,7 +12,7 @@ description: >
 
 - template_version: "2.1.0"
 - last_applied: timestamp
-- agents_configured: ["tanko", "mumuni", "abiba", "koby", "koonimo", "kagenz0"]
+- agents_configured: ["tanko", "mumuni", "abiba", "tdunna", "baggy", "kagenz0"]
 - agent_keys: map (see Agent Keys section)
 - infra_endpoints_verified: array
 
@@ -29,8 +29,8 @@ Sub-agent profiles inherit auth from the main config — no separate keys needed
 | Tanko | `tanko-*` | 192.168.68.122 | jerome@.122 | — |
 | Mumuni | `mumuni-jul2026` | 192.168.68.123 | root@.123 | 6 profiles ✱ |
 | Abiba | `abiba-*` | 192.168.68.24 | local | — |
-| Koby | `koby-*` | ? | Zulip | — |
-| Koonimo | `koonimo-*` | 192.168.68.114 | Zulip | — |
+| Tdunna | `tdunna-*` | ? | Zulip | — |
+| Baggy | `baggy-*` | ? | Zulip | — |
 | Kagenz0 | `kagenz0-*` | ? | Zulip | — |
 
 ✱ Mumuni sub-agents: syslog-code, syslog-devops, syslog-email, syslog-research,

hermes-key-enforcement.prose.md

@@ -74,12 +74,21 @@ model:
 Run on any Hermes host to detect violations:
 
 ```bash
-grep -rn 'api_key: sk-' /root/.hermes/ \
+# 1. Check config.yaml for hardcoded harness keys
+grep -rn 'api_key: sk-' /home/jerome/.hermes/ \
   --include='config.yaml' \
   | grep -v 'deepseek\|openai\|anthropic\|DEEPSEEK'
+
+# 2. Check systemd drop-ins for master key leaks (2026-07-05: Tanko had this)
+grep -rn 'LITELLM_API_KEY' /home/jerome/.config/systemd/user/ 2>/dev/null
+grep -rn 'LITELLM_API_KEY=sk-litellm-7f96080d' /home/jerome/.config/systemd/ 2>/dev/null
+
+# 3. Verify running process env matches dedicated key
+cat /proc/$(cat /home/jerome/.hermes/gateway.pid | python3 -c "import sys,json; print(json.load(sys.stdin)['pid'])")/environ \
+  | tr '\0' '\n' | grep LITELLM_API_KEY
 ```
 
-If any output — violation. Fix immediately.
+If any output from step 2 — **critical violation** (master key leaked). Fix immediately.
 
 ## Rotation Procedure
 
@@ -102,7 +111,7 @@ curl -s -H "Authorization: Bearer sk-NEW_KEY" http://192.168.68.116/v1/models
 **Keys are permanent and use bare agent name aliases.**
 
 - **Duration**: `null` — keys never expire. This is enforced by `default_key_generate_params` in `litellm_config.yaml`.
-- **Alias convention**: bare agent name only (e.g., `tanko`, `mumuni`, `koby`, `koonimo`). No dates, no versions. The alias IS the identity.
+- **Alias convention**: bare agent name only (e.g., `tanko`, `mumuni`, `tdunna`, `baggy`). No dates, no versions. The alias IS the identity.
 - **Rotation triggers**: compromise, personnel departure, or quarterly security hygiene. NOT calendar-driven.
 - **Max budget**: $100 per key (config default).
 
@@ -117,16 +126,16 @@ litellm_settings:
       purpose: "agent-inference"
 ```
 
-## Verified Agents (2026-07-04 final)
+## Verified Agents (2026-07-05 update)
 
-| Agent | CT | Status | LiteLLM Alias | Key Type | Systemd Source | Last Verified |
-|-------|-----|--------|---------------|----------|----------------|---------------|
-| Tanko | 112 | ✅ Compliant | `tanko` | Dedicated | `/etc/environment` only | 23:00 EDT |
-| Mumuni | 114 | ✅ Compliant | `mumuni` | Dedicated | `/etc/environment` only | 23:00 EDT |
-| Koby | 111 | ✅ Compliant | `koby` | Dedicated | User service + drop-in | 23:00 EDT |
-| Koonimo | 113 | ✅ Compliant | `koonimo` | Dedicated | System service + drop-in | 23:00 EDT |
-| Abiba | 100 | ✅ N/A (pi native) | — | — | — | 23:00 EDT |
-| Kagenz0 | 105 | ❌ DOWN | — | — | — | 19:14 EDT |
+| Agent | CT | IP | LiteLLM Alias | Key | Status | Systemd Source | Last Verified |
+|-------|-----|-----|---------------|-----|--------|----------------|---------------|
+| Tanko | 112 | .122 | `tanko` | `sk-CggiHWlamQyShxWC3Hx6uw` | ✅ Fixed | User drop-in `env.conf` | 20:17 UTC Jul 5 |
+| Mumuni | 114 | .123 | `mumuni` | `sk-XY2aUfvy2BIs6kp1ZPh6VA` | ⚠️ Unverified | `/etc/environment` | 23:00 EDT Jul 4 |
+| Tdunna | 111 | ? | `tdunna` | `sk-6sbCNjz2T6lTVDBdlNHXsA` | ✅ Fixed | `/etc/environment` + drop-in | 23:30 UTC Jul 5 |
+| Baggy | 113 | ? | `baggy` | `sk-krnw_zGBwvvL5b7l2t-s-A` | ✅ Fixed | `/etc/environment` | 23:30 UTC Jul 5 |
+| Abiba | 100 | .65 | — | — | ✅ N/A (pi native) | — | 19:44 UTC Jul 5 |
+| Kagenz0 | 105 | ? | — | — | ❌ DOWN | — | 19:14 EDT Jul 4 |
 
 ### Systemd Service Pattern (2026-07-04 fix)
 
@@ -180,3 +189,37 @@ auth → validate → lint → ai-review → gate
 - **Status check**: `PR Pipeline — Authorize → Validate → Review → Merge` required before merge
 - **Runner**: `runner-ct110` (Gitea Actions v0.6.1) on CT 110
 - **Config**: `.gitea/workflows/pr-pipeline.yaml`
+
+## Known Bug: auxiliary_client ignores api_key_env (2026-07-05)
+
+**Bug**: `_resolve_task_provider_model()` in `agent/auxiliary_client.py` reads
+`api_key` from auxiliary task configs (vision, compression, etc.) but does NOT
+resolve `api_key_env`. The custom provider resolution path handles `api_key_env`,
+but auxiliary tasks take a different code path that ignores it.
+
+**Impact**: Vision analysis and compression calls fall through to the `"no-key-required"`
+placeholder, causing 401 errors on LiteLLM/harness (which require `sk-*` keys).
+
+**Workaround**: Set `api_key` directly alongside `api_key_env` in each auxiliary
+task config:
+
+```yaml
+auxiliary:
+  vision:
+    api_key: sk-CggiHWlamQyShxWC3Hx6uw      # ← workaround
+    api_key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/v1
+    model: gemma-4-12b
+    provider: harness
+  compression:
+    api_key: sk-CggiHWlamQyShxWC3Hx6uw      # ← workaround
+    api_key_env: LITELLM_API_KEY
+    base_url: http://192.168.68.116/v1
+    model: gemma-4-12b
+    provider: harness
+```
+
+**Affected agents**: All Hermes agents with harness/LiteLLM provider and
+`api_key_env` in auxiliary configs (all 4 Hermes agents patched 2026-07-05).
+
+**Source location**: `agent/auxiliary_client.py` line 5478

infrastructure-control.prose.md

@@ -68,7 +68,7 @@ description: >
 | CT 116 (syslog-api) | SSH root | SSH key | ✅ |
 | Tanko CT (.122) | SSH jerome | id_ed25519 | ✅ |
 | Mumuni CT (.123) | SSH root | id_ed25519 | ✅ |
-| Koonimo CT (.114) | SSH jerome | ❌ no key access |
+| Baggy CT (113) | SSH jerome | ❌ no key access |
 | Netbird (.17) | SSH root | SSH key | ✅ |
 | Gitea | API token | abiba-bot token | ✅ |
 | Zulip | Bot API key | abiba-bot@chat.sysloggh.net | ✅ |
@@ -85,7 +85,7 @@ description: >
 
 ### Reachability Matrix
 
-| From / To | PVE API | docker-vm (.7) | CT 116 | Tanko (.122) | Mumuni (.123) | Koonimo (.114) |
+| From / To | PVE API | docker-vm (.7) | CT 116 | Tanko (.122) | Mumuni (.123) | Baggy (.114) |
 |-----------|---------|----------------|--------|-------------|---------------|----------------|
 | **Abiba** (.24) | ✅ :443 | ✅ SSH | ✅ SSH | ✅ SSH jerome | ✅ SSH root | ❌ SSH |
 | **Tanko** (.122) | ❌ | ❌ | ❌ via NetBird | ✅ | ❌ | ❌ |
@@ -585,7 +585,7 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
 | 110 | gitea | minipve | — | Git | ❌ |
 | 111 | tdunna | amdpve | — | ? | ❌ |
 | 112 | tanko | amdpve | .122 | Hermes agent | ✅ |
-| 113 | baggy/koonimo | amdpve | .114 | Hermes agent | ✅ |
+| 113 | baggy | amdpve | ? | Hermes agent | ✅ |
 | 114 | mumuni | minipve | .123 | Hermes agent | ✅ |
 | 115 | scottdenya | amdpve | — | ? | ❌ |
 | 116 | syslog-api | minipve | .116 | LiteLLM + Grafana | ❌ |
@@ -602,3 +602,59 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
 | docker-vm (.7) | Audiobookshelf | `/opt/audiobookshelf/docker-compose.yml` |
 | CT 116 (.116) | Inference Harness | `/opt/inference-harness/docker-compose.yml` |
 | Netbird (.17) | Netbird | Docker run (not compose) |
+
+## CT Access (pct-run — no IPs needed)
+
+All CTs are accessible via `pct-run <CT_ID> <command>`. No IP addresses required.
+Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run.sh`.
+
+| CT | Name | Node | pct-run |
+|-----|------|------|---------|
+| 100 | abiba | amdpve | `pct-run 100` |
+| 105 | kagentz | amdpve | `pct-run 105` |
+| 111 | tdunna | amdpve | `pct-run 111` |
+| 112 | tanko | amdpve | `pct-run 112` |
+| 113 | baggy | amdpve | `pct-run 113` |
+| 115 | scottdenya | amdpve | `pct-run 115` |
+| 104 | authentik | minipve | `pct-run 104` |
+| 110 | gitea | minipve | `pct-run 110` |
+| 114 | mumuni | minipve | `pct-run 114` |
+| 116 | syslog-api | minipve | `pct-run 116` |
+| 106 | ra-h-os | storepve | `pct-run 106` |
+| 107 | proxmox-backup | storepve | `pct-run 107` |
+| 108 | media | storepve | `pct-run 108` |
+| 117 | zulip | storepve | `pct-run 117` |
+| 102 | adguard | acerpve | `pct-run 102` |
+
+GPU bare-metal hosts (.8 acerpve, .110 ocupve, .15 amdpve) are NOT CTs — use SSH directly:
+```bash
+ssh root@192.168.68.8   # RTX 3090
+ssh root@192.168.68.110 # RTX 5070
+ssh root@192.168.68.15  # Strix Halo
+```
+
+## Section 7: Agent Health Check (consolidated — 2026-07-05)
+
+Replaces 7 scattered Zulip health scripts with a single non-disruptive check
+running every 10 minutes via cron (`/root/scripts/agent-health-check.py`).
+
+The script is **read-only** — it never restarts, kills, or modifies anything.
+Disruptive cron-based gateways restarts (like Mumuni's zulip-watchdog.sh,
+which was kill+nohup outside systemd) are banned by policy.
+
+### Checks Performed
+
+| Check | Frequency | What It Detects |
+|-------|-----------|-----------------|
+| LiteLLM key validation | 10 min | All 4 agent keys authenticate and return models |
+| GPU port conflict | 10 min | Ghost processes squatting port 8080 (ss vs systemd MainPID) |
+| Gateway liveness | 10 min | Gateway process running, state file readable |
+| Zulip streaming | 10 min | `edit_message` present in adapter (streaming supported) |
+| Recent errors | 10 min | Error count in journald for last 10 min |
+
+### Disabled Scripts
+
+| Script | Why Disabled |
+|--------|-------------|
+| `zulip-watchdog.sh` (Mumuni) | kill+nohup bypassed systemd, 27 restarts, pattern mismatch |
+| `zulip-monitor.sh` (Abiba) | Replaced by agent-health-check.py + PM2 auto-restart |

memory-audit-maintenance.prose.md

@@ -1,7 +1,7 @@
 ---
 name: memory-audit-maintenance
 kind: responsibility
-description: Shared memory audit and maintenance contract for all Hermes agents (Mumuni, Tanko, Koby, Koonimo). Each agent runs it against its own isolated memory files — no cross-agent access, no shared state. Detects staleness, enforces writer registry, and rotates canary tokens.
+description: Shared memory audit and maintenance contract for all Hermes agents (Mumuni, Tanko, Tdunna, Baggy). Each agent runs it against its own isolated memory files — no cross-agent access, no shared state. Detects staleness, enforces writer registry, and rotates canary tokens.
 id: 067NC4KG01RG50R40M30E20918
 ---
 
@@ -15,13 +15,13 @@ Autonomously audit and reorganize an agent's native memory (MEMORY.md, USER.md,
 
 ### Scope
 
-This contract is the **Hermes Agent standard** for memory maintenance. It is shared across all Hermes agents (Mumuni, Tanko, Koby, Koonimo). Each agent runs it against its own memory files only  no cross-agent access, no shared state, no shared ledger, no shared canary. The contract is the standard; each agent enforces it independently with fully isolated data.
+This contract is the **Hermes Agent standard** for memory maintenance. It is shared across all Hermes agents (Mumuni, Tanko, Tdunna, Baggy). Each agent runs it against its own memory files only  no cross-agent access, no shared state, no shared ledger, no shared canary. The contract is the standard; each agent enforces it independently with fully isolated data.
 
 **Agent Roster:**
 - Mumuni
 - Tanko
-- Koby
-- Koonimo
+- Tdunna
+- Baggy
 
 **Isolation Principle:** Each agent has its own:
 - `MEMORY.md` and `USER.md`
@@ -343,4 +343,4 @@ return {
 
 ### Per-Agent Notes
 
-Each Hermes agent (Mumuni, Tanko, Koby, Koonimo) runs this contract against its own `~/.hermes/memories/` directory. The contract is identical across agents, but all data is fully isolated: separate ledgers, separate writer registries, separate canaries. If a new agent is added to the roster, it must be listed in `### Scope` above and given its own isolated memory directory.
+Each Hermes agent (Mumuni, Tanko, Tdunna, Baggy) runs this contract against its own `~/.hermes/memories/` directory. The contract is identical across agents, but all data is fully isolated: separate ledgers, separate writer registries, separate canaries. If a new agent is added to the roster, it must be listed in `### Scope` above and given its own isolated memory directory.

scripts/agent-health-check.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""
+/root/scripts/agent-health-check.py — Consolidated Agent Health Verification
+
+Single non-disruptive health check replacing 7 scattered scripts.
+Verifies: LiteLLM keys, GPU port conflicts, agent Zulip streaming,
+gateway liveness, and gateway log health. NEVER restarts anything.
+
+Usage:
+  python3 /root/scripts/agent-health-check.py          # Full check
+  python3 /root/scripts/agent-health-check.py --json   # Machine-readable
+  python3 /root/scripts/agent-health-check.py --quiet  # Only output on failure
+
+Cron: */10 * * * * python3 /root/scripts/agent-health-check.py --quiet
+"""
+
+import subprocess, json, sys, os, time
+from datetime import datetime
+
+LITELLM = "http://192.168.68.116:80"
+
+AGENTS = {
+    "tanko":   {"ct": 112, "host": "192.168.68.122", "key": "sk-CggiHWlamQyShxWC3Hx6uw", "user": "jerome"},
+    "mumuni":  {"ct": 114, "host": "192.168.68.123", "key": "sk-VrqCNlwUgzoNGOpikJ7nwQ", "user": "root"},
+    "tdunna":  {"ct": 111, "host": None,              "key": "sk-6sbCNjz2T6lTVDBdlNHXsA", "user": None},
+    "baggy":   {"ct": 113, "host": None,              "key": "sk-krnw_zGBwvvL5b7l2t-s-A", "user": None},
+}
+
+GPU_HOSTS = {
+    "gpu-rtx3090 (.8)":     {"host": "192.168.68.8",   "port": 8080, "service": "llama-server"},
+    "gpu-rtx5070 (.110)":   {"host": "192.168.68.110", "port": 8080, "service": "llama-server"},
+    "gpu-strixhalo (.15)":  {"host": "192.168.68.15",  "port": 8080, "service": "ornith-server"},
+}
+
+FAIL = []
+
+def ssh(host, cmd, user="root"):
+    """Execute a command on a remote host, return stdout or None."""
+    try:
+        result = subprocess.run(
+            ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=8",
+             f"{user}@{host}", cmd],
+            capture_output=True, text=True, timeout=15
+        )
+        return result.stdout.strip() if result.returncode == 0 else None
+    except:
+        return None
+
+def http_get(url, headers=None, timeout=5):
+    """Return HTTP status code as string."""
+    try:
+        cmd = ["curl", "-sfk", "--connect-timeout", str(timeout), "-o", "/dev/null", "-w", "%{http_code}"]
+        if headers:
+            for k, v in headers.items():
+                cmd.extend(["-H", f"{k}: {v}"])
+        cmd.append(url)
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout+3)
+        return result.stdout.strip() or "000"
+    except:
+        return "timeout"
+
+def http_json(url, headers=None, timeout=5):
+    """Return parsed JSON from URL, or None."""
+    try:
+        cmd = ["curl", "-sfk", "--connect-timeout", str(timeout)]
+        if headers:
+            for k, v in headers.items():
+                cmd.extend(["-H", f"{k}: {v}"])
+        cmd.append(url)
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout+3)
+        return json.loads(result.stdout) if result.returncode == 0 and result.stdout else None
+    except:
+        return None
+
+
+# ═══════════════════════════════════════════════════════════════════
+# CHECK 1: LiteLLM Key Validation
+# ═══════════════════════════════════════════════════════════════════
+
+def check_keys():
+    for name, agent in AGENTS.items():
+        data = http_json(f"{LITELLM}/v1/models",
+                         headers={"Authorization": f"Bearer {agent['key']}"})
+        if data and data.get("data"):
+            model = data["data"][0].get("id", "?")
+            print(f"  ✅ {name}: key valid → {model}")
+        else:
+            print(f"  ❌ {name}: KEY FAILURE — auth rejected or unreachable")
+            FAIL.append(f"key:{name}")
+
+
+# ═══════════════════════════════════════════════════════════════════
+# CHECK 2: GPU Port Conflict Detection
+# ═══════════════════════════════════════════════════════════════════
+
+def check_gpu_ports():
+    for label, gpu in GPU_HOSTS.items():
+        host = gpu["host"]
+        port = gpu["port"]
+        svc = gpu["service"]
+
+        svc_status = ssh(host, f"systemctl is-active {svc}")
+        port_owner = ssh(host, f"ss -tlnp 2>/dev/null | grep -Po ':{port}\\s+.*pid=\\K[0-9]+' | head -1")
+
+        if not svc_status:
+            print(f"  ❌ {label}: UNREACHABLE")
+            FAIL.append(f"gpu-unreachable:{host}")
+            continue
+
+        if not port_owner:
+            print(f"  ❌ {label}: PORT {port} NOT LISTENING (svc={svc_status})")
+            FAIL.append(f"gpu-no-port:{label}")
+        elif svc_status != "active":
+            svc_pid = ssh(host, f"systemctl show {svc} -p MainPID 2>/dev/null | cut -d= -f2")
+            if svc_pid and port_owner != svc_pid:
+                print(f"  ❌ {label}: GHOST PROCESS — port owned by pid {port_owner}, svc pid {svc_pid} (svc={svc_status})")
+                FAIL.append(f"gpu-ghost:{label}:{port_owner}")
+            else:
+                print(f"  ⚠️  {label}: svc={svc_status}, port owned by {port_owner}")
+        else:
+            # Verify health endpoint
+            health = ssh(host, f"curl -s --max-time 5 http://localhost:{port}/health")
+            if health and '"status":"ok"' in health:
+                print(f"  ✅ {label}: healthy (pid={port_owner})")
+            elif health and '"status":"no slot available"' in health:
+                print(f"  ✅ {label}: healthy (loading, pid={port_owner})")
+            elif health and '"error"' in health.lower():
+                print(f"  ⚠️  {label}: error response (pid={port_owner}): {health[:80]}")
+            else:
+                print(f"  ⚠️  {label}: unknown health (pid={port_owner}): {str(health)[:80]}")
+
+
+# ═══════════════════════════════════════════════════════════════════
+# CHECK 3: Agent Gateway Liveness + Streaming
+# ═══════════════════════════════════════════════════════════════════
+
+def check_agents():
+    for name, agent in AGENTS.items():
+        host = agent.get("host")
+        user = agent.get("user")
+        ct = agent["ct"]
+
+        if not host or not user:
+            print(f"  ⬜ {name} (CT {ct}): cannot SSH — skip liveness check")
+            continue
+
+        # Gateway process
+        pid = ssh(host, "pgrep -f 'hermes_cli.main gateway run' | head -1", user=user)
+        if not pid:
+            print(f"  ❌ {name}: GATEWAY NOT RUNNING")
+            FAIL.append(f"gateway-down:{name}")
+            continue
+
+        # Gateway state file
+        state = ssh(host, "cat ~/.hermes/gateway_state.json 2>/dev/null", user=user)
+        if state:
+            try:
+                st = json.loads(state)
+                gw_state = st.get("gateway_state", "?")
+                zulip = st.get("platforms", {}).get("zulip", {}).get("state", "?")
+            except:
+                gw_state, zulip = "corrupt", "?"
+        else:
+            gw_state, zulip = "no-state-file", "?"
+
+        # Zulip streaming: does adapter have edit_message?
+        adapter_paths = [
+            "~/.hermes/plugins/zulip-platform/adapter.py",
+            "~/.hermes/plugins/platforms/zulip/adapter.py",
+        ]
+        streaming = "no"
+        for p in adapter_paths:
+            has_edit = ssh(host, f"grep -c 'async def edit_message' {p} 2>/dev/null", user=user)
+            if has_edit and has_edit != "0":
+                streaming = "yes"
+                break
+
+        # Recent errors
+        recent_errors = ssh(host,
+            r"journalctl --user -u hermes-gateway --since '10 min ago' -o cat --no-pager 2>/dev/null "
+            r"| grep -ci 'error\|traceback\|exception\|401\|403\|500' || echo 0",
+            user=user)
+        recent_errors = (recent_errors or "0").strip().split("\n")[-1]  # take last line
+
+        print(f"  {'✅' if gw_state == 'running' and zulip == 'connected' else '⚠️'} "
+              f"{name}: gw={gw_state} zulip={zulip} streaming={streaming} "
+              f"errors_10m={recent_errors.strip() or '0'} pid={pid}")
+
+
+# ═══════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════
+
+def main():
+    quiet = "--quiet" in sys.argv
+    as_json = "--json" in sys.argv
+
+    if not quiet:
+        print(f"🏥 Agent Health Check — {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}")
+        print()
+
+    print("🔑 LiteLLM Keys:")
+    check_keys()
+    print()
+
+    print("🎮 GPU Port Health:")
+    check_gpu_ports()
+    print()
+
+    print("🤖 Agent Gateways:")
+    check_agents()
+
+    if FAIL:
+        print(f"\n❌ {len(FAIL)} FAILURE(S): {' | '.join(FAIL)}")
+        if quiet:
+            # In quiet mode, only print failures as a single alert line
+            print(f"ALERT agent-health:{','.join(FAIL)}")
+    elif not quiet:
+        print("\n✅ All checks passed")
+
+    if as_json:
+        print(json.dumps({"timestamp": datetime.now().isoformat(),
+                          "failures": FAIL, "healthy": len(FAIL) == 0}))
+
+    sys.exit(1 if FAIL else 0)
+
+if __name__ == "__main__":
+    main()

scripts/pct-run.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# pct-run — Run commands inside Proxmox CTs by CT ID only. No IPs needed.
+# Usage: pct-run <CT_ID> [command...]
+# If no command given, opens a shell.
+#
+# Source of truth: Proxmox pct config on each node.
+# No hardcoded IPs. No prose contract drift.
+
+set -euo pipefail
+
+# ── CT ID → PVE Node mapping (maintained HERE, not in prose contracts) ──
+declare -A CT_NODES=(
+    # amdpve (192.168.68.15)
+    [100]=amdpve    # abiba
+    [105]=amdpve    # kagentz
+    [111]=amdpve    # tdunna
+    [112]=amdpve    # tanko
+    [113]=amdpve    # baggy
+    [115]=amdpve    # scottdenya
+    # minipve (192.168.68.12)
+    [104]=minipve   # authentik
+    [110]=minipve   # gitea
+    [114]=minipve   # mumuni
+    [116]=minipve   # syslog-api
+    # storepve (192.168.68.6)
+    [106]=storepve  # ra-h-os
+    [107]=storepve  # proxmox-backup
+    [108]=storepve  # media
+    [117]=storepve  # zulip
+    # acerpve (192.168.68.9)
+    [102]=acerpve   # adguard
+    # ocupve (192.168.68.5) — no CTs (bare metal GPU .110)
+)
+
+# Each node must be root-accessible via SSH hostname
+# (Ensure ~/.ssh/config or /etc/hosts has these resolvable)
+
+# ── PVE node → IP (needed only because SSH needs an address) ──
+declare -A NODE_IPS=(
+    [amdpve]=192.168.68.15
+    [minipve]=192.168.68.12
+    [storepve]=192.168.68.6
+    [acerpve]=192.168.68.9
+    [ocupve]=192.168.68.5
+)
+
+resolve_node() {
+    local ct_id="$1"
+    local node="${CT_NODES[$ct_id]:-}"
+    if [[ -z "$node" ]]; then
+        echo "ERROR: CT $ct_id not found in mapping" >&2
+        exit 1
+    fi
+    echo "$node"
+}
+
+node_ip() {
+    local node="$1"
+    local ip="${NODE_IPS[$node]:-}"
+    if [[ -z "$ip" ]]; then
+        echo "ERROR: Node $node not found in IP mapping" >&2
+        exit 1
+    fi
+    echo "$ip"
+}
+
+main() {
+    if [[ $# -lt 1 ]]; then
+        echo "Usage: pct-run <CT_ID> [command...]" >&2
+        echo "  pct-run 112 cat /etc/hostname" >&2
+        echo "  pct-run 114 systemctl status hermes-gateway" >&2
+        echo ""
+        echo "Known CTs:" >&2
+        for ct in $(echo "${!CT_NODES[@]}" | tr ' ' '\n' | sort -n); do
+            echo "  CT $ct → ${CT_NODES[$ct]}" >&2
+        done
+        exit 1
+    fi
+
+    local ct_id="$1"
+    shift
+    local node
+    node=$(resolve_node "$ct_id")
+    local ip
+    ip=$(node_ip "$node")
+
+    if [[ $# -eq 0 ]]; then
+        # Interactive shell
+        exec ssh -t "root@${ip}" "pct enter ${ct_id}"
+    else
+        # One-shot command
+        ssh "root@${ip}" "pct exec ${ct_id} -- $*"
+    fi
+}
+
+main "$@"

zulip-health.prose.md

@@ -77,6 +77,24 @@ Log the condition as "pending restart" with the timestamp. If the condition pers
 Log as "unreachable" — don't treat as critical unless it persists for 3+ consecutive checks.
 
 ### Severity escalation
+
+## Streaming Support (2026-07-05)
+
+Zulip agents now support progressive message editing during agent generation.
+When a Hermes agent (Tanko, Mumuni) processes a message, the response is
+streamed in real-time via Zulip's `PATCH /api/v1/messages/{id}` API:
+
+- Adapter implements `edit_message()` using `_api_patch()` helper
+- Gateway stream consumer progressively edits the Zulip message
+- User sees real-time agent thinking instead of waiting for full response
+- Verified: Tanko (CT 112) and Mumuni (CT 114) both have streaming active
+
+### Verification
+```bash
+# Check if agent has streaming:
+grep -c "async def edit_message" ~/.hermes/plugins/*/zulip*/adapter.py
+# Must return 1 — streaming is active
+```
 - Single agent warning → log only
 - Single agent critical → relay message to user
 - Two or more agents critical → immediate relay + attempt auto-recovery