Merge branch 'master' into fm/infra-maint-contract

no-mistakes(document): Reframed infra-maintenance contract as deliberate partition
no-mistakes(review): fix docker rollback command to pin digest then compose up
2026-07-18 23:12:20 +00:00 · 2026-07-18 22:16:09 +00:00 · 2026-07-18 22:07:15 +00:00 · 2026-07-18 22:06:48 +00:00 · 2026-07-18 22:06:34 +00:00 · 2026-07-18 22:02:45 +00:00
9 changed files with 381 additions and 333 deletions
@@ -1 +0,0 @@
-__pycache__/
@@ -128,3 +128,10 @@ safe-mutate --verify "CMD" [--expect "PATTERN"] --mutate "CMD" [--reason "WHY"]
 Read the [Authoring Guide](docs/AUTHORING-GUIDE.md) before writing any new contract.
 It covers the full process: verify → draft → lint → review → ship, with templates
 and style rules.
+
+## Maintaining this file
+
+Keep this file for knowledge useful to almost every future agent session in this project.
+Do not repeat what the codebase already shows; point to the authoritative file or command instead.
+Prefer rewriting or pruning existing entries over appending new ones.
+When updating this file, preserve this bar for all agents and keep entries concise.
@@ -0,0 +1 @@
+AGENTS.md
@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-"""
-Hermes Config Audit — validates a live config.yaml against the prose contract rules.
-
-Usage:
-    python3 audit-hermes-config.py <config.yaml>
-    python3 audit-hermes-config.py /root/.hermes/config.yaml
-
-Exit codes:
-    0 = all checks pass
-    1 = one or more contract violations found
-
-This script encodes every rule from hermes-config-template.prose.md so config
-changes can be verified before and after application. It is the single automated
-enforcement layer for the prose contract.
-
-Contract: /root/prose-contracts/hermes-config-template.prose.md
-"""
-
-import sys
-import yaml
-
-VIOLATIONS = []
-WARNINGS = []
-PASSES = []
-
-
-def check(condition, rule, message):
-    if condition:
-        PASSES.append(f"[{rule}] {message}")
-    else:
-        VIOLATIONS.append(f"[{rule}] {message}")
-
-
-def warn(rule, message):
-    WARNINGS.append(f"[{rule}] {message}")
-
-
-def audit(path):
-    with open(path) as f:
-        cfg = yaml.safe_load(f)
-
-    model = cfg.get("model", {})
-    fb = cfg.get("fallback_providers", {})
-    comp = cfg.get("compression", {})
-    aux = cfg.get("auxiliary", {})
-    deleg = cfg.get("delegation", {})
-    cps = cfg.get("custom_providers", [])
-    cp = cps[0] if cps else {}
-
-    # --- Rule 3: API Keys via Environment ---
-    check(
-        model.get("api_key") in ("", None),
-        "Rule 3",
-        f"model.api_key must be empty (got {model.get('api_key')!r}) — keys via env var, not hardcoded",
-    )
-    check(
-        model.get("api_key_env") == "LITELLM_API_KEY",
-        "Rule 3",
-        f"model.api_key_env must be LITELLM_API_KEY (got {model.get('api_key_env')!r})",
-    )
-
-    # --- Rule 5: Main Config Base URL ---
-    expected_base = "http://192.168.68.116/v1"
-    check(
-        model.get("base_url") == expected_base,
-        "Rule 5",
-        f"model.base_url must be {expected_base} (got {model.get('base_url')!r}) — /v1 not /litellm/v1",
-    )
-
-    # --- Rule 6: max_tokens Is Required ---
-    check(
-        isinstance(model.get("max_tokens"), int) and model.get("max_tokens") <= 8192,
-        "Rule 6",
-        f"model.max_tokens must be set and <= 8192 (got {model.get('max_tokens')!r}) — thermal safety",
-    )
-
-    # --- Rule 7: Auxiliary Model Consistency ---
-    check(
-        comp.get("model") == "syslog-auto",
-        "Rule 7",
-        f"compression.model must be syslog-auto (got {comp.get('model')!r}) — auto-routing to prevent Strix Halo overload",
-    )
-    aux_comp = aux.get("compression", {})
-    check(
-        aux_comp.get("model") == "syslog-auto",
-        "Rule 7",
-        f"auxiliary.compression.model must be syslog-auto (got {aux_comp.get('model')!r}) — must match compression.model",
-    )
-
-    # --- Rule 8: GPU Workload Distribution ---
-    check(
-        aux.get("vision", {}).get("model") == "gpu-light",
-        "Rule 8",
-        f"auxiliary.vision.model must be gpu-light (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
-    )
-    check(
-        aux.get("web_extract", {}).get("model") == "gpu-light",
-        "Rule 8",
-        f"auxiliary.web_extract.model must be gpu-light (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
-    )
-
-    # --- Rule 9: Compression Threshold ---
-    check(
-        comp.get("threshold") == 0.65,
-        "Rule 9",
-        f"compression.threshold must be 0.65 for 128K models (got {comp.get('threshold')!r})",
-    )
-    check(
-        comp.get("max_context_window") == 131072,
-        "Rule 9",
-        f"compression.max_context_window must be 131072 (got {comp.get('max_context_window')!r}) — matches 128K GPU capacity",
-    )
-
-    # --- Rule 10: Default Model Must Be syslog-auto ---
-    check(
-        model.get("default") == "syslog-auto",
-        "Rule 10",
-        f"model.default must be syslog-auto (got {model.get('default')!r}) — auto-routing default",
-    )
-
-    # --- Rule 14: Provider Name Must Match custom_providers Name ---
-    check(
-        model.get("provider") == "harness",
-        "Rule 14",
-        f"model.provider must be 'harness' (got {model.get('provider')!r}) — NOT 'custom'. "
-        f"provider: custom causes generic resolution path that ignores key_env → 'no-key-required' → 401",
-    )
-    check(
-        comp.get("provider") == "harness",
-        "Rule 14",
-        f"compression.provider must be 'harness' (got {comp.get('provider')!r})",
-    )
-    for aux_name in ("vision", "web_extract", "compression"):
-        aux_provider = aux.get(aux_name, {}).get("provider")
-        check(
-            aux_provider == "harness",
-            "Rule 14",
-            f"auxiliary.{aux_name}.provider must be 'harness' (got {aux_provider!r})",
-        )
-    check(
-        deleg.get("provider") == "harness",
-        "Rule 14",
-        f"delegation.provider must be 'harness' (got {deleg.get('provider')!r})",
-    )
-    check(
-        fb.get("provider") == "deepseek",
-        "Rule 14",
-        f"fallback_providers.provider must be 'deepseek' (got {fb.get('provider')!r}) — "
-        f"true fallback diversity, not same endpoint as primary",
-    )
-    check(
-        fb.get("model") == "deepseek-v4-flash",
-        "Rule 14",
-        f"fallback_providers.model must be 'deepseek-v4-flash' (got {fb.get('model')!r})",
-    )
-    check(
-        fb.get("api_key_env") == "DEEPSEEK_API_KEY",
-        "Rule 14",
-        f"fallback_providers.api_key_env must be DEEPSEEK_API_KEY (got {fb.get('api_key_env')!r})",
-    )
-
-    # --- custom_providers sanity ---
-    check(
-        cp.get("name") == "harness",
-        "custom_providers",
-        f"custom_providers[0].name must be 'harness' (got {cp.get('name')!r})",
-    )
-    check(
-        cp.get("key_env") == "LITELLM_API_KEY" or cp.get("api_key_env") == "LITELLM_API_KEY",
-        "custom_providers",
-        f"custom_providers[0] must have key_env or api_key_env = LITELLM_API_KEY "
-        f"(got key_env={cp.get('key_env')!r}, api_key_env={cp.get('api_key_env')!r})",
-    )
-    check(
-        cp.get("base_url", "").endswith("/v1"),
-        "custom_providers",
-        f"custom_providers[0].base_url must end with /v1 (got {cp.get('base_url')!r})",
-    )
-
-    # --- No raw model names (Rule 7/8 spirit) ---
-    raw_names = {"gemma-4-12b", "qwen3.6-27B-code", "qwen3.6-35B-udq4", "ornith-1.0-35b"}
-    for section_path, section_dict in [
-        ("model", model), ("compression", comp),
-        ("auxiliary.vision", aux.get("vision", {})),
-        ("auxiliary.web_extract", aux.get("web_extract", {})),
-        ("auxiliary.compression", aux.get("compression", {})),
-        ("delegation", deleg),
-    ]:
-        m = section_dict.get("model", "")
-        if m in raw_names:
-            warn(
-                "Rule 7/8",
-                f"{section_path}.model = {m!r} — raw model name, use stable alias instead "
-                f"(gpu-light, gpu-dense, strix-moe, syslog-auto)",
-            )
-
-    # --- Report ---
-    print(f"{'=' * 60}")
-    print(f"Hermes Config Audit: {path}")
-    print(f"{'=' * 60}")
-    print(f"\n✅ PASSED ({len(PASSES)}):")
-    for p in PASSES:
-        print(f"  ✅ {p}")
-
-    if WARNINGS:
-        print(f"\n⚠️  WARNINGS ({len(WARNINGS)}):")
-        for w in WARNINGS:
-            print(f"  ⚠️  {w}")
-
-    if VIOLATIONS:
-        print(f"\n❌ VIOLATIONS ({len(VIOLATIONS)}):")
-        for v in VIOLATIONS:
-            print(f"  ❌ {v}")
-        print(f"\n{'=' * 60}")
-        print(f"RESULT: FAIL — {len(VIOLATIONS)} violation(s) must be fixed")
-        print(f"{'=' * 60}")
-        return 1
-    else:
-        print(f"\n{'=' * 60}")
-        print(f"RESULT: PASS — all contract rules satisfied")
-        print(f"{'=' * 60}")
-        return 0
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python3 audit-hermes-config.py <config.yaml>")
-        sys.exit(2)
-    sys.exit(audit(sys.argv[1]))
@@ -22,6 +22,7 @@ owners:
 - abiba
 - mumuni
 - kwame
+- ops
 trigger_types:
 - scheduled
 - event_driven
@@ -58,6 +59,7 @@ index:
    - memory-audit-maintenance
    - gpu-fleet
    - infrastructure-update
+    - infrastructure-maintenance
    reference:
    - infrastructure-control
    - ra-h-os-custodianship-contract
@@ -90,6 +92,7 @@ index:
    - infrastructure-control
    - infrastructure-monitoring
    - infrastructure-update
+    - infrastructure-maintenance
    - pm2-self-heal
    - disk-gc-threat-response
    gpu:
@@ -130,7 +133,6 @@ index:
    - build-zulip-plugin
    - stirling-pdf-agent-access
    - gpu-fleet
-    - infrastructure-update
    - infrastructure-control
    - zulip-adapter-lessons
    - pi-approval-architecture
@@ -144,6 +146,9 @@ index:
    - mumuni-delegation
    kwame:
    - hello-world
+    ops:
+    - infrastructure-maintenance
+    - infrastructure-update
  by_trigger:
    scheduled:
    - hermes-key-enforcement
@@ -156,6 +161,7 @@ index:
    - litellm-health
    - memory-audit-maintenance
    - infrastructure-update
+    - infrastructure-maintenance
    event_driven:
    - litellm-self-heal
    - pm2-self-heal
@@ -197,6 +203,7 @@ index:
    - hermes-zulip-plugin
    - build-zulip-plugin
    - infrastructure-update
+    - infrastructure-maintenance
    - ra-h-os-custodianship-contract
    - mumuni-delegation
    normal:
@@ -1256,13 +1263,108 @@ contracts:
  last_run: null
  last_status: null
  drift_alerts: []
+- name: infrastructure-maintenance
+  file: infrastructure-maintenance.prose.md
+  kind: responsibility
+  category: maintenance
+  sensitivity: high
+  status: active
+  owner: ops
+  version: 1.0.0
+  trigger:
+    type: scheduled
+    cadence: 0 2 * * 0
+    description: Weekly host-level maintenance Sunday at 2am ET (replaces infrastructure-update
+      build-phase role; infra-update moves to ops)
+    cron_job_id: null
+  execution:
+    agent: ops
+    timeout: 3600
+    requires:
+    - infrastructure-monitoring run within last 30 minutes (pre-update health baseline)
+    - Proxmox snapshot of primary host OR /tmp backup dir created this run
+    - LiteLLM master key from Infisical vault for health verification
+    protocol:
+    - Load contract from prose-contracts/main
+    - Phase 0 preflight — capture health baseline, backup check, record image baseline
+    - Phase 1 apt update && apt upgrade -y on primary host
+    - Phase 2 docker compose pull for LiteLLM, SearXNG, and other running containers
+    - Phase 3 restart stacks one at a time with per-stack health verification
+    - Phase 4 verify every critical service (LiteLLM, SearXNG, Zulip, Gitea, PM2, Hermes gateways)
+    - On failure — rollback per protocol, escalate, do not loop beyond circuit breaker
+    - Log actions to ~/.hermes/runs/infrastructure-maintenance/
+  verification:
+    postconditions:
+    - check: all critical services running after update
+      verify: 'curl -sf http://192.168.68.116/litellm/v1/models && curl -sf https://chat.sysloggh.net/api/v1/server_settings && curl -sf https://git.sysloggh.net/api/v1/version && curl -sf http://192.168.68.7:8888 && pm2 jlist'
+      expect: all probes 200 OK / processes online
+    - check: no regressions from pre-update health baseline
+      verify: diff Phase 0 health-baseline against Phase 4 results
+      expect: no GREEN service turned RED
+    - check: docker containers on latest stable tags
+      verify: docker inspect --format '{{.Config.Image}}' <container> per service matches image-baseline.pulled_tag
+      expect: all containers running pulled tags
+    - check: APT packages up to date with no held broken packages
+      verify: apt list --upgradable 2>/dev/null | wc -l and apt-get -s upgrade | grep -ci broken
+      expect: upgradable == 0, broken == 0
+    artifact: maintenance run report with phase results and any rollback/escalation
+    verify_commands:
+    - curl -sf http://192.168.68.116/litellm/v1/models
+    - curl -sf http://192.168.68.7:8888
+    - curl -sf https://chat.sysloggh.net/api/v1/server_settings
+    - curl -sf https://git.sysloggh.net/api/v1/version
+    - pm2 jlist
+  receipt:
+    format: json
+    storage: ~/.hermes/runs/infrastructure-maintenance/
+    graph_node: true
+    schema:
+      contract: string
+      run_id: string
+      timestamp: ISO 8601
+      agent: string
+      status: pass|fail|escalated
+      phase: preflight|apt|images|restarts|verify|rollback|done|failed
+      actions_taken: array
+      postconditions: array
+      drift_alerts: array
+      evidence_path: string
+  escalation:
+    info:
+      action: log_to_receipt
+      notify: []
+    warning:
+      action: relay_alert
+      notify:
+      - abiba
+      - mumuni
+    critical:
+      action: relay_alert
+      notify:
+      - abiba
+      - mumuni
+    fatal:
+      action: relay_alert + pause + human_required
+      notify:
+      - abiba
+      - mumuni
+      - kwame
+  circuit_breaker:
+    max_retries: 2
+    window: 7200
+    trip_action: escalate_to_fatal
+  depends_on:
+  - infrastructure-monitoring
+  last_run: null
+  last_status: null
+  drift_alerts: []
 - name: infrastructure-update
  file: infrastructure-update.prose.md
  kind: responsibility
  category: maintenance
  sensitivity: high
  status: active
-  owner: abiba
+  owner: ops
  version: 1.0.0
  trigger:
    type: scheduled
@@ -130,7 +130,7 @@ mcp_servers:
 # ─── Compression ───
 compression:
  enabled: true
-  model: syslog-auto               # ⚠️ Must match auxiliary.compression.model. Stable alias (gpu-fleet § Stable Role-Based Aliases). NOT ornith-1.0-35b (LiteLLM does not serve that name).
+  model: strix-moe               # ⚠️ Must match auxiliary.compression.model. Stable alias (gpu-fleet § Stable Role-Based Aliases). NOT ornith-1.0-35b (LiteLLM does not serve that name).
  provider: harness
  max_context_window: 131072      # MUST match actual GPU capacity. All 3 GPUs are 128K (Jul 17).
  threshold: 0.65                 # Fires at ~170K for 262K window, ~85K for 128K
@@ -166,7 +166,7 @@ auxiliary:
    timeout: 30
  compression:
    provider: harness
-    model: syslog-auto              # MUST match compression.model above. Stable alias for Strix Halo (weighted pool).
+    model: strix-moe              # MUST match compression.model above. Stable alias for Strix Halo.
    base_url: http://192.168.68.116/v1   # Rule 5: /v1 NOT /litellm/v1
    api_key_env: LITELLM_API_KEY
    timeout: 300                 # gpu-fleet: 300s for large-history summarization (was 60)
@@ -252,11 +252,6 @@ The following MUST be identical across ALL profiles:
 - Compression uses `strix-moe` (stable alias for Strix Halo — 64GB, 128K ctx, compression-optimized)
 - **`strix-moe` is the only valid compression model name** — LiteLLM does NOT serve `ornith-1.0-35b`
  (it serves `strix-moe`, `qwen3.6-35B-udq4`, `gpu-dense`, `gpu-light`, `syslog-auto`, `gemma-4-12b`, `qwen3.6-27B-code`). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
- **OPERATIONAL DECISION (2026-07-23): Use `syslog-auto` for compression across all agents.**
-  The `syslog-auto` alias routes to the Strix Halo, but uses the weighted pool instead of pinning
-  to `strix-moe` directly. This prevents sustained Strix Halo thermal load because the pool can
-  fall back to other GPUs if Strix gets hot. Both `compression.model` and `auxiliary.compression.model`
-  MUST be `syslog-auto`.
 - All auxiliary services MUST use identical routing:
  - `base_url: http://192.168.68.116/v1`  (Rule 5: `/v1`, NOT `/litellm/v1`)
  - `api_key_env: LITELLM_API_KEY`
@@ -270,11 +265,11 @@ The following MUST be identical across ALL profiles:
 ### Rule 8: GPU Workload Distribution (UPDATED 2026-07-16)
 - **RTX 3090 (24GB, 128K ctx, qwen3.6-27B-code)**: Heavy reasoning, code gen, long conversations
 - **RTX 5070 (12GB, 128K ctx, gemma-4-12b)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
- **Strix Halo (64GB, 128K ctx, syslog-auto)**: Context compression, summarization, long docs
+- **Strix Halo (64GB, 128K ctx, strix-moe)**: Context compression, summarization, long docs
 - Agent profiles MUST route auxiliary tasks to the correct GPU:
  - `auxiliary.vision.model: gemma-4-12b` (RTX 5070)
  - `auxiliary.web_extract.model: gemma-4-12b` (RTX 5070)
-  - `auxiliary.compression.model: syslog-auto` (Strix Halo)
+  - `auxiliary.compression.model: strix-moe` (Strix Halo)
 - Default model (`model.default`) and custom_provider remain `syslog-auto` for auto-routing
 - For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
 - Do NOT use `threshold: 0.25` — this fires at 65K, causing premature context loss
@@ -376,26 +371,6 @@ curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $K" http://192.
 - `/etc/environment` is NO LONGER the canonical key source (stale values there caused 401s).
 - Do NOT leave a hardcoded stale key in `/etc/environment` — it shadows the drop-in/wrapper.

-### Rule 14: Provider Name Must Match custom_providers Name (ADDED 2026-07-19, WAL #1471)
-
- `model.provider` MUST be `harness` (the `custom_providers[0].name`), NOT the literal string `custom`
- When `provider: custom`, Hermes' `_get_named_custom_provider("custom")` returns None (no provider is
-  named "custom" — it is named "harness"), causing a fall-through to the generic resolution path
-  (`source: env/config`) at `runtime_provider.py:1156`
- The generic path builds `api_key_candidates` from `model.api_key` (empty), host-gated
-  OLLAMA/OPENAI/OPENROUTER keys, and `_host_derived_api_key` (returns "" for IP addresses)
- **The generic path does NOT resolve `model.api_key_env` or `custom_providers.key_env`** —
-  `LITELLM_API_KEY` is never read, producing `api_key = "no-key-required"` → HTTP 401
- The named custom provider path (`source: custom_provider:harness`) DOES read `key_env` —
-  but only triggers when `provider` matches the `custom_providers[0].name`
- All sections MUST use `provider: harness`: `model`, `compression`, `auxiliary.vision`,
-  `auxiliary.web_extract`, `auxiliary.compression`, `delegation`
- Only `fallback_providers` uses a different provider (`deepseek`) for true fallback diversity
- **Diagnostic**: If you see `source: env/config` in a request dump or log, the provider name
-  is wrong. It should be `source: custom_provider:harness`.
- **Audit script**: Run `python3 /root/prose-contracts/audit-hermes-config.py <config.yaml>`
-  before and after any config change to catch this and all other rule violations.
-
 ## Execution

 1. **Check current config** — Read the target agent's config.yaml
@@ -0,0 +1,200 @@
+---
+kind: responsibility
+name: infrastructure-maintenance
+description: >
+  Weekly system-level maintenance for the Syslog inference fleet: OS package
+  updates on the primary host, Docker image pulls for LiteLLM/SearXNG and other
+  running containers, container restarts with health verification, post-update
+  verification of every critical service (LiteLLM proxy, SearXNG, Zulip, Gitea,
+  PM2 processes, Hermes gateways), and rollback on failure. Consolidates the
+  raw shell scripts that previously did this piecemeal. This contract owns the
+  HOST-LEVEL weekly maintenance loop on the primary host plus Docker image
+  pulls ONLY for .116 and .7, while infrastructure-update owns the FULL-FLEET
+  cluster-wide wave (apt across the full PVE cluster + CTs/VMs AND its Docker
+  image Wave 3 across all stacks). Runs Sunday 2am ET. Owner:
+  ops (firstmate secondmate). Blast radius: an unverified image pull can break
+  LiteLLM (all agents lose inference) or SearXNG (search-stack down); a bad apt
+  upgrade can leave the host in a half-upgraded state. Pre-update backup check
+  and rollback are mandatory for this reason.
+agent: ops
+triggers:
+  - weekly (Sunday 02:00 ET) via cron
+  - on demand when ops/abiba triggers "infra maintenance"
+version: 1.0.0
+---
+
+## Maintains
+
+- maintenance-status: { phase: idle|preflight|apt|images|restarts|verify|rollback|done|failed, host, step, result, timestamp }
+- image-baseline: { service, current_tag, pulled_tag, digest, updated_at } — last known-good image per container
+- apt-state: { upgradable_before, upgradable_after, held_broken, kernel_reboot_required }
+- health-baseline: snapshot of critical-service health captured pre-update (used for regression check post-update)
+- rollback-snapshot: { backup_path, configs, image_digests, timestamp } — restore point created in preflight
+- maintenance-history: array of past runs with phase results and any escalations
+
+## Scope
+
+Primary host is the maintenance host where apt updates apply. Docker image pulls
+span the two Docker ecosystems that run critical services. infrastructure-update
+runs the full-fleet cluster-wide wave (including its Wave 3 Docker pulls across
+all stacks/hosts); this contract runs a narrower host-level weekly pull limited
+to .116 and .7. Topology, CT IDs, and IPs are live-state fields — verify against
+`infrastructure-control.prose.md` (the source of truth) and the live system
+before mutating.
+
+| Host | IP | Role | Trust |
+|------|----|------|-------|
+| CT 116 (syslog-api) | 192.168.68.116 | LiteLLM proxy + Grafana + Prometheus (inference harness) | ⚠️ VERIFY-BEFORE-USE |
+| VM 109 (docker-vm) | 192.168.68.7 | SearXNG + Firecrawl + home stack (Docker host) | ⚠️ VERIFY-BEFORE-USE |
+| CT 117 (zulip) | 192.168.68.19 | Zulip (storepve bridge IP .19) | ⚠️ VERIFY-BEFORE-USE |
+| Gitea | https://git.sysloggh.net | Prose-contracts + agent configs source control | ⚠️ VERIFY-BEFORE-USE |
+| CT 100 (abiba/pi) | 192.168.68.24 | PM2 processes (pi agent harness) | ⚠️ VERIFY-BEFORE-USE |
+
+> "Primary host" for the apt phase is the host the ops agent runs maintenance
+> from. Confirm which host that is against infrastructure-control before
+> running; do not assume. If the ops agent is containerized/CT-based, apt runs
+> inside that CT.
+
+## Requires
+
+- SSH/exec access to CT 116 (.116) and VM 109 (.7) for Docker operations
+- `apt`, `docker`, `docker compose` available on target hosts
+- LiteLLM master key available (Infisical vault, `LITELLM_API_KEY`) for health verification
+- `infrastructure-monitoring` run completed within the last 30 minutes — provides the pre-update health baseline used by the regression check
+- Writable backup directory `/tmp/infra-maintenance-backup-<date>/` on each mutated host
+- Proxmox snapshot of the primary host available (or confirmed not required) before apt phase
+
+## Continuity
+
+- Self-driven: weekly cron `0 2 * * 0` (Sunday 02:00 ET)
+- Also wakes on: explicit "infra maintenance" trigger from ops/abiba
+- Depends on `infrastructure-monitoring` for the pre-update health baseline — do not run if the last monitoring run is stale (>30 min) or RED; abort and escalate instead
+
+## Execution
+
+### Phase 0 — Preflight (snapshot/backup check + health baseline)
+
+1. **Capture health baseline** — run the `infrastructure-monitoring` postcondition checks (LiteLLM, Zulip, Gitea, SearXNG, Proxmox API) and record results as `health-baseline`. If any critical service is already down, **abort**: maintenance must not run on a degraded fleet.
+2. **Backup check** — confirm a Proxmox snapshot of the primary host exists OR `/tmp/infra-maintenance-backup-<date>/` was created this run. Snapshot critical config files into the backup dir:
+   - `/opt/inference-harness/docker-compose.yml`, `/opt/inference-harness/litellm_config.yaml` (CT 116)
+   - `/opt/search-stack/searxng/docker-compose.yml`, `/opt/search-stack/firecrawl-source/docker-compose.yaml` (VM 109)
+3. **Record image baseline** — `docker inspect --format '{{.Image}} {{.Config.Image}}' <container>` for every running container on .116 and .7; store digests in `image-baseline` so rollback can restore them.
+4. **Disk check** — `df -h` on each mutated host; abort if free space <20% (apt upgrade + image pulls need headroom).
+
+### Phase 1 — OS package updates (primary host)
+
+```bash
+# On the primary host only (VERIFY host against infrastructure-control first)
+apt update
+apt upgrade -y
+```
+
+- Capture `apt list --upgradable` before and after → store in `apt-state`.
+- If apt reports held/broken packages (`apt-get -s upgrade | grep -i broken`, or non-zero exit), **stop** — do not force. Record `held_broken` and go to rollback/escalate.
+- If `/var/run/reboot-required` exists after upgrade, flag `kernel_reboot_required: true` in `apt-state` but **do not reboot automatically** — that's a separate coordinated action (see infra-update Wave 4). Note it in the report.
+
+### Phase 2 — Docker image pulls
+
+Pull latest stable tags for every running container. Do NOT pin to `:main`/`:nightly` — use stable tags where the compose file specifies them; otherwise `latest`.
+
+```bash
+# CT 116 (.116) — inference harness
+cd /opt/inference-harness && docker compose pull
+
+# VM 109 (.7) — search + home stacks
+cd /opt/search-stack/searxng && docker compose pull
+cd /opt/search-stack/firecrawl-source && docker compose pull
+# any other running stacks on .7 (home stack, audiobookshelf) — pull per their compose files
+```
+
+- LiteLLM and SearXNG are the two explicitly required pulls; "any other running containers" means every stack with a compose file on .116 and .7.
+- Record pulled tag + digest per service in `image-baseline`.
+
+### Phase 3 — Container restarts with health verification
+
+Restart one stack at a time, verify health before moving to the next. Do not restart everything at once — a failure mid-wave must leave the rest running.
+
+```bash
+# CT 116
+cd /opt/inference-harness && docker compose up -d
+# VM 109
+cd /opt/search-stack/searxng && docker compose up -d
+cd /opt/search-stack/firecrawl-source && docker compose up -d
+```
+
+After each stack comes up, wait for health (max 120s):
+- `docker ps` shows the container `Up` (and `healthy` if a healthcheck is defined)
+- Service-specific probe passes (see Phase 4 probes)
+
+If a stack fails to come up within 120s, **stop the wave** and go to rollback for that stack only; do not proceed to the next.
+
+### Phase 4 — Post-update service verification
+
+After ALL updates (apt + images + restarts), verify every critical service is back up and matches the pre-update baseline. This is the regression gate.
+
+| Service | Probe | Expect |
+|---------|-------|--------|
+| LiteLLM proxy | `curl -sf http://192.168.68.116/litellm/v1/models` | 200 OK, models returned |
+| LiteLLM MCP gateway | `curl -sf http://192.168.68.116:4000/mcp-rest/tools/list -H "Authorization: Bearer $MASTER_KEY"` | 90 tools (23 RA-H OS + 67 GitHub) |
+| SearXNG | `curl -sf http://192.168.68.7:8888` | 200 OK |
+| Zulip | `curl -sf https://chat.sysloggh.net/api/v1/server_settings` | 200 OK |
+| Gitea | `curl -sf https://git.sysloggh.net/api/v1/version` | 200 OK |
+| PM2 processes | `pm2 jlist` (CT 100) | all pi-agent processes `online` |
+| Hermes gateways | SSH to Mumuni CT 114, Tanko CT 112; `systemctl is-active hermes-gateway` | `active` for each |
+
+Regression check: every service that was GREEN in `health-baseline` must still be GREEN. A service that was already RED (and caused a preflight abort) is excluded — but Phase 0 should have aborted before we got here.
+
+## Rollback Protocol
+
+If ANY service in Phase 4 fails to come back up (or regresses vs baseline):
+
+1. **Image rollback** — for the failing stack, restore the previous image:
+   ```bash
+   # Restore from recorded image-baseline digest
+   docker compose down
+   # Pin the service image to the recorded digest in compose, then recreate
+   #   image: <name>@sha256:<previous_digest>
+   docker compose pull && docker compose up -d
+   ```
+2. **APT rollback** — restore the primary host from the Proxmox snapshot taken/confirmed in Phase 0. If no snapshot, `apt install <pkg>=<old_version>` per package using apt history (`/var/log/apt/history.log`).
+3. **Config rollback** — restore configs from `/tmp/infra-maintenance-backup-<date>/`.
+4. **Re-verify** — re-run the Phase 4 probes on the rolled-back service. If still failing, escalate (do not loop — circuit breaker below).
+5. **Escalate** — send a Zulip DM to abiba + mumuni with: failing service, phase, baseline vs current, rollback actions taken, backup path.
+
+## Circuit Breaker
+
+- `max_retries: 2` per failing phase — after 2 rollback attempts on the same service, stop and escalate.
+- `window: 7200` seconds — no more than 2 retries within a 2-hour window.
+- `trip_action: escalate_to_fatal` — when tripped, escalate to fatal (abiba + mumuni + kwame) and pause; a human must clear before the next scheduled run.
+
+## Report
+
+After completion (or on abort), emit a receipt (JSON) to `~/.hermes/runs/infrastructure-maintenance/` and send a Zulip DM summary:
+
+```
+🛠 Infrastructure Maintenance — YYYY-MM-DD
+
+Phase: apt | images | restarts | verify | rollback
+Primary host: <host>
+APT: <N> packages upgraded, <M> held/broken, kernel_reboot_required=<bool>
+Images pulled: LiteLLM <tag>, SearXNG <tag>, <others>
+Services: all GREEN | <service> FAILED (rolled back)
+Baseline regression: none | <details>
+Backup: /tmp/infra-maintenance-backup-YYYYMMDD/
+Escalation: none | warning | critical | fatal
+```
+
+## Verification Postconditions
+
+- All critical services running after update (Phase 4 all GREEN)
+- No regressions from pre-update health baseline (Phase 0 baseline)
+- Docker containers on latest stable tags (`image-baseline.pulled_tag` recorded)
+- APT packages up to date with no held broken packages (`apt-state.held_broken == 0`)
+
+## Related Contracts
+
+- `infrastructure-update.prose.md` — owns the full-fleet cluster-wide wave INCLUDING its Wave 3 Docker image updates across all stacks (SearXNG, Firecrawl, Inference Harness on .116, home stack, audiobookshelf); infrastructure-maintenance is a deliberately narrower host-level weekly pull scoped to .116 and .7.
+- `infrastructure-monitoring.prose.md` — provides the pre-update health baseline (depends_on).
+- `infrastructure-control.prose.md` — topology source of truth (CT IDs, IPs, hostnames).
+- `litellm-health.prose.md` — LiteLLM probe details.
+- `proxmox-monitor.prose.md` — Docker stats + monitoring stack health.
@@ -1,71 +0,0 @@
---
-kind: pattern
-name: memory-fixer
-description: >
-  Auto-fix low-hanging fruit in the graph. No judgment calls — only deterministic Level 1 operations. 
-  Escalate anything that needs Kwame's input.
-version: 1.1.0
---
-
-# Memory Fixer
-
-## Purpose
-Auto-fix low-hanging fruit in the graph. No judgment calls — only deterministic Level 1 operations. Escalate anything that needs Kwame's input.
-
-## Level 0 Auto-Deletes (Allowed Without Approval)
-Ephemeral heartbeat and log nodes that violate "Logs NEVER go in the graph":
-
- `[LITELLM-HEALTH]`, `[GPU-SELF-HEAL]`, `[PM2-SELF-HEAL]`
- `[PROXMOX-MONITOR]`, `[GPU-MONITOR]`, `[INFRA-MONITOR]`, `[AGENT-HEALTH]`, `[DISK-GC]`
- `[WAL]` entries older than 30 days
-
-**Condition:** node must be an orphan (no edges). Deleting a connected node risks breaking other nodes.
-
-**Method:** direct SQLite on `.65` (MCP has no delete tool):
-```bash
-ssh root@192.168.68.65 "sqlite3 /root/.local/share/RA-H/db/rah.sqlite \"
-DELETE FROM nodes WHERE id IN (
-  SELECT id FROM nodes WHERE id NOT IN (SELECT from_node_id FROM edges)
-  AND id NOT IN (SELECT to_node_id FROM edges)
-  AND title LIKE '[LITELLM-HEALTH]%' -- add more prefixes as needed
-);\""
-```
-
-## Level 1 Auto-Fixes (No Judgment Required)
-
-### 1. Missing `type` Field
-For nodes with content but no `metadata.type`:
- Title contains "Proxmox" or "infrastructure" → `type: infrastructure`
- Title contains "skill" or "how to" or "guide" → `type: skill`
- Title contains "doc" or "template" or "brand" → `type: documentation`
- Title starts with "WAL:" or "TASK:" → `type: note`
- Title starts with "[LEARN]" → `type: documentation`
- Otherwise → `type: note` (default)
-
-### 2. Missing `tenant` / `namespace`
-For any node with NULL tenant or namespace:
-```sql
-UPDATE nodes 
-SET metadata = json_set(
-  COALESCE(metadata, '{}'),
-  '$.tenant', 'syslogsolution',
-  '$.namespace', 'syslogsolution'
-)
-WHERE json_extract(metadata, '$.tenant') IS NULL 
-   OR json_extract(metadata, '$.namespace') IS NULL;
-```
-
-### 3. Staleness State Transitions
-Using the type-based windows from the memory-monitor contract:
- Nodes stale > their window → transition to `state: review_pending`
- Nodes in `review_pending` for >7 days → escalate to Kwame (Level 2)
-
-## Level 2 Escalations (Kwame Decision Required)
-1. **Nodes in `review_pending` >7 days** — Archive, refresh, or keep?
-2. **Orphan Nodes >90 days old** — Delete or Connect?
-3. **Potential Duplicate Nodes** — Same title or >70% overlap. Merge or Keep?
-4. **Conflicting Metadata** — Content suggests one tenant but metadata says another.
-
-## Logging
-Every Level 1 fix logged to `~/.hermes/logs/memory-fixer/YYYY-MM-DD.md`
-Every Level 2 escalation logged and delivered to Kwame.
@@ -409,3 +409,68 @@ Backup v2 before starting: `cp index.js index.js.v2-backup-$(date +%Y%m%d-%H%M%S
 | Queue expiry handling | Crash | Auto re-register |
 | Busy worker deadlock | Router death | Worker SIGKILL + error DM |
 | PM2 restart exhaustion | Yes (max_restarts=10) | No (max_restarts=100 + watchdog) |
+
+---
+
+## Incident Log — 2026-07-18 Fleet-Wide Audit
+
+### Fleet State After Audit
+
+| Agent | Platform | Zulip State | Issues Found | Fix Applied |
+|-------|----------|-------------|--------------|-------------|
+| **Abiba** | pi (CT 100) | ✅ Connected | API key missing from Infisical injection; poll timeout noise | Added .env fallback; AbortError treated as empty poll (no retry); poll timeout 65s→90s |
+| **Tanko** | Hermes (CT 112) | ✅ Connected | Gateway disconnected since Jul 11; watchdog restart didn't re-establish Zulip | Full gateway restart (kill wrapper, let infisical-gateway.sh respawn) |
+| **Mumuni** | Hermes (CT 114) | ✅ Connected | No issues found | None needed |
+
+### Key Fixes Applied
+
+**1. Abiba — Credential Fallback (L4 Pattern)**
+- Root cause: `zulip.api_key` in config.yaml is `""` (expected from Infisical). Infisical vault `ABIBA_ZULIP_API_KEY` wasn't being injected into the process environment.
+- Fix: Added `.env` file fallback at `/root/.pi/agent/extensions/zulip/.env` with known-working key, sourced before the Infisical `exec`.
+- Lesson: Per L4 from gpu-self-heal, Infisical is not always available — always keep a local `.env` fallback.
+
+**2. Abiba — Poll Timeout Handling**
+- Root cause: Zulip long-poll uses `AbortSignal.timeout(65000)`. Zulip's default `event_queue_longpoll_timeout_seconds` can exceed 65s. When the signal fires, an `AbortError` is thrown and caught by the circuit breaker as a failure.
+- Fix: Caught `AbortError` inside `poll()` and return empty array (no events) instead of throwing. Extended timeout to 90s to match Zulip server default.
+- Reference: [Zulip Events System — long-poll timeout](https://zulip.readthedocs.io/en/11.6/subsystems/events-system.html)
+
+**3. Tanko — Gateway Restart**
+- Root cause: Gateway process was running but Zulip platform stayed in "disconnected" state since Jul 11, 2026. The wrapper script (`infisical-gateway.sh`) restarts on crash but the gateway wasn't re-establishing Zulip on restart.
+- Fix: Killed gateway PID to trigger wrapper restart. New gateway (PID 331991) established Zulip connection successfully.
+
+### Fleet-Wide Zulip Health Metrics (as of 2026-07-18)
+
+| Metric | Value |
+|--------|-------|
+| Zulip server | ✅ HTTP 200 |
+| Agents connected | 3/3 (Abiba, Tanko, Mumuni) |
+| Abiba circuit breaker | CLOSED (0 failures) |
+| Abiba uptime | 2D (post-restart) |
+| Tanko gateway uptime | Ongoing |
+| Mumuni gateway uptime | Ongoing |
+| Watchdog status | ✅ Online (2D uptime) |
+
+### Hermes Agent Zulip Plugin Improvements
+
+Based on the audit, improvements that should be ported to all Hermes Zulip adapters:
+
+1. **Circuit breaker pattern** — Already in Abiba's pi extension. Hermes adapters should add the same CLOSED→OPEN→HALF_OPEN state machine with exponential backoff.
+2. **Credential fallback** — All Hermes agents use Infisical for credentials. Add `.env` local fallback per L4 pattern for `ZULIP_API_KEY`.
+3. **Queue re-registration** — Handle `BAD_EVENT_QUEUE_ID` with automatic re-registration instead of gateway restart.
+4. **Supervisor watchdog** — Hermes uses PM2 which auto-restarts on crash, but has no health-check watchdog. Add lightweight external health checks.
+5. **Streaming** — All agents have `streaming: true` in their zulip config. Verify `edit_message()` is implemented in each adapter.
+
+### Abiba pi Zulip Extension v2 — Implemented Resilience Summary
+
+| Feature | Status | Notes |
+|---------|--------|-------|
+| Circuit breaker | ✅ | CLOSED→OPEN→HALF_OPEN; 50% failure threshold; 30s reset timeout |
+| Retry with jitter | ✅ | 2 attempts, 200ms base, 50-100% jitter |
+| Queue lifecycle | ✅ | 10min idle_queue_timeout; BAD_EVENT_QUEUE_ID handling |
+| Crash prevention | ✅ | uncaughtException + unhandledRejection recovery |
+| Worker timeout | ✅ | 5min busy timeout → SIGKILL + error DM |
+| Health endpoint | ✅ | :9200 with circuit breaker metrics |
+| Echo prevention | ✅ | Dynamic bot user resolution |
+| Poll timeout (AbortError) | ✅ v2.1 | Normal timeout returns [] instead of error |
+| Credential fallback | ✅ v2.1 | .env file before Infisical exec |
+| Provider auto-fix | ✅ | Detects reasoning_content models, switches to compatible |
Author	SHA1	Message	Date
jerome	5d2ecbace6	Merge branch 'master' into fm/infra-maint-contract PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 3s Details PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 0s Details	2026-07-18 23:12:20 +00:00
root	99789a00a1	no-mistakes(document): Reframed infra-maintenance contract as deliberate partition PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 3s Details PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 0s Details	2026-07-18 22:16:09 +00:00
root	a68879e904	no-mistakes(review): fix docker rollback command to pin digest then compose up	2026-07-18 22:07:15 +00:00
jerome	06d2bcbc9e	Merge pull request 'fix: fleet config issues from 2026-07-18 relay review' (#23 ) from fix/fleet-config-issues-20260718 into master PR Pipeline — Authorize → Validate → Review → Merge / auth (push) Successful in 0s Details PR Pipeline — Authorize → Validate → Review → Merge / validate (push) Successful in 1s Details PR Pipeline — Authorize → Validate → Review → Merge / lint (push) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / ai-review (push) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / gate (push) Successful in 0s Details Reviewed-on: #23	2026-07-18 22:06:48 +00:00
jerome	c4a8c45835	Merge pull request 'zulip-resilience: fleet-wide audit findings and fixes 2026-07-18' (#22 ) from feat/zulip-resilience-audit-20260718 into master PR Pipeline — Authorize → Validate → Review → Merge / auth (push) Successful in 1s Details PR Pipeline — Authorize → Validate → Review → Merge / validate (push) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / lint (push) Successful in 3s Details PR Pipeline — Authorize → Validate → Review → Merge / ai-review (push) Successful in 2s Details PR Pipeline — Authorize → Validate → Review → Merge / gate (push) Successful in 1s Details Reviewed-on: #22	2026-07-18 22:06:34 +00:00
root	1d027f71f6	feat(contracts): add infrastructure-maintenance contract New responsibility contract consolidating host-level system maintenance and Docker image lifecycle management, filling the gap left by infrastructure-update (which owns cluster-wide apt waves). Scope: - OS package updates on primary host with pre-update snapshot/backup check - Docker image pulls for LiteLLM, SearXNG, and other running containers - Container restarts with per-stack health verification - Post-update verification: LiteLLM, SearXNG, Zulip, Gitea, PM2, Hermes gateways - Rollback on failure (image/apt/config restore) with circuit breaker Owner: ops (firstmate secondmate). Trigger: weekly Sunday 2am ET. Escalation: warning/critical->abiba+mumuni, fatal->abiba+mumuni+kwame. circuit_breaker: max_retries 2, window 7200, trip_action escalate_to_fatal. depends_on: infrastructure-monitoring (pre-update health baseline). Registry: - Add infrastructure-maintenance to by_category.maintenance, by_domain.infrastructure, by_owner.ops (new), by_trigger.scheduled, by_sensitivity.high - Add 'ops' to owners list - Move infrastructure-update owner abiba -> ops (in contracts entry + by_owner index) Also adds '## Maintaining this file' section to AGENTS.md per fm-ensure-agents-md.	2026-07-18 22:02:45 +00:00
root	23f3f378c5	zulip-resilience: add fleet-wide audit findings and fixes from 2026-07-18 PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 4s Details PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 3s Details PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 6s Details PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 5s Details PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 0s Details - Added Incident Log section documenting fleet-wide Zulip audit - Abiba: poll timeout AbortError fix (returns [] instead of error) - Abiba: credential fallback .env file for Infisical outages - Tanko: full gateway restart to recover Zulip connection - Fleet health metrics summary table - Hermes agent improvement recommendations (circuit breaker, credential fallback, queue re-registration, watchdog) - Abiba pi extension v2.1 resilience feature matrix	2026-07-18 16:33:12 +00:00