diff --git a/disk-gc-threat-response.prose.md b/disk-gc-threat-response.prose.md index 7a61f5d..acb3ef8 100644 --- a/disk-gc-threat-response.prose.md +++ b/disk-gc-threat-response.prose.md @@ -1,17 +1,8 @@ --- report_only_agents: - koby # ⛔ KOBY IS NEVER REPAIRED (Rule 17, 2026-08-17) — detect + report, never fix on .129 -# ⛔ GUEST/HOST-KEYED report-only list. This is the gate the GC executor MUST honour. -# An agent-name marker above is NOT sufficient: the scan unit is a guest, and an agent -# marker can silently miss the guest it lives on. Key exclusions on the guest/host. -report_only_guests: - - guest: 111 - hostname: tdunna - ip: 192.168.68.129 - node: storepve - reason: > - Theo's box. Captain ruling 2026-08-17, re-confirmed 2026-09-10: Theo handles - CT 111 himself. DETECT-AND-REPORT-ONLY at every threat level. +# ⛔ The guest/host-keyed report-only gate the GC executor MUST honour lives in the body +# "Hard gate" YAML block below — that block is authoritative and is the only copy. kind: responsibility name: disk-gc-threat-response description: > @@ -37,7 +28,7 @@ logged within 5 minutes of discovery. ## Scope -All 20 Proxmox guests (17 LXC + 3 QEMU VMs) via `pct-run` + 3 GPU bare-metal hosts via direct SSH. +All 20 Proxmox guests (17 LXC via `pct-run` + 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts via direct SSH. Docker hosts get special attention: | Host | CT | Disk Risk | GC Strategy | @@ -128,7 +119,8 @@ agent-name marker can silently miss the guest it lives on — it must never be t **The authoritative machine-readable exclusion list is the YAML block below.** The executor reads it at run time; `scripts/disk-gc-plan.py` turns a fleet scan into the action plan using it. -Extend the list here, never by hand-maintaining a second copy. +Extend the list here, never by hand-maintaining a second copy. The Execution loop below MUST +call that planner and MUST NOT reimplement the gate. ```yaml # disk-gc report-only guests — authoritative. Keyed on guest/host, not agent. @@ -146,41 +138,32 @@ report_only_guests: let fleet = call disk-scanner scope: all -let report_only = load-report-only-guests() -- from the YAML block above +-- The report-only gate is IMPLEMENTED IN scripts/disk-gc-plan.py and MUST NOT be +-- reimplemented here. That planner reads the contract's `report_only_guests` YAML block +-- and matches on guest id OR hostname OR IP, so the tested gate is the executed gate. +let plan = call disk-gc-plan + fleet: fleet -let threats = [] -for ct in fleet: - if ct.usage_pct >= 95: - push threats { ct: ct.id, level: "CRITICAL", pct: ct.usage_pct } - else if ct.usage_pct >= 85: - push threats { ct: ct.id, level: "RED", pct: ct.usage_pct } - else if ct.usage_pct >= 75: - push threats { ct: ct.id, level: "AMBER", pct: ct.usage_pct } - --- sort by severity descending -sort threats by pct desc - -for threat in threats: - -- HARD GATE: an excluded guest is alerted and skipped. No gc-executor call is - -- constructed for it at any level, so no GC command can be emitted for it. - if threat.ct in report_only: +for row in plan: + if row.action == "report-only": + -- Excluded guest: alert only. No gc-executor call is constructed for it, at any level. call alerter - threat: threat - result: { action: "report-only", reason: report_only[threat.ct].reason } + threat: row + result: { action: "report-only", reason: row.reason } continue let result = call gc-executor - ct: threat.ct - level: threat.level - strategy: lookup-gc-strategy(threat.ct) + ct: row.target + level: row.level + strategy: lookup-gc-strategy(row.target) call alerter - threat: threat + threat: row result: result call summary-reporter fleet: fleet - threats: threats + plan: plan ``` ## GC Strategies by Host Type @@ -292,7 +275,7 @@ dangling images and orphaned build cache. No automated GC was in place. ## Incident Log: 2026-07-09 — amdpve docker bloat ### Discovery -Scheduled fleet disk scan via `pct-run` across all 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts. +Scheduled fleet disk scan across all 20 Proxmox guests (17 LXC via `pct-run`, 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts. > **Report-only gate applies to this scan:** CT 111 (`tdunna`, 192.168.68.129) is alerted but never > garbage-collected at any level. amdpve (.15) flagged at 78% (AMBER threshold: 75%). diff --git a/infrastructure-control.prose.md b/infrastructure-control.prose.md index d7a4a36..5c40667 100644 --- a/infrastructure-control.prose.md +++ b/infrastructure-control.prose.md @@ -16,8 +16,8 @@ description: > **Last verified:** 2026-08-15 — hwepve removed from Tabiri cluster (now 5 nodes: minipve, amdpve, storepve, acerpve, ocupve). hwepve (192.168.68.4) is a standalone PVE node + NetBird routing peer; - London relocation pending. CTs 100 (abiba) and 105 (kagentz) moved - to minipve. + London relocation pending. CT 100 (abiba) is on minipve; CT 105 + (kagentz) is on amdpve. --- # Infrastructure Control Pattern @@ -105,16 +105,16 @@ description: > | Node | IP | CPU | RAM | VMs/CTs | Role | |------|----|-----|-----|---------|------| -| minipve | .12 | 16C | 30GB | abiba, kagentz, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging | -| amdpve | .15 | 32C | 62GB | tanko, tdunna, baggy, scottdenya | Agents, compute | -| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip | Docker, storage, chat | +| minipve | .12 | 16C | 30GB | abiba, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging | +| amdpve | .15 | 32C | 62GB | kagentz, tanko, baggy, scottdenya, adguard2 | Agents, compute | +| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna | Docker, storage, chat | | acerpve | .9 | 28C | 31GB | llm-gpu | GPU VMs | | ocupve | .5 | 12C | 14GB | ocu-llm | GPU VMs | -> **Note:** CTs on storepve include jdownloader (CT 118). AdGuard (CT 102) is on -> minipve at .10, not acerpve. Abiba (CT 100) and kagentz (CT 105) are on -> minipve (moved from hwepve 2026-08-15). Mumuni runs inside Abiba CT100 -> (.24); CT 114 (mumuni) no longer exists in the cluster. +> **Note:** CTs on storepve include jdownloader (CT 118) and tdunna (CT 111). +> AdGuard (CT 102) is on minipve at .10, not acerpve. Abiba (CT 100) is on +> minipve (moved from hwepve 2026-08-15); kagentz (CT 105) is on amdpve. +> Mumuni runs inside Abiba CT100 (.24); CT 114 (mumuni) no longer exists in the cluster. > > **hwepve (192.168.68.4) — STANDALONE (removed from Tabiri 2026-08-15):** > Huawei MateBook 16 (KLVL-WXX9), pve-manager/9.2.10, kernel 7.0.14-8-pve. @@ -602,13 +602,13 @@ ssh root@192.168.68.110 "systemctl restart llama-server" | 102 | adguard | **minipve** | **.10** | DNS | ❌ | | 103 | ocu-llm | ocupve | .110 | GPU RTX 5070 | ❌ | | 104 | authentik | minipve | .11 | OIDC | ❌ | -| 105 | kagentz | minipve | — | Agent Zero | ✅ | +| 105 | kagentz | amdpve | — | Agent Zero | ✅ | | 106 | ra-h-os | storepve | .65 | KG bridge | ✅ MCP | | 107 | pbs | storepve | — | Backups | ❌ | | 108 | media | storepve | — | Media | ❌ | | 109 | docker-vm | storepve | .7 | Docker host | ❌ | | 110 | gitea | minipve | **.17** | Git | ❌ | -| 111 | tdunna | amdpve | .129 | Hermes agent | ✅ | +| 111 | tdunna | storepve | .129 | Hermes agent — ⛔ REPORT-ONLY (Theo's box, no GC) | ✅ | | 112 | tanko | amdpve | .122 | DSH (DeepSeek Harness) agent | ✅ | | 113 | baggy | amdpve | .114 | Hermes agent | ✅ | | 115 | scottdenya | amdpve | .75 | Denya OneCare | ❌ | @@ -616,6 +616,7 @@ ssh root@192.168.68.110 "systemctl restart llama-server" | 117 | zulip | storepve | .19 | Chat | ❌ | | 118 | jdownloader | storepve | .20 | JDownloader LXC (dedicated, migrated from docker-vm 2026-08-01) | ✅ | | 119 | infisical-vault | minipve | — | Vault | ❌ | +| 120 | adguard2 | amdpve | — | DNS (secondary AdGuard) | ❌ | ## Appendix C: Docker Compose Files Location @@ -636,8 +637,8 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run. | CT | Name | Node | pct-run | |-----|------|------|---------| | 100 | abiba | minipve | `pct-run 100` | -| 105 | kagentz | minipve | `pct-run 105` | -| 111 | tdunna | amdpve | `pct-run 111` | +| 105 | kagentz | amdpve | `pct-run 105` | +| 111 | tdunna | storepve | `pct-run 111` (⛔ report-only — no GC) | | 112 | tanko | amdpve | `pct-run 112` | | 113 | baggy | amdpve | `pct-run 113` | | 115 | scottdenya | amdpve | `pct-run 115` | @@ -648,6 +649,9 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run. | 107 | proxmox-backup | storepve | `pct-run 107` | | 108 | media | storepve | `pct-run 108` | | 117 | zulip | storepve | `pct-run 117` | +| 118 | jdownloader | storepve | `pct-run 118` | +| 119 | infisical-vault | minipve | `pct-run 119` | +| 120 | adguard2 | amdpve | `pct-run 120` | | 102 | adguard | **minipve** | `pct-run 102` | GPU bare-metal hosts (.8 acerpve, .110 ocupve, .15 amdpve) are NOT CTs — use SSH directly: diff --git a/scripts/disk-gc-plan.py b/scripts/disk-gc-plan.py old mode 100644 new mode 100755 index e6d53ec..f251268 --- a/scripts/disk-gc-plan.py +++ b/scripts/disk-gc-plan.py @@ -61,7 +61,8 @@ def load_report_only_guests(contract_path: pathlib.Path) -> list[dict]: def _keys(entry: dict) -> set[str]: - """Guest/host identity keys for an exclusion entry.""" + """Guest/host identity keys, shared by exclusions and scan entries so the two + sides of the gate can never key on different fields.""" out: set[str] = set() for field in ("guest", "id", "vmid", "hostname", "name", "ip"): value = entry.get(field) @@ -70,15 +71,6 @@ def _keys(entry: dict) -> set[str]: return out -def _entry_keys(scan_entry: dict) -> set[str]: - out: set[str] = set() - for field in ("id", "guest", "vmid", "hostname", "name", "ip"): - value = scan_entry.get(field) - if value is not None and str(value).strip(): - out.add(str(value).strip().lower()) - return out - - def level_for(pct: float) -> str | None: if pct >= CRITICAL: return "CRITICAL" @@ -99,7 +91,7 @@ def build_plan(scan: list[dict], report_only: list[dict]) -> list[dict]: level = level_for(float(pct)) if level is None: continue # GREEN: log only, no action - scan_keys = _entry_keys(entry) + scan_keys = _keys(entry) match = next((e for e, keys in excluded if keys & scan_keys), None) target = next( (entry.get(k) for k in ("id", "guest", "vmid", "hostname", "name", "ip") diff --git a/scripts/prose-ai-review.sh b/scripts/prose-ai-review.sh index cefc089..76c394a 100755 --- a/scripts/prose-ai-review.sh +++ b/scripts/prose-ai-review.sh @@ -47,17 +47,19 @@ You are a code reviewer for OpenProse infrastructure contracts in the Syslog Sol The infrastructure-control.prose.md contract is the canonical reference for the cluster topology: **Proxmox Cluster "Tabiri" (5 nodes):** -- amdpve (192.168.68.15): tanko, tdunna, baggy, scottdenya -- minipve (192.168.68.12): abiba, kagentz, adguard, authentik, gitea, syslog-api, infisical-vault -- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip +- amdpve (192.168.68.15): kagentz, tanko, baggy, scottdenya, adguard2 +- minipve (192.168.68.12): abiba, adguard, authentik, gitea, syslog-api, infisical-vault +- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna - acerpve (192.168.68.9): llm-gpu - ocupve (192.168.68.5): ocu-llm -**CT IDs (verified 2026-07-24 against PVE API):** +**CT IDs (verified 2026-09-12 against PVE API):** 100:abiba 102:adguard 104:authentik 105:kagentz 106:ra-h-os 107:pbs 108:media 110:gitea 111:tdunna 112:tanko 113:baggy 115:scottdenya 116:syslog-api 117:zulip -118:jdownloader 119:infisical-vault +118:jdownloader 119:infisical-vault 120:adguard2 + +**CT 111 (tdunna, 192.168.68.129) is REPORT-ONLY — Theo's box; alert only, never garbage-collect.** **NO CT 122, CT 123, or .19 exist in the cluster.** diff --git a/tests/test_disk_gc_report_only.py b/tests/test_disk_gc_report_only.py index 2df496a..5f1279b 100644 --- a/tests/test_disk_gc_report_only.py +++ b/tests/test_disk_gc_report_only.py @@ -78,12 +78,40 @@ def test_agent_name_alone_does_not_gate_a_guest(tmp_path): assert plan and plan[0]["action"] == "gc-executor" -def test_contract_carries_the_guest_keyed_exclusion(tmp_path): - """The authoritative gate must exist and identify CT 111 by guest/host.""" - plan = _plan([{"id": 111, "usage_pct": 95}], tmp_path) - reason = _actions_for(plan, 111)[0]["reason"] - assert reason, "the exclusion must carry a reason for the alert" - contract = (ROOT / "disk-gc-threat-response.prose.md").read_text() - assert "report_only_guests:" in contract - assert "192.168.68.129" in contract - assert "tdunna" in contract +def _plan_with_contract(scan, contract_text, tmp_path, name): + contract = tmp_path / name + contract.write_text(contract_text) + scan_file = tmp_path / f"scan-{name}.json" + scan_file.write_text(json.dumps(scan)) + proc = subprocess.run( + [sys.executable, str(PLAN), "--scan", str(scan_file), + "--contract", str(contract), "--json"], + capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_gate_is_read_from_the_contract_block(tmp_path): + """The gate is data-driven by the contract block: the planner excludes the guest + when the block names it and acts on it when the block does not. Executes the real + planner interface against both fixtures so the behaviour change is observable.""" + scan = [{"id": 111, "hostname": "tdunna", "ip": "192.168.68.129", "usage_pct": 95}] + with_gate = ( + "```yaml\n" + "report_only_guests:\n" + " - guest: 111\n" + " hostname: tdunna\n" + " ip: 192.168.68.129\n" + " reason: \"fixture reason\"\n" + "```\n" + ) + without_gate = "```yaml\nreport_only_guests: []\n```\n" + + gated = _plan_with_contract(scan, with_gate, tmp_path, "gated.prose.md") + assert gated[0]["action"] == "report-only", gated + assert gated[0]["reason"] == "fixture reason", gated + + ungated = _plan_with_contract(scan, without_gate, tmp_path, "ungated.prose.md") + assert ungated[0]["action"] == "gc-executor", ungated + assert ungated[0]["action"] != gated[0]["action"]