From 7b8cc5f9ace6e56dc1c4ee25aa9626448156f323 Mon Sep 17 00:00:00 2001 From: abiba Date: Sat, 12 Sep 2026 18:33:02 +0000 Subject: [PATCH] fix(disk-gc): hard guest-level report-only gate for CT 111/.129; correct stale fleet map CT 111 (tdunna, 192.168.68.129) is Theo's box and is report-only per the captain (2026-08-17, re-confirmed 2026-09-10). The contract defined AMBER as 'GC scheduled for next run' and its Execution loop called gc-executor for EVERY threat with no guest-level exclusion - so a single AMBER reading there would have scheduled apt clean / journal vacuum / log+tmp deletion against someone else's box. The only marker was frontmatter report_only_agents, which names an AGENT while the scan unit is a GUEST. - Gate the Execution loop on a guest/host-keyed report_only_guests block (guest id, hostname and IP all match); an excluded guest is alerted and skipped, so no gc-executor call is constructed for it at any level. - Carry the ruling in the contract body next to the loop, not only in frontmatter. - scripts/disk-gc-plan.py: executable planner that reads the contract's authoritative exclusion block and emits the action plan; tests/ covers it. - Correct the stale fleet map against pvesh /cluster/resources: CT 105 -> amdpve (was minipve), CT 111 -> storepve (was amdpve), add guests 118/119/120, and fix the '15 CTs' counts (20 guests: 17 LXC + 3 QEMU VMs). - scripts/pct-run.sh: same stale map (105/111 wrong node, 120 missing) - this is why pct-run 111/105 failed. - Fold in disk-gc-ct100-probe-gap-20260911: CT 100 verifiably works through pct-run now that the map is correct; documented that the scanner must probe it like any other guest, never via a local-only path (the scanner runs inside CT 100). --- disk-gc-threat-response.prose.md | 126 ++++++++++++++++++----- scripts/disk-gc-plan.py | 164 ++++++++++++++++++++++++++++++ scripts/pct-run.sh | 13 +-- tests/test_disk_gc_report_only.py | 89 ++++++++++++++++ 4 files changed, 358 insertions(+), 34 deletions(-) create mode 100644 scripts/disk-gc-plan.py create mode 100644 tests/test_disk_gc_report_only.py diff --git a/disk-gc-threat-response.prose.md b/disk-gc-threat-response.prose.md index 34243d6..7a61f5d 100644 --- a/disk-gc-threat-response.prose.md +++ b/disk-gc-threat-response.prose.md @@ -1,11 +1,23 @@ --- report_only_agents: - koby # ⛔ KOBY IS NEVER REPAIRED (Rule 17, 2026-08-17) — detect + report, never fix on .129 +# ⛔ GUEST/HOST-KEYED report-only list. This is the gate the GC executor MUST honour. +# An agent-name marker above is NOT sufficient: the scan unit is a guest, and an agent +# marker can silently miss the guest it lives on. Key exclusions on the guest/host. +report_only_guests: + - guest: 111 + hostname: tdunna + ip: 192.168.68.129 + node: storepve + reason: > + Theo's box. Captain ruling 2026-08-17, re-confirmed 2026-09-10: Theo handles + CT 111 himself. DETECT-AND-REPORT-ONLY at every threat level. kind: responsibility name: disk-gc-threat-response description: > Recurring disk health scan, garbage collection, and threat response - across 15 Proxmox CTs + 3 GPU bare-metal hosts. Triggered by incident + across 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts + (fleet verified against `pvesh get /cluster/resources` 2026-09-12). Triggered by incident 2026-07-04 where CT 105 (kagentz) hit 87% disk (49G/59G) from Docker image bloat — 5 dangling images, 15 build cache layers. Recovered 35.67GB. Second incident 2026-07-09: amdpve (.15) Docker @@ -25,7 +37,7 @@ logged within 5 minutes of discovery. ## Scope -All 15 CTs via `pct-run` + 3 GPU bare-metal hosts via direct SSH. +All 20 Proxmox guests (17 LXC + 3 QEMU VMs) via `pct-run` + 3 GPU bare-metal hosts via direct SSH. Docker hosts get special attention: | Host | CT | Disk Risk | GC Strategy | @@ -99,10 +111,43 @@ and escalation trail. ## Execution +### Hard gate: report-only guests (READ THIS BEFORE RUNNING GC) + +**CT 111 / hostname `tdunna` / 192.168.68.129 is DETECT-AND-REPORT-ONLY.** It belongs to Theo. +The captain ruled 2026-08-17 and re-confirmed 2026-09-10 that Theo handles CT 111 himself. +At **every** threat level — AMBER, RED, or CRITICAL — the executor must: + +- push the threat row and alert the owner, and +- **never** call `gc-executor`, and **never** run any GC command against that guest: no + `apt-get clean/autoremove`, no `journalctl --vacuum-*`, no `find /var/log -delete`, no + `/tmp`/`/var/tmp` deletion, no snap removal, no `docker system prune`. + +This gate is keyed on **guest id / hostname / IP**, not on an agent name. The frontmatter +`report_only_agents` marker (e.g. `koby`) names an AGENT while the scan unit is a GUEST, so an +agent-name marker can silently miss the guest it lives on — it must never be the only gate. + +**The authoritative machine-readable exclusion list is the YAML block below.** The executor +reads it at run time; `scripts/disk-gc-plan.py` turns a fleet scan into the action plan using it. +Extend the list here, never by hand-maintaining a second copy. + +```yaml +# disk-gc report-only guests — authoritative. Keyed on guest/host, not agent. +report_only_guests: + - guest: 111 + hostname: tdunna + ip: 192.168.68.129 + node: storepve + reason: "Theo's box — captain ruling 2026-08-17, re-confirmed 2026-09-10" +``` + +### Loop + ```prose let fleet = call disk-scanner scope: all +let report_only = load-report-only-guests() -- from the YAML block above + let threats = [] for ct in fleet: if ct.usage_pct >= 95: @@ -116,11 +161,19 @@ for ct in fleet: sort threats by pct desc for threat in threats: + -- HARD GATE: an excluded guest is alerted and skipped. No gc-executor call is + -- constructed for it at any level, so no GC command can be emitted for it. + if threat.ct in report_only: + call alerter + threat: threat + result: { action: "report-only", reason: report_only[threat.ct].reason } + continue + let result = call gc-executor ct: threat.ct level: threat.level strategy: lookup-gc-strategy(threat.ct) - + call alerter threat: threat result: result @@ -239,7 +292,9 @@ dangling images and orphaned build cache. No automated GC was in place. ## Incident Log: 2026-07-09 — amdpve docker bloat ### Discovery -Scheduled fleet disk scan via `pct-run` across all 15 CTs + 3 GPU bare-metal hosts. +Scheduled fleet disk scan via `pct-run` across all 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts. +> **Report-only gate applies to this scan:** CT 111 (`tdunna`, 192.168.68.129) is alerted but never +> garbage-collected at any level. amdpve (.15) flagged at 78% (AMBER threshold: 75%). ### Diagnosis @@ -272,25 +327,35 @@ one-off GPU builds. No automated post-migration cleanup was in place. - Contract now scans GPU bare-metal hosts alongside CTs - Access via `pct-run` script for all CTs (no hardcoded IPs) -## Access Matrix (documented 2026-07-09) +## Access Matrix (verified against `pvesh get /cluster/resources` 2026-09-12) -### CT Access (via pct-run) -| CT | Name | Node | Status | -|----|------|------|--------| -| 100 | abiba | minipve | local | -| 102 | adguard | minipve | ✅ reachable | -| 104 | authentik | minipve | ✅ reachable | -| 105 | kagentz | minipve | ✅ reachable | -| 106 | ra-h-os | storepve | ✅ reachable | -| 107 | pbs | storepve | ✅ reachable | -| 108 | media | storepve | ✅ reachable | -| 110 | gitea | minipve | ✅ reachable | -| 111 | tdunna | amdpve | ✅ reachable | -| 112 | tanko | amdpve | ✅ reachable | -| 113 | baggy | amdpve | ✅ reachable | -| 115 | scottdenya | amdpve | ✅ reachable | -| 116 | syslog-api | minipve | ✅ reachable | -| 117 | zulip | storepve | ✅ reachable | +### Guest Access (via `pct-run` — CT id only, node resolved by `scripts/pct-run.sh`) +| Guest | Name | Node | Type | Status | +|------|------|------|------|--------| +| 100 | abiba | minipve | lxc | ✅ reachable (probed via pct-run like any other guest; no local shortcut) | +| 102 | adguard | minipve | lxc | ✅ reachable | +| 104 | authentik | minipve | lxc | ✅ reachable | +| 105 | kagentz | **amdpve** | lxc | ✅ reachable (was documented as minipve — corrected) | +| 106 | ra-h-os | storepve | lxc | ✅ reachable | +| 107 | pbs | storepve | lxc | ✅ reachable | +| 108 | media | storepve | lxc | ✅ reachable | +| 110 | gitea | minipve | lxc | ✅ reachable | +| 111 | tdunna | **storepve** | lxc | ⛔ **REPORT-ONLY** (192.168.68.129, Theo's box — no GC at any level) | +| 112 | tanko | amdpve | lxc | ✅ reachable | +| 113 | baggy | amdpve | lxc | ✅ reachable | +| 115 | scottdenya | amdpve | lxc | ✅ reachable | +| 116 | syslog-api | minipve | lxc | ✅ reachable | +| 117 | zulip | storepve | lxc | ✅ reachable | +| 118 | jdownloader | storepve | lxc | ✅ reachable | +| 119 | infisical-vault | minipve | lxc | ✅ reachable | +| 120 | adguard2 | amdpve | lxc | ✅ reachable | + +### QEMU VMs (via direct SSH) +| VM | Name | Node | IP | Status | +|----|------|------|-----|--------| +| 101 | llm-gpu (workload now bare metal .8) | acerpve | — | ✅ reachable | +| 103 | ocu-llm (workload now bare metal .110) | ocupve | — | ✅ reachable | +| 109 | docker-vm | storepve | 192.168.68.7 | ✅ reachable | ### GPU Bare Metal (via direct SSH) | Host | IP | GPU | Status | @@ -299,9 +364,14 @@ one-off GPU builds. No automated post-migration cleanup was in place. | ocu-llm | 192.168.68.110 | RTX 5070 | ✅ reachable | | amdpve | 192.168.68.15 | Strix Halo | ✅ reachable | -### KVM VM (via direct SSH) -| Host | IP | Role | Status | -|------|-----|------|--------| -| docker-vm | 192.168.68.7 | 16 Docker containers, 4 stacks | ✅ reachable | - -> **Note:** CT 118 is now jdownloader (active on storepve). CT 119 (infisical-vault) added on minipve.\n> **Migrated:** CT 101 → .8, CT 103 → .110 (bare metal GPU).\n> **KVM VM:** CT 109 (docker-vm) is a KVM VM, not LXC — access via SSH .7. +> **Fleet count:** 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts. Corrected +> 2026-09-12: CT 105 → amdpve, CT 111 → storepve, and guests 118/119/120 were missing. +> +> **CT 100 probe gap (folded in):** CT 100 previously reported "unreachable (not reported)" every +> run. Root cause is the same stale access layer: `pct-run` resolves the guest's node from its map, +> and the map/contract must reflect `pvesh /cluster/resources`. Verified working from inside CT 100: +> `scripts/pct-run.sh 100 "df -P / | tail -1"` → `23% /`. Probe CT 100 through `pct-run` like any +> other guest — never through a local-only path, since the scanner itself runs inside CT 100 and a +> container has no `pct` binary. +> +> **KVM VM:** CT 109 (docker-vm) is a QEMU VM, not LXC — access via SSH .7. diff --git a/scripts/disk-gc-plan.py b/scripts/disk-gc-plan.py new file mode 100644 index 0000000..e6d53ec --- /dev/null +++ b/scripts/disk-gc-plan.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""disk-gc-plan — turn a fleet disk scan into the GC action plan. + +This is the executable side of `disk-gc-threat-response.prose.md`. It exists so the +report-only gate is enforced by code that can be tested, rather than by prose the +executor might misread. + +THE HARD GATE: guests listed in the contract's `report_only_guests` block are +DETECT-AND-REPORT-ONLY at EVERY level (AMBER, RED, CRITICAL). This tool will never +emit a `gc-executor` action for one, so no GC command can be constructed for it. + +The gate is keyed on GUEST identity — guest id, hostname, or IP — never on an agent +name. An agent-name marker can silently miss the guest it lives on; a guest marker +cannot. + +The authoritative exclusion list lives in the contract itself (the fenced ```yaml +block containing `report_only_guests:`). This tool reads it from there so there is +only ever one copy. + +Usage: + disk-gc-plan.py --scan scan.json # [{"id":111,"usage_pct":84}, ...] + cat scan.json | disk-gc-plan.py # same, via stdin + disk-gc-plan.py --scan scan.json --json # machine-readable plan + +Scan entries may carry any of: id / guest / vmid, hostname / name, ip. +Exit codes: 0 ok, 1 usage/parse error. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys + +import yaml + +REPO = pathlib.Path(__file__).resolve().parent.parent +DEFAULT_CONTRACT = REPO / "disk-gc-threat-response.prose.md" + +AMBER, RED, CRITICAL = 75, 85, 95 + + +def load_report_only_guests(contract_path: pathlib.Path) -> list[dict]: + """Read the authoritative report_only_guests block out of the contract. + + The contract carries it as a fenced ```yaml block. Parsing the declared, + machine-readable block is the intended interface — the contract owns the list. + """ + text = contract_path.read_text(encoding="utf-8") + for block in re.findall(r"```yaml\n(.*?)```", text, re.S): + if "report_only_guests:" in block: + data = yaml.safe_load(block) + guests = data.get("report_only_guests") or [] + if not isinstance(guests, list): + raise SystemExit("report_only_guests must be a list") + return guests + raise SystemExit( + f"no authoritative report_only_guests block found in {contract_path}" + ) + + +def _keys(entry: dict) -> set[str]: + """Guest/host identity keys for an exclusion entry.""" + out: set[str] = set() + for field in ("guest", "id", "vmid", "hostname", "name", "ip"): + value = entry.get(field) + if value is not None and str(value).strip(): + out.add(str(value).strip().lower()) + return out + + +def _entry_keys(scan_entry: dict) -> set[str]: + out: set[str] = set() + for field in ("id", "guest", "vmid", "hostname", "name", "ip"): + value = scan_entry.get(field) + if value is not None and str(value).strip(): + out.add(str(value).strip().lower()) + return out + + +def level_for(pct: float) -> str | None: + if pct >= CRITICAL: + return "CRITICAL" + if pct >= RED: + return "RED" + if pct >= AMBER: + return "AMBER" + return None + + +def build_plan(scan: list[dict], report_only: list[dict]) -> list[dict]: + excluded = [(e, _keys(e)) for e in report_only] + plan: list[dict] = [] + for entry in scan: + pct = entry.get("usage_pct") + if pct is None: + continue + level = level_for(float(pct)) + if level is None: + continue # GREEN: log only, no action + scan_keys = _entry_keys(entry) + match = next((e for e, keys in excluded if keys & scan_keys), None) + target = next( + (entry.get(k) for k in ("id", "guest", "vmid", "hostname", "name", "ip") + if entry.get(k) not in (None, "")), + "?", + ) + if match is not None: + plan.append({ + "target": target, + "level": level, + "pct": float(pct), + "action": "report-only", + "reason": match.get("reason", "").strip(), + }) + else: + plan.append({ + "target": target, + "level": level, + "pct": float(pct), + "action": "gc-executor", + }) + plan.sort(key=lambda row: row["pct"], reverse=True) + return plan + + +def main() -> int: + ap = argparse.ArgumentParser(description="Plan disk GC actions with the report-only gate.") + ap.add_argument("--scan", help="JSON file: list of {id|hostname|ip, usage_pct}") + ap.add_argument("--contract", default=str(DEFAULT_CONTRACT)) + ap.add_argument("--json", action="store_true", help="emit the plan as JSON") + args = ap.parse_args() + + raw = pathlib.Path(args.scan).read_text() if args.scan else sys.stdin.read() + try: + scan = json.loads(raw) + except json.JSONDecodeError as exc: + print(f"invalid scan JSON: {exc}", file=sys.stderr) + return 1 + if not isinstance(scan, list): + print("scan must be a JSON list", file=sys.stderr) + return 1 + + report_only = load_report_only_guests(pathlib.Path(args.contract)) + plan = build_plan(scan, report_only) + + if args.json: + print(json.dumps(plan, indent=2)) + return 0 + + if not plan: + print("no threats (nothing at or above 75%)") + return 0 + for row in plan: + if row["action"] == "report-only": + print(f" {row['target']} {row['level']} {row['pct']}% -> REPORT-ONLY (no GC) — {row['reason']}") + else: + print(f" {row['target']} {row['level']} {row['pct']}% -> gc-executor") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pct-run.sh b/scripts/pct-run.sh index afbf56a..d4239d5 100755 --- a/scripts/pct-run.sh +++ b/scripts/pct-run.sh @@ -11,11 +11,13 @@ set -euo pipefail # ── CT ID → PVE Node mapping (maintained HERE, not in prose contracts) ── declare -A CT_NODES=( # amdpve (192.168.68.15) - [111]=amdpve # tdunna + [105]=amdpve # kagentz (was hwepve — corrected 2026-09-12; live per pvesh) [112]=amdpve # tanko [113]=amdpve # baggy [115]=amdpve # scottdenya + [120]=amdpve # adguard2 (added 2026-09-12) # minipve (192.168.68.12) + [100]=minipve # abiba (was hwepve) [102]=minipve # adguard (was acerpve) [104]=minipve # authentik [110]=minipve # gitea @@ -25,17 +27,16 @@ declare -A CT_NODES=( [106]=storepve # ra-h-os [107]=storepve # proxmox-backup [108]=storepve # media + [111]=storepve # tdunna (was amdpve — corrected 2026-09-12; live per pvesh) [117]=storepve # zulip [118]=storepve # jdownloader # acerpve (192.168.68.9) — no CTs (bare metal GPU .8) - [100]=minipve # abiba (was hwepve) - [105]=minipve # kagentz (was hwepve) # ocupve (192.168.68.5) — no CTs (bare metal GPU .110) # # REMOVED CTs (migrated to bare metal, decommissioned, or VMs): - # 101 llm-gpu → bare metal 192.168.68.8 (RTX 3090) - # 103 ocu-llm → bare metal 192.168.68.110 (RTX 5070) - # 109 docker-vm → KVM VM 192.168.68.7 (use direct SSH) + # 101 llm-gpu → bare metal 192.168.68.8 (RTX 3090) [QEMU VM on acerpve] + # 103 ocu-llm → bare metal 192.168.68.110 (RTX 5070) [QEMU VM on ocupve] + # 109 docker-vm → KVM VM 192.168.68.7 (use direct SSH) [QEMU VM on storepve] ) # Each node must be root-accessible via SSH hostname diff --git a/tests/test_disk_gc_report_only.py b/tests/test_disk_gc_report_only.py new file mode 100644 index 0000000..2df496a --- /dev/null +++ b/tests/test_disk_gc_report_only.py @@ -0,0 +1,89 @@ +"""Regression tests for the disk-gc report-only gate (CT 111 / tdunna / .129). + +WHY THIS FILE EXISTS: `disk-gc-threat-response.prose.md` defined AMBER as "GC scheduled +for next run" and its Execution loop called `gc-executor` for EVERY threat, with no +guest-level exclusion. CT 111 (tdunna, 192.168.68.129) belongs to Theo and is +report-only per the captain (2026-08-17, re-confirmed 2026-09-10) — so a single AMBER +reading on that guest would have scheduled GC commands (apt clean, journal vacuum, +log/tmp deletion, snap removal) against someone else's box. The only marker was +frontmatter `report_only_agents`, which names an AGENT while the scan unit is a GUEST. + +These tests execute the real planner (`scripts/disk-gc-plan.py`) and assert observable +behaviour: an excluded guest never produces a `gc-executor` action at any level, while +our own guests still do. +""" +from __future__ import annotations + +import json +import pathlib +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +PLAN = ROOT / "scripts" / "disk-gc-plan.py" + + +def _plan(scan, tmp_path): + scan_file = tmp_path / "scan.json" + scan_file.write_text(json.dumps(scan)) + proc = subprocess.run( + [sys.executable, str(PLAN), "--scan", str(scan_file), "--json"], + capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def _actions_for(plan, target): + return [row for row in plan if str(row["target"]) == str(target)] + + +def test_excluded_guest_never_gets_gc_at_any_level(tmp_path): + """CT 111 at AMBER, RED and CRITICAL — always report-only, never gc-executor.""" + for pct, level in ((84, "AMBER"), (90, "RED"), (97, "CRITICAL")): + plan = _plan([{"id": 111, "hostname": "tdunna", "ip": "192.168.68.129", + "usage_pct": pct}], tmp_path) + rows = _actions_for(plan, 111) + assert rows, f"CT 111 must still be reported at {level}" + assert rows[0]["level"] == level + assert rows[0]["action"] == "report-only", rows + assert not any(r["action"] == "gc-executor" for r in rows) + + +def test_exclusion_matches_on_any_identity_key(tmp_path): + """The gate is keyed on guest/host, so id, hostname or IP all match.""" + for entry in ({"id": 111, "usage_pct": 95}, + {"hostname": "tdunna", "usage_pct": 95}, + {"ip": "192.168.68.129", "usage_pct": 95}): + plan = _plan([entry], tmp_path) + assert all(r["action"] == "report-only" for r in plan), (entry, plan) + + +def test_our_own_guests_still_get_gc(tmp_path): + """acerpve .9 and amdpve .15 are ours — they must still be acted on.""" + plan = _plan([{"hostname": "acerpve", "ip": "192.168.68.9", "usage_pct": 77}, + {"hostname": "amdpve", "ip": "192.168.68.15", "usage_pct": 76}], tmp_path) + assert len(plan) == 2 + assert all(r["action"] == "gc-executor" for r in plan), plan + + +def test_below_threshold_emits_nothing(tmp_path): + """GREEN guests produce no action at all.""" + assert _plan([{"id": 111, "usage_pct": 40}], tmp_path) == [] + + +def test_agent_name_alone_does_not_gate_a_guest(tmp_path): + """An agent-name marker must not be the gate: an unrelated guest still gets GC.""" + plan = _plan([{"id": 999, "hostname": "koby", "usage_pct": 95}], tmp_path) + assert plan and plan[0]["action"] == "gc-executor" + + +def test_contract_carries_the_guest_keyed_exclusion(tmp_path): + """The authoritative gate must exist and identify CT 111 by guest/host.""" + plan = _plan([{"id": 111, "usage_pct": 95}], tmp_path) + reason = _actions_for(plan, 111)[0]["reason"] + assert reason, "the exclusion must carry a reason for the alert" + contract = (ROOT / "disk-gc-threat-response.prose.md").read_text() + assert "report_only_guests:" in contract + assert "192.168.68.129" in contract + assert "tdunna" in contract