fix(disk-gc): hard guest-level report-only gate for CT 111/.129 + correct stale fleet map #81
@@ -1,17 +1,8 @@
|
||||
---
|
||||
report_only_agents:
|
||||
- koby # ⛔ KOBY IS NEVER REPAIRED (Rule 17, 2026-08-17) — detect + report, never fix on .129
|
||||
# ⛔ GUEST/HOST-KEYED report-only list. This is the gate the GC executor MUST honour.
|
||||
# An agent-name marker above is NOT sufficient: the scan unit is a guest, and an agent
|
||||
# marker can silently miss the guest it lives on. Key exclusions on the guest/host.
|
||||
report_only_guests:
|
||||
- guest: 111
|
||||
hostname: tdunna
|
||||
ip: 192.168.68.129
|
||||
node: storepve
|
||||
reason: >
|
||||
Theo's box. Captain ruling 2026-08-17, re-confirmed 2026-09-10: Theo handles
|
||||
CT 111 himself. DETECT-AND-REPORT-ONLY at every threat level.
|
||||
# ⛔ The guest/host-keyed report-only gate the GC executor MUST honour lives in the body
|
||||
# "Hard gate" YAML block below — that block is authoritative and is the only copy.
|
||||
kind: responsibility
|
||||
name: disk-gc-threat-response
|
||||
description: >
|
||||
@@ -37,7 +28,7 @@ logged within 5 minutes of discovery.
|
||||
|
||||
## Scope
|
||||
|
||||
All 20 Proxmox guests (17 LXC + 3 QEMU VMs) via `pct-run` + 3 GPU bare-metal hosts via direct SSH.
|
||||
All 20 Proxmox guests (17 LXC via `pct-run` + 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts via direct SSH.
|
||||
Docker hosts get special attention:
|
||||
|
||||
| Host | CT | Disk Risk | GC Strategy |
|
||||
@@ -128,7 +119,8 @@ agent-name marker can silently miss the guest it lives on — it must never be t
|
||||
|
||||
**The authoritative machine-readable exclusion list is the YAML block below.** The executor
|
||||
reads it at run time; `scripts/disk-gc-plan.py` turns a fleet scan into the action plan using it.
|
||||
Extend the list here, never by hand-maintaining a second copy.
|
||||
Extend the list here, never by hand-maintaining a second copy. The Execution loop below MUST
|
||||
call that planner and MUST NOT reimplement the gate.
|
||||
|
||||
```yaml
|
||||
# disk-gc report-only guests — authoritative. Keyed on guest/host, not agent.
|
||||
@@ -146,41 +138,32 @@ report_only_guests:
|
||||
let fleet = call disk-scanner
|
||||
scope: all
|
||||
|
||||
let report_only = load-report-only-guests() -- from the YAML block above
|
||||
-- The report-only gate is IMPLEMENTED IN scripts/disk-gc-plan.py and MUST NOT be
|
||||
-- reimplemented here. That planner reads the contract's `report_only_guests` YAML block
|
||||
-- and matches on guest id OR hostname OR IP, so the tested gate is the executed gate.
|
||||
let plan = call disk-gc-plan
|
||||
fleet: fleet
|
||||
|
||||
let threats = []
|
||||
for ct in fleet:
|
||||
if ct.usage_pct >= 95:
|
||||
push threats { ct: ct.id, level: "CRITICAL", pct: ct.usage_pct }
|
||||
else if ct.usage_pct >= 85:
|
||||
push threats { ct: ct.id, level: "RED", pct: ct.usage_pct }
|
||||
else if ct.usage_pct >= 75:
|
||||
push threats { ct: ct.id, level: "AMBER", pct: ct.usage_pct }
|
||||
|
||||
-- sort by severity descending
|
||||
sort threats by pct desc
|
||||
|
||||
for threat in threats:
|
||||
-- HARD GATE: an excluded guest is alerted and skipped. No gc-executor call is
|
||||
-- constructed for it at any level, so no GC command can be emitted for it.
|
||||
if threat.ct in report_only:
|
||||
for row in plan:
|
||||
if row.action == "report-only":
|
||||
-- Excluded guest: alert only. No gc-executor call is constructed for it, at any level.
|
||||
call alerter
|
||||
threat: threat
|
||||
result: { action: "report-only", reason: report_only[threat.ct].reason }
|
||||
threat: row
|
||||
result: { action: "report-only", reason: row.reason }
|
||||
continue
|
||||
|
||||
let result = call gc-executor
|
||||
ct: threat.ct
|
||||
level: threat.level
|
||||
strategy: lookup-gc-strategy(threat.ct)
|
||||
ct: row.target
|
||||
level: row.level
|
||||
strategy: lookup-gc-strategy(row.target)
|
||||
|
||||
call alerter
|
||||
threat: threat
|
||||
threat: row
|
||||
result: result
|
||||
|
||||
call summary-reporter
|
||||
fleet: fleet
|
||||
threats: threats
|
||||
plan: plan
|
||||
```
|
||||
|
||||
## GC Strategies by Host Type
|
||||
@@ -292,7 +275,7 @@ dangling images and orphaned build cache. No automated GC was in place.
|
||||
## Incident Log: 2026-07-09 — amdpve docker bloat
|
||||
|
||||
### Discovery
|
||||
Scheduled fleet disk scan via `pct-run` across all 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts.
|
||||
Scheduled fleet disk scan across all 20 Proxmox guests (17 LXC via `pct-run`, 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts.
|
||||
> **Report-only gate applies to this scan:** CT 111 (`tdunna`, 192.168.68.129) is alerted but never
|
||||
> garbage-collected at any level.
|
||||
amdpve (.15) flagged at 78% (AMBER threshold: 75%).
|
||||
|
||||
@@ -16,8 +16,8 @@ description: >
|
||||
**Last verified:** 2026-08-15 — hwepve removed from Tabiri cluster
|
||||
(now 5 nodes: minipve, amdpve, storepve, acerpve, ocupve). hwepve
|
||||
(192.168.68.4) is a standalone PVE node + NetBird routing peer;
|
||||
London relocation pending. CTs 100 (abiba) and 105 (kagentz) moved
|
||||
to minipve.
|
||||
London relocation pending. CT 100 (abiba) is on minipve; CT 105
|
||||
(kagentz) is on amdpve.
|
||||
---
|
||||
|
||||
# Infrastructure Control Pattern
|
||||
@@ -105,16 +105,16 @@ description: >
|
||||
|
||||
| Node | IP | CPU | RAM | VMs/CTs | Role |
|
||||
|------|----|-----|-----|---------|------|
|
||||
| minipve | .12 | 16C | 30GB | abiba, kagentz, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging |
|
||||
| amdpve | .15 | 32C | 62GB | tanko, tdunna, baggy, scottdenya | Agents, compute |
|
||||
| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip | Docker, storage, chat |
|
||||
| minipve | .12 | 16C | 30GB | abiba, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging |
|
||||
| amdpve | .15 | 32C | 62GB | kagentz, tanko, baggy, scottdenya, adguard2 | Agents, compute |
|
||||
| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna | Docker, storage, chat |
|
||||
| acerpve | .9 | 28C | 31GB | llm-gpu | GPU VMs |
|
||||
| ocupve | .5 | 12C | 14GB | ocu-llm | GPU VMs |
|
||||
|
||||
> **Note:** CTs on storepve include jdownloader (CT 118). AdGuard (CT 102) is on
|
||||
> minipve at .10, not acerpve. Abiba (CT 100) and kagentz (CT 105) are on
|
||||
> minipve (moved from hwepve 2026-08-15). Mumuni runs inside Abiba CT100
|
||||
> (.24); CT 114 (mumuni) no longer exists in the cluster.
|
||||
> **Note:** CTs on storepve include jdownloader (CT 118) and tdunna (CT 111).
|
||||
> AdGuard (CT 102) is on minipve at .10, not acerpve. Abiba (CT 100) is on
|
||||
> minipve (moved from hwepve 2026-08-15); kagentz (CT 105) is on amdpve.
|
||||
> Mumuni runs inside Abiba CT100 (.24); CT 114 (mumuni) no longer exists in the cluster.
|
||||
>
|
||||
> **hwepve (192.168.68.4) — STANDALONE (removed from Tabiri 2026-08-15):**
|
||||
> Huawei MateBook 16 (KLVL-WXX9), pve-manager/9.2.10, kernel 7.0.14-8-pve.
|
||||
@@ -602,13 +602,13 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
|
||||
| 102 | adguard | **minipve** | **.10** | DNS | ❌ |
|
||||
| 103 | ocu-llm | ocupve | .110 | GPU RTX 5070 | ❌ |
|
||||
| 104 | authentik | minipve | .11 | OIDC | ❌ |
|
||||
| 105 | kagentz | minipve | — | Agent Zero | ✅ |
|
||||
| 105 | kagentz | amdpve | — | Agent Zero | ✅ |
|
||||
| 106 | ra-h-os | storepve | .65 | KG bridge | ✅ MCP |
|
||||
| 107 | pbs | storepve | — | Backups | ❌ |
|
||||
| 108 | media | storepve | — | Media | ❌ |
|
||||
| 109 | docker-vm | storepve | .7 | Docker host | ❌ |
|
||||
| 110 | gitea | minipve | **.17** | Git | ❌ |
|
||||
| 111 | tdunna | amdpve | .129 | Hermes agent | ✅ |
|
||||
| 111 | tdunna | storepve | .129 | Hermes agent — ⛔ REPORT-ONLY (Theo's box, no GC) | ✅ |
|
||||
| 112 | tanko | amdpve | .122 | DSH (DeepSeek Harness) agent | ✅ |
|
||||
| 113 | baggy | amdpve | .114 | Hermes agent | ✅ |
|
||||
| 115 | scottdenya | amdpve | .75 | Denya OneCare | ❌ |
|
||||
@@ -616,6 +616,7 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
|
||||
| 117 | zulip | storepve | .19 | Chat | ❌ |
|
||||
| 118 | jdownloader | storepve | .20 | JDownloader LXC (dedicated, migrated from docker-vm 2026-08-01) | ✅ |
|
||||
| 119 | infisical-vault | minipve | — | Vault | ❌ |
|
||||
| 120 | adguard2 | amdpve | — | DNS (secondary AdGuard) | ❌ |
|
||||
|
||||
## Appendix C: Docker Compose Files Location
|
||||
|
||||
@@ -636,8 +637,8 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run.
|
||||
| CT | Name | Node | pct-run |
|
||||
|-----|------|------|---------|
|
||||
| 100 | abiba | minipve | `pct-run 100` |
|
||||
| 105 | kagentz | minipve | `pct-run 105` |
|
||||
| 111 | tdunna | amdpve | `pct-run 111` |
|
||||
| 105 | kagentz | amdpve | `pct-run 105` |
|
||||
| 111 | tdunna | storepve | `pct-run 111` (⛔ report-only — no GC) |
|
||||
| 112 | tanko | amdpve | `pct-run 112` |
|
||||
| 113 | baggy | amdpve | `pct-run 113` |
|
||||
| 115 | scottdenya | amdpve | `pct-run 115` |
|
||||
@@ -648,6 +649,9 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run.
|
||||
| 107 | proxmox-backup | storepve | `pct-run 107` |
|
||||
| 108 | media | storepve | `pct-run 108` |
|
||||
| 117 | zulip | storepve | `pct-run 117` |
|
||||
| 118 | jdownloader | storepve | `pct-run 118` |
|
||||
| 119 | infisical-vault | minipve | `pct-run 119` |
|
||||
| 120 | adguard2 | amdpve | `pct-run 120` |
|
||||
| 102 | adguard | **minipve** | `pct-run 102` |
|
||||
|
||||
GPU bare-metal hosts (.8 acerpve, .110 ocupve, .15 amdpve) are NOT CTs — use SSH directly:
|
||||
|
||||
Regular → Executable
+3
-11
@@ -61,7 +61,8 @@ def load_report_only_guests(contract_path: pathlib.Path) -> list[dict]:
|
||||
|
||||
|
||||
def _keys(entry: dict) -> set[str]:
|
||||
"""Guest/host identity keys for an exclusion entry."""
|
||||
"""Guest/host identity keys, shared by exclusions and scan entries so the two
|
||||
sides of the gate can never key on different fields."""
|
||||
out: set[str] = set()
|
||||
for field in ("guest", "id", "vmid", "hostname", "name", "ip"):
|
||||
value = entry.get(field)
|
||||
@@ -70,15 +71,6 @@ def _keys(entry: dict) -> set[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _entry_keys(scan_entry: dict) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for field in ("id", "guest", "vmid", "hostname", "name", "ip"):
|
||||
value = scan_entry.get(field)
|
||||
if value is not None and str(value).strip():
|
||||
out.add(str(value).strip().lower())
|
||||
return out
|
||||
|
||||
|
||||
def level_for(pct: float) -> str | None:
|
||||
if pct >= CRITICAL:
|
||||
return "CRITICAL"
|
||||
@@ -99,7 +91,7 @@ def build_plan(scan: list[dict], report_only: list[dict]) -> list[dict]:
|
||||
level = level_for(float(pct))
|
||||
if level is None:
|
||||
continue # GREEN: log only, no action
|
||||
scan_keys = _entry_keys(entry)
|
||||
scan_keys = _keys(entry)
|
||||
match = next((e for e, keys in excluded if keys & scan_keys), None)
|
||||
target = next(
|
||||
(entry.get(k) for k in ("id", "guest", "vmid", "hostname", "name", "ip")
|
||||
|
||||
@@ -47,17 +47,19 @@ You are a code reviewer for OpenProse infrastructure contracts in the Syslog Sol
|
||||
The infrastructure-control.prose.md contract is the canonical reference for the cluster topology:
|
||||
|
||||
**Proxmox Cluster "Tabiri" (5 nodes):**
|
||||
- amdpve (192.168.68.15): tanko, tdunna, baggy, scottdenya
|
||||
- minipve (192.168.68.12): abiba, kagentz, adguard, authentik, gitea, syslog-api, infisical-vault
|
||||
- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip
|
||||
- amdpve (192.168.68.15): kagentz, tanko, baggy, scottdenya, adguard2
|
||||
- minipve (192.168.68.12): abiba, adguard, authentik, gitea, syslog-api, infisical-vault
|
||||
- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna
|
||||
- acerpve (192.168.68.9): llm-gpu
|
||||
- ocupve (192.168.68.5): ocu-llm
|
||||
|
||||
**CT IDs (verified 2026-07-24 against PVE API):**
|
||||
**CT IDs (verified 2026-09-12 against PVE API):**
|
||||
100:abiba 102:adguard 104:authentik 105:kagentz 106:ra-h-os
|
||||
107:pbs 108:media 110:gitea 111:tdunna 112:tanko
|
||||
113:baggy 115:scottdenya 116:syslog-api 117:zulip
|
||||
118:jdownloader 119:infisical-vault
|
||||
118:jdownloader 119:infisical-vault 120:adguard2
|
||||
|
||||
**CT 111 (tdunna, 192.168.68.129) is REPORT-ONLY — Theo's box; alert only, never garbage-collect.**
|
||||
|
||||
**NO CT 122, CT 123, or .19 exist in the cluster.**
|
||||
|
||||
|
||||
@@ -78,12 +78,40 @@ def test_agent_name_alone_does_not_gate_a_guest(tmp_path):
|
||||
assert plan and plan[0]["action"] == "gc-executor"
|
||||
|
||||
|
||||
def test_contract_carries_the_guest_keyed_exclusion(tmp_path):
|
||||
"""The authoritative gate must exist and identify CT 111 by guest/host."""
|
||||
plan = _plan([{"id": 111, "usage_pct": 95}], tmp_path)
|
||||
reason = _actions_for(plan, 111)[0]["reason"]
|
||||
assert reason, "the exclusion must carry a reason for the alert"
|
||||
contract = (ROOT / "disk-gc-threat-response.prose.md").read_text()
|
||||
assert "report_only_guests:" in contract
|
||||
assert "192.168.68.129" in contract
|
||||
assert "tdunna" in contract
|
||||
def _plan_with_contract(scan, contract_text, tmp_path, name):
|
||||
contract = tmp_path / name
|
||||
contract.write_text(contract_text)
|
||||
scan_file = tmp_path / f"scan-{name}.json"
|
||||
scan_file.write_text(json.dumps(scan))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(PLAN), "--scan", str(scan_file),
|
||||
"--contract", str(contract), "--json"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def test_gate_is_read_from_the_contract_block(tmp_path):
|
||||
"""The gate is data-driven by the contract block: the planner excludes the guest
|
||||
when the block names it and acts on it when the block does not. Executes the real
|
||||
planner interface against both fixtures so the behaviour change is observable."""
|
||||
scan = [{"id": 111, "hostname": "tdunna", "ip": "192.168.68.129", "usage_pct": 95}]
|
||||
with_gate = (
|
||||
"```yaml\n"
|
||||
"report_only_guests:\n"
|
||||
" - guest: 111\n"
|
||||
" hostname: tdunna\n"
|
||||
" ip: 192.168.68.129\n"
|
||||
" reason: \"fixture reason\"\n"
|
||||
"```\n"
|
||||
)
|
||||
without_gate = "```yaml\nreport_only_guests: []\n```\n"
|
||||
|
||||
gated = _plan_with_contract(scan, with_gate, tmp_path, "gated.prose.md")
|
||||
assert gated[0]["action"] == "report-only", gated
|
||||
assert gated[0]["reason"] == "fixture reason", gated
|
||||
|
||||
ungated = _plan_with_contract(scan, without_gate, tmp_path, "ungated.prose.md")
|
||||
assert ungated[0]["action"] == "gc-executor", ungated
|
||||
assert ungated[0]["action"] != gated[0]["action"]
|
||||
|
||||
Reference in New Issue
Block a user