From 8ed3b9c6062a396b71e9e2fab7b9d840a1b16b03 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Sep 2026 00:41:34 +0000 Subject: [PATCH] fix: implement host-filesystem state file with transition detection The contract said the state file was written after every scan, but the script had no state-file logic at all. This PR adds: 1. Host filesystem scanning (probe_host_filesystems) - probes df on all PVE nodes 2. State file I/O (read_state_file/write_state_file) - absolute path from script location 3. Band classification (classify_band) - HOST-WARN/AMBER/RED thresholds 4. Transition detection (detect_transitions) - alerts on escalation/recovery 5. CLI flags (--hosts-only, --guests-only) to control which parts run The contract now specifies the state file path resolves from the script's own location (not CWD-relative), so two different execution contexts cannot write to two different places. --- disk-gc-threat-response.prose.md | 2 +- scripts/disk-gc-scan.py | 242 ++++++++++++++++++++++++++++++- state/host-disk-bands.json | 18 +++ 3 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 state/host-disk-bands.json diff --git a/disk-gc-threat-response.prose.md b/disk-gc-threat-response.prose.md index f5d72cf..2c879a1 100644 --- a/disk-gc-threat-response.prose.md +++ b/disk-gc-threat-response.prose.md @@ -65,7 +65,7 @@ Host filesystems have their own risk profile and their own bands. A host root ne **Escalations are STATE-CHANGE driven, not per-run.** A volume alerts ONCE when it enters a higher band (GREEN->WARN, WARN->AMBER, AMBER->RED) and ONCE when it drops back down (a recovery notice). While a volume stays in the same band, it is reported in the scan output only — no DM, no channel alert. This prevents the same 96% easystore2 from re-DMing the owner on every 6-hour scan and drowning a real warning in noise. -**State lives in a small JSON state file:** `state/host-disk-bands.json`, keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.) +**State lives in a small JSON state file:** the scanner resolves it to an absolute path from the script's location — `$(dirname "$0")/state/host-disk-bands.json` (i.e., the `state/` directory next to the `scripts/` directory in the prose-contracts repo). Keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.) **Volume naming rule:** Every host line MUST name the volume and what lives on it. Example output: diff --git a/scripts/disk-gc-scan.py b/scripts/disk-gc-scan.py index 8c5e218..e33947a 100644 --- a/scripts/disk-gc-scan.py +++ b/scripts/disk-gc-scan.py @@ -153,6 +153,25 @@ GPU_HOSTS = [ CONNECT_TIMEOUT = 5 SSH_OPTS = "-o BatchMode=yes -o ConnectTimeout=" + str(CONNECT_TIMEOUT) +# Host filesystem thresholds (from contract) +HOST_THRESHOLDS = { + "WARN": 85, + "AMBER": 90, + "RED": 95, +} + +# State file path (absolute, so execution context doesn't matter) +STATE_FILE = pathlib.Path(__file__).resolve().parent.parent / "state" / "host-disk-bands.json" + +# PVE nodes to probe for host filesystems +HOST_NODES = [ + {"hostname": "acerpve", "ip": "192.168.68.9"}, + {"hostname": "amdpve", "ip": "192.168.68.15"}, + {"hostname": "storepve", "ip": "192.168.68.6"}, + {"hostname": "minipve", "ip": "192.168.68.12"}, + {"hostname": "ocupve", "ip": "192.168.68.5"}, +] + def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]: """Run a command and return (exit_code, stdout, stderr).""" @@ -298,6 +317,151 @@ def scan_fleet() -> list[dict]: return results +def classify_band(usage_pct: float) -> str: + """Classify a percentage into a band.""" + if usage_pct >= HOST_THRESHOLDS["RED"]: + return "HOST-RED" + elif usage_pct >= HOST_THRESHOLDS["AMBER"]: + return "HOST-AMBER" + elif usage_pct >= HOST_THRESHOLDS["WARN"]: + return "HOST-WARN" + else: + return "GREEN" + + +def probe_host_filesystems() -> tuple[list[dict], dict[str, str]]: + """Probe host filesystems on all PVE nodes. + + Returns: + - List of host filesystem results + - Dict of volume_key -> current_band (for state file) + """ + results = [] + current_bands = {} + + for node in HOST_NODES: + ip = node["ip"] + hostname = node["hostname"] + + # Probe df for host filesystems + probe_cmd = f'ssh {SSH_OPTS} root@{ip} "df -hP / /media/* tank 2>/dev/null | tail -n +2"' + exit_code, stdout, stderr = run_cmd(probe_cmd, timeout=15) + + if exit_code != 0: + results.append({ + "target": f"{hostname} ({ip})", + "hostname": hostname, + "ip": ip, + "reachable": False, + "volumes": [], + "probe_cmd": probe_cmd, + "failure_kind": "ssh-error", + }) + continue + + # Parse df output and classify each volume + volumes = [] + for line in stdout.splitlines(): + parts = line.split() + if len(parts) < 6: + continue + + dev, size, used, avail, pct_str, mount = parts[:6] + pct = float(pct_str.rstrip("%")) + band = classify_band(pct) + + # Volume type classification + if mount.startswith("/media/"): + vol_type = "media" + elif mount == "/" or "pve" in dev: + vol_type = "host-root" + elif mount == "tank" or "tank" in mount: + vol_type = "pbs-datastore" + else: + vol_type = "other" + + # Volume key for state file (host/volume) + volume_key = f"{hostname}/{mount}" + current_bands[volume_key] = band + + volumes.append({ + "mount": mount, + "device": dev, + "size": size, + "used": used, + "avail": avail, + "pct": pct, + "band": band, + "type": vol_type, + }) + + results.append({ + "target": f"{hostname} ({ip})", + "hostname": hostname, + "ip": ip, + "reachable": True, + "volumes": volumes, + "probe_cmd": probe_cmd, + "failure_kind": None, + }) + + return results, current_bands + + +def read_state_file() -> Optional[dict[str, str]]: + """Read the state file if it exists.""" + if not STATE_FILE.exists(): + return None + try: + with open(STATE_FILE) as f: + return json.load(f) + except (json.JSONDecodeError, IOError) as e: + print(f"⚠️ State file exists but unreadable: {e}", file=sys.stderr) + return {} + + +def write_state_file(bands: dict[str, str]) -> None: + """Write the state file.""" + STATE_FILE.parent.mkdir(parents=True, exist_ok=True) + try: + with open(STATE_FILE, "w") as f: + json.dump(bands, f, indent=2) + except IOError as e: + print(f"⚠️ State file write failed: {e}", file=sys.stderr) + + +def detect_transitions(current_bands: dict[str, str], prior_bands: Optional[dict[str, str]]) -> list[dict]: + """Detect band transitions (current vs. prior).""" + if prior_bands is None: + # First run — no transitions, just establish baseline + return [] + + transitions = [] + # Check for volumes that moved to a higher band (escalation) + for volume, current_band in current_bands.items(): + prior_band = prior_bands.get(volume, "GREEN") + + # Band ordering: GREEN < HOST-WARN < HOST-AMBER < HOST-RED + band_order = {"GREEN": 0, "HOST-WARN": 1, "HOST-AMBER": 2, "HOST-RED": 3} + + if band_order[current_band] > band_order[prior_band]: + transitions.append({ + "type": "escalation", + "volume": volume, + "from": prior_band, + "to": current_band, + }) + elif band_order[current_band] < band_order[prior_band]: + transitions.append({ + "type": "recovery", + "volume": volume, + "from": prior_band, + "to": current_band, + }) + + return transitions + + def render_results(results: list[dict]) -> str: """Render scan results in human-readable format.""" lines = [] @@ -316,22 +480,88 @@ def render_results(results: list[dict]) -> str: return "\n".join(lines) +def render_host_results(results: list[dict], transitions: list[dict], prior_bands: Optional[dict[str, str]]) -> str: + """Render host filesystem results in human-readable format.""" + lines = [] + lines.append("") + lines.append("=== Host Filesystem Bands ===") + lines.append("") + + # Render transitions first (they're the actionable alerts) + if prior_bands is None: + lines.append(" (first run — recording baseline, no alerts)") + elif not transitions: + lines.append(" (no band changes since last scan)") + else: + for t in transitions: + volume, from_band, to_band = t["volume"], t["from"], t["to"] + if t["type"] == "escalation": + lines.append(f" ⚠️ {volume}: {from_band} → {to_band} (ESCALATION)") + else: + lines.append(f" ✅ {volume}: {from_band} → {to_band} (RECOVERY)") + + # Render all volumes with their bands + lines.append("") + for node_result in results: + if not node_result["reachable"]: + lines.append(f" ❌ {node_result['target']}: UNREACHABLE ({node_result['failure_kind']})") + continue + + lines.append(f" {node_result['target']}:") + for vol in node_result["volumes"]: + lines.append(f" {vol['mount']} ({vol['type']}): {vol['pct']}% ({vol['used']}/{vol['size']}, {vol['avail']} free) -> {vol['band']}") + + return "\n".join(lines) + + def main() -> int: import argparse - ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests.") + ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests and host filesystems.") ap.add_argument("--json", action="store_true", help="machine-readable output") + ap.add_argument("--hosts-only", action="store_true", help="scan host filesystems only") + ap.add_argument("--guests-only", action="store_true", help="scan guests only (skip host filesystems)") args = ap.parse_args() - results = scan_fleet() + # Scan guests (unless --hosts-only) + guest_results = [] + if not args.hosts_only: + guest_results = scan_fleet() + + # Scan host filesystems (unless --guests-only) + host_results = [] + current_bands = {} + if not args.guests_only: + host_results, current_bands = probe_host_filesystems() + + # Read prior state and detect transitions + prior_bands = read_state_file() + transitions = detect_transitions(current_bands, prior_bands) + + # Write new state + write_state_file(current_bands) + else: + prior_bands = None + transitions = [] if args.json: - print(json.dumps(results, indent=2)) + # JSON output + output = { + "guests": guest_results, + "hosts": host_results, + "transitions": transitions, + "prior_bands": prior_bands, + } + print(json.dumps(output, indent=2)) else: - print(render_results(results)) + # Human-readable output + if guest_results: + print(render_results(guest_results)) + + if host_results: + print(render_host_results(host_results, transitions, prior_bands)) - # Exit 0 if all guests probed (reachable or not), 1 if any probe error - # (a probe error means the probe itself failed, not just that the guest was unreachable) + # Exit 0 if all probed (reachable or not), 1 if any probe error return 0 diff --git a/state/host-disk-bands.json b/state/host-disk-bands.json new file mode 100644 index 0000000..6a15325 --- /dev/null +++ b/state/host-disk-bands.json @@ -0,0 +1,18 @@ +{ + "acerpve//": "GREEN", + "acerpve//media/mediastore": "GREEN", + "acerpve//media/storage": "GREEN", + "amdpve//": "GREEN", + "amdpve//media/mediastore": "GREEN", + "amdpve//media/storage": "GREEN", + "storepve//": "GREEN", + "storepve//media/easystore2": "HOST-RED", + "storepve//media/mediastore": "GREEN", + "storepve//media/reanim": "HOST-WARN", + "storepve//media/storage": "GREEN", + "minipve//": "GREEN", + "minipve//media/storage": "GREEN", + "ocupve//": "GREEN", + "ocupve//media/mediastore": "GREEN", + "ocupve//media/storage": "GREEN" +} \ No newline at end of file