.gitignore

@@ -1 +1,2 @@
 __pycache__/
+state/host-disk-bands.json

disk-gc-threat-response.prose.md

@@ -65,7 +65,7 @@ Host filesystems have their own risk profile and their own bands. A host root ne
 
 **Escalations are STATE-CHANGE driven, not per-run.** A volume alerts ONCE when it enters a higher band (GREEN->WARN, WARN->AMBER, AMBER->RED) and ONCE when it drops back down (a recovery notice). While a volume stays in the same band, it is reported in the scan output only — no DM, no channel alert. This prevents the same 96% easystore2 from re-DMing the owner on every 6-hour scan and drowning a real warning in noise.
 
-**State lives in a small JSON state file:** `state/host-disk-bands.json`, keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.)
+**State lives in a small JSON state file:** the scanner resolves it to an absolute path from the script's location — `$(dirname "$0")/state/host-disk-bands.json` (i.e., the `state/` directory next to the `scripts/` directory in the prose-contracts repo). The state file is gitignored runtime state — the scanner creates it on first run. Keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.)
 
 **Volume naming rule:** Every host line MUST name the volume and what lives on it. Example output:
 

scripts/disk-gc-scan.py

@@ -153,6 +153,25 @@ GPU_HOSTS = [
 CONNECT_TIMEOUT = 5
 SSH_OPTS = "-o BatchMode=yes -o ConnectTimeout=" + str(CONNECT_TIMEOUT)
 
+# Host filesystem thresholds (from contract)
+HOST_THRESHOLDS = {
+    "WARN": 85,
+    "AMBER": 90,
+    "RED": 95,
+}
+
+# State file path (absolute, so execution context doesn't matter)
+STATE_FILE = pathlib.Path(__file__).resolve().parent.parent / "state" / "host-disk-bands.json"
+
+# PVE nodes to probe for host filesystems
+HOST_NODES = [
+    {"hostname": "acerpve", "ip": "192.168.68.9"},
+    {"hostname": "amdpve", "ip": "192.168.68.15"},
+    {"hostname": "storepve", "ip": "192.168.68.6"},
+    {"hostname": "minipve", "ip": "192.168.68.12"},
+    {"hostname": "ocupve", "ip": "192.168.68.5"},
+]
+
 
 def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]:
     """Run a command and return (exit_code, stdout, stderr)."""
@@ -298,6 +317,151 @@ def scan_fleet() -> list[dict]:
     return results
 
 
+def classify_band(usage_pct: float) -> str:
+    """Classify a percentage into a band."""
+    if usage_pct >= HOST_THRESHOLDS["RED"]:
+        return "HOST-RED"
+    elif usage_pct >= HOST_THRESHOLDS["AMBER"]:
+        return "HOST-AMBER"
+    elif usage_pct >= HOST_THRESHOLDS["WARN"]:
+        return "HOST-WARN"
+    else:
+        return "GREEN"
+
+
+def probe_host_filesystems() -> tuple[list[dict], dict[str, str]]:
+    """Probe host filesystems on all PVE nodes.
+    
+    Returns:
+        - List of host filesystem results
+        - Dict of volume_key -> current_band (for state file)
+    """
+    results = []
+    current_bands = {}
+    
+    for node in HOST_NODES:
+        ip = node["ip"]
+        hostname = node["hostname"]
+        
+        # Probe df for host filesystems
+        probe_cmd = f'ssh {SSH_OPTS} root@{ip} "df -hP / /media/* tank 2>/dev/null | tail -n +2"'
+        exit_code, stdout, stderr = run_cmd(probe_cmd, timeout=15)
+        
+        if exit_code != 0:
+            results.append({
+                "target": f"{hostname} ({ip})",
+                "hostname": hostname,
+                "ip": ip,
+                "reachable": False,
+                "volumes": [],
+                "probe_cmd": probe_cmd,
+                "failure_kind": "ssh-error",
+            })
+            continue
+        
+        # Parse df output and classify each volume
+        volumes = []
+        for line in stdout.splitlines():
+            parts = line.split()
+            if len(parts) < 6:
+                continue
+            
+            dev, size, used, avail, pct_str, mount = parts[:6]
+            pct = float(pct_str.rstrip("%"))
+            band = classify_band(pct)
+            
+            # Volume type classification
+            if mount.startswith("/media/"):
+                vol_type = "media"
+            elif mount == "/" or "pve" in dev:
+                vol_type = "host-root"
+            elif mount == "tank" or "tank" in mount:
+                vol_type = "pbs-datastore"
+            else:
+                vol_type = "other"
+            
+            # Volume key for state file (host/volume)
+            volume_key = f"{hostname}/{mount}"
+            current_bands[volume_key] = band
+            
+            volumes.append({
+                "mount": mount,
+                "device": dev,
+                "size": size,
+                "used": used,
+                "avail": avail,
+                "pct": pct,
+                "band": band,
+                "type": vol_type,
+            })
+        
+        results.append({
+            "target": f"{hostname} ({ip})",
+            "hostname": hostname,
+            "ip": ip,
+            "reachable": True,
+            "volumes": volumes,
+            "probe_cmd": probe_cmd,
+            "failure_kind": None,
+        })
+    
+    return results, current_bands
+
+
+def read_state_file() -> Optional[dict[str, str]]:
+    """Read the state file if it exists."""
+    if not STATE_FILE.exists():
+        return None
+    try:
+        with open(STATE_FILE) as f:
+            return json.load(f)
+    except (json.JSONDecodeError, IOError) as e:
+        print(f"⚠️  State file exists but unreadable: {e}", file=sys.stderr)
+        return {}
+
+
+def write_state_file(bands: dict[str, str]) -> None:
+    """Write the state file."""
+    STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with open(STATE_FILE, "w") as f:
+            json.dump(bands, f, indent=2)
+    except IOError as e:
+        print(f"⚠️  State file write failed: {e}", file=sys.stderr)
+
+
+def detect_transitions(current_bands: dict[str, str], prior_bands: Optional[dict[str, str]]) -> list[dict]:
+    """Detect band transitions (current vs. prior)."""
+    if prior_bands is None:
+        # First run — no transitions, just establish baseline
+        return []
+    
+    transitions = []
+    # Check for volumes that moved to a higher band (escalation)
+    for volume, current_band in current_bands.items():
+        prior_band = prior_bands.get(volume, "GREEN")
+        
+        # Band ordering: GREEN < HOST-WARN < HOST-AMBER < HOST-RED
+        band_order = {"GREEN": 0, "HOST-WARN": 1, "HOST-AMBER": 2, "HOST-RED": 3}
+        
+        if band_order[current_band] > band_order[prior_band]:
+            transitions.append({
+                "type": "escalation",
+                "volume": volume,
+                "from": prior_band,
+                "to": current_band,
+            })
+        elif band_order[current_band] < band_order[prior_band]:
+            transitions.append({
+                "type": "recovery",
+                "volume": volume,
+                "from": prior_band,
+                "to": current_band,
+            })
+    
+    return transitions
+
+
 def render_results(results: list[dict]) -> str:
     """Render scan results in human-readable format."""
     lines = []
@@ -316,22 +480,88 @@ def render_results(results: list[dict]) -> str:
     return "\n".join(lines)
 
 
+def render_host_results(results: list[dict], transitions: list[dict], prior_bands: Optional[dict[str, str]]) -> str:
+    """Render host filesystem results in human-readable format."""
+    lines = []
+    lines.append("")
+    lines.append("=== Host Filesystem Bands ===")
+    lines.append("")
+    
+    # Render transitions first (they're the actionable alerts)
+    if prior_bands is None:
+        lines.append("  (first run — recording baseline, no alerts)")
+    elif not transitions:
+        lines.append("  (no band changes since last scan)")
+    else:
+        for t in transitions:
+            volume, from_band, to_band = t["volume"], t["from"], t["to"]
+            if t["type"] == "escalation":
+                lines.append(f"  ⚠️  {volume}: {from_band} → {to_band} (ESCALATION)")
+            else:
+                lines.append(f"  ✅ {volume}: {from_band} → {to_band} (RECOVERY)")
+    
+    # Render all volumes with their bands
+    lines.append("")
+    for node_result in results:
+        if not node_result["reachable"]:
+            lines.append(f"  ❌ {node_result['target']}: UNREACHABLE ({node_result['failure_kind']})")
+            continue
+        
+        lines.append(f"  {node_result['target']}:")
+        for vol in node_result["volumes"]:
+            lines.append(f"    {vol['mount']} ({vol['type']}): {vol['pct']}% ({vol['used']}/{vol['size']}, {vol['avail']} free) -> {vol['band']}")
+    
+    return "\n".join(lines)
+
+
 def main() -> int:
     import argparse
 
-    ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests.")
+    ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests and host filesystems.")
     ap.add_argument("--json", action="store_true", help="machine-readable output")
+    ap.add_argument("--hosts-only", action="store_true", help="scan host filesystems only")
+    ap.add_argument("--guests-only", action="store_true", help="scan guests only (skip host filesystems)")
     args = ap.parse_args()
 
-    results = scan_fleet()
+    # Scan guests (unless --hosts-only)
+    guest_results = []
+    if not args.hosts_only:
+        guest_results = scan_fleet()
+
+    # Scan host filesystems (unless --guests-only)
+    host_results = []
+    current_bands = {}
+    if not args.guests_only:
+        host_results, current_bands = probe_host_filesystems()
+        
+        # Read prior state and detect transitions
+        prior_bands = read_state_file()
+        transitions = detect_transitions(current_bands, prior_bands)
+        
+        # Write new state
+        write_state_file(current_bands)
+    else:
+        prior_bands = None
+        transitions = []
 
     if args.json:
-        print(json.dumps(results, indent=2))
+        # JSON output
+        output = {
+            "guests": guest_results,
+            "hosts": host_results,
+            "transitions": transitions,
+            "prior_bands": prior_bands,
+        }
+        print(json.dumps(output, indent=2))
     else:
-        print(render_results(results))
+        # Human-readable output
+        if guest_results:
+            print(render_results(guest_results))
+        
+        if host_results:
+            print(render_host_results(host_results, transitions, prior_bands))
 
-    # Exit 0 if all guests probed (reachable or not), 1 if any probe error
+    # Exit 0 if all probed (reachable or not), 1 if any probe error
-    # (a probe error means the probe itself failed, not just that the guest was unreachable)
     return 0