Compare commits

...
Author SHA1 Message Date
root 39209c7ac9 fix: untrack host-disk-bands.json and document gitignored status
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 7s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 8s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 0s
The state file is runtime state (rewritten every scan), so tracking it in git
means:
- every executor's clone becomes permanently dirty after one run
- a scan in one clone produces a merge conflict with a scan in another
- the committed baseline can be stale in a way nobody notices

Added to .gitignore and removed from the index. Contract updated to say
'the state file lives at <abs path> and is gitignored runtime state - the
scanner creates it on first run'.
2026-09-16 00:45:46 +00:00
root 8ed3b9c606 fix: implement host-filesystem state file with transition detection
The contract said the state file was written after every scan, but the script
had no state-file logic at all. This PR adds:

1. Host filesystem scanning (probe_host_filesystems) - probes df on all PVE nodes
2. State file I/O (read_state_file/write_state_file) - absolute path from script location
3. Band classification (classify_band) - HOST-WARN/AMBER/RED thresholds
4. Transition detection (detect_transitions) - alerts on escalation/recovery
5. CLI flags (--hosts-only, --guests-only) to control which parts run

The contract now specifies the state file path resolves from the script's own
location (not CWD-relative), so two different execution contexts cannot write
to two different places.
2026-09-16 00:41:34 +00:00
3 changed files with 238 additions and 7 deletions
+1
View File
@@ -1 +1,2 @@
__pycache__/
state/host-disk-bands.json
+1 -1
View File
@@ -65,7 +65,7 @@ Host filesystems have their own risk profile and their own bands. A host root ne
**Escalations are STATE-CHANGE driven, not per-run.** A volume alerts ONCE when it enters a higher band (GREEN->WARN, WARN->AMBER, AMBER->RED) and ONCE when it drops back down (a recovery notice). While a volume stays in the same band, it is reported in the scan output only — no DM, no channel alert. This prevents the same 96% easystore2 from re-DMing the owner on every 6-hour scan and drowning a real warning in noise.
**State lives in a small JSON state file:** `state/host-disk-bands.json`, keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.)
**State lives in a small JSON state file:** the scanner resolves it to an absolute path from the script's location — `$(dirname "$0")/state/host-disk-bands.json` (i.e., the `state/` directory next to the `scripts/` directory in the prose-contracts repo). The state file is gitignored runtime state — the scanner creates it on first run. Keyed by `host/volume` → last-seen band. The scanner reads the prior band, compares to the current band, and DMs only on a transition. The state file is written after every scan. (Chosen over a periodic digest because the scan already runs every 6h and a transition is genuinely new, actionable state that warrants an immediate DM — but only once.)
**Volume naming rule:** Every host line MUST name the volume and what lives on it. Example output:
+236 -6
View File
@@ -153,6 +153,25 @@ GPU_HOSTS = [
CONNECT_TIMEOUT = 5
SSH_OPTS = "-o BatchMode=yes -o ConnectTimeout=" + str(CONNECT_TIMEOUT)
# Host filesystem thresholds (from contract)
HOST_THRESHOLDS = {
"WARN": 85,
"AMBER": 90,
"RED": 95,
}
# State file path (absolute, so execution context doesn't matter)
STATE_FILE = pathlib.Path(__file__).resolve().parent.parent / "state" / "host-disk-bands.json"
# PVE nodes to probe for host filesystems
HOST_NODES = [
{"hostname": "acerpve", "ip": "192.168.68.9"},
{"hostname": "amdpve", "ip": "192.168.68.15"},
{"hostname": "storepve", "ip": "192.168.68.6"},
{"hostname": "minipve", "ip": "192.168.68.12"},
{"hostname": "ocupve", "ip": "192.168.68.5"},
]
def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]:
"""Run a command and return (exit_code, stdout, stderr)."""
@@ -298,6 +317,151 @@ def scan_fleet() -> list[dict]:
return results
def classify_band(usage_pct: float) -> str:
"""Classify a percentage into a band."""
if usage_pct >= HOST_THRESHOLDS["RED"]:
return "HOST-RED"
elif usage_pct >= HOST_THRESHOLDS["AMBER"]:
return "HOST-AMBER"
elif usage_pct >= HOST_THRESHOLDS["WARN"]:
return "HOST-WARN"
else:
return "GREEN"
def probe_host_filesystems() -> tuple[list[dict], dict[str, str]]:
"""Probe host filesystems on all PVE nodes.
Returns:
- List of host filesystem results
- Dict of volume_key -> current_band (for state file)
"""
results = []
current_bands = {}
for node in HOST_NODES:
ip = node["ip"]
hostname = node["hostname"]
# Probe df for host filesystems
probe_cmd = f'ssh {SSH_OPTS} root@{ip} "df -hP / /media/* tank 2>/dev/null | tail -n +2"'
exit_code, stdout, stderr = run_cmd(probe_cmd, timeout=15)
if exit_code != 0:
results.append({
"target": f"{hostname} ({ip})",
"hostname": hostname,
"ip": ip,
"reachable": False,
"volumes": [],
"probe_cmd": probe_cmd,
"failure_kind": "ssh-error",
})
continue
# Parse df output and classify each volume
volumes = []
for line in stdout.splitlines():
parts = line.split()
if len(parts) < 6:
continue
dev, size, used, avail, pct_str, mount = parts[:6]
pct = float(pct_str.rstrip("%"))
band = classify_band(pct)
# Volume type classification
if mount.startswith("/media/"):
vol_type = "media"
elif mount == "/" or "pve" in dev:
vol_type = "host-root"
elif mount == "tank" or "tank" in mount:
vol_type = "pbs-datastore"
else:
vol_type = "other"
# Volume key for state file (host/volume)
volume_key = f"{hostname}/{mount}"
current_bands[volume_key] = band
volumes.append({
"mount": mount,
"device": dev,
"size": size,
"used": used,
"avail": avail,
"pct": pct,
"band": band,
"type": vol_type,
})
results.append({
"target": f"{hostname} ({ip})",
"hostname": hostname,
"ip": ip,
"reachable": True,
"volumes": volumes,
"probe_cmd": probe_cmd,
"failure_kind": None,
})
return results, current_bands
def read_state_file() -> Optional[dict[str, str]]:
"""Read the state file if it exists."""
if not STATE_FILE.exists():
return None
try:
with open(STATE_FILE) as f:
return json.load(f)
except (json.JSONDecodeError, IOError) as e:
print(f"⚠️ State file exists but unreadable: {e}", file=sys.stderr)
return {}
def write_state_file(bands: dict[str, str]) -> None:
"""Write the state file."""
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
try:
with open(STATE_FILE, "w") as f:
json.dump(bands, f, indent=2)
except IOError as e:
print(f"⚠️ State file write failed: {e}", file=sys.stderr)
def detect_transitions(current_bands: dict[str, str], prior_bands: Optional[dict[str, str]]) -> list[dict]:
"""Detect band transitions (current vs. prior)."""
if prior_bands is None:
# First run — no transitions, just establish baseline
return []
transitions = []
# Check for volumes that moved to a higher band (escalation)
for volume, current_band in current_bands.items():
prior_band = prior_bands.get(volume, "GREEN")
# Band ordering: GREEN < HOST-WARN < HOST-AMBER < HOST-RED
band_order = {"GREEN": 0, "HOST-WARN": 1, "HOST-AMBER": 2, "HOST-RED": 3}
if band_order[current_band] > band_order[prior_band]:
transitions.append({
"type": "escalation",
"volume": volume,
"from": prior_band,
"to": current_band,
})
elif band_order[current_band] < band_order[prior_band]:
transitions.append({
"type": "recovery",
"volume": volume,
"from": prior_band,
"to": current_band,
})
return transitions
def render_results(results: list[dict]) -> str:
"""Render scan results in human-readable format."""
lines = []
@@ -316,22 +480,88 @@ def render_results(results: list[dict]) -> str:
return "\n".join(lines)
def render_host_results(results: list[dict], transitions: list[dict], prior_bands: Optional[dict[str, str]]) -> str:
"""Render host filesystem results in human-readable format."""
lines = []
lines.append("")
lines.append("=== Host Filesystem Bands ===")
lines.append("")
# Render transitions first (they're the actionable alerts)
if prior_bands is None:
lines.append(" (first run — recording baseline, no alerts)")
elif not transitions:
lines.append(" (no band changes since last scan)")
else:
for t in transitions:
volume, from_band, to_band = t["volume"], t["from"], t["to"]
if t["type"] == "escalation":
lines.append(f" ⚠️ {volume}: {from_band} → {to_band} (ESCALATION)")
else:
lines.append(f" ✅ {volume}: {from_band} → {to_band} (RECOVERY)")
# Render all volumes with their bands
lines.append("")
for node_result in results:
if not node_result["reachable"]:
lines.append(f" ❌ {node_result['target']}: UNREACHABLE ({node_result['failure_kind']})")
continue
lines.append(f" {node_result['target']}:")
for vol in node_result["volumes"]:
lines.append(f" {vol['mount']} ({vol['type']}): {vol['pct']}% ({vol['used']}/{vol['size']}, {vol['avail']} free) -> {vol['band']}")
return "\n".join(lines)
def main() -> int:
import argparse
ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests.")
ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests and host filesystems.")
ap.add_argument("--json", action="store_true", help="machine-readable output")
ap.add_argument("--hosts-only", action="store_true", help="scan host filesystems only")
ap.add_argument("--guests-only", action="store_true", help="scan guests only (skip host filesystems)")
args = ap.parse_args()
results = scan_fleet()
# Scan guests (unless --hosts-only)
guest_results = []
if not args.hosts_only:
guest_results = scan_fleet()
# Scan host filesystems (unless --guests-only)
host_results = []
current_bands = {}
if not args.guests_only:
host_results, current_bands = probe_host_filesystems()
# Read prior state and detect transitions
prior_bands = read_state_file()
transitions = detect_transitions(current_bands, prior_bands)
# Write new state
write_state_file(current_bands)
else:
prior_bands = None
transitions = []
if args.json:
print(json.dumps(results, indent=2))
# JSON output
output = {
"guests": guest_results,
"hosts": host_results,
"transitions": transitions,
"prior_bands": prior_bands,
}
print(json.dumps(output, indent=2))
else:
print(render_results(results))
# Human-readable output
if guest_results:
print(render_results(guest_results))
if host_results:
print(render_host_results(host_results, transitions, prior_bands))
# Exit 0 if all guests probed (reachable or not), 1 if any probe error
# (a probe error means the probe itself failed, not just that the guest was unreachable)
# Exit 0 if all probed (reachable or not), 1 if any probe error
return 0