|
|
|
@@ -153,6 +153,25 @@ GPU_HOSTS = [
|
|
|
|
|
CONNECT_TIMEOUT = 5
|
|
|
|
|
SSH_OPTS = "-o BatchMode=yes -o ConnectTimeout=" + str(CONNECT_TIMEOUT)
|
|
|
|
|
|
|
|
|
|
# Host filesystem thresholds (from contract)
|
|
|
|
|
HOST_THRESHOLDS = {
|
|
|
|
|
"WARN": 85,
|
|
|
|
|
"AMBER": 90,
|
|
|
|
|
"RED": 95,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# State file path (absolute, so execution context doesn't matter)
|
|
|
|
|
STATE_FILE = pathlib.Path(__file__).resolve().parent.parent / "state" / "host-disk-bands.json"
|
|
|
|
|
|
|
|
|
|
# PVE nodes to probe for host filesystems
|
|
|
|
|
HOST_NODES = [
|
|
|
|
|
{"hostname": "acerpve", "ip": "192.168.68.9"},
|
|
|
|
|
{"hostname": "amdpve", "ip": "192.168.68.15"},
|
|
|
|
|
{"hostname": "storepve", "ip": "192.168.68.6"},
|
|
|
|
|
{"hostname": "minipve", "ip": "192.168.68.12"},
|
|
|
|
|
{"hostname": "ocupve", "ip": "192.168.68.5"},
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]:
|
|
|
|
|
"""Run a command and return (exit_code, stdout, stderr)."""
|
|
|
|
@@ -298,6 +317,151 @@ def scan_fleet() -> list[dict]:
|
|
|
|
|
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def classify_band(usage_pct: float) -> str:
|
|
|
|
|
"""Classify a percentage into a band."""
|
|
|
|
|
if usage_pct >= HOST_THRESHOLDS["RED"]:
|
|
|
|
|
return "HOST-RED"
|
|
|
|
|
elif usage_pct >= HOST_THRESHOLDS["AMBER"]:
|
|
|
|
|
return "HOST-AMBER"
|
|
|
|
|
elif usage_pct >= HOST_THRESHOLDS["WARN"]:
|
|
|
|
|
return "HOST-WARN"
|
|
|
|
|
else:
|
|
|
|
|
return "GREEN"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def probe_host_filesystems() -> tuple[list[dict], dict[str, str]]:
|
|
|
|
|
"""Probe host filesystems on all PVE nodes.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
- List of host filesystem results
|
|
|
|
|
- Dict of volume_key -> current_band (for state file)
|
|
|
|
|
"""
|
|
|
|
|
results = []
|
|
|
|
|
current_bands = {}
|
|
|
|
|
|
|
|
|
|
for node in HOST_NODES:
|
|
|
|
|
ip = node["ip"]
|
|
|
|
|
hostname = node["hostname"]
|
|
|
|
|
|
|
|
|
|
# Probe df for host filesystems
|
|
|
|
|
probe_cmd = f'ssh {SSH_OPTS} root@{ip} "df -hP / /media/* tank 2>/dev/null | tail -n +2"'
|
|
|
|
|
exit_code, stdout, stderr = run_cmd(probe_cmd, timeout=15)
|
|
|
|
|
|
|
|
|
|
if exit_code != 0:
|
|
|
|
|
results.append({
|
|
|
|
|
"target": f"{hostname} ({ip})",
|
|
|
|
|
"hostname": hostname,
|
|
|
|
|
"ip": ip,
|
|
|
|
|
"reachable": False,
|
|
|
|
|
"volumes": [],
|
|
|
|
|
"probe_cmd": probe_cmd,
|
|
|
|
|
"failure_kind": "ssh-error",
|
|
|
|
|
})
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# Parse df output and classify each volume
|
|
|
|
|
volumes = []
|
|
|
|
|
for line in stdout.splitlines():
|
|
|
|
|
parts = line.split()
|
|
|
|
|
if len(parts) < 6:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
dev, size, used, avail, pct_str, mount = parts[:6]
|
|
|
|
|
pct = float(pct_str.rstrip("%"))
|
|
|
|
|
band = classify_band(pct)
|
|
|
|
|
|
|
|
|
|
# Volume type classification
|
|
|
|
|
if mount.startswith("/media/"):
|
|
|
|
|
vol_type = "media"
|
|
|
|
|
elif mount == "/" or "pve" in dev:
|
|
|
|
|
vol_type = "host-root"
|
|
|
|
|
elif mount == "tank" or "tank" in mount:
|
|
|
|
|
vol_type = "pbs-datastore"
|
|
|
|
|
else:
|
|
|
|
|
vol_type = "other"
|
|
|
|
|
|
|
|
|
|
# Volume key for state file (host/volume)
|
|
|
|
|
volume_key = f"{hostname}/{mount}"
|
|
|
|
|
current_bands[volume_key] = band
|
|
|
|
|
|
|
|
|
|
volumes.append({
|
|
|
|
|
"mount": mount,
|
|
|
|
|
"device": dev,
|
|
|
|
|
"size": size,
|
|
|
|
|
"used": used,
|
|
|
|
|
"avail": avail,
|
|
|
|
|
"pct": pct,
|
|
|
|
|
"band": band,
|
|
|
|
|
"type": vol_type,
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
results.append({
|
|
|
|
|
"target": f"{hostname} ({ip})",
|
|
|
|
|
"hostname": hostname,
|
|
|
|
|
"ip": ip,
|
|
|
|
|
"reachable": True,
|
|
|
|
|
"volumes": volumes,
|
|
|
|
|
"probe_cmd": probe_cmd,
|
|
|
|
|
"failure_kind": None,
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
return results, current_bands
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def read_state_file() -> Optional[dict[str, str]]:
|
|
|
|
|
"""Read the state file if it exists."""
|
|
|
|
|
if not STATE_FILE.exists():
|
|
|
|
|
return None
|
|
|
|
|
try:
|
|
|
|
|
with open(STATE_FILE) as f:
|
|
|
|
|
return json.load(f)
|
|
|
|
|
except (json.JSONDecodeError, IOError) as e:
|
|
|
|
|
print(f"⚠️ State file exists but unreadable: {e}", file=sys.stderr)
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write_state_file(bands: dict[str, str]) -> None:
|
|
|
|
|
"""Write the state file."""
|
|
|
|
|
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
try:
|
|
|
|
|
with open(STATE_FILE, "w") as f:
|
|
|
|
|
json.dump(bands, f, indent=2)
|
|
|
|
|
except IOError as e:
|
|
|
|
|
print(f"⚠️ State file write failed: {e}", file=sys.stderr)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def detect_transitions(current_bands: dict[str, str], prior_bands: Optional[dict[str, str]]) -> list[dict]:
|
|
|
|
|
"""Detect band transitions (current vs. prior)."""
|
|
|
|
|
if prior_bands is None:
|
|
|
|
|
# First run — no transitions, just establish baseline
|
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
transitions = []
|
|
|
|
|
# Check for volumes that moved to a higher band (escalation)
|
|
|
|
|
for volume, current_band in current_bands.items():
|
|
|
|
|
prior_band = prior_bands.get(volume, "GREEN")
|
|
|
|
|
|
|
|
|
|
# Band ordering: GREEN < HOST-WARN < HOST-AMBER < HOST-RED
|
|
|
|
|
band_order = {"GREEN": 0, "HOST-WARN": 1, "HOST-AMBER": 2, "HOST-RED": 3}
|
|
|
|
|
|
|
|
|
|
if band_order[current_band] > band_order[prior_band]:
|
|
|
|
|
transitions.append({
|
|
|
|
|
"type": "escalation",
|
|
|
|
|
"volume": volume,
|
|
|
|
|
"from": prior_band,
|
|
|
|
|
"to": current_band,
|
|
|
|
|
})
|
|
|
|
|
elif band_order[current_band] < band_order[prior_band]:
|
|
|
|
|
transitions.append({
|
|
|
|
|
"type": "recovery",
|
|
|
|
|
"volume": volume,
|
|
|
|
|
"from": prior_band,
|
|
|
|
|
"to": current_band,
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
return transitions
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def render_results(results: list[dict]) -> str:
|
|
|
|
|
"""Render scan results in human-readable format."""
|
|
|
|
|
lines = []
|
|
|
|
@@ -316,22 +480,88 @@ def render_results(results: list[dict]) -> str:
|
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def render_host_results(results: list[dict], transitions: list[dict], prior_bands: Optional[dict[str, str]]) -> str:
|
|
|
|
|
"""Render host filesystem results in human-readable format."""
|
|
|
|
|
lines = []
|
|
|
|
|
lines.append("")
|
|
|
|
|
lines.append("=== Host Filesystem Bands ===")
|
|
|
|
|
lines.append("")
|
|
|
|
|
|
|
|
|
|
# Render transitions first (they're the actionable alerts)
|
|
|
|
|
if prior_bands is None:
|
|
|
|
|
lines.append(" (first run — recording baseline, no alerts)")
|
|
|
|
|
elif not transitions:
|
|
|
|
|
lines.append(" (no band changes since last scan)")
|
|
|
|
|
else:
|
|
|
|
|
for t in transitions:
|
|
|
|
|
volume, from_band, to_band = t["volume"], t["from"], t["to"]
|
|
|
|
|
if t["type"] == "escalation":
|
|
|
|
|
lines.append(f" ⚠️ {volume}: {from_band} → {to_band} (ESCALATION)")
|
|
|
|
|
else:
|
|
|
|
|
lines.append(f" ✅ {volume}: {from_band} → {to_band} (RECOVERY)")
|
|
|
|
|
|
|
|
|
|
# Render all volumes with their bands
|
|
|
|
|
lines.append("")
|
|
|
|
|
for node_result in results:
|
|
|
|
|
if not node_result["reachable"]:
|
|
|
|
|
lines.append(f" ❌ {node_result['target']}: UNREACHABLE ({node_result['failure_kind']})")
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
lines.append(f" {node_result['target']}:")
|
|
|
|
|
for vol in node_result["volumes"]:
|
|
|
|
|
lines.append(f" {vol['mount']} ({vol['type']}): {vol['pct']}% ({vol['used']}/{vol['size']}, {vol['avail']} free) -> {vol['band']}")
|
|
|
|
|
|
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> int:
|
|
|
|
|
import argparse
|
|
|
|
|
|
|
|
|
|
ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests.")
|
|
|
|
|
ap = argparse.ArgumentParser(description="Deterministic disk usage probe for fleet guests and host filesystems.")
|
|
|
|
|
ap.add_argument("--json", action="store_true", help="machine-readable output")
|
|
|
|
|
ap.add_argument("--hosts-only", action="store_true", help="scan host filesystems only")
|
|
|
|
|
ap.add_argument("--guests-only", action="store_true", help="scan guests only (skip host filesystems)")
|
|
|
|
|
args = ap.parse_args()
|
|
|
|
|
|
|
|
|
|
results = scan_fleet()
|
|
|
|
|
# Scan guests (unless --hosts-only)
|
|
|
|
|
guest_results = []
|
|
|
|
|
if not args.hosts_only:
|
|
|
|
|
guest_results = scan_fleet()
|
|
|
|
|
|
|
|
|
|
# Scan host filesystems (unless --guests-only)
|
|
|
|
|
host_results = []
|
|
|
|
|
current_bands = {}
|
|
|
|
|
if not args.guests_only:
|
|
|
|
|
host_results, current_bands = probe_host_filesystems()
|
|
|
|
|
|
|
|
|
|
# Read prior state and detect transitions
|
|
|
|
|
prior_bands = read_state_file()
|
|
|
|
|
transitions = detect_transitions(current_bands, prior_bands)
|
|
|
|
|
|
|
|
|
|
# Write new state
|
|
|
|
|
write_state_file(current_bands)
|
|
|
|
|
else:
|
|
|
|
|
prior_bands = None
|
|
|
|
|
transitions = []
|
|
|
|
|
|
|
|
|
|
if args.json:
|
|
|
|
|
print(json.dumps(results, indent=2))
|
|
|
|
|
# JSON output
|
|
|
|
|
output = {
|
|
|
|
|
"guests": guest_results,
|
|
|
|
|
"hosts": host_results,
|
|
|
|
|
"transitions": transitions,
|
|
|
|
|
"prior_bands": prior_bands,
|
|
|
|
|
}
|
|
|
|
|
print(json.dumps(output, indent=2))
|
|
|
|
|
else:
|
|
|
|
|
print(render_results(results))
|
|
|
|
|
# Human-readable output
|
|
|
|
|
if guest_results:
|
|
|
|
|
print(render_results(guest_results))
|
|
|
|
|
|
|
|
|
|
if host_results:
|
|
|
|
|
print(render_host_results(host_results, transitions, prior_bands))
|
|
|
|
|
|
|
|
|
|
# Exit 0 if all guests probed (reachable or not), 1 if any probe error
|
|
|
|
|
# (a probe error means the probe itself failed, not just that the guest was unreachable)
|
|
|
|
|
# Exit 0 if all probed (reachable or not), 1 if any probe error
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|