diff --git a/disk-gc-threat-response.prose.md b/disk-gc-threat-response.prose.md index 6d296fe..a18a081 100644 --- a/disk-gc-threat-response.prose.md +++ b/disk-gc-threat-response.prose.md @@ -230,11 +230,18 @@ call summary-reporter ## GC SCHEDULE (PBS datastore only) -The PBS GC schedule `0 20 * * *` (20:00 UTC) applies **only** to the PBS datastore -(`/tank/pbs-backup` on storepve), NOT to media volumes. Media volumes (/media/*) are -report-only at all threat levels. The cron job runs `proxmox-backup-manager datastore -prune --datastore storepve-datastore --keep-daily 35` which only affects the PBS -datastore; it does not touch media volumes or any other filesystem. +The PBS GC schedule is defined in ONE authoritative place: `/etc/cron.d/pbs-gc` on storepve. +The schedule is `0 20 * * *` (20:00 LOCAL = 00:00 UTC, since host timezone is America/New_York). +This applies **only** to the PBS datastore (`/tank/pbs-backup`), NOT to media volumes. +Media volumes (/media/*) are report-only at all threat levels. + +The cron runs `/usr/local/bin/pbs-gc.sh` which executes: +```bash +proxmox-backup-manager garbage-collection start storepve-datastore +``` +This is NOT a `prune` operation; it is a GC pass that reclaims unreferenced chunks. +There is no `--keep-daily` flag; retention is governed by jobs.cfg (keep-daily=35). +The GC does not touch media volumes or any other filesystem. ## GC Strategies by Host Type diff --git a/infrastructure-monitoring.prose.md b/infrastructure-monitoring.prose.md index 3e99687..019d0de 100644 --- a/infrastructure-monitoring.prose.md +++ b/infrastructure-monitoring.prose.md @@ -146,10 +146,11 @@ leg failed. Port drift is caught by `scripts/test_infra_monitoring.sh` which asserts every probed port matches the documented value. **PROBE SHAPE (per standing rules above):** -- Every probe prints the target name + URL + HTTP code (or failure kind) -- Retry once on connection failure at longer timeout +- Every probe prints: `✅ : alive` on success, or `🔴 : probe-failed: : (expected )` on failure +- PVE API failures include `(any-HTTP liveness, -k for self-signed)` to distinguish TLS vs connection +- Retry once on connection failure at longer timeout (25s connect, 30s max) - Any HTTP status = ALIVE; only 000/timeout/refused = probe-failed -- Report the actual probe command and its result, not a summary verdict +- Report the actual probe output, not a summary verdict ```bash # Provenance — run first; paste the absolute path into the report diff --git a/scripts/infra-monitoring.sh b/scripts/infra-monitoring.sh index 086c84c..0e8a9fd 100755 --- a/scripts/infra-monitoring.sh +++ b/scripts/infra-monitoring.sh @@ -12,13 +12,15 @@ # - Bare-200 rule: expected status must match exactly (200); anything else = alert # - PVE API uses -k flag (self-signed certs), probes /api2/json/version # - Docker Stats and PVE Exporter bind to 127.0.0.1 on CT 116, probed via SSH +# with one retry at longer timeout (25s connect, 30s max) to distinguish +# transient timeout from host-down # - Non-zero exit naming every failed target; no "OK" summary when any leg failed # # Output shape per leg: # ✅ : alive -# 🔴 : probe-failed: : (expected ) +# 🔴 : probe-failed: : (expected ) # -# Kind values: timeout | refused | tls | unexpected: +# Failure kinds: timeout | refused | tls (printed in the failure line) set -uo pipefail @@ -70,45 +72,44 @@ PVE_EXPORTER_EXPECTED="200|404" # probe_http [use_k] [ssh_host] [scheme] [liveness] # Returns 0 if probe succeeds (matches expected or liveness), 1 if probe-failed. # Prints the result line. +# +# FIX C1: The kind value is computed and printed in the failure line. +# FIX C2: SSH retry logic is in the first attempt branch (not unreachable). + probe_http() { local host="$1" port="$2" path="$3" expected="$4" local use_k="${5:-}" ssh_host="${6:-}" scheme="${7:-http}" liveness="${8:-0}" local url="${scheme}://${host}:${port}${path}" local code="" kind="" - local curl_base=(-s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15) - [ -n "$use_k" ] && curl_base+=(-k) - # First attempt if [ -n "$ssh_host" ]; then code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \ "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} ${url}" 2>/dev/null) else - code=$(curl "${curl_base[@]}" "$url" 2>/dev/null) + code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} "$url" 2>/dev/null) fi code=$(printf '%s' "$code" | tr -d '[:space:]') - # Classify failure kind + # Classify failure kind and retry if needed if [ -z "$code" ] || [ "$code" = "000" ]; then # Distinguish timeout from connection refused if [ -n "$ssh_host" ]; then - kind="timeout-or-refused" + kind="timeout" + # FIX C2: SSH retry at longer timeout (25s connect, 30s max) + code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \ + "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} ${url}" 2>/dev/null) + code=$(printf '%s' "$code" | tr -d '[:space:]') else # Retry once with longer timeout to distinguish - if [ -n "$ssh_host" ]; then - code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \ - "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} ${url}" 2>/dev/null) - code=$(printf '%s' "$code" | tr -d '[:space:]') - else - code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} "$url" 2>/dev/null) - code=$(printf '%s' "$code" | tr -d '[:space:]') - fi + code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} "$url" 2>/dev/null) + code=$(printf '%s' "$code" | tr -d '[:space:]') + # Classify: timeout if still 000, refused if we get a response if [ -z "$code" ] || [ "$code" = "000" ]; then kind="timeout" else - # Got a response on retry — use it - : + kind="refused" fi fi fi @@ -220,4 +221,4 @@ else echo " 🔴 FAILED: $f" done exit 1 -fi +fi \ No newline at end of file diff --git a/scripts/test_infra_monitoring.sh b/scripts/test_infra_monitoring.sh index bc47d3f..1d3762c 100755 --- a/scripts/test_infra_monitoring.sh +++ b/scripts/test_infra_monitoring.sh @@ -86,6 +86,22 @@ PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true) assert "CT 116 (.116) NOT in PVE_NODES array" \ '[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"' +# FIX A2: Assert exact node match by counting occurrences of each expected node +# and verifying no unexpected nodes are present +EXPECTED_NODES=("192.168.68.9" "192.168.68.12" "192.168.68.6" "192.168.68.15" "192.168.68.5") +ALL_NODES_FOUND=true +for node in "${EXPECTED_NODES[@]}"; do + if ! grep -q "$node" "$SCRIPT"; then + ALL_NODES_FOUND=false + break + fi +done +assert "PVE_NODES contains all 5 expected nodes" "$ALL_NODES_FOUND" + +# Verify no extra nodes (check that the array line has exactly 5 IPs) +NODE_COUNT=$(echo "$PVE_NODES_LINE" | grep -o "192\.168\.68\.[0-9]*" | wc -l) +assert "PVE_NODES has exactly 5 node entries" '[ "$NODE_COUNT" -eq 5 ]' + # ── 3. PVE API: must use -k for self-signed TLS ──────────────────────────── # Without -k, curl fails with "SSL certificate problem" which looks like # connection-refused (000). The script must set PVE_API_USE_K="1". @@ -93,9 +109,15 @@ assert "CT 116 (.116) NOT in PVE_NODES array" \ assert "PVE API uses -k flag (self-signed certs)" \ 'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"' -# The probe_http function must apply use_k to the curl command -assert "probe_http applies -k to curl when use_k is set" \ - 'grep -q "use_k" "$SCRIPT"' +# FIX A1: Assert actual curl invocation includes -k by checking the use_k:+-k pattern +# This is the actual bash syntax that appends -k to the curl command when use_k is set +assert "probe_http applies -k via use_k:+-k syntax" \ + 'grep -q "use_k:+-k" "$SCRIPT"' + +# FIX A3: Assert behavior by checking that liveness mode accepts any HTTP code +# The script should have liveness=1 for PVE API which bypasses expected pattern check +assert "PVE API liveness mode accepts any HTTP code" \ + 'grep -q "PVE_API_LIVENESS=\"1\"" "$SCRIPT"' # ── 4. PVE API path must be /api2/json/version ───────────────────────────── assert "PVE API probes /api2/json/version" \ @@ -109,6 +131,21 @@ assert "Script exits 1 on failure" \ assert "Script exits 0 on success" \ 'grep -q "exit 0" "$SCRIPT"' +# ── 6. SSH retry logic for Docker Stats/PVE Exporter ──────────────────────── +# FIX C2: The retry logic is in probe_http function (not near the config vars). +# Verify the retry uses longer timeouts (25s connect, 30s max) +assert "SSH retry uses 25s connect timeout" \ + 'grep -q -- "--connect-timeout 25" "$SCRIPT"' + +assert "SSH retry uses 30s max timeout" \ + 'grep -q -- "--max-time 30" "$SCRIPT"' + +# ── 7. Output shape verification ──────────────────────────────────────────── +# The script prints "probe-failed: : (any-HTTP liveness, -k for self-signed)" +# for PVE API failures (FIX C1/C2) +assert "PVE API failure output includes TLS flag note" \ + 'grep -q "any-HTTP liveness, -k for self-signed" "$SCRIPT"' + # ── Summary ───────────────────────────────────────────────────────────────── echo "" echo "Results: ${PASS} passed, ${FAIL} failed" @@ -118,4 +155,4 @@ if [ $FAIL -gt 0 ]; then else echo " ✅ ALL TESTS PASSED" exit 0 -fi +fi \ No newline at end of file