fix: probe-drift round 2 — stale monitoring expectations (health check, PVE API, GPU port 80, report provenance)

Second probe-drift correction pass after #65/#66/#68. All four legs were stale
consumer expectations, not live faults.

1. scripts/agent-health-check.py (v4)
   - abiba declared pi-only runtime (harness purge): Hermes-era gateway, config
     and wrapper legs are skipped instead of failing.
   - koby declared report_only (captain ruling 2026-08-17, Rule 17): every koby
     leg is detected and reported, never counted as a fleet failure or repaired.
   - koby's CT 111 mapping corrected to storepve (.6); the old amdpve mapping
     made `pct status 111` fail and read as ct-unreachable.
   - wrapper check no longer FAILs .env-based wrappers that legitimately never
     invoke infisical (koonimo).
   - keys load in main() (load_agent_keys) so the module is importable/testable.
   - every run prints absolute execution provenance (script + cwd), in the header
     and in --json.
   Before: 6 FAILURE(S). After: 0 failures, koby reported read-only.

2. infrastructure-monitoring.prose.md
   - PVE API probe repointed from CT 116 (no pveproxy, 000) to the five real
     nodes on https://<node>:8006/api2/json/version, all 401 = alive.
   - any-HTTP-response liveness rule added (401/3xx alive; 000/timeout = DOWN).
   - LiteLLM health documented as 301 -> /litellm/health/liveliness, not bare 200.

3. gpu-monitor.prose.md
   - GPU health probes on :8080 (or router /health/unified); bare port 80 on a
     GPU host is forbidden (no listener -> false DEGRADED).
   - router /health/unified 301 -> /gpu/gpu-data documented as alive.
   - port-discipline + liveness rule + direct-fallback execution step.

4. Report provenance (all contracts)
   - docs/AUTHORING-GUIDE.md documents the rule; scripts/prose-lint.sh enforces
     that any **Report format** contract states an absolute path (pwd -P).
   - provenance added to gpu-monitor, infrastructure-monitoring, proxmox-monitor.

Tests: tests/test_probe_drift.py (23 passed); prose-lint.sh clean + shellcheck
clean; CI frontmatter validation passes. Evidence with per-leg before/after and
absolute paths: docs/probe-drift-round2-evidence.md.
This commit is contained in:
2026-09-10 01:23:54 +00:00
parent 532250b017
commit c66d9c1e20
8 changed files with 689 additions and 67 deletions
+152
View File
@@ -0,0 +1,152 @@
"""Regression tests for the 2026-09-09/10 probe-drift corrections.
WHY THIS FILE EXISTS: the monitoring contracts kept emitting false alarms from
stale expectations rather than live faults.
* agent-health-check v3 reported 6 failures that were all stale expectations:
abiba (pi-only since the harness purge) was tested as a Hermes host, koby
(report-only per the captain's 2026-08-17 ruling) was counted as repairable,
koby's CT 111 was probed on amdpve where it does not exist (it runs on
storepve .6), and a .env-based hermes wrapper was FAILed for not mentioning
infisical.
* gpu-monitor emitted "DEGRADED — GPU-rtx3090 000, GPU-rtx5070 000" three
times from probing bare port 80 on GPU hosts while :8080 answered 200.
* infrastructure-monitoring probed CT 116 for the PVE API (no pveproxy ->
000) instead of the five real cluster nodes, which answer 401 = alive.
These tests pin the corrections so the false alarms cannot return silently.
They are static/structural over the contracts plus an inert import of the health
script — no live network, vault, or SSH access is required.
"""
from __future__ import annotations
import importlib.util
import pathlib
import re
import pytest
ROOT = pathlib.Path(__file__).resolve().parents[1]
AHC = ROOT / "scripts" / "agent-health-check.py"
GPU = ROOT / "gpu-monitor.prose.md"
INFRA = ROOT / "infrastructure-monitoring.prose.md"
PROXMOX = ROOT / "proxmox-monitor.prose.md"
@pytest.fixture(scope="module")
def ahc():
"""Import agent-health-check.py without live network/SSH side effects."""
spec = importlib.util.spec_from_file_location("agent_health_check", AHC)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
# ── agent-health-check: stale-expectation legs ───────────────────────
def test_import_does_not_contact_vault(ahc):
# Keys are loaded in main() via load_agent_keys(); importing must stay inert.
assert callable(ahc.load_agent_keys)
assert all(agent.get("key") is None for agent in ahc.AGENTS.values())
def test_abiba_is_pi_only_runtime(ahc):
# .24 has run pi-only since the harness purge: no Hermes gateway, config, or
# wrapper. Probing those legs produced false failures.
assert ahc.AGENTS["abiba"]["runtime"] == "pi"
def test_koby_is_report_only(ahc):
# Captain's 2026-08-17 ruling (Rule 17): detect and report, never repair.
assert ahc.AGENTS["koby"]["report_only"] is True
def test_koby_ct111_is_on_storepve(ahc):
# Live-verified 2026-09-10: `pct status 111` = running on storepve (.6);
# amdpve has no lxc/111.conf, which is what false-failed before.
assert ahc.AGENTS["koby"]["pve"] == "storepve"
@pytest.mark.parametrize("agent", ["koby", "koonimo", "tanko"])
def test_report_only_legs_never_count_as_failures(ahc, agent):
ahc.FAIL.clear()
try:
ahc._fail(f"probe:{agent}", agent)
if ahc.AGENTS[agent].get("report_only"):
assert ahc.FAIL == []
else:
assert ahc.FAIL == [f"probe:{agent}"]
finally:
ahc.FAIL.clear()
def test_failure_recording_accepts_agentless_keys(ahc):
ahc.FAIL.clear()
try:
ahc._fail("gpu-no-port:gpu-rtx3090 (.8)")
assert ahc.FAIL == ["gpu-no-port:gpu-rtx3090 (.8)"]
finally:
ahc.FAIL.clear()
def test_script_reports_absolute_execution_provenance(ahc):
src = AHC.read_text()
assert "execution_path" in src
assert "os.getcwd()" in src
def test_wrapper_check_has_no_bare_infisical_expectation(ahc):
# A wrapper that never mentions infisical (koonimo sources ~/.hermes/.env)
# must not be FAILed merely for lacking an infisical reference.
assert "wrapper resolves creds without infisical" in AHC.read_text()
# ── item 4: every contract report carries its execution path ──────────
@pytest.mark.parametrize("contract", [GPU, INFRA, PROXMOX], ids=lambda p: p.name)
def test_report_format_requires_execution_provenance(contract):
text = contract.read_text()
assert "**Report format**" in text
assert re.search(r"absolute path|pwd -P|executed from", text)
# ── item 3: gpu-monitor probes the real GPU endpoints ────────────────
def test_gpu_monitor_probes_gpu_health_on_8080():
text = GPU.read_text()
assert "http://192.168.68.8:8080/health" in text
assert "http://192.168.68.110:8080/health" in text
def test_gpu_monitor_forbids_bare_port_80_on_gpu_hosts():
text = GPU.read_text()
assert re.search(r"never .{0,20}bare port 80", text, re.I)
# The false-DEGRADED symptom must be documented, not just implied.
assert "DEGRADED" in text
def test_gpu_monitor_documents_router_301_as_alive():
text = GPU.read_text()
assert "/gpu/gpu-data" in text
assert "301" in text
# ── item 2: infrastructure-monitoring PVE API vantage ────────────────
def test_infra_monitoring_does_not_probe_ct116_for_pve_api():
assert "https://192.168.68.116:8006" not in INFRA.read_text()
@pytest.mark.parametrize(
"node",
["192.168.68.9", "192.168.68.5", "192.168.68.15", "192.168.68.6", "192.168.68.12"],
)
def test_infra_monitoring_probes_every_real_pve_node(node):
assert node in INFRA.read_text()
def test_infra_monitoring_uses_any_http_liveness_rule():
text = INFRA.read_text()
assert "api2/json/version" in text
assert "ANY HTTP status" in text or "any-HTTP" in text