Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
edcf465831 | ||
|
|
6f40a3be60 | ||
|
|
d9368467ff | ||
|
|
876b011359 | ||
|
|
d23cce89e1 | ||
|
|
9ada2b23c7 | ||
|
|
bd0065bb31 | ||
|
|
f99f7e1e34 | ||
|
|
8210fd905c | ||
|
|
b3644f0292 | ||
|
|
672bf8a912 | ||
|
|
6e612ce37b | ||
|
|
357f808a82 | ||
|
|
6272d29978 | ||
|
|
4bd6132cf5 | ||
|
|
6d65cba064 | ||
|
|
de1428b4ae | ||
|
|
4ea2d0309f | ||
|
|
7b8cc5f9ac | ||
|
|
d9e06863d8 | ||
|
|
a2edc2f56f | ||
|
|
78b501798f | ||
|
|
1f1b47f59d | ||
|
|
3d55764799 | ||
|
|
ce48070f21 | ||
|
|
9e581ab203 | ||
|
|
9cac3589cf | ||
|
|
221f9f79f3 | ||
|
|
1dc040251d |
@@ -43,17 +43,21 @@ jobs:
|
||||
echo "=== Prose Contract Frontmatter Validation ==="
|
||||
FAILED=0
|
||||
for f in $(find . -name "*.prose.md" -not -path "./.git/*" -not -path "./runs/*"); do
|
||||
# NOTE: use herestrings, not `echo "$FM" | grep ...`. Under the runner's
|
||||
# `-e -o pipefail`, `grep -q` exits on first match and can SIGPIPE the
|
||||
# producer, making the pipeline report non-zero and raising a false
|
||||
# "Missing name/description" whose file set varies run to run.
|
||||
FM=$(sed -n '/^---$/,/^---$/p' "$f" | sed '1d;$d')
|
||||
[ -z "$FM" ] && { echo " ❌ $f: No YAML frontmatter"; FAILED=$((FAILED+1)); continue; }
|
||||
|
||||
KIND=$(echo "$FM" | grep '^kind:' | awk '{print $2}')
|
||||
KIND=$(grep '^kind:' <<< "$FM" | awk '{print $2}')
|
||||
case "$KIND" in
|
||||
function|responsibility|gateway|pattern|test|template|architecture|enforcement) echo " ✅ $f: kind=$KIND" ;;
|
||||
*) echo " ❌ $f: Invalid kind='$KIND'"; FAILED=$((FAILED+1)) ;;
|
||||
esac
|
||||
|
||||
echo "$FM" | grep -q '^name:' || { echo " ❌ $f: Missing name"; FAILED=$((FAILED+1)); }
|
||||
echo "$FM" | grep -q '^description:' || { echo " ❌ $f: Missing description"; FAILED=$((FAILED+1)); }
|
||||
grep -q '^name:' <<< "$FM" || { echo " ❌ $f: Missing name"; FAILED=$((FAILED+1)); }
|
||||
grep -q '^description:' <<< "$FM" || { echo " ❌ $f: Missing description"; FAILED=$((FAILED+1)); }
|
||||
done
|
||||
[ $FAILED -gt 0 ] && { echo "❌ FRONTMATTER FAILED ($FAILED error(s))"; exit 1; }
|
||||
echo "✅ Frontmatter validation passed"
|
||||
|
||||
@@ -55,7 +55,7 @@ Two incidents taught us this:
|
||||
### Stage 3 — AI Review
|
||||
- Diff is sent to `syslog-auto` model via LiteLLM
|
||||
- Review checks against infrastructure-control ground truth:
|
||||
- CT IDs match PVE cluster (100-117, no 122/123)
|
||||
- CT IDs match the PVE cluster inventory in `infrastructure-control.prose.md` Appendix B (100-120 with gaps; no 122/123)
|
||||
- Grafana is direct LAN :3001, NOT behind nginx
|
||||
- Zulip is CT 117 on storepve (bridge IP .19)
|
||||
- Strix Halo :8080 is firewalled to .116 only
|
||||
|
||||
@@ -88,7 +88,7 @@ prose run memory-audit-maintenance memory_threshold=90 verify_configs=true
|
||||
prose run hermes-config-template agent_name=syslog-devops default_model=claude-sonnet-4
|
||||
|
||||
# Configure an agent with a different auxiliary model
|
||||
prose run hermes-config-template agent_name=syslog-code default_model=qwen3.6-27B-code auxiliary_model=gpu-vision
|
||||
prose run hermes-config-template agent_name=syslog-code default_model=gpu-dense auxiliary_model=gpu-vision
|
||||
```
|
||||
|
||||
### Option B: Manual Execution
|
||||
|
||||
+85
-18
@@ -36,6 +36,59 @@ def warn(rule, message):
|
||||
WARNINGS.append(f"[{rule}] {message}")
|
||||
|
||||
|
||||
# Derivation rule: a model name is any scalar under a mapping key named `model` or
|
||||
# `model_name`, at any depth. The top-level `model:` SECTION is the exception where the model
|
||||
# name lives under `default`/`model`/`model_name` inside that section, so it is descended
|
||||
# specially. The only other exception is key `models` (litellm key-generation params carry a
|
||||
# list of model names). EXTEND THE ALLOWLIST for a new exception; do NOT add another field by
|
||||
# hand.
|
||||
MODEL_KEYS = ("model", "model_name")
|
||||
MODEL_SECTION_KEYS = ("default", "model", "model_name")
|
||||
MODEL_LIST_KEYS = ("models",)
|
||||
|
||||
|
||||
def _iter_model_values(node, path=""):
|
||||
"""Yield (path, value) for every model-name-bearing scalar in a config."""
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
child = f"{path}.{key}" if path else key
|
||||
if key in MODEL_KEYS:
|
||||
if isinstance(value, dict):
|
||||
for subkey in MODEL_SECTION_KEYS:
|
||||
subvalue = value.get(subkey)
|
||||
if isinstance(subvalue, str):
|
||||
yield (f"{child}.{subkey}", subvalue)
|
||||
for subkey, subvalue in value.items():
|
||||
if isinstance(subvalue, (dict, list)):
|
||||
yield from _iter_model_values(subvalue, f"{child}.{subkey}")
|
||||
elif isinstance(value, list):
|
||||
yield from _iter_model_values(value, child)
|
||||
else:
|
||||
yield (child, value)
|
||||
elif key in MODEL_LIST_KEYS:
|
||||
yield from _iter_model_list(value, child)
|
||||
elif isinstance(value, (dict, list)):
|
||||
yield from _iter_model_values(value, child)
|
||||
elif isinstance(node, list):
|
||||
for i, item in enumerate(node):
|
||||
yield from _iter_model_values(item, f"{path}[{i}]")
|
||||
|
||||
|
||||
def _iter_model_list(node, path):
|
||||
"""Yield scalars under an allowlisted `models` key (list of names or list of dicts)."""
|
||||
if isinstance(node, list):
|
||||
for i, item in enumerate(node):
|
||||
yield from _iter_model_list(item, f"{path}[{i}]")
|
||||
elif isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key in MODEL_KEYS and isinstance(value, str):
|
||||
yield (f"{path}.{key}", value)
|
||||
elif isinstance(value, (dict, list)):
|
||||
yield from _iter_model_list(value, f"{path}.{key}")
|
||||
else:
|
||||
yield (path, node)
|
||||
|
||||
|
||||
def audit(path):
|
||||
with open(path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
@@ -89,15 +142,16 @@ def audit(path):
|
||||
)
|
||||
|
||||
# --- Rule 8: GPU Workload Distribution ---
|
||||
# gpu-light (and gemma-4-12b) were retired 2026-09-12; the RTX 5070 stable alias is gpu-vision.
|
||||
check(
|
||||
aux.get("vision", {}).get("model") == "gpu-light",
|
||||
aux.get("vision", {}).get("model") == "gpu-vision",
|
||||
"Rule 8",
|
||||
f"auxiliary.vision.model must be gpu-light (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
f"auxiliary.vision.model must be gpu-vision (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
)
|
||||
check(
|
||||
aux.get("web_extract", {}).get("model") == "gpu-light",
|
||||
aux.get("web_extract", {}).get("model") == "gpu-vision",
|
||||
"Rule 8",
|
||||
f"auxiliary.web_extract.model must be gpu-light (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
f"auxiliary.web_extract.model must be gpu-vision (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
)
|
||||
|
||||
# --- Rule 9: Compression Threshold ---
|
||||
@@ -109,7 +163,7 @@ def audit(path):
|
||||
check(
|
||||
comp.get("max_context_window") == 131072,
|
||||
"Rule 9",
|
||||
f"compression.max_context_window must be 131072 (got {comp.get('max_context_window')!r}) — matches 128K GPU capacity",
|
||||
f"compression.max_context_window must be 131072 (got {comp.get('max_context_window')!r}) — syslog-auto pool floor (NVIDIA hosts 128K; Strix Halo 256K)",
|
||||
)
|
||||
|
||||
# --- Rule 10: Default Model Must Be syslog-auto ---
|
||||
@@ -178,21 +232,34 @@ def audit(path):
|
||||
f"custom_providers[0].base_url must end with /v1 (got {cp.get('base_url')!r})",
|
||||
)
|
||||
|
||||
# --- No raw model names (Rule 7/8 spirit) ---
|
||||
raw_names = {"gemma-4-12b", "qwen3.6-27B-code", "qwen3.6-35B-udq4", "ornith-1.0-35b"}
|
||||
for section_path, section_dict in [
|
||||
("model", model), ("compression", comp),
|
||||
("auxiliary.vision", aux.get("vision", {})),
|
||||
("auxiliary.web_extract", aux.get("web_extract", {})),
|
||||
("auxiliary.compression", aux.get("compression", {})),
|
||||
("delegation", deleg),
|
||||
]:
|
||||
m = section_dict.get("model", "")
|
||||
if m in raw_names:
|
||||
# --- Retired/raw model names (Rule 7/8 spirit) ---
|
||||
# The audit's job is to catch configs that are BROKEN, not to enforce a style preference.
|
||||
# NON-RESOLVING names (removed 2026-09-12, verified 400/403 via live LiteLLM) must hard-FAIL:
|
||||
# gpu-light -> gpu-vision ; gemma-4-12b -> gpu-vision
|
||||
# crew-auto -> syslog-auto (its 64K cap is retired; no cap in force) ; ornith-1.0-35b -> strix-moe
|
||||
# RESOLVING names (verified 200) are discouraged but working, so they only WARN:
|
||||
# qwen3.6-27B-code -> gpu-dense ; qwen3.6-35B-udq4 -> strix-moe
|
||||
# Failing a working alias would reject valid configs - the exact defect this change fixes.
|
||||
non_resolving = {
|
||||
"gpu-light": "gpu-vision",
|
||||
"gemma-4-12b": "gpu-vision",
|
||||
"crew-auto": "syslog-auto (its 64K cap is retired; no cap in force)",
|
||||
"ornith-1.0-35b": "strix-moe",
|
||||
"qwen3.6-27B-code": "gpu-dense",
|
||||
"qwen3.6-35B-udq4": "strix-moe",
|
||||
}
|
||||
raw_but_live = {}
|
||||
for field_path, value in _iter_model_values(cfg):
|
||||
if value in non_resolving:
|
||||
check(
|
||||
False,
|
||||
"Rule 7/8",
|
||||
f"{field_path} = {value!r} is retired and no longer resolves (2026-09-12) — use {non_resolving[value]}",
|
||||
)
|
||||
elif value in raw_but_live:
|
||||
warn(
|
||||
"Rule 7/8",
|
||||
f"{section_path}.model = {m!r} — raw model name, use stable alias instead "
|
||||
f"(gpu-light, gpu-dense, strix-moe, syslog-auto)",
|
||||
f"{field_path} = {value!r} is a raw-but-live model name — prefer the stable alias {raw_but_live[value]}",
|
||||
)
|
||||
|
||||
# --- Report ---
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
---
|
||||
report_only_agents:
|
||||
- koby # ⛔ KOBY IS NEVER REPAIRED (Rule 17, 2026-08-17) — detect + report, never fix on .129
|
||||
# ⛔ The guest/host-keyed report-only gate the GC executor MUST honour lives in the body
|
||||
# "Hard gate" YAML block below — that block is authoritative and is the only copy.
|
||||
kind: responsibility
|
||||
name: disk-gc-threat-response
|
||||
description: >
|
||||
Recurring disk health scan, garbage collection, and threat response
|
||||
across 15 Proxmox CTs + 3 GPU bare-metal hosts. Triggered by incident
|
||||
across 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts
|
||||
(fleet verified against `pvesh get /cluster/resources` 2026-09-12). Triggered by incident
|
||||
2026-07-04 where CT 105 (kagentz) hit 87% disk (49G/59G) from
|
||||
Docker image bloat — 5 dangling images, 15 build cache layers.
|
||||
Recovered 35.67GB. Second incident 2026-07-09: amdpve (.15) Docker
|
||||
@@ -25,7 +28,7 @@ logged within 5 minutes of discovery.
|
||||
|
||||
## Scope
|
||||
|
||||
All 15 CTs via `pct-run` + 3 GPU bare-metal hosts via direct SSH.
|
||||
All 20 Proxmox guests (17 LXC via `pct-run` + 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts via direct SSH.
|
||||
Docker hosts get special attention:
|
||||
|
||||
| Host | CT | Disk Risk | GC Strategy |
|
||||
@@ -99,35 +102,67 @@ and escalation trail.
|
||||
|
||||
## Execution
|
||||
|
||||
### Hard gate: report-only guests (READ THIS BEFORE RUNNING GC)
|
||||
|
||||
**CT 111 / hostname `tdunna` / 192.168.68.129 is DETECT-AND-REPORT-ONLY.** It belongs to Theo.
|
||||
The captain ruled 2026-08-17 and re-confirmed 2026-09-10 that Theo handles CT 111 himself.
|
||||
At **every** threat level — AMBER, RED, or CRITICAL — the executor must:
|
||||
|
||||
- push the threat row and alert the owner, and
|
||||
- **never** call `gc-executor`, and **never** run any GC command against that guest: no
|
||||
`apt-get clean/autoremove`, no `journalctl --vacuum-*`, no `find /var/log -delete`, no
|
||||
`/tmp`/`/var/tmp` deletion, no snap removal, no `docker system prune`.
|
||||
|
||||
This gate is keyed on **guest id / hostname / IP**, not on an agent name. The frontmatter
|
||||
`report_only_agents` marker (e.g. `koby`) names an AGENT while the scan unit is a GUEST, so an
|
||||
agent-name marker can silently miss the guest it lives on — it must never be the only gate.
|
||||
|
||||
**The authoritative machine-readable exclusion list is the YAML block below.** The executor
|
||||
reads it at run time; `scripts/disk-gc-plan.py` turns a fleet scan into the action plan using it.
|
||||
Extend the list here, never by hand-maintaining a second copy. The Execution loop below MUST
|
||||
call that planner and MUST NOT reimplement the gate.
|
||||
|
||||
```yaml
|
||||
# disk-gc report-only guests — authoritative. Keyed on guest/host, not agent.
|
||||
report_only_guests:
|
||||
- guest: 111
|
||||
hostname: tdunna
|
||||
ip: 192.168.68.129
|
||||
node: storepve
|
||||
reason: "Theo's box — captain ruling 2026-08-17, re-confirmed 2026-09-10"
|
||||
```
|
||||
|
||||
### Loop
|
||||
|
||||
```prose
|
||||
let fleet = call disk-scanner
|
||||
scope: all
|
||||
|
||||
let threats = []
|
||||
for ct in fleet:
|
||||
if ct.usage_pct >= 95:
|
||||
push threats { ct: ct.id, level: "CRITICAL", pct: ct.usage_pct }
|
||||
else if ct.usage_pct >= 85:
|
||||
push threats { ct: ct.id, level: "RED", pct: ct.usage_pct }
|
||||
else if ct.usage_pct >= 75:
|
||||
push threats { ct: ct.id, level: "AMBER", pct: ct.usage_pct }
|
||||
-- The report-only gate is IMPLEMENTED IN scripts/disk-gc-plan.py and MUST NOT be
|
||||
-- reimplemented here. That planner reads the contract's `report_only_guests` YAML block
|
||||
-- and matches on guest id OR hostname OR IP, so the tested gate is the executed gate.
|
||||
let plan = call disk-gc-plan
|
||||
fleet: fleet
|
||||
|
||||
-- sort by severity descending
|
||||
sort threats by pct desc
|
||||
for row in plan:
|
||||
if row.action == "report-only":
|
||||
-- Excluded guest: alert only. No gc-executor call is constructed for it, at any level.
|
||||
call alerter
|
||||
threat: row
|
||||
result: { action: "report-only", reason: row.reason }
|
||||
else:
|
||||
let result = call gc-executor
|
||||
ct: row.target
|
||||
level: row.level
|
||||
strategy: lookup-gc-strategy(row.target)
|
||||
|
||||
for threat in threats:
|
||||
let result = call gc-executor
|
||||
ct: threat.ct
|
||||
level: threat.level
|
||||
strategy: lookup-gc-strategy(threat.ct)
|
||||
|
||||
call alerter
|
||||
threat: threat
|
||||
result: result
|
||||
call alerter
|
||||
threat: row
|
||||
result: result
|
||||
|
||||
call summary-reporter
|
||||
fleet: fleet
|
||||
threats: threats
|
||||
plan: plan
|
||||
```
|
||||
|
||||
## GC Strategies by Host Type
|
||||
@@ -239,7 +274,9 @@ dangling images and orphaned build cache. No automated GC was in place.
|
||||
## Incident Log: 2026-07-09 — amdpve docker bloat
|
||||
|
||||
### Discovery
|
||||
Scheduled fleet disk scan via `pct-run` across all 15 CTs + 3 GPU bare-metal hosts.
|
||||
Scheduled fleet disk scan across all 20 Proxmox guests (17 LXC via `pct-run`, 3 QEMU VMs via direct SSH) + 3 GPU bare-metal hosts.
|
||||
> **Report-only gate applies to this scan:** CT 111 (`tdunna`, 192.168.68.129) is alerted but never
|
||||
> garbage-collected at any level.
|
||||
amdpve (.15) flagged at 78% (AMBER threshold: 75%).
|
||||
|
||||
### Diagnosis
|
||||
@@ -272,25 +309,35 @@ one-off GPU builds. No automated post-migration cleanup was in place.
|
||||
- Contract now scans GPU bare-metal hosts alongside CTs
|
||||
- Access via `pct-run` script for all CTs (no hardcoded IPs)
|
||||
|
||||
## Access Matrix (documented 2026-07-09)
|
||||
## Access Matrix (verified against `pvesh get /cluster/resources` 2026-09-12)
|
||||
|
||||
### CT Access (via pct-run)
|
||||
| CT | Name | Node | Status |
|
||||
|----|------|------|--------|
|
||||
| 100 | abiba | minipve | local |
|
||||
| 102 | adguard | minipve | ✅ reachable |
|
||||
| 104 | authentik | minipve | ✅ reachable |
|
||||
| 105 | kagentz | minipve | ✅ reachable |
|
||||
| 106 | ra-h-os | storepve | ✅ reachable |
|
||||
| 107 | pbs | storepve | ✅ reachable |
|
||||
| 108 | media | storepve | ✅ reachable |
|
||||
| 110 | gitea | minipve | ✅ reachable |
|
||||
| 111 | tdunna | amdpve | ✅ reachable |
|
||||
| 112 | tanko | amdpve | ✅ reachable |
|
||||
| 113 | baggy | amdpve | ✅ reachable |
|
||||
| 115 | scottdenya | amdpve | ✅ reachable |
|
||||
| 116 | syslog-api | minipve | ✅ reachable |
|
||||
| 117 | zulip | storepve | ✅ reachable |
|
||||
### Guest Access (via `pct-run` — CT id only, node resolved by `scripts/pct-run.sh`)
|
||||
| Guest | Name | Node | Type | Status |
|
||||
|------|------|------|------|--------|
|
||||
| 100 | abiba | minipve | lxc | ✅ reachable (probed via pct-run like any other guest; no local shortcut) |
|
||||
| 102 | adguard | minipve | lxc | ✅ reachable |
|
||||
| 104 | authentik | minipve | lxc | ✅ reachable |
|
||||
| 105 | kagentz | **amdpve** | lxc | ✅ reachable (was documented as minipve — corrected) |
|
||||
| 106 | ra-h-os | storepve | lxc | ✅ reachable |
|
||||
| 107 | pbs | storepve | lxc | ✅ reachable |
|
||||
| 108 | media | storepve | lxc | ✅ reachable |
|
||||
| 110 | gitea | minipve | lxc | ✅ reachable |
|
||||
| 111 | tdunna | **storepve** | lxc | ⛔ **REPORT-ONLY** (192.168.68.129, Theo's box — no GC at any level) |
|
||||
| 112 | tanko | amdpve | lxc | ✅ reachable |
|
||||
| 113 | baggy | amdpve | lxc | ✅ reachable |
|
||||
| 115 | scottdenya | amdpve | lxc | ✅ reachable |
|
||||
| 116 | syslog-api | minipve | lxc | ✅ reachable |
|
||||
| 117 | zulip | storepve | lxc | ✅ reachable |
|
||||
| 118 | jdownloader | storepve | lxc | ✅ reachable |
|
||||
| 119 | infisical-vault | minipve | lxc | ✅ reachable |
|
||||
| 120 | adguard2 | amdpve | lxc | ✅ reachable |
|
||||
|
||||
### QEMU VMs (via direct SSH)
|
||||
| VM | Name | Node | IP | Status |
|
||||
|----|------|------|-----|--------|
|
||||
| 101 | llm-gpu (workload now bare metal .8) | acerpve | — | ✅ reachable |
|
||||
| 103 | ocu-llm (workload now bare metal .110) | ocupve | — | ✅ reachable |
|
||||
| 109 | docker-vm | storepve | 192.168.68.7 | ✅ reachable |
|
||||
|
||||
### GPU Bare Metal (via direct SSH)
|
||||
| Host | IP | GPU | Status |
|
||||
@@ -299,9 +346,14 @@ one-off GPU builds. No automated post-migration cleanup was in place.
|
||||
| ocu-llm | 192.168.68.110 | RTX 5070 | ✅ reachable |
|
||||
| amdpve | 192.168.68.15 | Strix Halo | ✅ reachable |
|
||||
|
||||
### KVM VM (via direct SSH)
|
||||
| Host | IP | Role | Status |
|
||||
|------|-----|------|--------|
|
||||
| docker-vm | 192.168.68.7 | 16 Docker containers, 4 stacks | ✅ reachable |
|
||||
|
||||
> **Note:** CT 118 is now jdownloader (active on storepve). CT 119 (infisical-vault) added on minipve.\n> **Migrated:** CT 101 → .8, CT 103 → .110 (bare metal GPU).\n> **KVM VM:** CT 109 (docker-vm) is a KVM VM, not LXC — access via SSH .7.
|
||||
> **Fleet count:** 20 Proxmox guests (17 LXC + 3 QEMU VMs) + 3 GPU bare-metal hosts. Corrected
|
||||
> 2026-09-12: CT 105 → amdpve, CT 111 → storepve, and guests 118/119/120 were missing.
|
||||
>
|
||||
> **CT 100 probe gap (folded in):** CT 100 previously reported "unreachable (not reported)" every
|
||||
> run. Root cause is the same stale access layer: `pct-run` resolves the guest's node from its map,
|
||||
> and the map/contract must reflect `pvesh /cluster/resources`. Verified working from inside CT 100:
|
||||
> `scripts/pct-run.sh 100 "df -P / | tail -1"` → `23% /`. Probe CT 100 through `pct-run` like any
|
||||
> other guest — never through a local-only path, since the scanner itself runs inside CT 100 and a
|
||||
> container has no `pct` binary.
|
||||
>
|
||||
> **KVM VM:** CT 109 (docker-vm) is a QEMU VM, not LXC — access via SSH .7.
|
||||
|
||||
@@ -261,6 +261,10 @@ repaired.
|
||||
not exist` on .15. `agent-health-check.py` now carries the live-verified
|
||||
`storepve` mapping (the script is not the topology source of truth); the
|
||||
CRITICAL contract itself needs an authorized correction.
|
||||
**✅ Resolved 2026-09-12:** the topology was corrected in its owner,
|
||||
`infrastructure-control.prose.md` (CT 111 → storepve, CT 105 → amdpve), and
|
||||
`scripts/pct-run.sh` now matches. This snapshot is left as observed; treat
|
||||
those owner documents as authoritative.
|
||||
2. **Strix Halo `:8080` firewall claim is stale.** `prose-ai-review.sh`
|
||||
ground-truth rule #4 and `gpu-monitor.prose.md` say `:8080` is firewalled to
|
||||
`.116` only and `.24` cannot probe it. Live on .15:
|
||||
|
||||
+23
-23
@@ -8,12 +8,12 @@ description: >
|
||||
UPDATED 2026-07-15: Stable role-based aliases introduced: strix-moe,
|
||||
gpu-dense, gpu-vision (gpu-light was superseded by gpu-vision on 2026-09-12).
|
||||
These never change — only the underlying model does.
|
||||
Strix Halo: strix-moe → unsloth/Qwen3.6-35B-A3B-MTP (UD-Q4_K_M, 22GB).
|
||||
Strix Halo: strix-moe → Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf (22GB, 256K ctx).
|
||||
RTX 5070: gpu-vision — IQ4_NL + MTP draft (~122 tok/s, 2x faster).
|
||||
UPDATED 2026-07-17: Context reduced fleet-wide from 256K to 128K for stability.
|
||||
Strix Halo model swapped to qwen3.6-35B-udq4 (22GB, strix-moe alias).
|
||||
Instability observed near 100K at 256K (now all GPUs at 128K). 128K is the stable ceiling.
|
||||
For larger context needs → fall back to external providers (deepseek).
|
||||
UPDATED 2026-07-17: NVIDIA host context reduced from 256K to 128K for stability.
|
||||
Strix Halo model: Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf (alias strix-moe, 256K context).
|
||||
Strix Halo runs 256K (n_ctx 262144, --kv-unified); RTX 3090 and RTX 5070 remain at 128K.
|
||||
For >128K on NVIDIA hosts → fall back to external providers (deepseek).
|
||||
VRAM headroom improved: RTX 3090 ~70%, RTX 5070 ~65%.
|
||||
agent: abiba
|
||||
triggers:
|
||||
@@ -91,8 +91,8 @@ Single source of truth for models, aliases, rpm caps, weights and fallback chain
|
||||
CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not duplicate those values in
|
||||
contracts — read them there.
|
||||
|
||||
**Backward compatibility**: Old model-specific names (qwen3.6-27B-code, qwen3.5-9b-it) still work
|
||||
but are deprecated for agent configs. Only the stable aliases survive model swaps. `gemma-4-12b`
|
||||
**No backward compatibility**: Old model-specific names (qwen3.6-27B-code, qwen3.5-9b-it) are retired as of 2026-09-12
|
||||
and no longer resolve; do not use them in agent configs. Only the stable aliases survive model swaps. `gemma-4-12b`
|
||||
is retired and returns 400 `Invalid model name`.
|
||||
|
||||
## Routing Configuration (LiteLLM — July 2026)
|
||||
@@ -190,7 +190,7 @@ If no SSH access, send Zulip DM via abiba-bot with vault update instructions.
|
||||
| `/root/scripts/gpu-saturation-watchdog.py` | pi (.24) | Auto-restart stuck llama-server |
|
||||
| `/root/dashboard/gpu-fleet.html` | pi (.24) | Live HTML dashboard |
|
||||
| `/etc/systemd/system/llama-server.service` | .8, .110 | llama-server daemons (Nvidia GPUs) |
|
||||
| `/etc/systemd/system/strix-server.service` | .15 (amdpve) | llama-server daemon (Vulkan, Strix Halo) running unsloth/Qwen3.6-35B-A3B-MTP-GGUF. Note: `llama-server.service` and `llama-server@.service` are **masked** on .15 to prevent port 8080 collisions. |
|
||||
| `/etc/systemd/system/strix-server.service` | .15 (amdpve) | llama-server daemon (Vulkan, Strix Halo) running Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf (256K ctx). Note: `llama-server.service` and `llama-server@.service` are **masked** on .15 to prevent port 8080 collisions. |
|
||||
|
||||
## Prometheus & Grafana
|
||||
|
||||
@@ -207,7 +207,7 @@ If no SSH access, send Zulip DM via abiba-bot with vault update instructions.
|
||||
- **Router startup race**: Compose router.py doesn't call load_roster(). Reload thread sleeps 30s first.
|
||||
Fix: trigger roster reload via SSH after restart, or rebuild image with startup load_roster().
|
||||
- **LiteLLM /metrics**: Requires auth. Prometheus uses `/health/liveliness` as workaround.
|
||||
- **Strix Halo GPU**: Vulkan is the working backend (ROCm/HIP path abandoned — HSA runtime blocked on Debian 13). Build at `/root/llama.cpp/build-vk/`, commit `4fc4ec5` (2026-07-01), ggml 0.15.3 shared-lib arch. Mesa RADV 25.0.7, KHR_coopmat fast path active. ~70 tok/s gen, 532 tok/s prompt. Service: `strix-server.service` on port 8080, model: `qwen3.6-35B-udq4`, alias `strix-moe`, 128K context, flash-attn + q4 KV, multimodal (mmproj loaded).
|
||||
- **Strix Halo GPU**: Vulkan is the working backend (ROCm/HIP path abandoned — HSA runtime blocked on Debian 13). Build at `/root/llama.cpp/build-vk/`, commit `4fc4ec5` (2026-07-01), ggml 0.15.3 shared-lib arch. Mesa RADV 25.0.7, KHR_coopmat fast path active. ~70 tok/s gen, 532 tok/s prompt. Service: `strix-server.service` on port 8080, model: `Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf`, alias `strix-moe`, 256K context (n_ctx 262144), --parallel 2 --kv-unified, flash-attn + q4 KV, multimodal (mmproj loaded).
|
||||
- **Port conflict detection (2026-07-05)**: All 3 GPU wrappers now detect ghost processes squatting port 8080 before starting. `.8` and `.110` use inline pre-start check in `llama-wrapper.sh`; `.15` uses `/usr/local/bin/port-cleanup.sh` ExecStartPre. Replaces the blanket `pkill -9 -x llama-server` on .15 which would kill ALL llama-server instances regardless of port. Ghost detection was the root cause of .8 crash-looping for 27+ restarts (stale pid 25836 squatting 8080 after OOM kill).
|
||||
- **Strix Halo thermal safeguard (2026-07-02)**: `strix-server.service` has `-n 8192` (hard generation cap per request). Without it, `--predict` defaults to -1 (infinity) — a runaway request from .123 (old Mumuni CT114 — now inside Abiba CT100 at .24) decoded 39,868 tokens over 24 min, pushing Tctl to 98°C (crit 89.8°C) and throttling 70→29 t/s. The cap bounds worst-case generation to ~5 min. Do NOT remove `-n` without a replacement ceiling. Sustained load hits ~84°C even at 92s; the APU is fanless/low-flow. Clients MUST also set `max_tokens`.
|
||||
- **Port 8080 firewall**: amdpve iptables restricts 8080 to 192.168.68.116 (LiteLLM/router host) only. All inbound connections are from .116 (LiteLLM proxied via nginx). Localhost curls hang (SYN dropped). Always test from .116.
|
||||
@@ -216,17 +216,17 @@ If no SSH access, send Zulip DM via abiba-bot with vault update instructions.
|
||||
- **Alert migration**: All alerts now go to `#agent-hub` topics (`alerts-gpu`, `alerts-pm2`, `alerts-infra`) instead of DMs. Cross-agent visibility enabled.
|
||||
- **tok/s benchmarks**: Measured every 5 min via LiteLLM proxy. Baselines tracked with 30%/50% degradation thresholds.
|
||||
- **NetBird 502**: Tanko routes through NetBird for litellm.sysloggh.net. Use direct IP if NetBird down.
|
||||
- **Alias-retirement follow-up (2026-09-12)**: the agent-facing templates (`hermes-config-template.prose.md`, `hermes-agent-baseline.prose.md`), `litellm-api-keys.prose.md`, and the executable `audit-hermes-config.py` still reference the retired `gpu-light`/`gemma-4-12b` names, and `audit-hermes-config.py` Rule 8 currently fails a config whose vision/web_extract model is `gpu-vision`. These are tracked separately and are intentionally NOT updated in this change.
|
||||
- **Alias-retirement sweep (2026-09-12)**: `gemma-4-12b`, `gpu-light` and `crew-auto` are retired and replaced by `gpu-vision` / no cap respectively. The agent-facing templates (`hermes-config-template.prose.md`, `hermes-agent-baseline.prose.md`), `litellm-api-keys.prose.md`, `gpu-self-heal.prose.md`, `hermes-key-enforcement.prose.md`, `inference-optimization.prose.md`, `litellm-client-timeouts.prose.md` and the executable `audit-hermes-config.py` were all updated to the live canonical alias in the same change. **koby's config on .129 still names `gpu-light` (and `gemma-4-E4B`); .129 is report-only, so that is recorded for its owner and NOT edited here.**
|
||||
|
||||
## GPU Inference Benchmarks (Current)
|
||||
|
||||
| GPU | Model | Gen tok/s | Prompt tok/s | Baseline | Context |
|
||||
|-----|-------|-----------|--------------|----------|---------|
|
||||
| RTX 3090 (.8) | Qwen3.8-27B-Uncensored-Q4_K_M | **TBD** | — | — | **128K** |
|
||||
| Strix Halo (.15) | qwen3.6-35B-udq4 | **65** | 140 | — | **128K** |
|
||||
| Strix Halo (.15) | Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf (strix-moe) | **65** | 140 | — | **256K** |
|
||||
|
||||
Benchmarks from 2026-07-17. Strix Halo model: qwen3.6-35B-udq4. RTX 5070 MTP provides 2.7x speedup over pre-upgrade 70 tok/s.
|
||||
All 3 GPUs now at 128K context (2026-07-17, reduced from 256K for stability).
|
||||
Benchmarks from 2026-07-17. Strix Halo model: Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf (alias strix-moe), n_ctx 262144, --parallel 2 --kv-unified. RTX 5070 MTP provides 2.7x speedup over pre-upgrade 70 tok/s.
|
||||
GPU contexts: RTX 3090 (.8) and RTX 5070 (.110) at 128K; Strix Halo (.15) at 256K (2026-09-12).
|
||||
|
||||
Benchmarks run through LiteLLM proxy (192.168.68.116:4000) every 5 minutes.
|
||||
Degradation alerts fire at 30% (warning) and 50% (critical) below baseline.
|
||||
@@ -239,7 +239,7 @@ History stored at `/root/data/toks-history.json` with 7-day rolling window.
|
||||
### Stable Aliases — CRITICAL
|
||||
|
||||
All agent configs MUST use stable role-based aliases, never model-specific names:
|
||||
- `compression.model: strix-moe`
|
||||
- `compression.model: syslog-auto`
|
||||
- `auxiliary.vision.model: gpu-vision`
|
||||
- `delegation.model: gpu-dense`
|
||||
- `auxiliary.web_extract.model: gpu-vision`
|
||||
@@ -247,27 +247,27 @@ All agent configs MUST use stable role-based aliases, never model-specific names
|
||||
When the underlying model is swapped, only the LiteLLM config changes — agent configs are untouched.
|
||||
|
||||
### Context Windows
|
||||
- RTX 3090: **128K** (reduced from 256K 2026-07-17) | RTX 5070: **128K** (reduced from 256K) | Strix Halo: **128K**
|
||||
- **All agents**: 128K ceiling — stable margin. For >128K workloads, use external providers (deepseek)
|
||||
- Compression threshold 0.60: fires at ~77K (~51K headroom before 128K ceiling)
|
||||
- RTX 3090: **128K** (reduced from 256K 2026-07-17) | RTX 5070: **128K** (reduced from 256K) | Strix Halo: **256K** (2026-09-12)
|
||||
- **Agents via `syslog-auto`**: 128K ceiling — the pool's safe floor (NVIDIA hosts are 128K). For >128K workloads, use external providers (deepseek)
|
||||
- Compression threshold 0.65 (audit Rule 9): fires at ~85K (~43K headroom before 128K ceiling)
|
||||
- **Pi agents (Abiba)**: `compaction.reserveTokens: 52739` (≈60% of 128K)
|
||||
- Mumuni compression model alias: `strix-moe`
|
||||
- Mumuni compression model alias: `syslog-auto`
|
||||
|
||||
### Mumuni Agent Profile
|
||||
|
||||
Mumuni (kagentz CT105, 192.168.68.14 — migrated from CT100 2026-08-29) is the primary business assistant. This profile is the reference for all agent configs:
|
||||
Mumuni (kagentz CT105, 192.168.68.14 — migrated from CT100 2026-08-29) is the primary business assistant. This profile is the reference for all agent configs. The compression values below are the current required values per template Rules 7/9 and `audit-hermes-config.py`; whether Mumuni's LIVE config currently complies is a separate operational question.
|
||||
|
||||
| Setting | Value | Notes |
|
||||
|---------|-------|-------|
|
||||
| `model.default` | `syslog-auto` | Balanced default (pool router) |
|
||||
| `model.provider` | `custom:litellm` | LiteLLM on CT116 |
|
||||
| `compression.model` | `strix-moe` | Stable alias — survives model swaps |
|
||||
| `aux.compression.model` | `strix-moe` | Compression auxiliary model |
|
||||
| `compression.model` | `syslog-auto` | Rule 7: auto-routing, prevents Strix Halo overload |
|
||||
| `aux.compression.model` | `syslog-auto` | Must match `compression.model` (Rule 7) |
|
||||
| `aux.vision.model` | `gpu-vision` | Vision tasks (RTX 5070) |
|
||||
| `aux.web_extract.model` | `gpu-vision` | Web extraction |
|
||||
| `delegation.model` | `gpu-dense` | Sub-agent reasoning (RTX 3090) |
|
||||
| `context.max_context_window` | 131072 (128K) | Reduced from 256K 2026-07-17 — stable 128K ceiling |
|
||||
| `compression.threshold` | 0.60 | Triggers at ~77K (~60% of 128K) — optimized for 128K context |
|
||||
| `context.max_context_window` | 131072 (128K) | Conservative `syslog-auto` pool floor (NVIDIA hosts 128K; Strix Halo 256K) |
|
||||
| `compression.threshold` | 0.65 | Rule 9: triggers at ~85K for a 128K window |
|
||||
| `compression.target_ratio` | 0.3 | Compresses to ~38K |
|
||||
| `compression.protect_last_n` | 40 | Preserves last 40 messages |
|
||||
| `memory.memory_char_limit` | 800 | Brief memory entries |
|
||||
|
||||
@@ -27,8 +27,8 @@ agent: abiba
|
||||
┌──────┐ ┌──────┐ ┌────────┐
|
||||
│.8:8080│ │.110 │ │.116:80 │
|
||||
│RTX3090│ │:8080 │ │nginx │
|
||||
│gemma │ │RTX5070│ │router │
|
||||
└──────┘ │qwen27B│ │LiteLLM │
|
||||
│qwen │ │RTX5070│ │router │
|
||||
└──────┘ │vision │ │LiteLLM │
|
||||
└──────┘ │dashboard│
|
||||
└────────┘
|
||||
```
|
||||
|
||||
+13
-12
@@ -12,7 +12,7 @@ description: >
|
||||
Router (port 9000) DECOMMISSIONED 2026-09-11; references replaced with direct GPU routing.
|
||||
Benchmark baselines refreshed to live values.
|
||||
Prometheus exporters removed — not deployed; fall back to direct sidecar probes.
|
||||
Stable role-based aliases (strix-moe, gpu-dense, gpu-light) from gpu-fleet.
|
||||
Stable role-based aliases (strix-moe, gpu-dense, gpu-vision) from gpu-fleet.
|
||||
agent: abiba
|
||||
depends_on:
|
||||
- gpu-monitor.prose.md (live data source on .24:9100)
|
||||
@@ -48,12 +48,12 @@ depends_on:
|
||||
|
||||
| Alias | GPU | Host | Model | VRAM | Ctx | tok/s | Role |
|
||||
|-------|-----|------|-------|------|-----|-------|------|
|
||||
| `strix-moe` | Strix Halo 64GB | ct15 (.15:8080) | qwen3.6-35B-udq4 | ~10/64GB (16%) | 128K | 62.9 | Compression, summarization, long docs |
|
||||
| `strix-moe` | Strix Halo 64GB | ct15 (.15:8080) | Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf | ~10/64GB (16%) | 256K | 62.9 | Compression, summarization, long docs |
|
||||
|
||||
Key notes:
|
||||
- All models use direct GPU routing via LiteLLM (`api_key: not-needed`). Router (port 9000) was decommissioned 2026-09-11 and is NOT in the inference path.
|
||||
- Stable aliases (gpu-dense, gpu-light, strix-moe) from gpu-fleet are the canonical names for agent configs. Model-specific names still work but are deprecated.
|
||||
- RTX 5070 tok/s is ~145 for Qwen3.5-9B — gpu-light is the fastest endpoint. Route vision/web/light work there first. NOTE: Qwen3.5-9B is multimodal (image+text), NOT text-only like gemma-4-12b was.
|
||||
- Stable aliases (gpu-dense, gpu-vision, strix-moe) from gpu-fleet are the canonical names for agent configs — use these, not model-specific names. The retired names `gpu-light` and `gemma-4-12b` were superseded by `gpu-vision` on 2026-09-12 and no longer resolve (400 `Invalid model name`).
|
||||
- The RTX 5070 is the fastest endpoint per token — `gpu-vision` is its canonical alias. Route vision/web/light work there first. The RTX 5070 model is multimodal (image+text).
|
||||
- Strix Halo is 62.9 tok/s (89% of 70.5 baseline) — below optimal but stable. Check for competing workloads.
|
||||
- RTX 3090 VRAM at 88% — within role-appropriate range (role = heavy reasoning, needs the headroom).
|
||||
- RTX 5070 VRAM at 83% — role-appropriate for vision/web (smaller batch sizes).
|
||||
@@ -64,7 +64,7 @@ Key notes:
|
||||
- **Detect**: Any GPU temp >85°C sustained for 2+ consecutive polls
|
||||
- **Fix**:
|
||||
1. Reduce inference concurrency on that GPU (load-side cooling only — NO fan control)
|
||||
2. Redirect new requests to cooler GPUs via LiteLLM fallback chains (gemma → qwen, qwen → gemma)
|
||||
2. Redirect new requests to cooler GPUs via LiteLLM fallback chains (gpu-vision → gpu-dense, gpu-dense → gpu-vision)
|
||||
3. If all GPUs hot, alert about cooling infrastructure
|
||||
- **Verify**: Temp drops below 80°C within 5 minutes
|
||||
- **Escalate after**: 3 verification failures → Zulip alert
|
||||
@@ -142,10 +142,10 @@ Key notes:
|
||||
- **Escalate**: If Tier 2 triggers and temp still rising after 5 min → possible hardware failure
|
||||
|
||||
### Rule 9: Context Window Optimization
|
||||
- **Detect**: Benchmark tok/s vs baseline for each GPU at current context (all 128K)
|
||||
- **Detect**: Benchmark tok/s vs baseline for each GPU at its current context
|
||||
- RTX 3090 (128K ctx, ThinkingCap): baseline 74.8 tok/s — currently at 74.9 (100%)
|
||||
- RTX 5070 (128K ctx, HauhauCS QAT): baseline 165.2 tok/s — currently at 169.6 (103%)
|
||||
- Strix Halo (128K ctx, qwen3.6-35B-udq4): baseline 70.5 tok/s — currently at 62.9 (89%)
|
||||
- Strix Halo (256K ctx, Carnice-Qwen3.6-MoE-35B-A3B-Q4_K_M.gguf): baseline 70.5 tok/s — currently at 62.9 (89%)
|
||||
- **Fix**:
|
||||
- If tok/s > baseline → context has headroom, consider increasing
|
||||
- If tok/s < 90% baseline → reduce context by 25% and retest
|
||||
@@ -157,13 +157,14 @@ Key notes:
|
||||
### Rule 10: Workload Distribution Optimization (updated 2026-07-18)
|
||||
- **Detect**: GPU roles misaligned with hardware capabilities
|
||||
- **Target distribution**:
|
||||
- RTX 3090 (gpu-dense, 24GB, 74.9 tok/s) → Heavy reasoning, code gen, long conversations (slowest per-token but largest context capacity). Weight: 0.55 (LiteLLM).
|
||||
- RTX 5070 (gpu-light, 12GB, ~145 tok/s) → Vision (image+text), web search, lightweight tasks. Weight: 0.15 (LiteLLM).
|
||||
- Strix Halo (strix-moe, 64GB, 62.9 tok/s) → Context compression, summarization, long docs (MoE model). Weight: 0.30 (LiteLLM).
|
||||
- RTX 3090 (gpu-dense, 24GB, 74.9 tok/s) → Heavy reasoning, code gen, long conversations (slowest per-token but largest context capacity).
|
||||
- RTX 5070 (gpu-vision, 12GB, ~145 tok/s) → Vision (image+text), web search, lightweight tasks.
|
||||
- Strix Halo (strix-moe, 64GB, 62.9 tok/s) → Context compression, summarization, long docs (MoE model).
|
||||
- **Note**: RTX 5070 is the fastest endpoint per token. Route high-volume, low-complexity work there first.
|
||||
- **Weights are not restated here** — the live `syslog-auto` pool weights and rpm caps live in CT 116 `/opt/inference-harness/litellm_config.yaml`, the single source of truth.
|
||||
- **Fix**:
|
||||
- Alert if any GPU is handling workload outside its designated role
|
||||
- Recommend agent alias updates to match workload to GPU role (use stable aliases: gpu-dense, gpu-light, strix-moe)
|
||||
- Recommend agent alias updates to match workload to GPU role (use stable aliases: gpu-dense, gpu-vision, strix-moe)
|
||||
- Track per-GPU request distribution via LiteLLM spend logs
|
||||
- **Verify**: Each GPU's request pattern matches its designated role within 24h
|
||||
- **Escalate**: If role mismatch persists >48h → agent alias audit needed
|
||||
@@ -311,6 +312,6 @@ Pushed to `SyslogSolution/health-logs/gpu/{run_id}.json` — versioned, searchab
|
||||
If monitor response > 1MB, log a warning and skip the cycle rather than crashing.
|
||||
|
||||
### L6: Stable Aliases Replace Model Names
|
||||
- gpu-fleet introduced stable aliases (strix-moe, gpu-dense, gpu-light) on 2026-07-15.
|
||||
- gpu-fleet introduced stable aliases (strix-moe, gpu-dense, gpu-light) on 2026-07-15; `gpu-light` was superseded by `gpu-vision` on 2026-09-12.
|
||||
- Self-heal must use aliases for reporting and alerting, not model-specific names.
|
||||
- **Rule**: All alert messages and KG nodes use the stable alias as the GPU identifier.
|
||||
|
||||
@@ -5,7 +5,7 @@ version: 1.0.0
|
||||
description: >
|
||||
Canonical known-good baseline for all Syslog Hermes agents. Captures the exact
|
||||
configuration state, keys, workarounds, and audit procedure. When an agent's
|
||||
configuration goes sideways, restore from this baseline. Last verified 2026-07-16. All GPUs 128K context (reduced from 256K for stability Jul 2026) (RTX 3090 .8, RTX 5070 .110, Strix Halo .15). Parallel 1 fleet-wide (Strix Halo handles compression solo).
|
||||
configuration goes sideways, restore from this baseline. Last verified 2026-07-16. RTX 3090/5070 at 128K (reduced from 256K for stability Jul 2026); Strix Halo at 256K (2026-09-12) (RTX 3090 .8, RTX 5070 .110, Strix Halo .15). Parallel 1 fleet-wide (Strix Halo handles compression solo).
|
||||
author: Abiba (pi agent)
|
||||
---
|
||||
|
||||
@@ -24,11 +24,12 @@ done
|
||||
|
||||
| Agent | CT | Node | IP | LiteLLM Alias | Key Source | Platform |
|
||||
|-------|-----|------|-----|---------------|------------|----------|
|
||||
| Koby | 111 | amdpve | .129 | `koby` | Infisical vault | **Hermes** |
|
||||
| Koby | 111 | storepve | .129 | `koby` | Infisical vault | **Hermes** |
|
||||
| Koonimo | 113 | amdpve | .114 | `koonimo` | Infisical vault | Hermes |
|
||||
| Shumba | — | 192.168.68.119 | N/A | N/A (DeepSeek) | Hermes (RETIRED — CT119 now Infisical vault) |
|
||||
|
||||
> **Note**: CT hostnames (tdunna→CT111, baggy→CT113) differ from agent identities (koby, koonimo).
|
||||
> CT 111 (tdunna, 192.168.68.129, storepve) is report-only — Theo's box; alert only, never garbage-collect.
|
||||
|
||||
Access: `pct-run <CT_ID> <command>` — no IPs needed. GPU hosts (.8, .110, .15) use SSH.
|
||||
Keys are stored in Infisical vault (project=agents, env=production) and injected at
|
||||
@@ -80,7 +81,7 @@ custom_providers:
|
||||
auxiliary:
|
||||
vision:
|
||||
provider: harness
|
||||
model: gemma-4-12b # or syslog-auto
|
||||
model: gpu-vision # RTX 5070 stable alias (Rule 8; do not use syslog-auto for aux)
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
api_key_env: LITELLM_API_KEY
|
||||
api_key: <value from: infisical secrets get LITELLM_API_KEY --project=agents --env=production> # ← MANDATORY workaround
|
||||
@@ -95,7 +96,7 @@ auxiliary:
|
||||
threshold: 0.65
|
||||
target_ratio: 0.3
|
||||
provider: harness
|
||||
model: syslog-auto # or gemma-4-12b
|
||||
model: syslog-auto # Rule 7: compression must be syslog-auto
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
api_key_env: LITELLM_API_KEY
|
||||
api_key: <value from: infisical secrets get LITELLM_API_KEY --project=agents --env=production> # ← MANDATORY workaround
|
||||
@@ -185,7 +186,9 @@ Abiba (CT100) runs pi via PM2 with the Zulip extension.
|
||||
Config files: `~/.pi/agent/models.json`, `~/.pi/agent/settings.json`.
|
||||
|
||||
**models.json** — Must only list models authorized for the agent's LiteLLM key.
|
||||
Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
`/v1/models` is key-scoped and the live registry is CT 116
|
||||
`/opt/inference-harness/litellm_config.yaml`; treat the list below as a snapshot and re-read
|
||||
the registry before applying. Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
@@ -197,9 +200,7 @@ Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
{ "id": "syslog-auto" },
|
||||
{ "id": "strix-moe" },
|
||||
{ "id": "gpu-dense" },
|
||||
{ "id": "gpu-light" },
|
||||
{ "id": "qwen3.6-27B-code" },
|
||||
{ "id": "gemma-4-12b" }
|
||||
{ "id": "gpu-vision" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,17 +92,17 @@ work immediately after restart.
|
||||
```yaml
|
||||
# ─── Model Selection ───
|
||||
model:
|
||||
default: <agent_model> # e.g., strix-moe, qwen3.6-27B-code, syslog-auto
|
||||
default: <agent_model> # e.g., strix-moe, gpu-dense, syslog-auto
|
||||
provider: harness
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated path; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY # Injected via infisical run -- wrapper
|
||||
max_tokens: 4096 # ⚠️ CRITICAL: Prevents unbounded generation
|
||||
context_length: 131072 # For syslog-auto (all GPUs at 128K for stability).
|
||||
context_length: 131072 # Conservative floor for syslog-auto (NVIDIA hosts 128K; Strix Halo 256K).
|
||||
# ⚠️ MANDATORY: Hermes probes unknown models from 256K
|
||||
# and falls back to 256K when /v1/models lacks a context
|
||||
# field (llama-server does). Without this override, agents
|
||||
# silently run syslog-auto at 256K (verified 2026-08-09).
|
||||
# Set 65536 if using gemma-4-12b directly (tight VRAM).
|
||||
# Set 65536 if pinning a single model directly (tight VRAM).
|
||||
|
||||
fallback_providers:
|
||||
provider: deepseek
|
||||
@@ -133,7 +133,7 @@ compression:
|
||||
enabled: true
|
||||
model: syslog-auto # ⚠️ Must match auxiliary.compression.model. Stable alias (gpu-fleet § Stable Role-Based Aliases). NOT ornith-1.0-35b (LiteLLM does not serve that name).
|
||||
provider: harness
|
||||
max_context_window: 131072 # MUST match actual GPU capacity. All 3 GPUs are 128K (Jul 17).
|
||||
max_context_window: 131072 # MUST stay at the syslog-auto pool floor: NVIDIA hosts are 128K, Strix Halo 256K (2026-09-12).
|
||||
threshold: 0.65 # Fires at ~170K for 262K window, ~85K for 128K
|
||||
target_ratio: 0.30
|
||||
protect_last_n: 40
|
||||
@@ -143,25 +143,25 @@ compression:
|
||||
|
||||
# ─── Auxiliary Tasks (CONSISTENCY RULE) ───
|
||||
# All auxiliary services MUST use identical model, base_url, and api_key_env:
|
||||
# model: gpu-light # stable alias (NOT raw "gemma-4-12b")
|
||||
# model: gpu-vision # stable alias (NOT a raw model name)
|
||||
# base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
# api_key_env: LITELLM_API_KEY
|
||||
# Do NOT use syslog-auto for auxiliary tasks — it routes to the primary GPU.
|
||||
# gpu-light = RTX 5070 (12B), freeing the Strix Halo for agent reasoning.
|
||||
# gpu-vision = RTX 5070 (12B), freeing the Strix Halo for agent reasoning.
|
||||
# Heavy aux (delegation, x_search) use gpu-dense (RTX 3090) instead.
|
||||
# NEVER use raw model names (gemma-4-12b, qwen3.6-27B-code, qwen3.6-35B-udq4)
|
||||
# in agent configs — use the stable aliases so model swaps don't break agents.
|
||||
# NEVER use retired model names (qwen3.6-27B-code, qwen3.6-35B-udq4; gemma-4-12b is retired
|
||||
# and no longer resolves) in agent configs — use the stable aliases so model swaps don't break agents.
|
||||
auxiliary:
|
||||
vision:
|
||||
provider: harness
|
||||
model: gpu-light # stable alias for RTX 5070 (was raw gemma-4-12b)
|
||||
model: gpu-vision # stable alias for RTX 5070
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY
|
||||
timeout: 60
|
||||
download_timeout: 30
|
||||
web_extract:
|
||||
provider: harness
|
||||
model: gpu-light # stable alias for RTX 5070
|
||||
model: gpu-vision # stable alias for RTX 5070
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY
|
||||
timeout: 30
|
||||
@@ -173,10 +173,10 @@ auxiliary:
|
||||
timeout: 300 # gpu-fleet: 300s for large-history summarization (was 60)
|
||||
|
||||
# ─── Delegation / Heavy Aux (use gpu-dense = RTX 3090) ───
|
||||
# delegation.model and x_search.model use gpu-dense (NOT raw qwen3.6-27B-code).
|
||||
# delegation.model and x_search.model use gpu-dense (NOT retired raw name).
|
||||
|
||||
delegation:
|
||||
model: gpu-dense # stable alias for RTX 3090 (was raw qwen3.6-27B-code)
|
||||
model: gpu-dense # stable alias for RTX 3090
|
||||
provider: harness
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY
|
||||
@@ -252,10 +252,11 @@ The following MUST be identical across ALL profiles:
|
||||
- For agents needing longer outputs: raise to 8192, but never omit
|
||||
|
||||
### Rule 7: Auxiliary Model Consistency (UPDATED 2026-07-16)
|
||||
- Vision and web_extract use `gemma-4-12b` (RTX 5070 — 12GB, vision-optimized)
|
||||
- Compression uses `strix-moe` (stable alias for Strix Halo — 64GB, 128K ctx, compression-optimized)
|
||||
- **`strix-moe` is the only valid compression model name** — LiteLLM does NOT serve `ornith-1.0-35b`
|
||||
(it serves `strix-moe`, `qwen3.6-35B-udq4`, `gpu-dense`, `gpu-light`, `syslog-auto`, `gemma-4-12b`, `qwen3.6-27B-code`). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
||||
- Vision and web_extract use `gpu-vision` (RTX 5070 — 12GB, vision-optimized)
|
||||
- Compression uses `syslog-auto` — the Strix Halo weighted pool (64GB, 256K ctx, compression-optimized); do NOT pin `compression.model` to `strix-moe` (audit Rule 7 rejects it)
|
||||
- **`ornith-1.0-35b` is NOT a valid compression model name** — LiteLLM does not serve it
|
||||
(do not restate the served model list here — CT 116 `/opt/inference-harness/litellm_config.yaml`
|
||||
is the single source of truth for models, aliases, weights and fallbacks). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
||||
- **OPERATIONAL DECISION (2026-07-23): Use `syslog-auto` for compression across all agents.**
|
||||
The `syslog-auto` alias routes to the Strix Halo, but uses the weighted pool instead of pinning
|
||||
to `strix-moe` directly. This prevents sustained Strix Halo thermal load because the pool can
|
||||
@@ -264,41 +265,41 @@ The following MUST be identical across ALL profiles:
|
||||
- All auxiliary services MUST use identical routing:
|
||||
- `base_url: http://192.168.68.116/litellm/v1` (Rule 5, 2026-08-09: canonical authenticated; `/v1` also OK)
|
||||
- `api_key_env: LITELLM_API_KEY`
|
||||
- **Do NOT use `syslog-auto`** for auxiliary tasks — it routes unpredictably
|
||||
- **Do NOT use `syslog-auto` for `vision`/`web_extract`** — it routes unpredictably; compression is the deliberate exception (see the OPERATIONAL DECISION above)
|
||||
- **Compression on Strix Halo**: The strix-moe alias routes to Strix Halo
|
||||
(64GB UMA, 128K context) — the designated compression GPU. This frees the
|
||||
(64GB UMA, 256K context) — the designated compression GPU. This frees the
|
||||
RTX 5070 for vision and web search, and the RTX 3090 for heavy reasoning.
|
||||
- The `compression:` block's `model` MUST match `auxiliary: compression: model`
|
||||
- The `compression: max_context_window: 131072` MUST match actual GPU capacity (128K)
|
||||
- The `compression: max_context_window: 131072` MUST stay at the syslog-auto pool floor (NVIDIA hosts 128K; Strix Halo 256K)
|
||||
|
||||
### Rule 8: GPU Workload Distribution (UPDATED 2026-07-16)
|
||||
- **RTX 3090 (24GB, 128K ctx, qwen3.6-27B-code)**: Heavy reasoning, code gen, long conversations
|
||||
- **RTX 5070 (12GB, 128K ctx, gemma-4-12b)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
||||
- **Strix Halo (64GB, 128K ctx, syslog-auto)**: Context compression, summarization, long docs
|
||||
- **RTX 3090 (24GB, 128K ctx, gpu-dense)**: Heavy reasoning, code gen, long conversations
|
||||
- **RTX 5070 (12GB, 128K ctx, gpu-vision)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
||||
- **Strix Halo (64GB, 256K ctx, syslog-auto)**: Context compression, summarization, long docs
|
||||
- Agent profiles MUST route auxiliary tasks to the correct GPU:
|
||||
- `auxiliary.vision.model: gemma-4-12b` (RTX 5070)
|
||||
- `auxiliary.web_extract.model: gemma-4-12b` (RTX 5070)
|
||||
- `auxiliary.vision.model: gpu-vision` (RTX 5070)
|
||||
- `auxiliary.web_extract.model: gpu-vision` (RTX 5070)
|
||||
- `auxiliary.compression.model: syslog-auto` (Strix Halo)
|
||||
- Default model (`model.default`) and custom_provider remain `syslog-auto` for auto-routing
|
||||
- For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
|
||||
- Do NOT use `threshold: 0.25` — this fires at 65K, causing premature context loss
|
||||
- Do NOT use `threshold: 0.80` — this delays until ~105K, leaving only 23K margin
|
||||
- `max_context_window: 131072` MUST match the model's actual capacity (128K)
|
||||
- `max_context_window: 131072` MUST stay at the pool floor (NVIDIA hosts 128K; Strix Halo 256K)
|
||||
- See `devops-hermes-compression` skill for full reference
|
||||
|
||||
### Rule 9: Compression Threshold for 128K Models
|
||||
- For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
|
||||
- Do NOT use `threshold: 0.25` — this fires at 65K, causing premature context loss
|
||||
- Do NOT use `threshold: 0.80` — this delays until ~105K, leaving only 23K margin
|
||||
- `max_context_window: 131072` MUST match the model's actual capacity (all GPUs = 128K)
|
||||
- `max_context_window: 131072` MUST stay at the pool floor (NVIDIA hosts 128K; Strix Halo 256K)
|
||||
- See `devops-hermes-compression` skill for full reference
|
||||
|
||||
### Rule 10: Default Model Must Be `syslog-auto` (All Agents)
|
||||
- **Koby Exception**: Per captain ruling 2026-08-11, Koby is a DeepSeek-primary external agent; its primary model remains `deepseek-v4-flash` (via api.deepseek.com to preserve DeepSeek-specific reasoning, while other sections follow Rule 10.
|
||||
- **Hermes agents**: `model.default: syslog-auto`, `custom_providers[0].model: syslog-auto`
|
||||
- **pi agents**: `defaultModel: syslog-auto` in `settings.json`, first model in `models.json`
|
||||
- `syslog-auto` is the LiteLLM routing model — it load-balances between strix-moe
|
||||
and qwen3.6-27B-code, with gemma-4-12b as fallback. Using it protects against:
|
||||
- `syslog-auto` is the LiteLLM routing model — it load-balances across the live pool
|
||||
(see CT 116 `/opt/inference-harness/litellm_config.yaml` for the current members and weights). Using it protects against:
|
||||
- Model name typos that cause 403 errors and silent worker failures
|
||||
- Single GPU downtime (routing falls back automatically)
|
||||
- Key/model authorization mismatches
|
||||
@@ -321,7 +322,8 @@ When an agent shows "context issues" (premature compression, 401s, 504s, DeepSee
|
||||
verify ALL FOUR of these against the live config. They are the only root causes found in production:
|
||||
|
||||
1. **max_context_window correct?** — BOTH `compression.max_context_window` AND `context.max_context_window`
|
||||
MUST be `131072` (all GPUs are 128K). A value of `262144` causes instability near 100K and must NOT be used.
|
||||
MUST be `131072` (the syslog-auto pool floor: NVIDIA hosts are 128K; Strix Halo is 256K).
|
||||
A `262144` client window can route to a 128K NVIDIA host and fail, so it must NOT be used.
|
||||
~83K instead of ~170K. Check: `grep -n max_context_window ~/.hermes/config.yaml`
|
||||
2. **base_url uses authenticated path?** — `custom_providers[0].base_url`, `delegation.base_url`,
|
||||
and ALL `auxiliary.*.base_url` MUST be `http://192.168.68.116/litellm/v1` (Rule 5, canonical)
|
||||
|
||||
@@ -169,16 +169,20 @@ The agent picks up the new key via `infisical run --` at gateway startup.
|
||||
|
||||
**Keys are permanent and use bare agent name aliases.**
|
||||
|
||||
- **Duration**: `null` — keys never expire. This is enforced by `default_key_generate_params` in `litellm_config.yaml`.
|
||||
- **Duration**: `null` — keys never expire. NOT enforced today: CT 116 `litellm_config.yaml` has no `default_key_generate_params` block, and a key generated with no explicit models comes back with an empty models list. OPEN policy question: should agent keys expire by default? (captain security-policy decision, raised separately.)
|
||||
- **Alias convention**: bare agent name only (e.g., `tanko`, `mumuni`, `koby`, `koonimo`). No dates, no versions. The alias IS the identity.
|
||||
- **Rotation triggers**: compromise, personnel departure, or quarterly security hygiene. NOT calendar-driven.
|
||||
- **Max budget**: $100 per key (config default).
|
||||
|
||||
```yaml
|
||||
# In litellm_config.yaml — ensures all future keys inherit these defaults:
|
||||
# NOT currently set in the authority; recommended value. CT 116 litellm_config.yaml has no
|
||||
# default_key_generate_params block today, and a key generated with no explicit models comes back
|
||||
# with an EMPTY models list. `models` is a literal key-generation parameter, so this is a value to
|
||||
# ADD — re-read the live registry at CT 116 /opt/inference-harness/litellm_config.yaml and
|
||||
# re-verify before applying.
|
||||
litellm_settings:
|
||||
default_key_generate_params:
|
||||
models: ["syslog-auto", "qwen3.6-27B-code", "gemma-4-12b"]
|
||||
models: ["syslog-auto", "gpu-dense", "gpu-vision", "strix-moe"]
|
||||
duration: null # ← permanent
|
||||
max_budget: 100
|
||||
metadata:
|
||||
@@ -286,13 +290,13 @@ auxiliary:
|
||||
api_key: sk-<agent-key-from-vault> # ← workaround (get via: infisical secrets get LITELLM_API_KEY --project=agents --env=production --plain)
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
model: gemma-4-12b
|
||||
model: gpu-vision
|
||||
provider: harness
|
||||
compression:
|
||||
api_key: sk-<agent-key-from-vault> # ← workaround (same as above)
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
model: gemma-4-12b
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
```
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ connectivity recovery including end-to-end DM validation.
|
||||
|
||||
## Requires
|
||||
|
||||
- SSH access to target host (direct or via amdpve for CTs)
|
||||
- SSH access to target host (direct, or via the guest's Proxmox node for CTs)
|
||||
- Git repo at `https://git.sysloggh.net/SyslogSolution/zulip-platform-plugins.git`
|
||||
- Python 3 with `httpx` installed on target
|
||||
|
||||
@@ -56,7 +56,7 @@ connectivity recovery including end-to-end DM validation.
|
||||
| Host | CT | Proxmox | IP (direct) | Hermes Home | User |
|
||||
|------|-----|---------|-------------|-------------|------|
|
||||
| Tanko | CT112 | amdpve | 192.168.68.122 | /home/jerome/.hermes | jerome | *(DSH since 2026-08-27 — historical, plugin retired on this host)* |
|
||||
| Koby | CT111 | amdpve | 192.168.68.129 | /root/.hermes | root |
|
||||
| Koby | CT111 | storepve | 192.168.68.129 | /root/.hermes | root |
|
||||
| Shumba | — | — | 192.168.68.119 | /home/lucky/.hermes | lucky |
|
||||
|
||||
| Field | Value | Trust |
|
||||
@@ -72,7 +72,7 @@ connectivity recovery including end-to-end DM validation.
|
||||
### Step 1: Resolve Target
|
||||
|
||||
Map `target` to host, CT ID, hermes_home, and user from the live-state table.
|
||||
For CT112 and CT111, route through `ssh root@amdpve` then `pct exec <id>`.
|
||||
For CT112 route through `ssh root@amdpve`; for CT111 route through `ssh root@storepve` — then `pct exec <id>`.
|
||||
|
||||
### Step 2: Pull Latest Plugin Source
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ gateway restart, and connection validation.
|
||||
|
||||
## Requires
|
||||
|
||||
- SSH access to target host (direct or via amdpve for CTs)
|
||||
- SSH access to target host (direct, or via the guest's Proxmox node for CTs)
|
||||
- Git repo at `https://git.sysloggh.net/SyslogSolution/zulip-platform-plugins.git`
|
||||
- Python 3 with `httpx` installed on target
|
||||
- Zulip server accessible at `https://chat.sysloggh.net`
|
||||
@@ -52,7 +52,7 @@ gateway restart, and connection validation.
|
||||
|
||||
| Host | CT | Proxmox | IP (direct) | Hermes Home | User |
|
||||
|------|-----|---------|-------------|-------------|------|
|
||||
| Koby | CT111 | amdpve | 192.168.68.129 | /root/.hermes | root |
|
||||
| Koby | CT111 | storepve | 192.168.68.129 | /root/.hermes | root |
|
||||
| Shumba | — | — | 192.168.68.119 | /home/lucky/.hermes | lucky |
|
||||
|
||||
| Field | Value | Trust |
|
||||
@@ -67,7 +67,7 @@ gateway restart, and connection validation.
|
||||
### Step 1: Locate Target
|
||||
|
||||
Map `target` to connectivity parameters from the live-state table above.
|
||||
For CT112 and CT111, route through `ssh root@amdpve` then `pct exec <id>`.
|
||||
For CT112 route through `ssh root@amdpve`; for CT111 route through `ssh root@storepve` — then `pct exec <id>`.
|
||||
|
||||
### Step 2: Deploy Zulip Adapter
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ kind: responsibility
|
||||
description: >
|
||||
Optimizes the full Syslog inference stack — LiteLLM routing weights, GPU model
|
||||
assignments, agent context management, and prompt caching — to reduce response
|
||||
times to sub-15s average. All GPUs now at 128K context (stable ceiling).
|
||||
times to sub-15s average. NVIDIA GPUs at 128K context; Strix Halo at 256K (2026-09-12).
|
||||
id: 067NC6KP02RG60S50M40E30928
|
||||
---
|
||||
|
||||
@@ -23,7 +23,7 @@ management, and prompt caching — without sacrificing agent capability.
|
||||
.123, any others on .129/.122) including compression, model, context_window,
|
||||
prompt_caching, memory settings
|
||||
- `gpu-health`: health check response from all 3 GPU backends (strix-moe .15:8080,
|
||||
qwen .8:8080, gemma .110:8080)
|
||||
gpu-dense .8:8080, gpu-vision .110:8080)
|
||||
|
||||
### Maintains
|
||||
|
||||
@@ -56,14 +56,14 @@ duration.
|
||||
**Context is the root cause.** Every ~46K prompt token costs ~87s of
|
||||
prefill time at 532 tok/s. Fix context first, routing second.
|
||||
|
||||
- **Route by task**: qwen for code/standard queries; gemma for
|
||||
compression/auxiliary; strix-moe for compression tasks.
|
||||
- **Route by task**: gpu-dense for code/standard queries; gpu-vision for
|
||||
vision/web-auxiliary; syslog-auto for compression.
|
||||
- **Compress aggressively**: threshold at 40% (not 65%) — a 128K window should
|
||||
compact at 51K, not 85K. Target 15% tail (not 30%).
|
||||
- **Cache everything repeated**: system prompts, skill docs, AGENTS.md — these
|
||||
never change between turns. Single-digit cache hit rate is unacceptable.
|
||||
- **Lower context ceiling**: 128K window is the stable ceiling for agent conversations.
|
||||
GPUs reduced from 256K to 128K (2026-07-17). 128K window should compact at 85K (0.65 threshold). For larger contexts, route to external providers.
|
||||
GPUs reduced from 256K to 128K (2026-07-17) for the NVIDIA hosts; Strix Halo runs 256K (2026-09-12). 128K window should compact at 85K (0.65 threshold). For larger contexts, route to external providers.
|
||||
|
||||
### Shape
|
||||
|
||||
@@ -96,8 +96,10 @@ call enable-prompt-caching
|
||||
hosts: [192.168.68.15, 192.168.68.8, 192.168.68.110]
|
||||
|
||||
-- Phase 5: Verify end-to-end latency
|
||||
-- `models` is a literal verification parameter (a snapshot only): the authoritative registry is
|
||||
-- CT 116 /opt/inference-harness/litellm_config.yaml; re-read it before use.
|
||||
|
||||
call verify-latency
|
||||
host: 192.168.68.116
|
||||
models: [syslog-auto, qwen3.6-27B-code, gemma-4-12b]
|
||||
models: [syslog-auto, gpu-dense, gpu-vision, strix-moe]
|
||||
```
|
||||
|
||||
@@ -16,8 +16,8 @@ description: >
|
||||
**Last verified:** 2026-08-15 — hwepve removed from Tabiri cluster
|
||||
(now 5 nodes: minipve, amdpve, storepve, acerpve, ocupve). hwepve
|
||||
(192.168.68.4) is a standalone PVE node + NetBird routing peer;
|
||||
London relocation pending. CTs 100 (abiba) and 105 (kagentz) moved
|
||||
to minipve.
|
||||
London relocation pending. CT 100 (abiba) is on minipve; CT 105
|
||||
(kagentz) is on amdpve.
|
||||
---
|
||||
|
||||
# Infrastructure Control Pattern
|
||||
@@ -105,16 +105,16 @@ description: >
|
||||
|
||||
| Node | IP | CPU | RAM | VMs/CTs | Role |
|
||||
|------|----|-----|-----|---------|------|
|
||||
| minipve | .12 | 16C | 30GB | abiba, kagentz, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging |
|
||||
| amdpve | .15 | 32C | 62GB | tanko, tdunna, baggy, scottdenya | Agents, compute |
|
||||
| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip | Docker, storage, chat |
|
||||
| minipve | .12 | 16C | 30GB | abiba, authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging |
|
||||
| amdpve | .15 | 32C | 62GB | kagentz, tanko, baggy, scottdenya, adguard2 | Agents, compute |
|
||||
| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna | Docker, storage, chat |
|
||||
| acerpve | .9 | 28C | 31GB | llm-gpu | GPU VMs |
|
||||
| ocupve | .5 | 12C | 14GB | ocu-llm | GPU VMs |
|
||||
|
||||
> **Note:** CTs on storepve include jdownloader (CT 118). AdGuard (CT 102) is on
|
||||
> minipve at .10, not acerpve. Abiba (CT 100) and kagentz (CT 105) are on
|
||||
> minipve (moved from hwepve 2026-08-15). Mumuni runs inside Abiba CT100
|
||||
> (.24); CT 114 (mumuni) no longer exists in the cluster.
|
||||
> **Note:** CTs on storepve include jdownloader (CT 118) and tdunna (CT 111).
|
||||
> AdGuard (CT 102) is on minipve at .10, not acerpve. Abiba (CT 100) is on
|
||||
> minipve (moved from hwepve 2026-08-15); kagentz (CT 105) is on amdpve.
|
||||
> Mumuni runs inside Abiba CT100 (.24); CT 114 (mumuni) no longer exists in the cluster.
|
||||
>
|
||||
> **hwepve (192.168.68.4) — STANDALONE (removed from Tabiri 2026-08-15):**
|
||||
> Huawei MateBook 16 (KLVL-WXX9), pve-manager/9.2.10, kernel 7.0.14-8-pve.
|
||||
@@ -222,8 +222,8 @@ description: >
|
||||
|
||||
**Prometheus targets**:
|
||||
- 192.168.68.8:9400 (RTX 3090 — qwen)
|
||||
- 192.168.68.110:9400 (RTX 5070 — gemma)
|
||||
- 192.168.68.15:9400 (Strix Halo — qwen3.6-35B-udq4)
|
||||
- 192.168.68.110:9400 (RTX 5070 — gpu-vision)
|
||||
- 192.168.68.15:9400 (Strix Halo — strix-moe)
|
||||
- harness-litellm:4000 (LiteLLM health)
|
||||
|
||||
### Ecosystem C: Netbird (72.61.0.17 — Hostinger srv1079750.hstgr.cloud)
|
||||
@@ -602,13 +602,13 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
|
||||
| 102 | adguard | **minipve** | **.10** | DNS | ❌ |
|
||||
| 103 | ocu-llm | ocupve | .110 | GPU RTX 5070 | ❌ |
|
||||
| 104 | authentik | minipve | .11 | OIDC | ❌ |
|
||||
| 105 | kagentz | minipve | — | Agent Zero | ✅ |
|
||||
| 105 | kagentz | amdpve | — | Agent Zero | ✅ |
|
||||
| 106 | ra-h-os | storepve | .65 | KG bridge | ✅ MCP |
|
||||
| 107 | pbs | storepve | — | Backups | ❌ |
|
||||
| 108 | media | storepve | — | Media | ❌ |
|
||||
| 109 | docker-vm | storepve | .7 | Docker host | ❌ |
|
||||
| 110 | gitea | minipve | **.17** | Git | ❌ |
|
||||
| 111 | tdunna | amdpve | .129 | Hermes agent | ✅ |
|
||||
| 111 | tdunna | storepve | .129 | Hermes agent — ⛔ REPORT-ONLY (Theo's box, no GC) | ✅ |
|
||||
| 112 | tanko | amdpve | .122 | DSH (DeepSeek Harness) agent | ✅ |
|
||||
| 113 | baggy | amdpve | .114 | Hermes agent | ✅ |
|
||||
| 115 | scottdenya | amdpve | .75 | Denya OneCare | ❌ |
|
||||
@@ -616,6 +616,7 @@ ssh root@192.168.68.110 "systemctl restart llama-server"
|
||||
| 117 | zulip | storepve | .19 | Chat | ❌ |
|
||||
| 118 | jdownloader | storepve | .20 | JDownloader LXC (dedicated, migrated from docker-vm 2026-08-01) | ✅ |
|
||||
| 119 | infisical-vault | minipve | — | Vault | ❌ |
|
||||
| 120 | adguard2 | amdpve | — | DNS (secondary AdGuard) | ❌ |
|
||||
|
||||
## Appendix C: Docker Compose Files Location
|
||||
|
||||
@@ -636,8 +637,8 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run.
|
||||
| CT | Name | Node | pct-run |
|
||||
|-----|------|------|---------|
|
||||
| 100 | abiba | minipve | `pct-run 100` |
|
||||
| 105 | kagentz | minipve | `pct-run 105` |
|
||||
| 111 | tdunna | amdpve | `pct-run 111` |
|
||||
| 105 | kagentz | amdpve | `pct-run 105` |
|
||||
| 111 | tdunna | storepve | `pct-run 111` (⛔ report-only — no GC) |
|
||||
| 112 | tanko | amdpve | `pct-run 112` |
|
||||
| 113 | baggy | amdpve | `pct-run 113` |
|
||||
| 115 | scottdenya | amdpve | `pct-run 115` |
|
||||
@@ -648,6 +649,9 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run.
|
||||
| 107 | proxmox-backup | storepve | `pct-run 107` |
|
||||
| 108 | media | storepve | `pct-run 108` |
|
||||
| 117 | zulip | storepve | `pct-run 117` |
|
||||
| 118 | jdownloader | storepve | `pct-run 118` |
|
||||
| 119 | infisical-vault | minipve | `pct-run 119` |
|
||||
| 120 | adguard2 | amdpve | `pct-run 120` |
|
||||
| 102 | adguard | **minipve** | `pct-run 102` |
|
||||
|
||||
GPU bare-metal hosts (.8 acerpve, .110 ocupve, .15 amdpve) are NOT CTs — use SSH directly:
|
||||
|
||||
@@ -129,10 +129,15 @@ the any-HTTP rule. On those — the authenticated Zulip POST and the router
|
||||
pwd -P
|
||||
|
||||
# Zulip API health (POST ping)
|
||||
source /etc/litellm-monitor.env
|
||||
ZULIP_USER="abiba-bot@chat.sysloggh.net"
|
||||
curl -s -o /dev/null -w '%{http_code}' -X POST https://chat.sysloggh.net/api/v1/messages -u "${ZULIP_USER}:${ZULIP_BOT_KEY}"
|
||||
# Expected: 200 (HTTP 000 = unreachable/cache)
|
||||
# NOTE: /etc/litellm-monitor.env exists only on CT 116, retrieve keys from CT 116 via:
|
||||
zulip_key=$(ssh root@192.168.68.116 "grep ZULIP_BOT_KEY /etc/litellm-monitor.env | cut -d= -f2")
|
||||
if [ -z "$zulip_key" ]; then
|
||||
echo "credential-missing: ZULIP_BOT_KEY not found in /etc/litellm-monitor.env"
|
||||
else
|
||||
ZULIP_USER="abiba-bot@chat.sysloggh.net"
|
||||
curl -s -o /dev/null -w '%{http_code}' -X POST https://chat.sysloggh.net/api/v1/messages -u "${ZULIP_USER}:${zulip_key}"
|
||||
# Expected: 200 (HTTP 000 = unreachable/cache)
|
||||
fi
|
||||
|
||||
# PM2 process health
|
||||
pm2 jlist
|
||||
|
||||
@@ -67,8 +67,10 @@ description: >
|
||||
4. **If action == "create"**:
|
||||
- Generate new key with key_alias: "{agent_name}" (e.g., "tanko" — bare name, no date)
|
||||
- Set metadata: { "agent": "{agent_name}", "purpose": "agent-inference" }
|
||||
- Duration is null (permanent) — inherited from litellm default_key_generate_params
|
||||
- Set models: ["syslog-auto", "qwen3.6-27B-code", "gemma-4-12b", "strix-moe", "gpu-dense", "gpu-light", "qwen3.6-35B-udq4"]
|
||||
- Duration is whatever the caller passes; NO default enforcement exists today (CT 116 `litellm_config.yaml` has no `default_key_generate_params` block, and a key with no explicit models returns an empty models list). Agent keys are permanent by policy, not by that block. OPEN policy question: should agent keys expire by default? (captain security-policy decision, raised separately.)
|
||||
- Set models: read the live key-scoped set rather than hardcoding one — `/v1/models` is key-scoped,
|
||||
and the authoritative registry is CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not add
|
||||
retired names (`gemma-4-12b`, `gpu-light`, `crew-auto` — all retired 2026-09-12).
|
||||
- Note: `ornith-1.0-35b` is NOT a valid LiteLLM model name (use `strix-moe`, the stable alias). qwen3.6-35B-A3B removed from fleet (was never deployed).
|
||||
- Return the new key
|
||||
5. **If action == "rotate"**:
|
||||
|
||||
@@ -26,9 +26,8 @@ description: >
|
||||
| Model | avg latency | avg TTFT | p-profile (24h) |
|
||||
|---|---|---|---|
|
||||
| syslog-auto | 28.8s | 25.4s | 68 calls took 30-120s; tail to ~300s under load |
|
||||
| qwen3.6-27B-code | 23.0s | — | same backend class as syslog-auto |
|
||||
| strix-moe | 7.5s | — | Strix Halo, healthy |
|
||||
| gemma-4-12b | 2.6s | — | RTX 5070, healthy |
|
||||
| gpu-vision (retired gemma-4-12b, RTX 5070) | 2.6s | — | RTX 5070, healthy |
|
||||
|
||||
Sep 6 incident timeline: failures 04:00-07:00 EDT (0% GPU util = wedged
|
||||
backend), full recovery 07:00-08:00 with ZERO client failures once requests
|
||||
@@ -53,13 +52,11 @@ proxy queuing.
|
||||
|
||||
### 2. Auxiliary tasks — keep template timeouts, one correction
|
||||
|
||||
- vision: 60s (keep), web_extract: 30s (keep) — gemma-4-12b averages 2.6s;
|
||||
these are fine.
|
||||
- vision: 60s (keep), web_extract: 30s (keep) — the 2.6s average was measured on `gemma-4-12b` (retired 2026-09-12); the live RTX 5070 alias is `gpu-vision`.
|
||||
- compression: 300s (keep — this was already raised from 60 per gpu-fleet).
|
||||
- **gpu-dense delegation/x_search: set timeout >= 120s.** The RTX 3090
|
||||
(qwen3.6-27B-code backend, 23.0s avg) is the same speed class as
|
||||
syslog-auto; delegation defaults that assume fast responses will 408 the
|
||||
same way.
|
||||
(gpu-dense backend) is the same speed class as syslog-auto; delegation
|
||||
defaults that assume fast responses will 408 the same way.
|
||||
|
||||
### 3. Retry policy — backoff, not repetition
|
||||
|
||||
|
||||
+25
-6
@@ -44,7 +44,7 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
**What changed (v3.2.0 → v4.0.0 — 2026-07-08)**:
|
||||
- Router REMOVED from request path — LiteLLM proxies directly to GPU
|
||||
- All GPUs at parallel 2 (was parallel 1)
|
||||
- NVIDIA context reduced 256K→128K to free VRAM — now the stable ceiling across all GPUs (2026-07-17)
|
||||
- NVIDIA context reduced 256K→128K to free VRAM — the stable NVIDIA ceiling (2026-07-17); Strix Halo runs 256K (2026-09-12)
|
||||
- Timeouts and fallback chains are config state — read them from CT 116 `/opt/inference-harness/litellm_config.yaml`; they are not duplicated here.
|
||||
|
||||
## Parameters
|
||||
@@ -69,7 +69,16 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
- Network access to public_url, auth_host, and gpu_dashboard_url
|
||||
- LiteLLM master key for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`)
|
||||
- The dedicated `monitor` agent key on CT 116 at `/etc/litellm-monitor.env` (root-only 0600) for
|
||||
model inference checks — the master key must never be used for inference
|
||||
model inference checks, scoped for every alias step 7 probes (`gpu-dense`, `gpu-vision`,
|
||||
`strix-moe`, `syslog-auto`). Retrieve from the executor's host via:
|
||||
|
||||
```
|
||||
monitor key: ssh root@192.168.68.116 "grep LITELLM_MONITOR_KEY /etc/litellm-monitor.env | cut -d= -f2"
|
||||
master key: ssh root@192.168.68.116 "docker exec harness-litellm printenv LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
||||
If credentials are missing or unreadable, the probe must report `credential-missing` (not bare 401 or "0 keys").
|
||||
The master key must never be used for inference.
|
||||
|
||||
## GPU Fleet Topology
|
||||
|
||||
@@ -144,12 +153,19 @@ contracts — read them there. Do not re-add retired names (`gemma-4-12b`, `gpu-
|
||||
7. **Check model inference via LiteLLM** — Test one model on each GPU host. The health
|
||||
check runs on the **backend edge**, not the public edge, so these paths carry the
|
||||
`/litellm/` prefix:
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=qwen3.6-27B-code → expect 200 (RTX 3090, .8)
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=gpu-dense → expect 200 (RTX 3090, .8)
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=gpu-vision → expect 200 (RTX 5070, .110)
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=strix-moe → expect 200 (Strix Halo, .15)
|
||||
- Auth uses the dedicated `monitor` agent key, read on CT 116 from
|
||||
`/etc/litellm-monitor.env` (root-only 0600). Do NOT use the master key for inference —
|
||||
the master key is for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`).
|
||||
- Auth uses the dedicated `monitor` agent key. Retrieve via:
|
||||
`ssh root@192.168.68.116 "grep LITELLM_MONITOR_KEY /etc/litellm-monitor.env | cut -d= -f2"`
|
||||
Do NOT use the master key for inference — the master key is for admin endpoints only
|
||||
(`/key/list`, `/key/generate`, `/key/info`). Retrieve master key via:
|
||||
`ssh root@192.168.68.116 "docker exec harness-litellm printenv LITELLM_MASTER_KEY"`
|
||||
- KEY SCOPE: the `monitor` key MUST be scoped for the three probed aliases (`gpu-dense`,
|
||||
`gpu-vision`, `strix-moe`) plus the `syslog-auto` fallback pool, otherwise the probe
|
||||
returns 403 and the host is not covered.
|
||||
If a probe returns 403, widen the monitor key's model list on CT 116 (add the missing
|
||||
alias) and re-run — never drop the host from the probe to make the check pass.
|
||||
- `gemma-4-12b` was retired and returns 400 `Invalid model name` — do not re-add it to
|
||||
this list. The RTX 5070 host now serves `gpu-vision`.
|
||||
- `/v1/models` is **key-scoped**: a model is only visible to keys allowed to use it, so
|
||||
@@ -161,6 +177,9 @@ contracts — read them there. Do not re-add retired names (`gemma-4-12b`, `gpu-
|
||||
|
||||
8. **Check agent keys**:
|
||||
- GET http://{{backend_host}}/litellm/key/list with master key (admin endpoint) → verify all 6 agents have keys
|
||||
- **IMPORTANT**: Run the curl on the CT 116 HOST, not inside the container. The `harness-litellm` container has no curl/wget. Use:
|
||||
`ssh root@192.168.68.116 "curl -s -H 'Authorization: Bearer $MASTER_KEY' http://127.0.0.1:4000/key/list"`
|
||||
- If the response is empty or unparseable, report `admin-call-failed` (not "0 agent keys")
|
||||
|
||||
9. **Check Grafana**:
|
||||
- GET {{grafana_url}}/api/health → expect 200
|
||||
|
||||
@@ -90,8 +90,8 @@ raw data never provided.
|
||||
|
||||
| Worker | Model | Toolsets | Role | Use When |
|
||||
|--------|-------|----------|------|----------|
|
||||
| `syslog-code` | qwen3.6-27B-code | terminal, file, web, memory, skills | Code patches, automation, scripts | Writing/modifying code, creating scripts, debugging, reading/writing files |
|
||||
| `syslog-devops` | qwen3.6-27B-code | terminal, file, web, memory, skills | Infrastructure, DB, bridge, Proxmox | Server ops, SSH, Docker, Proxmox, DB queries, hardware checks |
|
||||
| `syslog-code` | gpu-dense | terminal, file, web, memory, skills | Code patches, automation, scripts | Writing/modifying code, creating scripts, debugging, reading/writing files |
|
||||
| `syslog-devops` | gpu-dense | terminal, file, web, memory, skills | Infrastructure, DB, bridge, Proxmox | Server ops, SSH, Docker, Proxmox, DB queries, hardware checks |
|
||||
| `syslog-email` | strix-moe | terminal, file, web, memory, skills | Email automation, mail operations | Sending/receiving email, inbox management, SMTP operations |
|
||||
| `syslog-research` | strix-moe | terminal, file, web, memory, skills, **browser** | Analysis, classification, data processing | Web research, browser tasks, data analysis, classification, reading docs |
|
||||
| `syslog-review` | strix-moe | terminal, file, web, memory, skills | Verification, QA, audit validation | **ALWAYS** verify worker output before delivery — especially for infra changes, code builds, and research findings |
|
||||
|
||||
@@ -9,7 +9,6 @@ description: >
|
||||
- abiba-telegram: { status: "online", uptime: string, restarts: number }
|
||||
- abiba-zulip: { status: "online", uptime: string, restarts: number }
|
||||
- gitea-runner: { status: "online", uptime: string, restarts: number }
|
||||
- spoton-service: { status: "online", uptime: string, restarts: number }
|
||||
- zulip-watchdog: { status: "online", uptime: string, restarts: number }
|
||||
- last_check: timestamp
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ agent: abiba
|
||||
| storepve | 192.168.68.6 | PVE |
|
||||
| acerpve | 192.168.68.9 | PVE (hosts llm-gpu qemu/101) |
|
||||
| minipve | 192.168.68.12 | PVE |
|
||||
| amdpve | 192.168.68.15 | PVE + Strix Halo LLM (qwen3.6-35B-udq4, strix-moe) |
|
||||
| amdpve | 192.168.68.15 | PVE + Strix Halo LLM (strix-moe) |
|
||||
|
||||
## Operations
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
disk-gc report-only verification — CT 111 / tdunna / 192.168.68.129
|
||||
date: 2026-09-12T19:11:36Z
|
||||
host: abiba (this scanner runs INSIDE CT 100 / abiba)
|
||||
branch head: 6e612ce37b1b9a9688b04e7f816848e7a4185fca
|
||||
command: scripts/disk-gc-plan.py --scan <live fleet scan>
|
||||
|
||||
PURPOSE: prove that on a REAL fleet scan, CT 111 is alerted and NO gc-executor action
|
||||
is emitted for it at any level. No GC command was executed against .129.
|
||||
|
||||
--- live fleet scan (df -P / via scripts/pct-run.sh for LXC, direct SSH for hosts) ---
|
||||
tdunna 84%
|
||||
acerpve 192.168.68.9 77%
|
||||
amdpve 192.168.68.15 76%
|
||||
ocu-llm 192.168.68.110 69%
|
||||
storepve 192.168.68.6 65%
|
||||
kagentz 61%
|
||||
tanko 56%
|
||||
minipve 192.168.68.12 49%
|
||||
authentik 45%
|
||||
infisical-vault 39%
|
||||
adguard 38%
|
||||
ocupve 192.168.68.5 38%
|
||||
scottdenya 35%
|
||||
syslog-api 34%
|
||||
llm-gpu 192.168.68.8 25%
|
||||
abiba 23%
|
||||
baggy 22%
|
||||
jdownloader 21%
|
||||
gitea 16%
|
||||
ra-h-os 13%
|
||||
zulip 12%
|
||||
docker-vm 192.168.68.7 11%
|
||||
adguard2 10%
|
||||
media 9%
|
||||
proxmox-backup-server 4%
|
||||
|
||||
--- planner output (action plan) ---
|
||||
111 AMBER 84.0% -> REPORT-ONLY (no GC) — Theo's box — captain ruling 2026-08-17, re-confirmed 2026-09-10
|
||||
acerpve AMBER 77.0% -> gc-executor
|
||||
amdpve AMBER 76.0% -> gc-executor
|
||||
|
||||
--- verdict ---
|
||||
CT 111 (tdunna) 84% AMBER -> report-only; no gc-executor row emitted; no GC run on .129.
|
||||
Owned hosts acerpve .9 (77%) and amdpve .15 (76%) -> gc-executor (ours).
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""disk-gc-plan — turn a fleet disk scan into the GC action plan.
|
||||
|
||||
This is the executable side of `disk-gc-threat-response.prose.md`. It exists so the
|
||||
report-only gate is enforced by code that can be tested, rather than by prose the
|
||||
executor might misread.
|
||||
|
||||
THE HARD GATE: guests listed in the contract's `report_only_guests` block are
|
||||
DETECT-AND-REPORT-ONLY at EVERY level (AMBER, RED, CRITICAL). This tool will never
|
||||
emit a `gc-executor` action for one, so no GC command can be constructed for it.
|
||||
|
||||
The gate is keyed on GUEST identity — guest id, hostname, or IP — never on an agent
|
||||
name. An agent-name marker can silently miss the guest it lives on; a guest marker
|
||||
cannot.
|
||||
|
||||
The authoritative exclusion list lives in the contract itself (the fenced ```yaml
|
||||
block containing `report_only_guests:`). This tool reads it from there so there is
|
||||
only ever one copy.
|
||||
|
||||
Usage:
|
||||
disk-gc-plan.py --scan scan.json # [{"id":111,"usage_pct":84}, ...]
|
||||
cat scan.json | disk-gc-plan.py # same, via stdin
|
||||
disk-gc-plan.py --scan scan.json --json # machine-readable plan
|
||||
|
||||
Scan entries may carry any of: id / guest / vmid / ct / ctid, hostname / name, ip.
|
||||
A threshold-crossing entry with no recognizable identity is reported, never GC'd.
|
||||
Exit codes: 0 ok, 1 usage/parse error.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
REPO = pathlib.Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONTRACT = REPO / "disk-gc-threat-response.prose.md"
|
||||
|
||||
AMBER, RED, CRITICAL = 75, 85, 95
|
||||
|
||||
IDENTITY_FIELDS = ("id", "guest", "vmid", "ct", "ctid", "hostname", "name", "ip")
|
||||
IDENTITY_TYPE_PREFIX = re.compile(r"^(?:lxc|qemu)/")
|
||||
UNIDENTIFIED_REASON = "unidentified target - refusing to schedule GC"
|
||||
|
||||
|
||||
def load_report_only_guests(contract_path: pathlib.Path) -> list[dict]:
|
||||
"""Read the authoritative report_only_guests block out of the contract.
|
||||
|
||||
The contract carries it as a fenced ```yaml block. Parsing the declared,
|
||||
machine-readable block is the intended interface — the contract owns the list.
|
||||
"""
|
||||
text = contract_path.read_text(encoding="utf-8")
|
||||
for block in re.findall(r"```yaml\n(.*?)```", text, re.S):
|
||||
if "report_only_guests:" in block:
|
||||
data = yaml.safe_load(block)
|
||||
guests = data.get("report_only_guests") or []
|
||||
if not isinstance(guests, list):
|
||||
raise SystemExit("report_only_guests must be a list")
|
||||
if not guests:
|
||||
raise SystemExit(
|
||||
"report_only_guests is empty or missing - refusing to plan GC "
|
||||
"without the report-only gate"
|
||||
)
|
||||
for guest in guests:
|
||||
if not isinstance(guest, dict) or not _keys(guest):
|
||||
raise SystemExit(
|
||||
"report_only_guests entry has no recognizable identity key "
|
||||
f"(expected one of: {', '.join(IDENTITY_FIELDS)}): {guest!r}"
|
||||
)
|
||||
return guests
|
||||
raise SystemExit(
|
||||
f"no authoritative report_only_guests block found in {contract_path}"
|
||||
)
|
||||
|
||||
|
||||
def _canonical_number(number: float) -> str:
|
||||
if float(number).is_integer():
|
||||
return str(int(number))
|
||||
return str(number).strip().lower()
|
||||
|
||||
|
||||
def _normalize_identity(value: object) -> str:
|
||||
"""Canonicalise a guest identity so differently-encoded ids compare equal:
|
||||
numeric and numeric-string ids collapse to an integer string, Proxmox
|
||||
type prefixes and leading zeros are stripped, and hostnames/IPs are only
|
||||
trimmed and lowercased."""
|
||||
if isinstance(value, bool):
|
||||
return str(value).strip().lower()
|
||||
if isinstance(value, (int, float)):
|
||||
return _canonical_number(float(value))
|
||||
text = str(value).strip().lower()
|
||||
text = IDENTITY_TYPE_PREFIX.sub("", text)
|
||||
try:
|
||||
return _canonical_number(float(text))
|
||||
except ValueError:
|
||||
return text
|
||||
|
||||
|
||||
def _keys(entry: dict) -> set[str]:
|
||||
"""Guest/host identity keys, shared by exclusions and scan entries so the two
|
||||
sides of the gate can never key on different fields."""
|
||||
out: set[str] = set()
|
||||
for field in IDENTITY_FIELDS:
|
||||
value = entry.get(field)
|
||||
if value is None:
|
||||
continue
|
||||
key = _normalize_identity(value)
|
||||
if key:
|
||||
out.add(key)
|
||||
return out
|
||||
|
||||
|
||||
def level_for(pct: float) -> str | None:
|
||||
if pct >= CRITICAL:
|
||||
return "CRITICAL"
|
||||
if pct >= RED:
|
||||
return "RED"
|
||||
if pct >= AMBER:
|
||||
return "AMBER"
|
||||
return None
|
||||
|
||||
|
||||
def build_plan(scan: list[dict], report_only: list[dict]) -> list[dict]:
|
||||
excluded = [(e, _keys(e)) for e in report_only]
|
||||
plan: list[dict] = []
|
||||
for entry in scan:
|
||||
pct = entry.get("usage_pct")
|
||||
if pct is None:
|
||||
continue
|
||||
level = level_for(float(pct))
|
||||
if level is None:
|
||||
continue # GREEN: log only, no action
|
||||
scan_keys = _keys(entry)
|
||||
target = next(
|
||||
(entry.get(k) for k in IDENTITY_FIELDS if entry.get(k) not in (None, "")),
|
||||
"?",
|
||||
)
|
||||
if not scan_keys:
|
||||
plan.append({
|
||||
"target": target,
|
||||
"level": level,
|
||||
"pct": float(pct),
|
||||
"action": "report-only",
|
||||
"reason": UNIDENTIFIED_REASON,
|
||||
})
|
||||
continue
|
||||
match = next((e for e, keys in excluded if keys & scan_keys), None)
|
||||
if match is not None:
|
||||
plan.append({
|
||||
"target": target,
|
||||
"level": level,
|
||||
"pct": float(pct),
|
||||
"action": "report-only",
|
||||
"reason": match.get("reason", "").strip(),
|
||||
})
|
||||
else:
|
||||
plan.append({
|
||||
"target": target,
|
||||
"level": level,
|
||||
"pct": float(pct),
|
||||
"action": "gc-executor",
|
||||
})
|
||||
plan.sort(key=lambda row: row["pct"], reverse=True)
|
||||
return plan
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Plan disk GC actions with the report-only gate.")
|
||||
ap.add_argument("--scan", help="JSON file: list of {id|ct|hostname|ip, usage_pct}")
|
||||
ap.add_argument("--contract", default=str(DEFAULT_CONTRACT))
|
||||
ap.add_argument("--json", action="store_true", help="emit the plan as JSON")
|
||||
args = ap.parse_args()
|
||||
|
||||
raw = pathlib.Path(args.scan).read_text() if args.scan else sys.stdin.read()
|
||||
try:
|
||||
scan = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"invalid scan JSON: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not isinstance(scan, list):
|
||||
print("scan must be a JSON list", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
report_only = load_report_only_guests(pathlib.Path(args.contract))
|
||||
plan = build_plan(scan, report_only)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(plan, indent=2))
|
||||
return 0
|
||||
|
||||
if not plan:
|
||||
print("no threats (nothing at or above 75%)")
|
||||
return 0
|
||||
for row in plan:
|
||||
if row["action"] == "report-only":
|
||||
print(f" {row['target']} {row['level']} {row['pct']}% -> REPORT-ONLY (no GC) — {row['reason']}")
|
||||
else:
|
||||
print(f" {row['target']} {row['level']} {row['pct']}% -> gc-executor")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+7
-6
@@ -11,11 +11,13 @@ set -euo pipefail
|
||||
# ── CT ID → PVE Node mapping (maintained HERE, not in prose contracts) ──
|
||||
declare -A CT_NODES=(
|
||||
# amdpve (192.168.68.15)
|
||||
[111]=amdpve # tdunna
|
||||
[105]=amdpve # kagentz (was hwepve — corrected 2026-09-12; live per pvesh)
|
||||
[112]=amdpve # tanko
|
||||
[113]=amdpve # baggy
|
||||
[115]=amdpve # scottdenya
|
||||
[120]=amdpve # adguard2 (added 2026-09-12)
|
||||
# minipve (192.168.68.12)
|
||||
[100]=minipve # abiba (was hwepve)
|
||||
[102]=minipve # adguard (was acerpve)
|
||||
[104]=minipve # authentik
|
||||
[110]=minipve # gitea
|
||||
@@ -25,17 +27,16 @@ declare -A CT_NODES=(
|
||||
[106]=storepve # ra-h-os
|
||||
[107]=storepve # proxmox-backup
|
||||
[108]=storepve # media
|
||||
[111]=storepve # tdunna (was amdpve — corrected 2026-09-12; live per pvesh)
|
||||
[117]=storepve # zulip
|
||||
[118]=storepve # jdownloader
|
||||
# acerpve (192.168.68.9) — no CTs (bare metal GPU .8)
|
||||
[100]=minipve # abiba (was hwepve)
|
||||
[105]=minipve # kagentz (was hwepve)
|
||||
# ocupve (192.168.68.5) — no CTs (bare metal GPU .110)
|
||||
#
|
||||
# REMOVED CTs (migrated to bare metal, decommissioned, or VMs):
|
||||
# 101 llm-gpu → bare metal 192.168.68.8 (RTX 3090)
|
||||
# 103 ocu-llm → bare metal 192.168.68.110 (RTX 5070)
|
||||
# 109 docker-vm → KVM VM 192.168.68.7 (use direct SSH)
|
||||
# 101 llm-gpu → bare metal 192.168.68.8 (RTX 3090) [QEMU VM on acerpve]
|
||||
# 103 ocu-llm → bare metal 192.168.68.110 (RTX 5070) [QEMU VM on ocupve]
|
||||
# 109 docker-vm → KVM VM 192.168.68.7 (use direct SSH) [QEMU VM on storepve]
|
||||
)
|
||||
|
||||
# Each node must be root-accessible via SSH hostname
|
||||
|
||||
@@ -47,17 +47,19 @@ You are a code reviewer for OpenProse infrastructure contracts in the Syslog Sol
|
||||
The infrastructure-control.prose.md contract is the canonical reference for the cluster topology:
|
||||
|
||||
**Proxmox Cluster "Tabiri" (5 nodes):**
|
||||
- amdpve (192.168.68.15): tanko, tdunna, baggy, scottdenya
|
||||
- minipve (192.168.68.12): abiba, kagentz, adguard, authentik, gitea, syslog-api, infisical-vault
|
||||
- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip
|
||||
- amdpve (192.168.68.15): kagentz, tanko, baggy, scottdenya, adguard2
|
||||
- minipve (192.168.68.12): abiba, adguard, authentik, gitea, syslog-api, infisical-vault
|
||||
- storepve (192.168.68.6): docker-vm, ra-h-os, PBS, media, jdownloader, zulip, tdunna
|
||||
- acerpve (192.168.68.9): llm-gpu
|
||||
- ocupve (192.168.68.5): ocu-llm
|
||||
|
||||
**CT IDs (verified 2026-07-24 against PVE API):**
|
||||
**CT IDs (verified 2026-09-12 against PVE API):**
|
||||
100:abiba 102:adguard 104:authentik 105:kagentz 106:ra-h-os
|
||||
107:pbs 108:media 110:gitea 111:tdunna 112:tanko
|
||||
113:baggy 115:scottdenya 116:syslog-api 117:zulip
|
||||
118:jdownloader 119:infisical-vault
|
||||
118:jdownloader 119:infisical-vault 120:adguard2
|
||||
|
||||
**CT 111 (tdunna, 192.168.68.129) is REPORT-ONLY — Theo's box; alert only, never garbage-collect.**
|
||||
|
||||
**NO CT 122, CT 123, or .19 exist in the cluster.**
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Regression tests for the 2026-09-12 retired-alias sweep in audit-hermes-config.py.
|
||||
|
||||
WHY THIS FILE EXISTS: the executable audit pushed agent configs toward a DEAD alias.
|
||||
Rule 8 required `auxiliary.vision.model == "gpu-light"` and
|
||||
`auxiliary.web_extract.model == "gpu-light"`, but `gpu-light` (and its raw predecessor
|
||||
`gemma-4-12b`) were retired on 2026-09-12 and now return 400 `Invalid model name`; the
|
||||
live RTX 5070 alias is `gpu-vision`. A config that adopted the correct canonical alias
|
||||
therefore FAILED our own audit, so the audit was actively enforcing a broken config.
|
||||
|
||||
These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert
|
||||
observable behaviour — exit code and the emitted rule message — for the live alias and
|
||||
for both retired names. No network, vault, or SSH access is required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
AUDIT = ROOT / "audit-hermes-config.py"
|
||||
|
||||
BASE = """
|
||||
model:
|
||||
api_key: ""
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
max_tokens: 4096
|
||||
default: syslog-auto
|
||||
provider: harness
|
||||
fallback_providers:
|
||||
provider: deepseek
|
||||
model: deepseek-v4-flash
|
||||
api_key_env: DEEPSEEK_API_KEY
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
threshold: 0.65
|
||||
max_context_window: 131072
|
||||
auxiliary:
|
||||
vision:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
web_extract:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
delegation:
|
||||
provider: harness
|
||||
custom_providers:
|
||||
- name: harness
|
||||
key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
"""
|
||||
|
||||
|
||||
def _run_config(tmp_path, name, text):
|
||||
cfg = tmp_path / name
|
||||
cfg.write_text(text)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(AUDIT), str(cfg)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return proc.returncode, proc.stdout
|
||||
|
||||
|
||||
def _run(tmp_path, alias):
|
||||
return _run_config(tmp_path, f"{alias}.yaml", BASE.format(alias=alias))
|
||||
|
||||
|
||||
def test_live_canonical_alias_passes(tmp_path):
|
||||
"""The RTX 5070 alias that actually resolves must satisfy Rule 8."""
|
||||
code, out = _run(tmp_path, "gpu-vision")
|
||||
assert code == 0, out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_gpu_light_is_rejected(tmp_path):
|
||||
"""A config pinned to the retired alias must fail, not pass."""
|
||||
code, out = _run(tmp_path, "gpu-light")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_gemma_is_rejected(tmp_path):
|
||||
"""The retired raw model name must fail Rule 8 as well."""
|
||||
code, out = _run(tmp_path, "gemma-4-12b")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_corrected_compression_example_passes(tmp_path):
|
||||
"""The corrected workaround (vision=gpu-vision, compression=syslog-auto) must PASS."""
|
||||
code, out = _run(tmp_path, "gpu-vision")
|
||||
assert code == 0, out
|
||||
assert "[Rule 7] compression.model must be syslog-auto (got 'syslog-auto')" in out
|
||||
assert "[Rule 7] auxiliary.compression.model must be syslog-auto (got 'syslog-auto')" in out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_delegation_is_rejected(tmp_path):
|
||||
"""delegation.model has no dedicated value rule, so a retired name there used to PASS."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"delegation-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\n model: gpu-light",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "delegation.model = 'gpu-light' is retired" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_custom_providers_is_rejected(tmp_path):
|
||||
"""custom_providers[*].model is model-bearing; a retired name there must fail."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"custom-provider-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
" - name: harness\n key_env: LITELLM_API_KEY",
|
||||
" - name: harness\n model: gpu-light\n key_env: LITELLM_API_KEY",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "custom_providers[0].model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_raw_name_fails(tmp_path):
|
||||
"""Retired raw names no longer resolve (400), so they fail; the 2026-09-12 registry change moved qwen3.6-27B-code from raw-but-live to non-resolving."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"retired-qwen.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\n model: qwen3.6-27B-code",
|
||||
),
|
||||
)
|
||||
assert code != 0, out
|
||||
assert "delegation.model = 'qwen3.6-27B-code' is retired and no longer resolves" in out
|
||||
assert "use gpu-dense" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_fallback_providers_is_rejected(tmp_path):
|
||||
"""fallback_providers.model is model-bearing; a retired name there must fail."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"fallback-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(" model: deepseek-v4-flash", " model: gpu-light"),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "fallback_providers.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_x_search_is_rejected(tmp_path):
|
||||
"""x_search.model was previously not enumerated; the derivation must catch it."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"x-search-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\nx_search:\n model: gpu-light",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "x_search.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_nested_auxiliary_block_is_rejected(tmp_path):
|
||||
"""A nested auxiliary sub-block outside the named three must still be derived."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"nested-aux-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
" compression:\n model: syslog-auto\n provider: harness\ndelegation:",
|
||||
" compression:\n model: syslog-auto\n provider: harness\n"
|
||||
" tasks:\n summarize:\n model: gpu-light\ndelegation:",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "auxiliary.tasks.summarize.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Regression tests for the disk-gc report-only gate (CT 111 / tdunna / .129).
|
||||
|
||||
WHY THIS FILE EXISTS: `disk-gc-threat-response.prose.md` defined AMBER as "GC scheduled
|
||||
for next run" and its Execution loop called `gc-executor` for EVERY threat, with no
|
||||
guest-level exclusion. CT 111 (tdunna, 192.168.68.129) belongs to Theo and is
|
||||
report-only per the captain (2026-08-17, re-confirmed 2026-09-10) — so a single AMBER
|
||||
reading on that guest would have scheduled GC commands (apt clean, journal vacuum,
|
||||
log/tmp deletion, snap removal) against someone else's box. The only marker was
|
||||
frontmatter `report_only_agents`, which names an AGENT while the scan unit is a GUEST.
|
||||
|
||||
These tests execute the real planner (`scripts/disk-gc-plan.py`) and assert observable
|
||||
behaviour: an excluded guest never produces a `gc-executor` action at any level, while
|
||||
our own guests still do.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
PLAN = ROOT / "scripts" / "disk-gc-plan.py"
|
||||
|
||||
|
||||
def _plan(scan, tmp_path):
|
||||
scan_file = tmp_path / "scan.json"
|
||||
scan_file.write_text(json.dumps(scan))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(PLAN), "--scan", str(scan_file), "--json"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def _actions_for(plan, target):
|
||||
return [row for row in plan if str(row["target"]) == str(target)]
|
||||
|
||||
|
||||
def test_excluded_guest_never_gets_gc_at_any_level(tmp_path):
|
||||
"""CT 111 at AMBER, RED and CRITICAL — always report-only, never gc-executor."""
|
||||
for pct, level in ((84, "AMBER"), (90, "RED"), (97, "CRITICAL")):
|
||||
plan = _plan([{"id": 111, "hostname": "tdunna", "ip": "192.168.68.129",
|
||||
"usage_pct": pct}], tmp_path)
|
||||
rows = _actions_for(plan, 111)
|
||||
assert rows, f"CT 111 must still be reported at {level}"
|
||||
assert rows[0]["level"] == level
|
||||
assert rows[0]["action"] == "report-only", rows
|
||||
assert not any(r["action"] == "gc-executor" for r in rows)
|
||||
|
||||
|
||||
def test_exclusion_matches_on_any_identity_key(tmp_path):
|
||||
"""The gate is keyed on guest/host, so id, ct, ctid, hostname or IP all match."""
|
||||
for entry in ({"id": 111, "usage_pct": 95},
|
||||
{"ct": 111, "usage_pct": 95},
|
||||
{"ctid": 111, "usage_pct": 95},
|
||||
{"hostname": "tdunna", "usage_pct": 95},
|
||||
{"ip": "192.168.68.129", "usage_pct": 95}):
|
||||
plan = _plan([entry], tmp_path)
|
||||
assert all(r["action"] == "report-only" for r in plan), (entry, plan)
|
||||
|
||||
|
||||
def test_exclusion_matches_encoded_identities(tmp_path):
|
||||
"""A differently-encoded CT 111 id must not slip past the gate to gc-executor."""
|
||||
encodings = ({"id": 111.0},
|
||||
{"id": "111.0"},
|
||||
{"id": "lxc/111"},
|
||||
{"id": "qemu/111"},
|
||||
{"id": "0111"},
|
||||
{"id": " 111 "})
|
||||
for alias in encodings:
|
||||
plan = _plan([{**alias, "usage_pct": 97}], tmp_path)
|
||||
assert plan, alias
|
||||
assert plan[0]["action"] == "report-only", (alias, plan)
|
||||
|
||||
|
||||
def test_our_own_guests_still_get_gc(tmp_path):
|
||||
"""acerpve .9 and amdpve .15 are ours — they must still be acted on."""
|
||||
plan = _plan([{"hostname": "acerpve", "ip": "192.168.68.9", "usage_pct": 77},
|
||||
{"hostname": "amdpve", "ip": "192.168.68.15", "usage_pct": 76}], tmp_path)
|
||||
assert len(plan) == 2
|
||||
assert all(r["action"] == "gc-executor" for r in plan), plan
|
||||
|
||||
|
||||
def test_below_threshold_emits_nothing(tmp_path):
|
||||
"""GREEN guests produce no action at all."""
|
||||
assert _plan([{"id": 111, "usage_pct": 40}], tmp_path) == []
|
||||
|
||||
|
||||
def test_agent_name_alone_does_not_gate_a_guest(tmp_path):
|
||||
"""An agent-name marker must not be the gate: an unrelated guest still gets GC."""
|
||||
plan = _plan([{"id": 999, "hostname": "koby", "usage_pct": 95}], tmp_path)
|
||||
assert plan and plan[0]["action"] == "gc-executor"
|
||||
|
||||
|
||||
def test_unidentified_threat_fails_closed(tmp_path):
|
||||
"""A threshold-crossing entry with no recognized identity must not schedule GC."""
|
||||
plan = _plan([{"usage_pct": 97}], tmp_path)
|
||||
assert plan, "an unidentified threat must still be reported"
|
||||
assert plan[0]["action"] == "report-only", plan
|
||||
assert plan[0]["reason"], plan
|
||||
|
||||
|
||||
def test_exclusion_entry_without_identity_fails_closed(tmp_path):
|
||||
"""A mis-typed exclusion entry must break the run, never silently disable the gate."""
|
||||
contract = tmp_path / "broken.prose.md"
|
||||
contract.write_text(
|
||||
"```yaml\n"
|
||||
"report_only_guests:\n"
|
||||
" - node: storepve\n"
|
||||
" reason: \"typo - no guest identity\"\n"
|
||||
"```\n"
|
||||
)
|
||||
scan_file = tmp_path / "scan.json"
|
||||
scan_file.write_text(json.dumps([{"ct": 111, "usage_pct": 97}]))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(PLAN), "--scan", str(scan_file),
|
||||
"--contract", str(contract), "--json"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert proc.returncode != 0, proc.stdout
|
||||
assert "gc-executor" not in proc.stdout
|
||||
assert "identity" in proc.stderr.lower(), proc.stderr
|
||||
|
||||
|
||||
def test_empty_exclusion_block_fails_closed(tmp_path):
|
||||
"""An emptied report_only_guests list must break the run, not disable the gate."""
|
||||
contract = tmp_path / "empty.prose.md"
|
||||
contract.write_text("```yaml\nreport_only_guests: []\n```\n")
|
||||
scan_file = tmp_path / "scan.json"
|
||||
scan_file.write_text(json.dumps([{"ct": 111, "usage_pct": 97}]))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(PLAN), "--scan", str(scan_file),
|
||||
"--contract", str(contract), "--json"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert proc.returncode != 0, proc.stdout
|
||||
assert "gc-executor" not in proc.stdout
|
||||
assert "report-only gate" in proc.stderr.lower(), proc.stderr
|
||||
|
||||
|
||||
def _plan_with_contract(scan, contract_text, tmp_path, name):
|
||||
contract = tmp_path / name
|
||||
contract.write_text(contract_text)
|
||||
scan_file = tmp_path / f"scan-{name}.json"
|
||||
scan_file.write_text(json.dumps(scan))
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(PLAN), "--scan", str(scan_file),
|
||||
"--contract", str(contract), "--json"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def test_gate_is_read_from_the_contract_block(tmp_path):
|
||||
"""The gate is data-driven by the contract block: the planner excludes the guest
|
||||
when the block names it and acts on it when the block does not. Executes the real
|
||||
planner interface against both fixtures so the behaviour change is observable."""
|
||||
scan = [{"id": 111, "hostname": "tdunna", "ip": "192.168.68.129", "usage_pct": 95}]
|
||||
with_gate = (
|
||||
"```yaml\n"
|
||||
"report_only_guests:\n"
|
||||
" - guest: 111\n"
|
||||
" hostname: tdunna\n"
|
||||
" ip: 192.168.68.129\n"
|
||||
" reason: \"fixture reason\"\n"
|
||||
"```\n"
|
||||
)
|
||||
without_gate = (
|
||||
"```yaml\n"
|
||||
"report_only_guests:\n"
|
||||
" - guest: 999\n"
|
||||
" reason: \"fixture excludes a different guest\"\n"
|
||||
"```\n"
|
||||
)
|
||||
|
||||
gated = _plan_with_contract(scan, with_gate, tmp_path, "gated.prose.md")
|
||||
assert gated[0]["action"] == "report-only", gated
|
||||
assert gated[0]["reason"] == "fixture reason", gated
|
||||
|
||||
ungated = _plan_with_contract(scan, without_gate, tmp_path, "ungated.prose.md")
|
||||
assert ungated[0]["action"] == "gc-executor", ungated
|
||||
assert ungated[0]["action"] != gated[0]["action"]
|
||||
Reference in New Issue
Block a user