Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2edc2f56f | ||
|
|
78b501798f | ||
|
|
1f1b47f59d | ||
|
|
3d55764799 | ||
|
|
ce48070f21 | ||
|
|
9e581ab203 | ||
|
|
9cac3589cf | ||
|
|
221f9f79f3 | ||
|
|
1dc040251d | ||
|
|
21f7b6171c | ||
|
|
bb17c2120f | ||
|
|
dc42ecc235 | ||
|
|
baaac9d7c6 | ||
|
|
2f65c38213 | ||
|
|
d9eb18c024 | ||
|
|
3f07b9bbcc | ||
|
|
1a598d0fcb | ||
|
|
e3752af162 |
@@ -43,17 +43,21 @@ jobs:
|
||||
echo "=== Prose Contract Frontmatter Validation ==="
|
||||
FAILED=0
|
||||
for f in $(find . -name "*.prose.md" -not -path "./.git/*" -not -path "./runs/*"); do
|
||||
# NOTE: use herestrings, not `echo "$FM" | grep ...`. Under the runner's
|
||||
# `-e -o pipefail`, `grep -q` exits on first match and can SIGPIPE the
|
||||
# producer, making the pipeline report non-zero and raising a false
|
||||
# "Missing name/description" whose file set varies run to run.
|
||||
FM=$(sed -n '/^---$/,/^---$/p' "$f" | sed '1d;$d')
|
||||
[ -z "$FM" ] && { echo " ❌ $f: No YAML frontmatter"; FAILED=$((FAILED+1)); continue; }
|
||||
|
||||
KIND=$(echo "$FM" | grep '^kind:' | awk '{print $2}')
|
||||
KIND=$(grep '^kind:' <<< "$FM" | awk '{print $2}')
|
||||
case "$KIND" in
|
||||
function|responsibility|gateway|pattern|test|template|architecture|enforcement) echo " ✅ $f: kind=$KIND" ;;
|
||||
*) echo " ❌ $f: Invalid kind='$KIND'"; FAILED=$((FAILED+1)) ;;
|
||||
esac
|
||||
|
||||
echo "$FM" | grep -q '^name:' || { echo " ❌ $f: Missing name"; FAILED=$((FAILED+1)); }
|
||||
echo "$FM" | grep -q '^description:' || { echo " ❌ $f: Missing description"; FAILED=$((FAILED+1)); }
|
||||
grep -q '^name:' <<< "$FM" || { echo " ❌ $f: Missing name"; FAILED=$((FAILED+1)); }
|
||||
grep -q '^description:' <<< "$FM" || { echo " ❌ $f: Missing description"; FAILED=$((FAILED+1)); }
|
||||
done
|
||||
[ $FAILED -gt 0 ] && { echo "❌ FRONTMATTER FAILED ($FAILED error(s))"; exit 1; }
|
||||
echo "✅ Frontmatter validation passed"
|
||||
|
||||
@@ -88,7 +88,7 @@ prose run memory-audit-maintenance memory_threshold=90 verify_configs=true
|
||||
prose run hermes-config-template agent_name=syslog-devops default_model=claude-sonnet-4
|
||||
|
||||
# Configure an agent with a different auxiliary model
|
||||
prose run hermes-config-template agent_name=syslog-code default_model=qwen3.6-27B-code auxiliary_model=gemma-4-12b
|
||||
prose run hermes-config-template agent_name=syslog-code default_model=qwen3.6-27B-code auxiliary_model=gpu-vision
|
||||
```
|
||||
|
||||
### Option B: Manual Execution
|
||||
@@ -116,7 +116,7 @@ Run on trigger or schedule. Maintain persistent world-model state across runs.
|
||||
| `zulip-health` | Zulip | Checks Zulip connectivity, message flow, and bot responsiveness. |
|
||||
| `zulip-mention-reliability` | Zulip | Diagnoses and fixes @mention detection issues in Zulip. |
|
||||
| `zulip-approval-fix` | Zulip | Fixes broken /approve and /deny slash commands for Hermes agents. |
|
||||
| `litellm-self-heal` | LiteLLM | Consolidated health check + self-healing for the full nginx → LiteLLM → GPU chain. Verifies 8 containers, 3 GPUs, model inference, and agent keys. Applies 9 remediation rules. (litellm-health merged into this contract 2026-07-09.) |
|
||||
| `litellm-self-heal` | LiteLLM | Applies remediation rules for LiteLLM stack failures detected by `litellm-health` (full nginx → LiteLLM → GPU chain). 9 remediation rules. |
|
||||
| `gpu-fleet` | GPU | Manages the GPU inference fleet: model deployment, registration, health checks, LiteLLM sync. |
|
||||
| `gpu-monitor` | GPU | Comprehensive GPU fleet monitor — polls sidecars, router, LiteLLM every 15s, renders SSE dashboard. |
|
||||
| `proxmox-monitor` | Infra | Proxmox cluster + Docker monitoring via the existing Grafana/Prometheus stack on CT 116. |
|
||||
@@ -149,7 +149,7 @@ Called on-demand as single-render tools.
|
||||
| Contract | Description |
|
||||
|---|---|
|
||||
| `litellm-api-keys` | Manages LiteLLM API keys for agent identity. Create, rotate, verify, and list agent keys. References gpu-fleet for current key inventory. |
|
||||
| `litellm-health` | ⚠️ **DEPRECATED** — consolidated into `litellm-self-heal` (2026-07-09). Retained for reference only. |
|
||||
| `litellm-health` | LiteLLM health check: public vs backend surfaces, CT 116 containers, GPU fleet, one model per GPU host, and agent keys. Owner of the probes; `litellm-self-heal` owns remediation. |
|
||||
| `infrastructure-monitoring` | Target-state for Prometheus + GPU exporters + Grafana. Core stack deployed, GPU exporters NOT live. |
|
||||
| `stirling-pdf-agent-access` | Documents the Stirling-PDF API access pattern for agents — global API key, 12 operations, curl examples. Agents use the `stirling-pdf-api` shared skill for templates. |
|
||||
| `hello-world` | Minimal test contract — verifies the OpenProse execution pipeline works. |
|
||||
|
||||
+85
-17
@@ -36,6 +36,59 @@ def warn(rule, message):
|
||||
WARNINGS.append(f"[{rule}] {message}")
|
||||
|
||||
|
||||
# Derivation rule: a model name is any scalar under a mapping key named `model` or
|
||||
# `model_name`, at any depth. The top-level `model:` SECTION is the exception where the model
|
||||
# name lives under `default`/`model`/`model_name` inside that section, so it is descended
|
||||
# specially. The only other exception is key `models` (litellm key-generation params carry a
|
||||
# list of model names). EXTEND THE ALLOWLIST for a new exception; do NOT add another field by
|
||||
# hand.
|
||||
MODEL_KEYS = ("model", "model_name")
|
||||
MODEL_SECTION_KEYS = ("default", "model", "model_name")
|
||||
MODEL_LIST_KEYS = ("models",)
|
||||
|
||||
|
||||
def _iter_model_values(node, path=""):
|
||||
"""Yield (path, value) for every model-name-bearing scalar in a config."""
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
child = f"{path}.{key}" if path else key
|
||||
if key in MODEL_KEYS:
|
||||
if isinstance(value, dict):
|
||||
for subkey in MODEL_SECTION_KEYS:
|
||||
subvalue = value.get(subkey)
|
||||
if isinstance(subvalue, str):
|
||||
yield (f"{child}.{subkey}", subvalue)
|
||||
for subkey, subvalue in value.items():
|
||||
if isinstance(subvalue, (dict, list)):
|
||||
yield from _iter_model_values(subvalue, f"{child}.{subkey}")
|
||||
elif isinstance(value, list):
|
||||
yield from _iter_model_values(value, child)
|
||||
else:
|
||||
yield (child, value)
|
||||
elif key in MODEL_LIST_KEYS:
|
||||
yield from _iter_model_list(value, child)
|
||||
elif isinstance(value, (dict, list)):
|
||||
yield from _iter_model_values(value, child)
|
||||
elif isinstance(node, list):
|
||||
for i, item in enumerate(node):
|
||||
yield from _iter_model_values(item, f"{path}[{i}]")
|
||||
|
||||
|
||||
def _iter_model_list(node, path):
|
||||
"""Yield scalars under an allowlisted `models` key (list of names or list of dicts)."""
|
||||
if isinstance(node, list):
|
||||
for i, item in enumerate(node):
|
||||
yield from _iter_model_list(item, f"{path}[{i}]")
|
||||
elif isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key in MODEL_KEYS and isinstance(value, str):
|
||||
yield (f"{path}.{key}", value)
|
||||
elif isinstance(value, (dict, list)):
|
||||
yield from _iter_model_list(value, f"{path}.{key}")
|
||||
else:
|
||||
yield (path, node)
|
||||
|
||||
|
||||
def audit(path):
|
||||
with open(path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
@@ -89,15 +142,16 @@ def audit(path):
|
||||
)
|
||||
|
||||
# --- Rule 8: GPU Workload Distribution ---
|
||||
# gpu-light (and gemma-4-12b) were retired 2026-09-12; the RTX 5070 stable alias is gpu-vision.
|
||||
check(
|
||||
aux.get("vision", {}).get("model") == "gpu-light",
|
||||
aux.get("vision", {}).get("model") == "gpu-vision",
|
||||
"Rule 8",
|
||||
f"auxiliary.vision.model must be gpu-light (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
f"auxiliary.vision.model must be gpu-vision (got {aux.get('vision', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
)
|
||||
check(
|
||||
aux.get("web_extract", {}).get("model") == "gpu-light",
|
||||
aux.get("web_extract", {}).get("model") == "gpu-vision",
|
||||
"Rule 8",
|
||||
f"auxiliary.web_extract.model must be gpu-light (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
f"auxiliary.web_extract.model must be gpu-vision (got {aux.get('web_extract', {}).get('model')!r}) — RTX 5070 stable alias",
|
||||
)
|
||||
|
||||
# --- Rule 9: Compression Threshold ---
|
||||
@@ -178,21 +232,35 @@ def audit(path):
|
||||
f"custom_providers[0].base_url must end with /v1 (got {cp.get('base_url')!r})",
|
||||
)
|
||||
|
||||
# --- No raw model names (Rule 7/8 spirit) ---
|
||||
raw_names = {"gemma-4-12b", "qwen3.6-27B-code", "qwen3.6-35B-udq4", "ornith-1.0-35b"}
|
||||
for section_path, section_dict in [
|
||||
("model", model), ("compression", comp),
|
||||
("auxiliary.vision", aux.get("vision", {})),
|
||||
("auxiliary.web_extract", aux.get("web_extract", {})),
|
||||
("auxiliary.compression", aux.get("compression", {})),
|
||||
("delegation", deleg),
|
||||
]:
|
||||
m = section_dict.get("model", "")
|
||||
if m in raw_names:
|
||||
# --- Retired/raw model names (Rule 7/8 spirit) ---
|
||||
# The audit's job is to catch configs that are BROKEN, not to enforce a style preference.
|
||||
# NON-RESOLVING names (removed 2026-09-12, verified 400/403 via live LiteLLM) must hard-FAIL:
|
||||
# gpu-light -> gpu-vision ; gemma-4-12b -> gpu-vision
|
||||
# crew-auto -> syslog-auto (its 64K cap is retired; no cap in force) ; ornith-1.0-35b -> strix-moe
|
||||
# RESOLVING names (verified 200) are discouraged but working, so they only WARN:
|
||||
# qwen3.6-27B-code -> gpu-dense ; qwen3.6-35B-udq4 -> strix-moe
|
||||
# Failing a working alias would reject valid configs - the exact defect this change fixes.
|
||||
non_resolving = {
|
||||
"gpu-light": "gpu-vision",
|
||||
"gemma-4-12b": "gpu-vision",
|
||||
"crew-auto": "syslog-auto (its 64K cap is retired; no cap in force)",
|
||||
"ornith-1.0-35b": "strix-moe",
|
||||
}
|
||||
raw_but_live = {
|
||||
"qwen3.6-27B-code": "gpu-dense",
|
||||
"qwen3.6-35B-udq4": "strix-moe",
|
||||
}
|
||||
for field_path, value in _iter_model_values(cfg):
|
||||
if value in non_resolving:
|
||||
check(
|
||||
False,
|
||||
"Rule 7/8",
|
||||
f"{field_path} = {value!r} is retired and no longer resolves (2026-09-12) — use {non_resolving[value]}",
|
||||
)
|
||||
elif value in raw_but_live:
|
||||
warn(
|
||||
"Rule 7/8",
|
||||
f"{section_path}.model = {m!r} — raw model name, use stable alias instead "
|
||||
f"(gpu-light, gpu-dense, strix-moe, syslog-auto)",
|
||||
f"{field_path} = {value!r} is a raw-but-live model name — prefer the stable alias {raw_but_live[value]}",
|
||||
)
|
||||
|
||||
# --- Report ---
|
||||
|
||||
@@ -693,8 +693,8 @@ contracts:
|
||||
version: 1.0.0
|
||||
trigger:
|
||||
type: scheduled
|
||||
cadence: '*/10 * * * *'
|
||||
description: "Every 10 minutes \u2014 LiteLLM proxy health"
|
||||
cadence: '5 3,7,11,15,19,23 * * *'
|
||||
description: "4-hourly staggered dispatch via fm-send (run contract litellm-health)"
|
||||
cron_job_id: null
|
||||
execution:
|
||||
agent: abiba
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
|
||||
Generated: 2026-07-13 20:59:18 ET
|
||||
|
||||
> **Point-in-time snapshot.** Schedules and cadences are authoritative in
|
||||
> `contract-registry.yaml`; any schedule quoted below may be stale. Do not use
|
||||
> this file as the source of truth for a contract's trigger.
|
||||
|
||||
---
|
||||
|
||||
## hermes-key-enforcement
|
||||
@@ -442,7 +446,7 @@ IMPORTANT: If the contract file does not exist in prose-contracts/main, report f
|
||||
|
||||
## litellm-health
|
||||
|
||||
**Category:** monitoring | **Domain:** litellm | **Owner:** abiba | **Schedule:** */10 * * * *
|
||||
**Category:** monitoring | **Domain:** litellm | **Owner:** abiba | **Schedule:** see contract-registry.yaml (authoritative)
|
||||
|
||||
```
|
||||
Contract Enforcement: litellm-health
|
||||
@@ -450,7 +454,7 @@ Contract Enforcement: litellm-health
|
||||
Category: monitoring
|
||||
Domain: litellm
|
||||
Owner: abiba
|
||||
Schedule: Every 10 minutes — LiteLLM proxy health
|
||||
Schedule: see contract-registry.yaml (authoritative)
|
||||
|
||||
This is a monitoring contract. Execute the monitoring checks defined in the contract. Report any deviations from expected state.
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ the description:
|
||||
|
||||
1. **What system does this contract touch?** Name the hosts, CTs, containers,
|
||||
and services explicitly. "The inference fleet" is vague. "GPU .8 (RTX 3090,
|
||||
qwen), .110 (RTX 5070, gemma), .15 (Strix Halo, strix-moe), and LiteLLM on CT
|
||||
qwen), .110 (RTX 5070, gpu-vision), .15 (Strix Halo, strix-moe), and LiteLLM on CT
|
||||
116" is specific.
|
||||
|
||||
2. **Who runs this contract, and when?** State the agent, the trigger (cron,
|
||||
|
||||
+33
-61
@@ -6,9 +6,10 @@ description: >
|
||||
registration, health checks, LiteLLM sync, agent key management, GPU
|
||||
saturation watchdog, Prometheus/Grafana monitoring, and self-healing.
|
||||
UPDATED 2026-07-15: Stable role-based aliases introduced: strix-moe,
|
||||
gpu-dense, gpu-light. These never change — only the underlying model does.
|
||||
gpu-dense, gpu-vision (gpu-light was superseded by gpu-vision on 2026-09-12).
|
||||
These never change — only the underlying model does.
|
||||
Strix Halo: strix-moe → unsloth/Qwen3.6-35B-A3B-MTP (UD-Q4_K_M, 22GB).
|
||||
RTX 5070: gemma-4-12b Q4_K_M → IQ4_NL + MTP draft (122 tok/s, 2x faster).
|
||||
RTX 5070: gpu-vision — IQ4_NL + MTP draft (~122 tok/s, 2x faster).
|
||||
UPDATED 2026-07-17: Context reduced fleet-wide from 256K to 128K for stability.
|
||||
Strix Halo model swapped to qwen3.6-35B-udq4 (22GB, strix-moe alias).
|
||||
Instability observed near 100K at 256K (now all GPUs at 128K). 128K is the stable ceiling.
|
||||
@@ -24,7 +25,7 @@ triggers:
|
||||
|
||||
## Maintains
|
||||
|
||||
- gpu_roster: { models: map, hosts: map } — Single source of truth for all GPU models
|
||||
- gpu_roster: { models: map, hosts: map } — GPU host/model roster; the authoritative alias/weight/fallback registry is CT 116 `litellm_config.yaml`
|
||||
- router: { status: "healthy", roster_loaded: bool, models: array }
|
||||
- litellm: { status: "healthy", keys: array, models: array }
|
||||
- agent_keys: { agent: api_key } — All agent API keys registered in LiteLLM DB
|
||||
@@ -79,54 +80,28 @@ triggers:
|
||||
Agent configs, cron jobs, and workflows MUST use these aliases, never model-specific names.
|
||||
When a model is swapped on a GPU, ONLY the infrastructure layer changes — agent configs are untouched.
|
||||
|
||||
| Alias | GPU | Current Model | Will Route To |
|
||||
|-------|-----|---------------|---------------|
|
||||
| `strix-moe` | Strix Halo (.15) | qwen3.6-35B-udq4 | Whatever runs on Strix Halo |
|
||||
| `gpu-dense` | RTX 3090 (.8) | Qwen3.8-27B-Uncensored-Q4_K_M | Whatever runs on RTX 3090 |
|
||||
| `gpu-light` | RTX 5070 (.110) | gemma-4-12b | Whatever runs on RTX 5070 |
|
||||
| Alias | Serves | Where | Kind |
|
||||
|-------|--------|-------|------|
|
||||
| `gpu-dense` | heavy reasoning | RTX 3090 (192.168.68.8) | direct alias |
|
||||
| `gpu-vision` | vision / web extract / light tasks | RTX 5070 (192.168.68.110) | direct alias AND `syslog-auto` pool member |
|
||||
| `strix-moe` | compression (MoE) | Strix Halo (192.168.68.15) | direct alias |
|
||||
| `syslog-auto` | balanced default | weighted pool across the three GPU hosts | pool router |
|
||||
|
||||
**Backward compatibility**: Old model-specific names (qwen3.6-27B-code, gemma-4-12b, qwen3.5-9b-it) still work
|
||||
but are deprecated for agent configs. Only the stable aliases survive model swaps.
|
||||
Single source of truth for models, aliases, rpm caps, weights and fallback chains:
|
||||
CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not duplicate those values in
|
||||
contracts — read them there.
|
||||
|
||||
## Current Model Assignments (2026-07-15)
|
||||
|
||||
| Model | GPU | Host | VRAM | Ctx | KV Cache | Parallel | Batch/Ubatch | Status |
|
||||
|-------|-----|------|------|-----|----------|----------|-------------|--------|
|
||||
**Backward compatibility**: Old model-specific names (qwen3.6-27B-code, qwen3.5-9b-it) still work
|
||||
but are deprecated for agent configs. Only the stable aliases survive model swaps. `gemma-4-12b`
|
||||
is retired and returns 400 `Invalid model name`.
|
||||
|
||||
## Routing Configuration (LiteLLM — July 2026)
|
||||
|
||||
### syslog-auto Weighted Pool (Direct GPU — bypasses router)
|
||||
|
||||
| Model | GPU | Weight | RPM Cap | Timeout |
|
||||
|-------|-----|--------|---------|---------|
|
||||
| Qwen3.8-27B-Uncensored-Q4_K_M | RTX 3090 (.8:8080) | **0.55** | 500 | **300s** |
|
||||
Model, alias, rpm/weight and fallback values are owned by CT 116
|
||||
`/opt/inference-harness/litellm_config.yaml` (see § Stable Role-Based Aliases above).
|
||||
|
||||
Note: All syslog-auto entries route directly to GPUs with `api_key: not-needed`. The router (port 9000) was decommissioned 2026-09-11 and is NOT in the inference path.
|
||||
|
||||
### Direct Model Endpoints
|
||||
|
||||
| Model | RPM Cap | Notes |
|
||||
|-------|---------|-------|
|
||||
|
||||
### Stable Aliases (for agent configs — never change)
|
||||
|
||||
| Alias | RPM Cap | Routes To | Purpose |
|
||||
|-------|---------|-----------|---------|
|
||||
| `strix-moe` | 40 | Strix Halo | Compression tasks (MoE models) |
|
||||
| `gpu-dense` | 500 | RTX 3090 | Heavy reasoning |
|
||||
| `gpu-light` | 500 | RTX 5070 | Vision, web extract, light tasks |
|
||||
|
||||
### Fallback Chains
|
||||
- gemma → qwen
|
||||
- qwen → gemma
|
||||
- strix-moe → qwen → gemma
|
||||
- syslog-auto → qwen → gemma → qwen3.6-35B-udq4
|
||||
|
||||
### Why Strix Halo RPM Is Capped
|
||||
- Direct (strix-moe): 40 RPM (tight) — Strix Halo is shared with compression tasks
|
||||
- Via syslog-auto: 60 RPM (moderate) — prevents flooding when multiple agents use syslog-auto simultaneously
|
||||
- Combined max: ~100 RPM across both paths — Strix Halo can sustain this at 80°C
|
||||
|
||||
## Operations
|
||||
|
||||
### add-model
|
||||
@@ -176,9 +151,7 @@ Show full fleet status: GPUs, models, VRAM, context windows, parallel slots, act
|
||||
2. Check llama-server processes: `ps aux | grep llama-server` on all 3 hosts
|
||||
3. Check LiteLLM: `curl http://192.168.68.116/health` (expect "I'm alive!")
|
||||
4. Check LiteLLM models: `curl -H "Authorization: Bearer $MASTER_KEY" http://192.168.68.116/v1/models`
|
||||
5. Check LiteLLM timeouts: `grep -n 'timeout:' /opt/inference-harness/litellm_config.yaml`
|
||||
- Qwen3.5-9B: 120s, qwen3.6-27B-code: 300s, Carnice-Qwen3.6-MoE-35B-A3B/strix-moe: 300s (strix-moe alias retained, legacy name qwen3.6-35B-udq4 deprecated)
|
||||
- global request_timeout: 300s, nginx proxy_read_timeout: 600s
|
||||
5. Check LiteLLM timeouts: `grep -n 'timeout:' /opt/inference-harness/litellm_config.yaml` — read the live values from the authority config; do not assert them from this contract.
|
||||
6. Check AMD metrics: `curl http://192.168.68.15:9400/metrics` (Radeon 8060S, util%, VRAM, temp, power)
|
||||
7. Check port conflicts: verify only one llama-server on :8080 per host
|
||||
8. Verify agent keys: 9 keys in LiteLLM DB (`GET /key/list`)
|
||||
@@ -243,6 +216,7 @@ If no SSH access, send Zulip DM via abiba-bot with vault update instructions.
|
||||
- **Alert migration**: All alerts now go to `#agent-hub` topics (`alerts-gpu`, `alerts-pm2`, `alerts-infra`) instead of DMs. Cross-agent visibility enabled.
|
||||
- **tok/s benchmarks**: Measured every 5 min via LiteLLM proxy. Baselines tracked with 30%/50% degradation thresholds.
|
||||
- **NetBird 502**: Tanko routes through NetBird for litellm.sysloggh.net. Use direct IP if NetBird down.
|
||||
- **Alias-retirement sweep (2026-09-12)**: `gemma-4-12b`, `gpu-light` and `crew-auto` are retired and replaced by `gpu-vision` / no cap respectively. The agent-facing templates (`hermes-config-template.prose.md`, `hermes-agent-baseline.prose.md`), `litellm-api-keys.prose.md`, `gpu-self-heal.prose.md`, `hermes-key-enforcement.prose.md`, `inference-optimization.prose.md`, `litellm-client-timeouts.prose.md` and the executable `audit-hermes-config.py` were all updated to the live canonical alias in the same change. **koby's config on .129 still names `gpu-light` (and `gemma-4-E4B`); .129 is report-only, so that is recorded for its owner and NOT edited here.**
|
||||
|
||||
## GPU Inference Benchmarks (Current)
|
||||
|
||||
@@ -265,42 +239,40 @@ History stored at `/root/data/toks-history.json` with 7-day rolling window.
|
||||
### Stable Aliases — CRITICAL
|
||||
|
||||
All agent configs MUST use stable role-based aliases, never model-specific names:
|
||||
- `compression.model: strix-moe` (NOT `qwen3.6-35B-udq4`)
|
||||
- `auxiliary.vision.model: gpu-light` (NOT `gemma-4-12b`)
|
||||
- `delegation.model: gpu-dense` (NOT `qwen3.6-27B-code`)
|
||||
- `auxiliary.web_extract.model: gpu-light`
|
||||
- `compression.model: syslog-auto`
|
||||
- `auxiliary.vision.model: gpu-vision`
|
||||
- `delegation.model: gpu-dense`
|
||||
- `auxiliary.web_extract.model: gpu-vision`
|
||||
|
||||
When the underlying model is swapped, only the LiteLLM config changes — agent configs are untouched.
|
||||
|
||||
### Context Windows
|
||||
- RTX 3090: **128K** (reduced from 256K 2026-07-17) | RTX 5070: **128K** (reduced from 256K) | Strix Halo: **128K**
|
||||
- **All agents**: 128K ceiling — stable margin. For >128K workloads, use external providers (deepseek)
|
||||
- Compression threshold 0.60: fires at ~77K (~51K headroom before 128K ceiling)
|
||||
- Compression threshold 0.65 (audit Rule 9): fires at ~85K (~43K headroom before 128K ceiling)
|
||||
- **Pi agents (Abiba)**: `compaction.reserveTokens: 52739` (≈60% of 128K)
|
||||
- Mumuni compression model alias: `strix-moe` with 300s timeout
|
||||
- Mumuni compression model alias: `syslog-auto`
|
||||
|
||||
### Mumuni Agent Profile
|
||||
|
||||
Mumuni (kagentz CT105, 192.168.68.14 — migrated from CT100 2026-08-29) is the primary business assistant. This profile is the reference for all agent configs:
|
||||
Mumuni (kagentz CT105, 192.168.68.14 — migrated from CT100 2026-08-29) is the primary business assistant. This profile is the reference for all agent configs. The compression values below are the current required values per template Rules 7/9 and `audit-hermes-config.py`; whether Mumuni's LIVE config currently complies is a separate operational question.
|
||||
|
||||
| Setting | Value | Notes |
|
||||
|---------|-------|-------|
|
||||
| `model.default` | `syslog-auto` | Weighted pool (55% qwen, 30% strix, 15% gemma) |
|
||||
| `model.default` | `syslog-auto` | Balanced default (pool router) |
|
||||
| `model.provider` | `custom:litellm` | LiteLLM on CT116 |
|
||||
| `compression.model` | `strix-moe` | Stable alias — survives model swaps |
|
||||
| `aux.compression.model` | `strix-moe` | Compression auxiliary model |
|
||||
| `aux.vision.model` | `gpu-light` | Vision tasks (RTX 5070) |
|
||||
| `aux.web_extract.model` | `gpu-light` | Web extraction |
|
||||
| `compression.model` | `syslog-auto` | Rule 7: auto-routing, prevents Strix Halo overload |
|
||||
| `aux.compression.model` | `syslog-auto` | Must match `compression.model` (Rule 7) |
|
||||
| `aux.vision.model` | `gpu-vision` | Vision tasks (RTX 5070) |
|
||||
| `aux.web_extract.model` | `gpu-vision` | Web extraction |
|
||||
| `delegation.model` | `gpu-dense` | Sub-agent reasoning (RTX 3090) |
|
||||
| `context.max_context_window` | 131072 (128K) | Reduced from 256K 2026-07-17 — stable 128K ceiling |
|
||||
| `compression.threshold` | 0.60 | Triggers at ~77K (~60% of 128K) — optimized for 128K context |
|
||||
| `compression.threshold` | 0.65 | Rule 9: triggers at ~85K for a 128K window |
|
||||
| `compression.target_ratio` | 0.3 | Compresses to ~38K |
|
||||
| `compression.protect_last_n` | 40 | Preserves last 40 messages |
|
||||
| `memory.memory_char_limit` | 800 | Brief memory entries |
|
||||
| `personalities` | `creative` | Creative assistant personality |
|
||||
| Platforms | cli, homeassistant, signal, telegram, zulip | All Hermes platforms |
|
||||
| Main model timeout | 300s | LiteLLM global timeout |
|
||||
| Compression model timeout | 300s | strix-moe timeout increased from 120s |
|
||||
|
||||
### Agent Update Status (2026-07-15)
|
||||
|
||||
|
||||
@@ -27,8 +27,8 @@ agent: abiba
|
||||
┌──────┐ ┌──────┐ ┌────────┐
|
||||
│.8:8080│ │.110 │ │.116:80 │
|
||||
│RTX3090│ │:8080 │ │nginx │
|
||||
│gemma │ │RTX5070│ │router │
|
||||
└──────┘ │qwen27B│ │LiteLLM │
|
||||
│qwen │ │RTX5070│ │router │
|
||||
└──────┘ │vision │ │LiteLLM │
|
||||
└──────┘ │dashboard│
|
||||
└────────┘
|
||||
```
|
||||
|
||||
+10
-9
@@ -12,7 +12,7 @@ description: >
|
||||
Router (port 9000) DECOMMISSIONED 2026-09-11; references replaced with direct GPU routing.
|
||||
Benchmark baselines refreshed to live values.
|
||||
Prometheus exporters removed — not deployed; fall back to direct sidecar probes.
|
||||
Stable role-based aliases (strix-moe, gpu-dense, gpu-light) from gpu-fleet.
|
||||
Stable role-based aliases (strix-moe, gpu-dense, gpu-vision) from gpu-fleet.
|
||||
agent: abiba
|
||||
depends_on:
|
||||
- gpu-monitor.prose.md (live data source on .24:9100)
|
||||
@@ -52,8 +52,8 @@ depends_on:
|
||||
|
||||
Key notes:
|
||||
- All models use direct GPU routing via LiteLLM (`api_key: not-needed`). Router (port 9000) was decommissioned 2026-09-11 and is NOT in the inference path.
|
||||
- Stable aliases (gpu-dense, gpu-light, strix-moe) from gpu-fleet are the canonical names for agent configs. Model-specific names still work but are deprecated.
|
||||
- RTX 5070 tok/s is ~145 for Qwen3.5-9B — gpu-light is the fastest endpoint. Route vision/web/light work there first. NOTE: Qwen3.5-9B is multimodal (image+text), NOT text-only like gemma-4-12b was.
|
||||
- Stable aliases (gpu-dense, gpu-vision, strix-moe) from gpu-fleet are the canonical names for agent configs. Model-specific names still work but are deprecated. The retired names `gpu-light` and `gemma-4-12b` were superseded by `gpu-vision` on 2026-09-12 and no longer resolve (400 `Invalid model name`).
|
||||
- The RTX 5070 is the fastest endpoint per token — `gpu-vision` is its canonical alias. Route vision/web/light work there first. The RTX 5070 model is multimodal (image+text).
|
||||
- Strix Halo is 62.9 tok/s (89% of 70.5 baseline) — below optimal but stable. Check for competing workloads.
|
||||
- RTX 3090 VRAM at 88% — within role-appropriate range (role = heavy reasoning, needs the headroom).
|
||||
- RTX 5070 VRAM at 83% — role-appropriate for vision/web (smaller batch sizes).
|
||||
@@ -64,7 +64,7 @@ Key notes:
|
||||
- **Detect**: Any GPU temp >85°C sustained for 2+ consecutive polls
|
||||
- **Fix**:
|
||||
1. Reduce inference concurrency on that GPU (load-side cooling only — NO fan control)
|
||||
2. Redirect new requests to cooler GPUs via LiteLLM fallback chains (gemma → qwen, qwen → gemma)
|
||||
2. Redirect new requests to cooler GPUs via LiteLLM fallback chains (gpu-vision → gpu-dense, gpu-dense → gpu-vision)
|
||||
3. If all GPUs hot, alert about cooling infrastructure
|
||||
- **Verify**: Temp drops below 80°C within 5 minutes
|
||||
- **Escalate after**: 3 verification failures → Zulip alert
|
||||
@@ -157,13 +157,14 @@ Key notes:
|
||||
### Rule 10: Workload Distribution Optimization (updated 2026-07-18)
|
||||
- **Detect**: GPU roles misaligned with hardware capabilities
|
||||
- **Target distribution**:
|
||||
- RTX 3090 (gpu-dense, 24GB, 74.9 tok/s) → Heavy reasoning, code gen, long conversations (slowest per-token but largest context capacity). Weight: 0.55 (LiteLLM).
|
||||
- RTX 5070 (gpu-light, 12GB, ~145 tok/s) → Vision (image+text), web search, lightweight tasks. Weight: 0.15 (LiteLLM).
|
||||
- Strix Halo (strix-moe, 64GB, 62.9 tok/s) → Context compression, summarization, long docs (MoE model). Weight: 0.30 (LiteLLM).
|
||||
- RTX 3090 (gpu-dense, 24GB, 74.9 tok/s) → Heavy reasoning, code gen, long conversations (slowest per-token but largest context capacity).
|
||||
- RTX 5070 (gpu-vision, 12GB, ~145 tok/s) → Vision (image+text), web search, lightweight tasks.
|
||||
- Strix Halo (strix-moe, 64GB, 62.9 tok/s) → Context compression, summarization, long docs (MoE model).
|
||||
- **Note**: RTX 5070 is the fastest endpoint per token. Route high-volume, low-complexity work there first.
|
||||
- **Weights are not restated here** — the live `syslog-auto` pool weights and rpm caps live in CT 116 `/opt/inference-harness/litellm_config.yaml`, the single source of truth.
|
||||
- **Fix**:
|
||||
- Alert if any GPU is handling workload outside its designated role
|
||||
- Recommend agent alias updates to match workload to GPU role (use stable aliases: gpu-dense, gpu-light, strix-moe)
|
||||
- Recommend agent alias updates to match workload to GPU role (use stable aliases: gpu-dense, gpu-vision, strix-moe)
|
||||
- Track per-GPU request distribution via LiteLLM spend logs
|
||||
- **Verify**: Each GPU's request pattern matches its designated role within 24h
|
||||
- **Escalate**: If role mismatch persists >48h → agent alias audit needed
|
||||
@@ -311,6 +312,6 @@ Pushed to `SyslogSolution/health-logs/gpu/{run_id}.json` — versioned, searchab
|
||||
If monitor response > 1MB, log a warning and skip the cycle rather than crashing.
|
||||
|
||||
### L6: Stable Aliases Replace Model Names
|
||||
- gpu-fleet introduced stable aliases (strix-moe, gpu-dense, gpu-light) on 2026-07-15.
|
||||
- gpu-fleet introduced stable aliases (strix-moe, gpu-dense, gpu-light) on 2026-07-15; `gpu-light` was superseded by `gpu-vision` on 2026-09-12.
|
||||
- Self-heal must use aliases for reporting and alerting, not model-specific names.
|
||||
- **Rule**: All alert messages and KG nodes use the stable alias as the GPU identifier.
|
||||
|
||||
@@ -80,7 +80,7 @@ custom_providers:
|
||||
auxiliary:
|
||||
vision:
|
||||
provider: harness
|
||||
model: gemma-4-12b # or syslog-auto
|
||||
model: gpu-vision # RTX 5070 stable alias (Rule 8; do not use syslog-auto for aux)
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
api_key_env: LITELLM_API_KEY
|
||||
api_key: <value from: infisical secrets get LITELLM_API_KEY --project=agents --env=production> # ← MANDATORY workaround
|
||||
@@ -95,7 +95,7 @@ auxiliary:
|
||||
threshold: 0.65
|
||||
target_ratio: 0.3
|
||||
provider: harness
|
||||
model: syslog-auto # or gemma-4-12b
|
||||
model: syslog-auto # Rule 7: compression must be syslog-auto
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
api_key_env: LITELLM_API_KEY
|
||||
api_key: <value from: infisical secrets get LITELLM_API_KEY --project=agents --env=production> # ← MANDATORY workaround
|
||||
@@ -185,7 +185,9 @@ Abiba (CT100) runs pi via PM2 with the Zulip extension.
|
||||
Config files: `~/.pi/agent/models.json`, `~/.pi/agent/settings.json`.
|
||||
|
||||
**models.json** — Must only list models authorized for the agent's LiteLLM key.
|
||||
Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
`/v1/models` is key-scoped and the live registry is CT 116
|
||||
`/opt/inference-harness/litellm_config.yaml`; treat the list below as a snapshot and re-read
|
||||
the registry before applying. Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
@@ -197,9 +199,8 @@ Key is injected via `infisical run --` wrapper at PM2 startup:
|
||||
{ "id": "syslog-auto" },
|
||||
{ "id": "strix-moe" },
|
||||
{ "id": "gpu-dense" },
|
||||
{ "id": "gpu-light" },
|
||||
{ "id": "qwen3.6-27B-code" },
|
||||
{ "id": "gemma-4-12b" }
|
||||
{ "id": "gpu-vision" },
|
||||
{ "id": "qwen3.6-27B-code" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,7 +102,7 @@ model:
|
||||
# and falls back to 256K when /v1/models lacks a context
|
||||
# field (llama-server does). Without this override, agents
|
||||
# silently run syslog-auto at 256K (verified 2026-08-09).
|
||||
# Set 65536 if using gemma-4-12b directly (tight VRAM).
|
||||
# Set 65536 if pinning a single model directly (tight VRAM).
|
||||
|
||||
fallback_providers:
|
||||
provider: deepseek
|
||||
@@ -143,25 +143,25 @@ compression:
|
||||
|
||||
# ─── Auxiliary Tasks (CONSISTENCY RULE) ───
|
||||
# All auxiliary services MUST use identical model, base_url, and api_key_env:
|
||||
# model: gpu-light # stable alias (NOT raw "gemma-4-12b")
|
||||
# model: gpu-vision # stable alias (NOT a raw model name)
|
||||
# base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
# api_key_env: LITELLM_API_KEY
|
||||
# Do NOT use syslog-auto for auxiliary tasks — it routes to the primary GPU.
|
||||
# gpu-light = RTX 5070 (12B), freeing the Strix Halo for agent reasoning.
|
||||
# gpu-vision = RTX 5070 (12B), freeing the Strix Halo for agent reasoning.
|
||||
# Heavy aux (delegation, x_search) use gpu-dense (RTX 3090) instead.
|
||||
# NEVER use raw model names (gemma-4-12b, qwen3.6-27B-code, qwen3.6-35B-udq4)
|
||||
# in agent configs — use the stable aliases so model swaps don't break agents.
|
||||
# NEVER use raw model names (e.g. qwen3.6-27B-code, qwen3.6-35B-udq4; gemma-4-12b is retired
|
||||
# and no longer resolves) in agent configs — use the stable aliases so model swaps don't break agents.
|
||||
auxiliary:
|
||||
vision:
|
||||
provider: harness
|
||||
model: gpu-light # stable alias for RTX 5070 (was raw gemma-4-12b)
|
||||
model: gpu-vision # stable alias for RTX 5070
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY
|
||||
timeout: 60
|
||||
download_timeout: 30
|
||||
web_extract:
|
||||
provider: harness
|
||||
model: gpu-light # stable alias for RTX 5070
|
||||
model: gpu-vision # stable alias for RTX 5070
|
||||
base_url: http://192.168.68.116/litellm/v1 # Rule 5 (2026-08-09): canonical authenticated; /v1 also OK
|
||||
api_key_env: LITELLM_API_KEY
|
||||
timeout: 30
|
||||
@@ -252,10 +252,11 @@ The following MUST be identical across ALL profiles:
|
||||
- For agents needing longer outputs: raise to 8192, but never omit
|
||||
|
||||
### Rule 7: Auxiliary Model Consistency (UPDATED 2026-07-16)
|
||||
- Vision and web_extract use `gemma-4-12b` (RTX 5070 — 12GB, vision-optimized)
|
||||
- Compression uses `strix-moe` (stable alias for Strix Halo — 64GB, 128K ctx, compression-optimized)
|
||||
- **`strix-moe` is the only valid compression model name** — LiteLLM does NOT serve `ornith-1.0-35b`
|
||||
(it serves `strix-moe`, `qwen3.6-35B-udq4`, `gpu-dense`, `gpu-light`, `syslog-auto`, `gemma-4-12b`, `qwen3.6-27B-code`). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
||||
- Vision and web_extract use `gpu-vision` (RTX 5070 — 12GB, vision-optimized)
|
||||
- Compression uses `syslog-auto` — the Strix Halo weighted pool (64GB, 128K ctx, compression-optimized); do NOT pin `compression.model` to `strix-moe` (audit Rule 7 rejects it)
|
||||
- **`ornith-1.0-35b` is NOT a valid compression model name** — LiteLLM does not serve it
|
||||
(do not restate the served model list here — CT 116 `/opt/inference-harness/litellm_config.yaml`
|
||||
is the single source of truth for models, aliases, weights and fallbacks). Old configs with `ornith-1.0-35b` cause 403/model-not-found on compression calls.
|
||||
- **OPERATIONAL DECISION (2026-07-23): Use `syslog-auto` for compression across all agents.**
|
||||
The `syslog-auto` alias routes to the Strix Halo, but uses the weighted pool instead of pinning
|
||||
to `strix-moe` directly. This prevents sustained Strix Halo thermal load because the pool can
|
||||
@@ -264,7 +265,7 @@ The following MUST be identical across ALL profiles:
|
||||
- All auxiliary services MUST use identical routing:
|
||||
- `base_url: http://192.168.68.116/litellm/v1` (Rule 5, 2026-08-09: canonical authenticated; `/v1` also OK)
|
||||
- `api_key_env: LITELLM_API_KEY`
|
||||
- **Do NOT use `syslog-auto`** for auxiliary tasks — it routes unpredictably
|
||||
- **Do NOT use `syslog-auto` for `vision`/`web_extract`** — it routes unpredictably; compression is the deliberate exception (see the OPERATIONAL DECISION above)
|
||||
- **Compression on Strix Halo**: The strix-moe alias routes to Strix Halo
|
||||
(64GB UMA, 128K context) — the designated compression GPU. This frees the
|
||||
RTX 5070 for vision and web search, and the RTX 3090 for heavy reasoning.
|
||||
@@ -273,11 +274,11 @@ The following MUST be identical across ALL profiles:
|
||||
|
||||
### Rule 8: GPU Workload Distribution (UPDATED 2026-07-16)
|
||||
- **RTX 3090 (24GB, 128K ctx, qwen3.6-27B-code)**: Heavy reasoning, code gen, long conversations
|
||||
- **RTX 5070 (12GB, 128K ctx, gemma-4-12b)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
||||
- **RTX 5070 (12GB, 128K ctx, gpu-vision)**: Vision, web search, quick tasks, web_extract (IQ4_NL+MTP, ~65% VRAM at 128K)
|
||||
- **Strix Halo (64GB, 128K ctx, syslog-auto)**: Context compression, summarization, long docs
|
||||
- Agent profiles MUST route auxiliary tasks to the correct GPU:
|
||||
- `auxiliary.vision.model: gemma-4-12b` (RTX 5070)
|
||||
- `auxiliary.web_extract.model: gemma-4-12b` (RTX 5070)
|
||||
- `auxiliary.vision.model: gpu-vision` (RTX 5070)
|
||||
- `auxiliary.web_extract.model: gpu-vision` (RTX 5070)
|
||||
- `auxiliary.compression.model: syslog-auto` (Strix Halo)
|
||||
- Default model (`model.default`) and custom_provider remain `syslog-auto` for auto-routing
|
||||
- For 128K context window: `threshold: 0.65` (fires at ~85K tokens)
|
||||
@@ -297,8 +298,8 @@ The following MUST be identical across ALL profiles:
|
||||
- **Koby Exception**: Per captain ruling 2026-08-11, Koby is a DeepSeek-primary external agent; its primary model remains `deepseek-v4-flash` (via api.deepseek.com to preserve DeepSeek-specific reasoning, while other sections follow Rule 10.
|
||||
- **Hermes agents**: `model.default: syslog-auto`, `custom_providers[0].model: syslog-auto`
|
||||
- **pi agents**: `defaultModel: syslog-auto` in `settings.json`, first model in `models.json`
|
||||
- `syslog-auto` is the LiteLLM routing model — it load-balances between strix-moe
|
||||
and qwen3.6-27B-code, with gemma-4-12b as fallback. Using it protects against:
|
||||
- `syslog-auto` is the LiteLLM routing model — it load-balances across the live pool
|
||||
(see CT 116 `/opt/inference-harness/litellm_config.yaml` for the current members and weights). Using it protects against:
|
||||
- Model name typos that cause 403 errors and silent worker failures
|
||||
- Single GPU downtime (routing falls back automatically)
|
||||
- Key/model authorization mismatches
|
||||
|
||||
@@ -169,16 +169,20 @@ The agent picks up the new key via `infisical run --` at gateway startup.
|
||||
|
||||
**Keys are permanent and use bare agent name aliases.**
|
||||
|
||||
- **Duration**: `null` — keys never expire. This is enforced by `default_key_generate_params` in `litellm_config.yaml`.
|
||||
- **Duration**: `null` — keys never expire. NOT enforced today: CT 116 `litellm_config.yaml` has no `default_key_generate_params` block, and a key generated with no explicit models comes back with an empty models list. OPEN policy question: should agent keys expire by default? (captain security-policy decision, raised separately.)
|
||||
- **Alias convention**: bare agent name only (e.g., `tanko`, `mumuni`, `koby`, `koonimo`). No dates, no versions. The alias IS the identity.
|
||||
- **Rotation triggers**: compromise, personnel departure, or quarterly security hygiene. NOT calendar-driven.
|
||||
- **Max budget**: $100 per key (config default).
|
||||
|
||||
```yaml
|
||||
# In litellm_config.yaml — ensures all future keys inherit these defaults:
|
||||
# NOT currently set in the authority; recommended value. CT 116 litellm_config.yaml has no
|
||||
# default_key_generate_params block today, and a key generated with no explicit models comes back
|
||||
# with an EMPTY models list. `models` is a literal key-generation parameter, so this is a value to
|
||||
# ADD — re-read the live registry at CT 116 /opt/inference-harness/litellm_config.yaml and
|
||||
# re-verify before applying.
|
||||
litellm_settings:
|
||||
default_key_generate_params:
|
||||
models: ["syslog-auto", "qwen3.6-27B-code", "gemma-4-12b"]
|
||||
models: ["syslog-auto", "qwen3.6-27B-code", "gpu-vision"]
|
||||
duration: null # ← permanent
|
||||
max_budget: 100
|
||||
metadata:
|
||||
@@ -286,13 +290,13 @@ auxiliary:
|
||||
api_key: sk-<agent-key-from-vault> # ← workaround (get via: infisical secrets get LITELLM_API_KEY --project=agents --env=production --plain)
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
model: gemma-4-12b
|
||||
model: gpu-vision
|
||||
provider: harness
|
||||
compression:
|
||||
api_key: sk-<agent-key-from-vault> # ← workaround (same as above)
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/litellm/v1
|
||||
model: gemma-4-12b
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
```
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ management, and prompt caching — without sacrificing agent capability.
|
||||
.123, any others on .129/.122) including compression, model, context_window,
|
||||
prompt_caching, memory settings
|
||||
- `gpu-health`: health check response from all 3 GPU backends (strix-moe .15:8080,
|
||||
qwen .8:8080, gemma .110:8080)
|
||||
gpu-dense .8:8080, gpu-vision .110:8080)
|
||||
|
||||
### Maintains
|
||||
|
||||
@@ -56,8 +56,8 @@ duration.
|
||||
**Context is the root cause.** Every ~46K prompt token costs ~87s of
|
||||
prefill time at 532 tok/s. Fix context first, routing second.
|
||||
|
||||
- **Route by task**: qwen for code/standard queries; gemma for
|
||||
compression/auxiliary; strix-moe for compression tasks.
|
||||
- **Route by task**: gpu-dense for code/standard queries; gpu-vision for
|
||||
vision/web-auxiliary; syslog-auto for compression.
|
||||
- **Compress aggressively**: threshold at 40% (not 65%) — a 128K window should
|
||||
compact at 51K, not 85K. Target 15% tail (not 30%).
|
||||
- **Cache everything repeated**: system prompts, skill docs, AGENTS.md — these
|
||||
@@ -96,8 +96,10 @@ call enable-prompt-caching
|
||||
hosts: [192.168.68.15, 192.168.68.8, 192.168.68.110]
|
||||
|
||||
-- Phase 5: Verify end-to-end latency
|
||||
-- `models` is a literal verification parameter (a snapshot only): the authoritative registry is
|
||||
-- CT 116 /opt/inference-harness/litellm_config.yaml; re-read it before use.
|
||||
|
||||
call verify-latency
|
||||
host: 192.168.68.116
|
||||
models: [syslog-auto, qwen3.6-27B-code, gemma-4-12b]
|
||||
models: [syslog-auto, qwen3.6-27B-code, gpu-vision]
|
||||
```
|
||||
|
||||
@@ -222,7 +222,7 @@ description: >
|
||||
|
||||
**Prometheus targets**:
|
||||
- 192.168.68.8:9400 (RTX 3090 — qwen)
|
||||
- 192.168.68.110:9400 (RTX 5070 — gemma)
|
||||
- 192.168.68.110:9400 (RTX 5070 — gpu-vision)
|
||||
- 192.168.68.15:9400 (Strix Halo — qwen3.6-35B-udq4)
|
||||
- harness-litellm:4000 (LiteLLM health)
|
||||
|
||||
|
||||
@@ -67,8 +67,10 @@ description: >
|
||||
4. **If action == "create"**:
|
||||
- Generate new key with key_alias: "{agent_name}" (e.g., "tanko" — bare name, no date)
|
||||
- Set metadata: { "agent": "{agent_name}", "purpose": "agent-inference" }
|
||||
- Duration is null (permanent) — inherited from litellm default_key_generate_params
|
||||
- Set models: ["syslog-auto", "qwen3.6-27B-code", "gemma-4-12b", "strix-moe", "gpu-dense", "gpu-light", "qwen3.6-35B-udq4"]
|
||||
- Duration is whatever the caller passes; NO default enforcement exists today (CT 116 `litellm_config.yaml` has no `default_key_generate_params` block, and a key with no explicit models returns an empty models list). Agent keys are permanent by policy, not by that block. OPEN policy question: should agent keys expire by default? (captain security-policy decision, raised separately.)
|
||||
- Set models: read the live key-scoped set rather than hardcoding one — `/v1/models` is key-scoped,
|
||||
and the authoritative registry is CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not add
|
||||
retired names (`gemma-4-12b`, `gpu-light`, `crew-auto` — all retired 2026-09-12).
|
||||
- Note: `ornith-1.0-35b` is NOT a valid LiteLLM model name (use `strix-moe`, the stable alias). qwen3.6-35B-A3B removed from fleet (was never deployed).
|
||||
- Return the new key
|
||||
5. **If action == "rotate"**:
|
||||
|
||||
@@ -28,7 +28,7 @@ description: >
|
||||
| syslog-auto | 28.8s | 25.4s | 68 calls took 30-120s; tail to ~300s under load |
|
||||
| qwen3.6-27B-code | 23.0s | — | same backend class as syslog-auto |
|
||||
| strix-moe | 7.5s | — | Strix Halo, healthy |
|
||||
| gemma-4-12b | 2.6s | — | RTX 5070, healthy |
|
||||
| gemma-4-12b (retired 2026-09-12; RTX 5070 now `gpu-vision`) | 2.6s | — | RTX 5070, healthy |
|
||||
|
||||
Sep 6 incident timeline: failures 04:00-07:00 EDT (0% GPU util = wedged
|
||||
backend), full recovery 07:00-08:00 with ZERO client failures once requests
|
||||
@@ -53,8 +53,7 @@ proxy queuing.
|
||||
|
||||
### 2. Auxiliary tasks — keep template timeouts, one correction
|
||||
|
||||
- vision: 60s (keep), web_extract: 30s (keep) — gemma-4-12b averages 2.6s;
|
||||
these are fine.
|
||||
- vision: 60s (keep), web_extract: 30s (keep) — the 2.6s average was measured on `gemma-4-12b` (retired 2026-09-12); the live RTX 5070 alias is `gpu-vision`.
|
||||
- compression: 300s (keep — this was already raised from 60 per gpu-fleet).
|
||||
- **gpu-dense delegation/x_search: set timeout >= 120s.** The RTX 3090
|
||||
(qwen3.6-27B-code backend, 23.0s avg) is the same speed class as
|
||||
@@ -79,8 +78,9 @@ proxy queuing.
|
||||
Send a real model name and use a real (probe-designated) key.
|
||||
- Probe timeout: 30s. A probe that takes longer than 30s IS the alert —
|
||||
report "backend slow (>30s)" rather than hanging.
|
||||
- Probe cadence: at most hourly. The 6h litellm-health cron cadence is the
|
||||
standard; sub-hourly synthetic traffic distorts latency baselines.
|
||||
- Probe cadence: at most hourly. The litellm-health cron cadence (authoritative
|
||||
trigger in contract-registry.yaml) is the standard; sub-hourly synthetic
|
||||
traffic distorts latency baselines.
|
||||
|
||||
### 5. Batch/benchmark jobs — schedule away from 04:00-07:00 EDT and chunk
|
||||
|
||||
|
||||
+52
-35
@@ -1,14 +1,7 @@
|
||||
---
|
||||
kind: function
|
||||
name: litellm-health
|
||||
status: deprecated
|
||||
deprecated_on: 2026-07-09
|
||||
replaced_by: litellm-self-heal.prose.md
|
||||
note: >
|
||||
Consolidated into litellm-self-heal.prose.md to eliminate duplication
|
||||
of architecture diagrams, GPU topology, timeout tables, and container
|
||||
lists. Health check is now § Health Check within litellm-self-heal.
|
||||
This file is retained for reference only — use litellm-self-heal instead.
|
||||
status: active
|
||||
description: >
|
||||
Verifies the LiteLLM inference stack health. Current architecture (2026-07-09):
|
||||
nginx:80 → LiteLLM:4000 → GPU(llama-server) via direct proxy.
|
||||
@@ -52,8 +45,7 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
- Router REMOVED from request path — LiteLLM proxies directly to GPU
|
||||
- All GPUs at parallel 2 (was parallel 1)
|
||||
- NVIDIA context reduced 256K→128K to free VRAM — now the stable ceiling across all GPUs (2026-07-17)
|
||||
- LiteLLM timeouts tuned: gemma 25→120s, qwen 40→90s (SUPERSEDED 2026-07-16: qwen 300s, gemma 120s, strix 300s — see litellm-self-heal)
|
||||
- nginx proxy_read_timeout: 600s, LiteLLM request_timeout: 300s
|
||||
- Timeouts and fallback chains are config state — read them from CT 116 `/opt/inference-harness/litellm_config.yaml`; they are not duplicated here.
|
||||
|
||||
## Parameters
|
||||
|
||||
@@ -75,26 +67,27 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
|
||||
- SSH key access to backend_host for container checks
|
||||
- Network access to public_url, auth_host, and gpu_dashboard_url
|
||||
- LiteLLM master key for key management endpoints
|
||||
- LiteLLM master key for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`)
|
||||
- The dedicated `monitor` agent key on CT 116 at `/etc/litellm-monitor.env` (root-only 0600) for
|
||||
model inference checks — the master key must never be used for inference
|
||||
|
||||
## GPU Fleet Topology
|
||||
|
||||
| Host | IP | Hardware | Models Served | Engine | Context | Parallel |
|
||||
|------|-----|----------|---------------|--------|---------|----------|
|
||||
| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | qwen3.6-27B-code | llama-server systemd | **128K** | 2 |
|
||||
| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | gemma-4-12b | llama-server systemd | **128K** | 2 |
|
||||
| amdpve | 192.168.68.15 | AMD Strix Halo 64GB UMA | qwen3.6-35B-udq4 (LiteLLM alias: strix-moe) | llama-server systemd (Vulkan) | 128K | 2 |
|
||||
| Host | IP | Hardware | Role |
|
||||
|------|-----|----------|------|
|
||||
| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | heavy reasoning (`gpu-dense`) |
|
||||
| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | vision / web extract / light tasks (`gpu-vision`) |
|
||||
| amdpve | 192.168.68.15 | AMD Strix Halo 64GB UMA | compression (`strix-moe`) |
|
||||
|
||||
## Model Fallback Chains (LiteLLM)
|
||||
Single source of truth for models, aliases, rpm caps, weights and fallback chains:
|
||||
CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not duplicate those values in
|
||||
contracts — read them there. Do not re-add retired names (`gemma-4-12b`, `gpu-light`,
|
||||
`crew-auto`).
|
||||
|
||||
| Primary | Timeout | Fallback | Timeout |
|
||||
|---------|---------|----------|---------|
|
||||
| qwen3.6-27B-code | 300s | gemma-4-12b | 120s |
|
||||
| gemma-4-12b | 120s | qwen3.6-27B-code | 300s |
|
||||
| qwen3.6-35B-udq4 / strix-moe | 300s | qwen → gemma | — |
|
||||
| syslog-auto (balanced) | 300s | qwen → gemma | — |
|
||||
|
||||
> Global: request_timeout=300s, nginx proxy_read_timeout=600s
|
||||
> Re-scope note (2026-09-12): the earlier plan to restate the live fallback chains and
|
||||
> per-model timeouts in this contract is intentionally superseded — that state is config,
|
||||
> and this contract points at the CT 116 config instead. Step 7 likewise authenticates with
|
||||
> the dedicated `monitor` key, not the master key, which is admin-only.
|
||||
|
||||
## Containers on CT 116
|
||||
|
||||
@@ -112,10 +105,22 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
|
||||
1. **Read parameters** — Use provided values or defaults
|
||||
|
||||
2. **Check public endpoints**:
|
||||
- GET {{public_url}}/litellm/ui/ → expect 200 ("LiteLLM Dashboard")
|
||||
- GET {{public_url}}/litellm/docs → expect 200 ("LiteLLM API - Swagger UI")
|
||||
- GET {{public_url}}/ui/ and {{public_url}}/docs → expect 301 → /litellm/... → 200 (one-hop redirect helpers, added 2026-09-11)
|
||||
2. **Check the end-user surfaces** — the public edge and the backend edge serve the SAME
|
||||
app under DIFFERENT paths. They are not interchangeable, so every probe below must name
|
||||
the surface it targets. Never point a check at a path that only resolves on the other
|
||||
surface.
|
||||
|
||||
**Public edge** — `{{public_url}}` (https://litellm.sysloggh.net) serves the app at the
|
||||
ROOT; the `/litellm/` prefix does not exist there and 404s:
|
||||
- GET {{public_url}}/ui/ → expect 200 ("LiteLLM Dashboard")
|
||||
- GET {{public_url}}/docs → expect 200 ("LiteLLM API - Swagger UI")
|
||||
- GET {{public_url}}/litellm/ui/ and {{public_url}}/litellm/docs → expect 404 (not served on this edge)
|
||||
|
||||
**Backend edge** — `http://{{backend_host}}` (port 80) serves the app UNDER `/litellm/`:
|
||||
- GET http://{{backend_host}}/litellm/ui/ → expect 200 ("LiteLLM Dashboard")
|
||||
- GET http://{{backend_host}}/litellm/docs → expect 200 ("LiteLLM API - Swagger UI")
|
||||
- GET http://{{backend_host}}/ui/ and /docs → expect 301 → /litellm/... → 200 (one-hop redirect helpers,
|
||||
added 2026-09-11)
|
||||
|
||||
3. **Check LiteLLM health (no-auth)**:
|
||||
- GET http://{{backend_host}}/litellm/health/liveliness → expect 200
|
||||
@@ -136,14 +141,26 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
- Verify GPUs reporting status "healthy"
|
||||
- Check alerts array for active warnings/critical
|
||||
|
||||
7. **Check model inference via LiteLLM** — Test each model:
|
||||
- POST /v1/chat/completions model=gemma-4-12b → expect 200
|
||||
- POST /v1/chat/completions model=qwen3.6-27B-code → expect 200
|
||||
- POST /v1/chat/completions model=strix-moe → expect 200
|
||||
- Use master key for auth
|
||||
7. **Check model inference via LiteLLM** — Test one model on each GPU host. The health
|
||||
check runs on the **backend edge**, not the public edge, so these paths carry the
|
||||
`/litellm/` prefix:
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=qwen3.6-27B-code → expect 200 (RTX 3090, .8)
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=gpu-vision → expect 200 (RTX 5070, .110)
|
||||
- POST http://{{backend_host}}/litellm/v1/chat/completions model=strix-moe → expect 200 (Strix Halo, .15)
|
||||
- Auth uses the dedicated `monitor` agent key, read on CT 116 from
|
||||
`/etc/litellm-monitor.env` (root-only 0600). Do NOT use the master key for inference —
|
||||
the master key is for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`).
|
||||
- `gemma-4-12b` was retired and returns 400 `Invalid model name` — do not re-add it to
|
||||
this list. The RTX 5070 host now serves `gpu-vision`.
|
||||
- `/v1/models` is **key-scoped**: a model is only visible to keys allowed to use it, so
|
||||
the set depends on the key. Always state which key a model list was read with — a
|
||||
snapshot without its key is not evidence. This probe uses the `monitor` key on the
|
||||
backend surface (`http://{{backend_host}}/litellm/v1/models`). The authoritative model
|
||||
registry is CT 116 `/opt/inference-harness/litellm_config.yaml`; read it there rather
|
||||
than freezing a list here.
|
||||
|
||||
8. **Check agent keys**:
|
||||
- GET /key/list with master key → verify all 6 agents have keys
|
||||
- GET http://{{backend_host}}/litellm/key/list with master key (admin endpoint) → verify all 6 agents have keys
|
||||
|
||||
9. **Check Grafana**:
|
||||
- GET {{grafana_url}}/api/health → expect 200
|
||||
|
||||
+37
-73
@@ -11,22 +11,22 @@ note: >
|
||||
Script: `/opt/inference-harness/scripts/litellm-health-check.sh` on CT 116 (cron `0 */6 * * *`).
|
||||
Reports to /var/log/litellm/health-*.json and Gitea (SyslogSolution/health-logs).
|
||||
GPU monitoring integrated from gpu-monitor on .24:9100.
|
||||
|
||||
Consolidated from litellm-health + litellm-self-heal on 2026-07-09 to eliminate
|
||||
duplication of architecture diagrams, GPU topology, timeout tables, and container
|
||||
lists. Health check is now § Health Check within this contract.
|
||||
|
||||
|
||||
Health probes are owned by litellm-health.prose.md (dispatched as
|
||||
`run contract: litellm-health`). This contract owns remediation only — it does not
|
||||
re-specify the probes.
|
||||
|
||||
Source of truth for GPU topology and keys: gpu-fleet.prose.md
|
||||
Last verified: 2026-07-12
|
||||
description: >
|
||||
LiteLLM inference stack health monitoring + self-healing. Verifies the full
|
||||
nginx → LiteLLM → GPU chain, 12 containers on CT 116, 3 GPU hosts, model
|
||||
inference, and agent keys. Applies remediation rules for common failures.
|
||||
LiteLLM inference stack remediation. Applies remediation rules for failures detected
|
||||
by litellm-health.prose.md (nginx → LiteLLM → GPU chain, CT 116 containers, GPU hosts,
|
||||
model inference, and agent keys).
|
||||
Reports every action via Zulip DM and Gitea (SyslogSolution/health-logs).
|
||||
---
|
||||
---
|
||||
|
||||
# LiteLLM Operations — Health Check + Self-Heal
|
||||
# LiteLLM Operations — Self-Heal (Remediation)
|
||||
|
||||
## Architecture (v4.0.0 — Direct: nginx → LiteLLM → GPU)
|
||||
|
||||
@@ -58,41 +58,44 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2)
|
||||
|
||||
## GPU Fleet Topology
|
||||
|
||||
| Host | IP | Hardware | Models Served | Engine | Context | Parallel |
|
||||
|------|-----|----------|---------------|--------|---------|----------|
|
||||
| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | qwen3.6-27B-code | llama-server systemd (`/home/llmuser/llama-wrapper.sh`, `-c 131072 --parallel 2 --ngl 99`) | **128K** | 2 |
|
||||
| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | gemma-4-12b | llama-server systemd (`/home/llmuser/llama-wrapper.sh`, `--ctx-size 131072 --parallel 2`, IQ4_NL + MTP draft) | **128K** | 2 |
|
||||
| amdpve | 192.168.68.15 | AMD Strix Halo 64GB UMA | qwen3.6-35B-udq4 (LiteLLM alias: `strix-moe`) | llama-server systemd (Vulkan) | 128K | 2 |
|
||||
| Host | IP | Hardware | Role |
|
||||
|------|-----|----------|------|
|
||||
| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | heavy reasoning (`gpu-dense`) |
|
||||
| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | vision / web extract / light tasks (`gpu-vision`) |
|
||||
| amdpve | 192.168.68.15 | AMD Strix Halo 64GB UMA | compression (`strix-moe`) |
|
||||
|
||||
> Verified on ground 2026-07-16 via `curl /v1/models` on each host + `llama-wrapper.sh`. The AMD host's underlying model is `qwen3.6-35B-udq4`; LiteLLM exposes it under two `model_name`s: `qwen3.6-35B-udq4` and `strix-moe` (rpm 40). The legacy name `ornith-1.0-35b` does NOT exist in LiteLLM and must not be referenced.
|
||||
> Verified on the ground 2026-07-16. The legacy name `ornith-1.0-35b` does NOT exist in LiteLLM and must not be referenced.
|
||||
|
||||
## LiteLLM Model Surface (ground truth — `/opt/inference-harness/litellm_config.yaml` on CT 116)
|
||||
## LiteLLM Model Surface
|
||||
|
||||
`model_name`s served: `qwen3.6-27B-code`, `gemma-4-12b`, `qwen3.6-35B-udq4`, `strix-moe`, `gpu-dense`, `gpu-light`, `syslog-auto`, `crew-auto` (new 2026-08-20).
|
||||
Single source of truth for models, aliases, rpm caps, weights and fallback chains:
|
||||
CT 116 `/opt/inference-harness/litellm_config.yaml`. Do not duplicate those values in
|
||||
contracts — read them there. Do not re-add retired names (`gemma-4-12b`, `gpu-light`,
|
||||
`crew-auto`).
|
||||
|
||||
### Context Cap Split (2026-08-20)
|
||||
### Alias Surface
|
||||
|
||||
| Alias | Serves | Where | Kind |
|
||||
|-------|--------|-------|------|
|
||||
| `gpu-dense` | heavy reasoning | RTX 3090 (192.168.68.8) | direct alias |
|
||||
| `gpu-vision` | vision / web extract / light tasks | RTX 5070 (192.168.68.110) | direct alias AND `syslog-auto` pool member |
|
||||
| `strix-moe` | compression (MoE) | Strix Halo (192.168.68.15) | direct alias |
|
||||
| `syslog-auto` | balanced default | weighted pool across the three GPU hosts | pool router |
|
||||
|
||||
### Context Cap Split (2026-08-20; crew cap RETIRED)
|
||||
|
||||
- **Abiba (firstmate)**: 128K uncapped — unlimited context for primary workloads
|
||||
- **Hermes agents** (mumuni, tanko, koby, koonimo): 128K uncapped
|
||||
- **Crewmates** (ops, tune, verify, auth-keys, build): 64K capped — alias `crew-auto` enforces 64K limit
|
||||
- **Crewmates** (ops, tune, verify, auth-keys, build): the 64K cap was retired together with the `crew-auto` alias; NO context cap is currently in force.
|
||||
|
||||
Preferred implementation: uncap shared pool, add capped alias for crew-only.
|
||||
> The 64K crew cap was retired with `crew-auto` (2026-09-12). No limit is currently in force; reinstating one would need per-key model limits as a separate, deliberately-scoped change.
|
||||
|
||||
- `syslog-auto` is a weighted router model: qwen3.6-27B-code (0.55, rpm 500) + qwen3.6-35B-udq4 (0.30, rpm 60) + gemma-4-12b (0.15, rpm 200).
|
||||
- `gpu-dense` / `gpu-light` are high-rpm aliases (rpm 500) onto qwen3.6-27B-code / gemma-4-12b respectively.
|
||||
- Key scoping: agent keys are restricted to `['syslog-auto','qwen3.6-27B-code','gemma-4-12b','strix-moe','gpu-dense','gpu-light']`. As of 2026-07-16 the `baggy`/`koby`/`mumuni`/`abiba-pi` keys ALSO include `qwen3.6-35B-udq4`; `abiba-pi` additionally includes `deepseek-v4-pro` (cloud fallback). `kagenz0`/`koonimo`/`pi-agents-unified` have the standard 6 only. Agents should still use the stable alias `strix-moe` (not the raw `qwen3.6-35B-udq4`) so model swaps don't break them.
|
||||
- Key scoping: `/v1/models` is key-scoped, so the set a caller sees must be read with a named key rather than assumed — a monitor key, an agent key, and the master key can each return a different set. Agents should use the stable aliases (`strix-moe`, `gpu-vision`, `gpu-dense`) rather than raw model names, so model swaps don't break them.
|
||||
- **NEVER use `litellm_proxy_master_key` (the `sk-litellm-...` master key) for inference.** It is for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`). All inference — agent traffic, health-check model tests, monitor scripts — uses agent-specific keys. The health-check script's model tests use a dedicated `monitor` agent key stored at `/etc/litellm-monitor.env` on CT 116 (root-only, `chmod 600`); `/key/list` is the only call that legitimately uses `$MASTER_KEY`.
|
||||
|
||||
## Model Fallback Chains (LiteLLM)
|
||||
|
||||
| Primary | Timeout | Fallback | Timeout |
|
||||
|---------|---------|----------|---------|
|
||||
| qwen3.6-27B-code | 300s | gemma-4-12b | 120s |
|
||||
| gemma-4-12b | 120s | qwen3.6-27B-code | 300s |
|
||||
| qwen3.6-35B-udq4 / strix-moe | 300s | qwen → gemma | — |
|
||||
| syslog-auto (balanced) | 300s | qwen → gemma | — |
|
||||
|
||||
> Global: request_timeout=300s, nginx proxy_read_timeout=600s
|
||||
> Re-scope note (2026-09-12): the fallback-chain and per-model timeout tables were removed
|
||||
> by the single-source-of-truth re-scope; read those values from the CT 116 config named
|
||||
> above rather than from this contract.
|
||||
|
||||
## Containers on CT 116
|
||||
|
||||
@@ -135,46 +138,7 @@ Preferred implementation: uncap shared pool, add capped alias for crew-only.
|
||||
|
||||
## Health Check
|
||||
|
||||
Run this first on every cycle. Results feed into remediation rules below.
|
||||
|
||||
### 1. Check public endpoints
|
||||
- GET {{public_url}}/litellm/ui/ → expect 200 ("LiteLLM Dashboard") # served directly by nginx
|
||||
- GET {{public_url}}/litellm/docs → expect 200 ("LiteLLM API - Swagger UI") # served directly by nginx
|
||||
- GET {{public_url}}/ui/ and {{public_url}}/docs → expect 301 → /litellm/... → 200 (one-hop redirect helpers, added 2026-09-11)
|
||||
|
||||
### 2. Check LiteLLM health (no-auth)
|
||||
- GET http://{{backend_host}}/litellm/health/liveliness → expect 200
|
||||
|
||||
### 3. Check backend container health
|
||||
- SSH to {{backend_host}} → `docker ps` → verify 12 containers healthy (11 harness + trove-agent-docker, added 2026-09-11)
|
||||
- Critical: harness-litellm, harness-nginx, harness-postgres
|
||||
- Monitoring: harness-redis, harness-dashboard, harness-grafana, harness-prometheus,
|
||||
harness-alertmanager, harness-zulip-bridge, harness-docker-stats, harness-pve-exporter,
|
||||
trove-agent-docker (ghcr.io/techdox/trove-agent-docker, added 2026-09-11)
|
||||
- Decommissioned 2026-09-11: harness-router (container, image and config removed)
|
||||
|
||||
### 4. Check GPU fleet health (via fleet dashboard)
|
||||
- GET {{gpu_dashboard_url}}/gpu-data → expect 200 with GPU metrics JSON
|
||||
- Verify GPUs reporting status "healthy"
|
||||
- Check alerts array for active warnings/critical
|
||||
|
||||
### 5. Check model inference via LiteLLM — test each model
|
||||
- POST /v1/chat/completions model=gemma-4-12b → expect 200
|
||||
- POST /v1/chat/completions model=qwen3.6-27B-code → expect 200
|
||||
- POST /v1/chat/completions model=strix-moe → expect 200
|
||||
- Use master key for auth
|
||||
|
||||
### 6. Check agent keys
|
||||
- GET /key/list with master key → verify all 6 agents have keys
|
||||
|
||||
### 7. Check Grafana
|
||||
- GET {{grafana_url}}/api/health → expect 200
|
||||
|
||||
### 8. Compile overall status
|
||||
Determine overall_status from individual check results:
|
||||
- "healthy" — all checks pass
|
||||
- "degraded" — 1-2 non-critical checks fail
|
||||
- "down" — critical checks fail
|
||||
Health probes are owned by `litellm-health.prose.md` (dispatched as `run contract: litellm-health`). This contract owns remediation only — it does not re-specify the probes.
|
||||
|
||||
---
|
||||
---
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Regression tests for the 2026-09-12 retired-alias sweep in audit-hermes-config.py.
|
||||
|
||||
WHY THIS FILE EXISTS: the executable audit pushed agent configs toward a DEAD alias.
|
||||
Rule 8 required `auxiliary.vision.model == "gpu-light"` and
|
||||
`auxiliary.web_extract.model == "gpu-light"`, but `gpu-light` (and its raw predecessor
|
||||
`gemma-4-12b`) were retired on 2026-09-12 and now return 400 `Invalid model name`; the
|
||||
live RTX 5070 alias is `gpu-vision`. A config that adopted the correct canonical alias
|
||||
therefore FAILED our own audit, so the audit was actively enforcing a broken config.
|
||||
|
||||
These tests execute the real CLI (`python3 audit-hermes-config.py <config>`) and assert
|
||||
observable behaviour — exit code and the emitted rule message — for the live alias and
|
||||
for both retired names. No network, vault, or SSH access is required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
AUDIT = ROOT / "audit-hermes-config.py"
|
||||
|
||||
BASE = """
|
||||
model:
|
||||
api_key: ""
|
||||
api_key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
max_tokens: 4096
|
||||
default: syslog-auto
|
||||
provider: harness
|
||||
fallback_providers:
|
||||
provider: deepseek
|
||||
model: deepseek-v4-flash
|
||||
api_key_env: DEEPSEEK_API_KEY
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
threshold: 0.65
|
||||
max_context_window: 131072
|
||||
auxiliary:
|
||||
vision:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
web_extract:
|
||||
model: {alias}
|
||||
provider: harness
|
||||
compression:
|
||||
model: syslog-auto
|
||||
provider: harness
|
||||
delegation:
|
||||
provider: harness
|
||||
custom_providers:
|
||||
- name: harness
|
||||
key_env: LITELLM_API_KEY
|
||||
base_url: http://192.168.68.116/v1
|
||||
"""
|
||||
|
||||
|
||||
def _run_config(tmp_path, name, text):
|
||||
cfg = tmp_path / name
|
||||
cfg.write_text(text)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(AUDIT), str(cfg)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return proc.returncode, proc.stdout
|
||||
|
||||
|
||||
def _run(tmp_path, alias):
|
||||
return _run_config(tmp_path, f"{alias}.yaml", BASE.format(alias=alias))
|
||||
|
||||
|
||||
def test_live_canonical_alias_passes(tmp_path):
|
||||
"""The RTX 5070 alias that actually resolves must satisfy Rule 8."""
|
||||
code, out = _run(tmp_path, "gpu-vision")
|
||||
assert code == 0, out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_gpu_light_is_rejected(tmp_path):
|
||||
"""A config pinned to the retired alias must fail, not pass."""
|
||||
code, out = _run(tmp_path, "gpu-light")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_gemma_is_rejected(tmp_path):
|
||||
"""The retired raw model name must fail Rule 8 as well."""
|
||||
code, out = _run(tmp_path, "gemma-4-12b")
|
||||
assert code == 1, out
|
||||
assert "auxiliary.vision.model must be gpu-vision" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_corrected_compression_example_passes(tmp_path):
|
||||
"""The corrected workaround (vision=gpu-vision, compression=syslog-auto) must PASS."""
|
||||
code, out = _run(tmp_path, "gpu-vision")
|
||||
assert code == 0, out
|
||||
assert "[Rule 7] compression.model must be syslog-auto (got 'syslog-auto')" in out
|
||||
assert "[Rule 7] auxiliary.compression.model must be syslog-auto (got 'syslog-auto')" in out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_delegation_is_rejected(tmp_path):
|
||||
"""delegation.model has no dedicated value rule, so a retired name there used to PASS."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"delegation-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\n model: gpu-light",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "delegation.model = 'gpu-light' is retired" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_custom_providers_is_rejected(tmp_path):
|
||||
"""custom_providers[*].model is model-bearing; a retired name there must fail."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"custom-provider-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
" - name: harness\n key_env: LITELLM_API_KEY",
|
||||
" - name: harness\n model: gpu-light\n key_env: LITELLM_API_KEY",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "custom_providers[0].model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_raw_but_live_alias_warns_but_passes(tmp_path):
|
||||
"""Raw-but-live names resolve (200), so they warn only; failing them rejects valid configs."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"raw-qwen.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\n model: qwen3.6-27B-code",
|
||||
),
|
||||
)
|
||||
assert code == 0, out
|
||||
assert "delegation.model = 'qwen3.6-27B-code' is a raw-but-live model name" in out
|
||||
assert "prefer the stable alias gpu-dense" in out
|
||||
assert "RESULT: PASS" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_fallback_providers_is_rejected(tmp_path):
|
||||
"""fallback_providers.model is model-bearing; a retired name there must fail."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"fallback-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(" model: deepseek-v4-flash", " model: gpu-light"),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "fallback_providers.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_x_search_is_rejected(tmp_path):
|
||||
"""x_search.model was previously not enumerated; the derivation must catch it."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"x-search-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
"delegation:\n provider: harness",
|
||||
"delegation:\n provider: harness\nx_search:\n model: gpu-light",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "x_search.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
|
||||
|
||||
def test_retired_alias_in_nested_auxiliary_block_is_rejected(tmp_path):
|
||||
"""A nested auxiliary sub-block outside the named three must still be derived."""
|
||||
code, out = _run_config(
|
||||
tmp_path,
|
||||
"nested-aux-gpu-light.yaml",
|
||||
BASE.format(alias="gpu-vision").replace(
|
||||
" compression:\n model: syslog-auto\n provider: harness\ndelegation:",
|
||||
" compression:\n model: syslog-auto\n provider: harness\n"
|
||||
" tasks:\n summarize:\n model: gpu-light\ndelegation:",
|
||||
),
|
||||
)
|
||||
assert code == 1, out
|
||||
assert "auxiliary.tasks.summarize.model = 'gpu-light'" in out
|
||||
assert "RESULT: FAIL" in out
|
||||
Reference in New Issue
Block a user