diff --git a/hermes-config-template.prose.md b/hermes-config-template.prose.md index 2dbef5d..cd95030 100644 --- a/hermes-config-template.prose.md +++ b/hermes-config-template.prose.md @@ -143,23 +143,25 @@ compression: # ─── Auxiliary Tasks (CONSISTENCY RULE) ─── # All auxiliary services MUST use identical model, base_url, and api_key_env: -# model: gemma-4-12b +# model: gpu-light # stable alias (NOT raw "gemma-4-12b") # base_url: http://192.168.68.116/v1 # api_key_env: LITELLM_API_KEY # Do NOT use syslog-auto for auxiliary tasks — it routes to the primary GPU. -# gemma-4-12b is a lightweight 12B model on the RTX 5070, freeing the Strix Halo -# for agent reasoning. +# gpu-light = RTX 5070 (12B), freeing the Strix Halo for agent reasoning. +# Heavy aux (delegation, x_search) use gpu-dense (RTX 3090) instead. +# NEVER use raw model names (gemma-4-12b, qwen3.6-27B-code, qwen3.6-35B-udq4) +# in agent configs — use the stable aliases so model swaps don't break agents. auxiliary: vision: provider: harness - model: gemma-4-12b + model: gpu-light # stable alias for RTX 5070 (was raw gemma-4-12b) base_url: http://192.168.68.116/v1 api_key_env: LITELLM_API_KEY timeout: 60 download_timeout: 30 web_extract: provider: harness - model: gemma-4-12b + model: gpu-light # stable alias for RTX 5070 base_url: http://192.168.68.116/v1 api_key_env: LITELLM_API_KEY timeout: 30 @@ -170,10 +172,19 @@ auxiliary: api_key_env: LITELLM_API_KEY timeout: 300 # gpu-fleet: 300s for large-history summarization (was 60) +# ─── Delegation / Heavy Aux (use gpu-dense = RTX 3090) ─── +# delegation.model and x_search.model use gpu-dense (NOT raw qwen3.6-27B-code). + +delegation: + model: gpu-dense # stable alias for RTX 3090 (was raw qwen3.6-27B-code) + provider: harness + base_url: http://192.168.68.116/v1 + api_key_env: LITELLM_API_KEY + # ─── Custom Provider ─── custom_providers: - name: harness - model: + model: syslog-auto # weighted pool (default) base_url: http://192.168.68.116/v1 api_key_env: LITELLM_API_KEY api_mode: chat_completions diff --git a/litellm-self-heal.prose.md b/litellm-self-heal.prose.md index 07398e3..71ade65 100644 --- a/litellm-self-heal.prose.md +++ b/litellm-self-heal.prose.md @@ -69,7 +69,8 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2) - `syslog-auto` is a weighted router model: qwen3.6-27B-code (0.55, rpm 500) + qwen3.6-35B-udq4 (0.30, rpm 60) + gemma-4-12b (0.15, rpm 200). - `gpu-dense` / `gpu-light` are high-rpm aliases (rpm 500) onto qwen3.6-27B-code / gemma-4-12b respectively. -- Key scoping: agent keys are restricted to `['syslog-auto','qwen3.6-27B-code','gemma-4-12b','strix-moe','gpu-dense','gpu-light']`. Note `qwen3.6-35B-udq4` is NOT in the agent-key allowlist — agents must use `strix-moe` to reach the AMD model. +- Key scoping: agent keys are restricted to `['syslog-auto','qwen3.6-27B-code','gemma-4-12b','strix-moe','gpu-dense','gpu-light']`. As of 2026-07-16 the `baggy`/`koby`/`mumuni`/`abiba-pi` keys ALSO include `qwen3.6-35B-udq4`; `abiba-pi` additionally includes `deepseek-v4-pro` (cloud fallback). `kagenz0`/`koonimo`/`pi-agents-unified` have the standard 6 only. Agents should still use the stable alias `strix-moe` (not the raw `qwen3.6-35B-udq4`) so model swaps don't break them. +- **NEVER use `litellm_proxy_master_key` (the `sk-litellm-...` master key) for inference.** It is for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`). All inference — agent traffic, health-check model tests, monitor scripts — uses agent-specific keys. The health-check script's model tests use a dedicated `monitor` agent key stored at `/etc/litellm-monitor.env` on CT 116 (root-only, `chmod 600`); `/key/list` is the only call that legitimately uses `$MASTER_KEY`. ## Model Fallback Chains (LiteLLM)