diff --git a/hermes-config-template.prose.md b/hermes-config-template.prose.md index 2292a2a..275e846 100644 --- a/hermes-config-template.prose.md +++ b/hermes-config-template.prose.md @@ -99,8 +99,8 @@ model: base_url: http://192.168.68.116/v1 api_key_env: LITELLM_API_KEY # Injected via infisical run -- wrapper max_tokens: 4096 # ⚠️ CRITICAL: Prevents unbounded generation - context_length: 262144 # For syslog-auto (ornith route supports 256K). - # Set 131072 if using qwen3.6-27B-code or gemma-4-12b directly. + context_length: 262144 # For syslog-auto (all GPUs support 256K). + # Set 131072 if using gemma-4-12b directly (12GB VRAM constraint). fallback_providers: provider: deepseek @@ -129,10 +129,10 @@ mcp_servers: # ─── Compression ─── compression: enabled: true - model: gemma-4-12b # ⚠️ Must match auxiliary.compression.model + model: ornith-1.0-35b # ⚠️ Must match auxiliary.compression.model provider: harness - max_context_window: 262144 # For syslog-auto (ornith supports 256K). - # Set 131072 if using qwen or gemma directly. + max_context_window: 262144 # For Strix Halo compression (256K ctx). + # Set 131072 if using gemma directly. threshold: 0.65 # Fires at ~170K for 262K window, ~85K for 128K target_ratio: 0.30 protect_last_n: 40 @@ -164,7 +164,7 @@ auxiliary: timeout: 30 compression: provider: harness - model: gemma-4-12b + model: ornith-1.0-35b base_url: http://192.168.68.116/v1 api_key_env: LITELLM_API_KEY timeout: 60 diff --git a/litellm-self-heal.prose.md b/litellm-self-heal.prose.md index b92e057..f39acdf 100644 --- a/litellm-self-heal.prose.md +++ b/litellm-self-heal.prose.md @@ -56,8 +56,8 @@ Request → nginx:80 → LiteLLM:4000 → GPU(llama-server, parallel 2) | Host | IP | Hardware | Models Served | Engine | Context | Parallel | |------|-----|----------|---------------|--------|---------|----------| -| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | qwen3.6-27B-code | llama-server systemd | 128K | 2 | -| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | gemma-4-12b | llama-server systemd | 128K | 2 | +| llm-gpu | 192.168.68.8 | NVIDIA RTX 3090 (24 GB) | qwen3.6-27B-code | llama-server systemd | **256K** | 1 | +| ocu-llm | 192.168.68.110 | NVIDIA RTX 5070 (12 GB) | gemma-4-12b | llama-server systemd | 131K | 2 | | amdpve | 192.168.68.15 | AMD Strix Halo 64GB UMA | ornith-1.0-35b | llama-server systemd (Vulkan) | 256K | 2 | ## Model Fallback Chains (LiteLLM)