diff --git a/infrastructure-monitoring.prose.md b/infrastructure-monitoring.prose.md index 66f9ed5..68544a7 100644 --- a/infrastructure-monitoring.prose.md +++ b/infrastructure-monitoring.prose.md @@ -130,11 +130,14 @@ pwd -P # Zulip API health (POST ping) # NOTE: /etc/litellm-monitor.env exists only on CT 116, retrieve keys from CT 116 via: -monitor_key=$(ssh root@192.168.68.116 "grep LITELLM_MONITOR_KEY /etc/litellm-monitor.env | cut -d= -f2") -zulip_key=$(ssh root@192.168.68.116 "grep ZULIP_BOT_KEY /etc/zulip-bot.env | cut -d= -f2" 2>/dev/null || echo "not-found") -ZULIP_USER="abiba-bot@chat.sysloggh.net" -curl -s -o /dev/null -w '%{http_code}' -X POST https://chat.sysloggh.net/api/v1/messages -u "${ZULIP_USER}:${zulip_key}" -# Expected: 200 (HTTP 000 = unreachable/cache) +zulip_key=$(ssh root@192.168.68.116 "grep ZULIP_BOT_KEY /etc/litellm-monitor.env | cut -d= -f2") +if [ -z "$zulip_key" ]; then + echo "credential-missing: ZULIP_BOT_KEY not found in /etc/litellm-monitor.env" +else + ZULIP_USER="abiba-bot@chat.sysloggh.net" + curl -s -o /dev/null -w '%{http_code}' -X POST https://chat.sysloggh.net/api/v1/messages -u "${ZULIP_USER}:${zulip_key}" + # Expected: 200 (HTTP 000 = unreachable/cache) +fi # PM2 process health pm2 jlist diff --git a/litellm-health.prose.md b/litellm-health.prose.md index bd11d32..5ac5853 100644 --- a/litellm-health.prose.md +++ b/litellm-health.prose.md @@ -161,8 +161,9 @@ contracts — read them there. Do not re-add retired names (`gemma-4-12b`, `gpu- Do NOT use the master key for inference — the master key is for admin endpoints only (`/key/list`, `/key/generate`, `/key/info`). Retrieve master key via: `ssh root@192.168.68.116 "docker exec harness-litellm printenv LITELLM_MASTER_KEY"` - - KEY SCOPE: the `monitor` key MUST be scoped for all three probed aliases (`gpu-dense`, - `gpu-vision`, `strix-moe`), otherwise the probe returns 403 and the host is not covered. + - KEY SCOPE: the `monitor` key MUST be scoped for the three probed aliases (`gpu-dense`, + `gpu-vision`, `strix-moe`) plus the `syslog-auto` fallback pool, otherwise the probe + returns 403 and the host is not covered. If a probe returns 403, widen the monitor key's model list on CT 116 (add the missing alias) and re-run — never drop the host from the probe to make the check pass. - `gemma-4-12b` was retired and returns 400 `Invalid model name` — do not re-add it to