71 lines
2.1 KiB
Bash
Executable File
71 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# daily-health-digest.sh — Daily Fleet Health Digest
|
|
# Runs all major health checks and summarizes results
|
|
#
|
|
# Legs:
|
|
# - infrastructure-monitoring (13 legs)
|
|
# - gpu-monitor (6 legs)
|
|
# - litellm-health (11 checks)
|
|
# - agent-health-check (4 agents + 3 GPUs + 4 LiteLLM keys)
|
|
# - proxmox-monitor (4 exporters)
|
|
# - pm2-self-heal (PM2 services)
|
|
# - zulip-health (Zulip mesh)
|
|
#
|
|
# Exit 0 if all pass, 1 if any fail.
|
|
#
|
|
# Run: bash scripts/daily-health-digest.sh
|
|
|
|
set -uo pipefail
|
|
|
|
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
|
|
SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
FAILED=()
|
|
|
|
echo "=== Daily Health Digest — $TIMESTAMP ==="
|
|
echo "Executed from: $(pwd -P)"
|
|
echo ""
|
|
|
|
# Run each health check and capture exit code
|
|
run_check() {
|
|
local name="$1"
|
|
local script="$2"
|
|
local runner="${3:-bash}"
|
|
local output
|
|
|
|
echo "── Running $name ──"
|
|
output=$(cd "$SCRIPTS_DIR/.." && $runner "$script" 2>&1)
|
|
local exit_code=$?
|
|
|
|
if [ $exit_code -eq 0 ]; then
|
|
# Extract just the summary lines (lines with ✅ or "All")
|
|
echo "$output" | grep -E "✅|All legs OK|All checks passed|All healthy" | tail -3
|
|
echo ""
|
|
else
|
|
echo " ❌ $name failed (exit $exit_code)"
|
|
echo "$output" | grep -E "🔴|❌" | tail -5
|
|
echo ""
|
|
FAILED+=("$name")
|
|
fi
|
|
}
|
|
|
|
# Run all checks
|
|
run_check "Infrastructure Monitoring" "scripts/infra-monitoring.sh"
|
|
run_check "GPU Monitor" "scripts/gpu-monitor.sh"
|
|
run_check "LiteLLM Health" "scripts/litellm-health-check.py" "python3"
|
|
run_check "Agent Health Check" "scripts/agent-health-check.py" "python3"
|
|
run_check "Proxmox Monitor" "scripts/proxmox-monitor.sh"
|
|
run_check "PM2 Self-Heal" "scripts/pm2-self-heal.sh"
|
|
run_check "Zulip Health" "scripts/zulip-monitor.sh"
|
|
|
|
# Summary
|
|
echo "─────────────────────────────────────────────"
|
|
if [ ${#FAILED[@]} -eq 0 ]; then
|
|
echo "✅ ALL HEALTH CHECKS PASSED"
|
|
exit 0
|
|
else
|
|
echo "❌ ${#FAILED[@]} HEALTH CHECK(S) FAILED:"
|
|
for f in "${FAILED[@]}"; do
|
|
echo " - $f"
|
|
done
|
|
exit 1
|
|
fi |