#!/bin/bash # contract-run.sh — Deterministic contract execution from machine scheduler # # Takes a contract name, resolves its script, runs it with timeout, # logs output to $CONTRACT_RUN_LOG_DIR (default: /var/log/contract-runs/), # and alerts on failure. # # Environment: # CONTRACT_RUN_LOG_DIR Override the log directory (default: /var/log/contract-runs) # # Usage: bash scripts/contract-run.sh # # Contract names map to scripts as follows: # infrastructure-monitoring -> scripts/infra-monitoring.sh # proxmox-monitor -> scripts/proxmox-monitor.sh # zulip-health -> scripts/zulip-monitor.sh # agent-health-check -> scripts/agent-health-check.py # litellm-health -> scripts/litellm-health-check.py # disk-gc-threat-response -> scripts/disk-gc-scan.py # pm2-self-heal -> scripts/pm2-self-heal.sh # # Exit codes: # 0 = contract passed # 1 = contract failed (alert sent) # 2 = probe failed (script missing, timeout, etc.) set -uo pipefail CONTRACT_NAME="$1" SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" LOG_DIR="${CONTRACT_RUN_LOG_DIR:-/var/log/contract-runs}" TIMESTAMP=$(date -u '+%Y%m%d-%H%M%S') LOG_FILE="${LOG_DIR}/${CONTRACT_NAME}-${TIMESTAMP}.log" # Ensure log directory exists mkdir -p "$LOG_DIR" # Map contract name to script path case "$CONTRACT_NAME" in infrastructure-monitoring) SCRIPT_PATH="${SCRIPTS_DIR}/infra-monitoring.sh" INTERPRETER="bash" ;; proxmox-monitor) SCRIPT_PATH="${SCRIPTS_DIR}/proxmox-monitor.sh" INTERPRETER="bash" ;; zulip-health) SCRIPT_PATH="${SCRIPTS_DIR}/zulip-monitor.sh" INTERPRETER="bash" ;; agent-health-check) SCRIPT_PATH="${SCRIPTS_DIR}/agent-health-check.py" INTERPRETER="python3" ;; litellm-health) SCRIPT_PATH="${SCRIPTS_DIR}/litellm-health-check.py" INTERPRETER="python3" ;; pm2-self-heal) SCRIPT_PATH="${SCRIPTS_DIR}/pm2-self-heal.sh" INTERPRETER="bash" ;; disk-gc-threat-response) SCRIPT_PATH="${SCRIPTS_DIR}/disk-gc-scan.py" INTERPRETER="python3" ;; search-stack-visibility) SCRIPT_PATH="${SCRIPTS_DIR}/search-stack-check.py" INTERPRETER="python3" ;; *) echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE" # Send alert for unknown contract ALERT_MSG="🔴 Contract $CONTRACT_NAME: unknown contract name. Log: $LOG_FILE" ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true fi exit 2 ;; esac # Check if script exists if [ ! -f "$SCRIPT_PATH" ]; then echo "Script not found: $SCRIPT_PATH" | tee -a "$LOG_FILE" # Send alert for missing script ALERT_MSG="🔴 Contract $CONTRACT_NAME: script not found at $SCRIPT_PATH. Log: $LOG_FILE" ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true fi exit 2 fi # Run the script with timeout and capture output echo "=== Contract: $CONTRACT_NAME ===" | tee "$LOG_FILE" echo "Started: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | tee -a "$LOG_FILE" echo "Script: $SCRIPT_PATH" | tee -a "$LOG_FILE" echo "" | tee -a "$LOG_FILE" # Use timeout to prevent hangs (10 minutes default) TIMEOUT=600 timeout "$TIMEOUT" $INTERPRETER "$SCRIPT_PATH" 2>&1 | tee -a "$LOG_FILE" EXIT_CODE=${PIPESTATUS[0]} # If timeout killed the process, EXIT_CODE will be 124 if [ $EXIT_CODE -eq 124 ]; then echo "⏰ TIMEOUT: script exceeded ${TIMEOUT}s limit" | tee -a "$LOG_FILE" fi echo "" | tee -a "$LOG_FILE" if [ $EXIT_CODE -eq 0 ]; then echo "✅ VERDICT: PASS" | tee -a "$LOG_FILE" exit 0 else echo "🔴 VERDICT: FAIL (exit code $EXIT_CODE)" | tee -a "$LOG_FILE" # Send alert (Zulip DM to user 9 + stream agent-hub topic alerts-infra) # Using the same alert path as other monitors ALERT_MSG="🔴 Contract $CONTRACT_NAME failed (exit $EXIT_CODE). Log: $LOG_FILE" ALERT_SENT=false # Take credentials from environment (ZULIP_API_KEY required) ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then # DM to user 9 DM_EXIT=0 curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || DM_EXIT=$? # Stream agent-hub topic alerts-infra STREAM_EXIT=0 curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=stream" \ -d "to=agent-hub" \ -d "topic=alerts-infra" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || STREAM_EXIT=$? if [ $DM_EXIT -eq 0 ] || [ $STREAM_EXIT -eq 0 ]; then ALERT_SENT=true else echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT FAILURE: DM exit=$DM_EXIT, stream exit=$STREAM_EXIT" >> "$LOG_FILE" fi else echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT SKIPPED: no ZULIP_API_KEY or curl" >> "$LOG_FILE" fi exit 1 fi