prose-contracts/scripts/contract-run.sh

#!/bin/bash
# contract-run.sh — Deterministic contract execution from machine scheduler
#
# Takes a contract name, resolves its script, runs it with timeout,
# logs output to $CONTRACT_RUN_LOG_DIR (default: /var/log/contract-runs/),
# and alerts on failure.
#
# Environment:
#   CONTRACT_RUN_LOG_DIR  Override the log directory (default: /var/log/contract-runs)
#
# Usage: bash scripts/contract-run.sh <contract-name>
#
# Contract names map to scripts as follows:
#   infrastructure-monitoring -> scripts/infra-monitoring.sh
#   proxmox-monitor           -> scripts/proxmox-monitor.sh
#   zulip-health              -> scripts/zulip-monitor.sh
#   agent-health-check        -> scripts/agent-health-check.py
#   litellm-health            -> scripts/litellm-health-check.py
#   disk-gc-threat-response   -> scripts/disk-gc-scan.py
#   pm2-self-heal             -> scripts/pm2-self-heal.sh
#
# Exit codes:
#   0 = contract passed
#   1 = contract failed (alert sent)
#   2 = probe failed (script missing, timeout, etc.)

set -uo pipefail

CONTRACT_NAME="$1"
SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
LOG_DIR="${CONTRACT_RUN_LOG_DIR:-/var/log/contract-runs}"
TIMESTAMP=$(date -u '+%Y%m%d-%H%M%S')
LOG_FILE="${LOG_DIR}/${CONTRACT_NAME}-${TIMESTAMP}.log"

# Ensure log directory exists
mkdir -p "$LOG_DIR"

# Map contract name to script path
case "$CONTRACT_NAME" in
    infrastructure-monitoring)
        SCRIPT_PATH="${SCRIPTS_DIR}/infra-monitoring.sh"
        INTERPRETER="bash"
        ;;
    proxmox-monitor)
        SCRIPT_PATH="${SCRIPTS_DIR}/proxmox-monitor.sh"
        INTERPRETER="bash"
        ;;
    zulip-health)
        SCRIPT_PATH="${SCRIPTS_DIR}/zulip-monitor.sh"
        INTERPRETER="bash"
        ;;
    agent-health-check)
        SCRIPT_PATH="${SCRIPTS_DIR}/agent-health-check.py"
        INTERPRETER="python3"
        ;;
    litellm-health)
        SCRIPT_PATH="${SCRIPTS_DIR}/litellm-health-check.py"
        INTERPRETER="python3"
        ;;
    pm2-self-heal)
        SCRIPT_PATH="${SCRIPTS_DIR}/pm2-self-heal.sh"
        INTERPRETER="bash"
        ;;
    disk-gc-threat-response)
        SCRIPT_PATH="${SCRIPTS_DIR}/disk-gc-scan.py"
        INTERPRETER="python3"
        ;;
    search-stack-visibility)
        SCRIPT_PATH="${SCRIPTS_DIR}/search-stack-check.py"
        INTERPRETER="python3"
        ;;
    *)
        echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE"
        # Send alert for unknown contract
        ALERT_MSG="🔴 Contract $CONTRACT_NAME: unknown contract name. Log: $LOG_FILE"
        ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}"
        ZULIP_API_KEY="${ZULIP_API_KEY:-}"
        ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}"
        if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then
            curl -sf -X POST "${ZULIP_API_URL}/messages" \
                -u "${ZULIP_USER}:${ZULIP_API_KEY}" \
                -d "type=private" \
                -d "to=9" \
                -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true
        fi
        exit 2
        ;;
esac

# Check if script exists
if [ ! -f "$SCRIPT_PATH" ]; then
    echo "Script not found: $SCRIPT_PATH" | tee -a "$LOG_FILE"
    # Send alert for missing script
    ALERT_MSG="🔴 Contract $CONTRACT_NAME: script not found at $SCRIPT_PATH. Log: $LOG_FILE"
    ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}"
    ZULIP_API_KEY="${ZULIP_API_KEY:-}"
    ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}"
    if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then
        curl -sf -X POST "${ZULIP_API_URL}/messages" \
            -u "${ZULIP_USER}:${ZULIP_API_KEY}" \
            -d "type=private" \
            -d "to=9" \
            -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true
    fi
    exit 2
fi

# Run the script with timeout and capture output
echo "=== Contract: $CONTRACT_NAME ===" | tee "$LOG_FILE"
echo "Started: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | tee -a "$LOG_FILE"
echo "Script: $SCRIPT_PATH" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"

# Use timeout to prevent hangs (10 minutes default)
TIMEOUT=600
timeout "$TIMEOUT" $INTERPRETER "$SCRIPT_PATH" 2>&1 | tee -a "$LOG_FILE"
EXIT_CODE=${PIPESTATUS[0]}

# If timeout killed the process, EXIT_CODE will be 124
if [ $EXIT_CODE -eq 124 ]; then
    echo "⏰ TIMEOUT: script exceeded ${TIMEOUT}s limit" | tee -a "$LOG_FILE"
fi

echo "" | tee -a "$LOG_FILE"
if [ $EXIT_CODE -eq 0 ]; then
    echo "✅ VERDICT: PASS" | tee -a "$LOG_FILE"
    exit 0
else
    echo "🔴 VERDICT: FAIL (exit code $EXIT_CODE)" | tee -a "$LOG_FILE"
    
    # Send alert (Zulip DM to user 9 + stream agent-hub topic alerts-infra)
    # Using the same alert path as other monitors
    ALERT_MSG="🔴 Contract $CONTRACT_NAME failed (exit $EXIT_CODE). Log: $LOG_FILE"
    ALERT_SENT=false
    
    # Take credentials from environment (ZULIP_API_KEY required)
    ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}"
    ZULIP_API_KEY="${ZULIP_API_KEY:-}"
    ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}"
    
    if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then
        # DM to user 9
        DM_EXIT=0
        curl -sf -X POST "${ZULIP_API_URL}/messages" \
            -u "${ZULIP_USER}:${ZULIP_API_KEY}" \
            -d "type=private" \
            -d "to=9" \
            -d "content=${ALERT_MSG}" > /dev/null 2>&1 || DM_EXIT=$?
        
        # Stream agent-hub topic alerts-infra
        STREAM_EXIT=0
        curl -sf -X POST "${ZULIP_API_URL}/messages" \
            -u "${ZULIP_USER}:${ZULIP_API_KEY}" \
            -d "type=stream" \
            -d "to=agent-hub" \
            -d "topic=alerts-infra" \
            -d "content=${ALERT_MSG}" > /dev/null 2>&1 || STREAM_EXIT=$?
        
        if [ $DM_EXIT -eq 0 ] || [ $STREAM_EXIT -eq 0 ]; then
            ALERT_SENT=true
        else
            echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT FAILURE: DM exit=$DM_EXIT, stream exit=$STREAM_EXIT" >> "$LOG_FILE"
        fi
    else
        echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT SKIPPED: no ZULIP_API_KEY or curl" >> "$LOG_FILE"
    fi
    
    exit 1
fi