#!/bin/bash # contract-run.sh — Deterministic contract execution from machine scheduler # # Takes a contract name, resolves its script, runs it with timeout, # logs output to $CONTRACT_RUN_LOG_DIR (default: /var/log/contract-runs/), # and alerts on failure. # # Environment: # CONTRACT_RUN_LOG_DIR Override the log directory (default: /var/log/contract-runs) # # Usage: bash scripts/contract-run.sh # # Contract names map to scripts as follows: # infrastructure-monitoring -> scripts/infra-monitoring.sh # proxmox-monitor -> scripts/proxmox-monitor.sh # zulip-health -> scripts/zulip-monitor.sh # agent-health-check -> scripts/agent-health-check.py # litellm-health -> scripts/litellm-health-check.py # disk-gc-threat-response -> scripts/disk-gc-scan.py # pm2-self-heal -> scripts/pm2-self-heal.sh # search-stack-visibility -> scripts/search-stack-check.py # # Execution copy: every contract pins the clone this script lives in (see # docs/contract-execution-pinning.md). Before a contract runs, this wrapper # proves the script it is about to execute byte-matches origin/master: # CONTRACT_REVISION_PREFLIGHT=enforce (default) refuse to report on mismatch # CONTRACT_REVISION_PREFLIGHT=warn log the mismatch and continue # CONTRACT_REVISION_PREFLIGHT=off skip the check entirely # # Exit codes: # 0 = contract passed # 1 = contract failed (alert sent) # 2 = probe failed (script missing, timeout, etc.) set -uo pipefail CONTRACT_NAME="$1" SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" LOG_DIR="${CONTRACT_RUN_LOG_DIR:-/var/log/contract-runs}" TIMESTAMP=$(date -u '+%Y%m%d-%H%M%S') LOG_FILE="${LOG_DIR}/${CONTRACT_NAME}-${TIMESTAMP}.log" # Ensure log directory exists mkdir -p "$LOG_DIR" # Map contract name to script path case "$CONTRACT_NAME" in infrastructure-monitoring) SCRIPT_PATH="${SCRIPTS_DIR}/infra-monitoring.sh" INTERPRETER="bash" ;; proxmox-monitor) SCRIPT_PATH="${SCRIPTS_DIR}/proxmox-monitor.sh" INTERPRETER="bash" ;; zulip-health) SCRIPT_PATH="${SCRIPTS_DIR}/zulip-monitor.sh" INTERPRETER="bash" ;; agent-health-check) SCRIPT_PATH="${SCRIPTS_DIR}/agent-health-check.py" INTERPRETER="python3" ;; litellm-health) SCRIPT_PATH="${SCRIPTS_DIR}/litellm-health-check.py" INTERPRETER="python3" ;; pm2-self-heal) SCRIPT_PATH="${SCRIPTS_DIR}/pm2-self-heal.sh" INTERPRETER="bash" ;; disk-gc-threat-response) SCRIPT_PATH="${SCRIPTS_DIR}/disk-gc-scan.py" INTERPRETER="python3" ;; search-stack-visibility) SCRIPT_PATH="${SCRIPTS_DIR}/search-stack-check.py" INTERPRETER="python3" ;; *) echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE" # Send alert for unknown contract ALERT_MSG="🔴 Contract $CONTRACT_NAME: unknown contract name. Log: $LOG_FILE" ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true fi exit 2 ;; esac # Check if script exists if [ ! -f "$SCRIPT_PATH" ]; then echo "Script not found: $SCRIPT_PATH" | tee -a "$LOG_FILE" # Send alert for missing script ALERT_MSG="🔴 Contract $CONTRACT_NAME: script not found at $SCRIPT_PATH. Log: $LOG_FILE" ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true fi exit 2 fi # Run the script with timeout and capture output echo "=== Contract: $CONTRACT_NAME ===" | tee "$LOG_FILE" echo "Started: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | tee -a "$LOG_FILE" echo "Script: $SCRIPT_PATH" | tee -a "$LOG_FILE" echo "" | tee -a "$LOG_FILE" # ── Revision preflight ─────────────────────────────────────────────────────── # A verdict is only meaningful if it came from the merged copy. Refuse to report # one from a mismatched or unverifiable copy; that is a probe failure (exit 2), # not a contract verdict, because the result would be untrustworthy. # See docs/contract-execution-pinning.md. REVISION_PREFLIGHT_MODE="${CONTRACT_REVISION_PREFLIGHT:-enforce}" REPO_ROOT="$(cd "${SCRIPTS_DIR}/.." && pwd)" if [ "$REVISION_PREFLIGHT_MODE" != "off" ] && [ -x "${SCRIPTS_DIR}/revision-preflight.sh" ]; then if "${SCRIPTS_DIR}/revision-preflight.sh" "$SCRIPT_PATH" "$REPO_ROOT" 2>&1 | tee -a "$LOG_FILE"; then : elif [ "$REVISION_PREFLIGHT_MODE" = "warn" ]; then echo "⚠️ revision preflight failed — continuing because CONTRACT_REVISION_PREFLIGHT=warn" | tee -a "$LOG_FILE" else echo "🚫 VERDICT WITHHELD: executing copy does not match the merged revision" | tee -a "$LOG_FILE" ALERT_MSG="🔴 Contract $CONTRACT_NAME: revision mismatch — verdict withheld. Log: $LOG_FILE" ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || true fi exit 2 fi fi # Use timeout to prevent hangs (10 minutes default) TIMEOUT=600 timeout "$TIMEOUT" $INTERPRETER "$SCRIPT_PATH" 2>&1 | tee -a "$LOG_FILE" EXIT_CODE=${PIPESTATUS[0]} # If timeout killed the process, EXIT_CODE will be 124 if [ $EXIT_CODE -eq 124 ]; then echo "⏰ TIMEOUT: script exceeded ${TIMEOUT}s limit" | tee -a "$LOG_FILE" fi echo "" | tee -a "$LOG_FILE" if [ $EXIT_CODE -eq 0 ]; then echo "✅ VERDICT: PASS" | tee -a "$LOG_FILE" exit 0 else echo "🔴 VERDICT: FAIL (exit code $EXIT_CODE)" | tee -a "$LOG_FILE" # Send alert (Zulip DM to user 9 + stream agent-hub topic alerts-infra) # Using the same alert path as other monitors ALERT_MSG="🔴 Contract $CONTRACT_NAME failed (exit $EXIT_CODE). Log: $LOG_FILE" ALERT_SENT=false # Take credentials from environment (ZULIP_API_KEY required) ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}" ZULIP_API_KEY="${ZULIP_API_KEY:-}" ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}" if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then # DM to user 9 DM_EXIT=0 curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=private" \ -d "to=9" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || DM_EXIT=$? # Stream agent-hub topic alerts-infra STREAM_EXIT=0 curl -sf -X POST "${ZULIP_API_URL}/messages" \ -u "${ZULIP_USER}:${ZULIP_API_KEY}" \ -d "type=stream" \ -d "to=agent-hub" \ -d "topic=alerts-infra" \ -d "content=${ALERT_MSG}" > /dev/null 2>&1 || STREAM_EXIT=$? if [ $DM_EXIT -eq 0 ] || [ $STREAM_EXIT -eq 0 ]; then ALERT_SENT=true else echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT FAILURE: DM exit=$DM_EXIT, stream exit=$STREAM_EXIT" >> "$LOG_FILE" fi else echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT SKIPPED: no ZULIP_API_KEY or curl" >> "$LOG_FILE" fi exit 1 fi