root
748ea389be
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 4s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 11s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 6s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 4s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 2s
1. Replace '(## Execution' + ')' with '## Execution' in 6 contract files 2. Make LOG_DIR honor CONTRACT_RUN_LOG_DIR env var (default: /var/log/contract-runs) 3. Fix Zulip alert: correct URL (https://chat.sysloggh.net/api/v1), user (abiba-bot@chat.sysloggh.net), take ZULIP_API_KEY from environment, and write alert failures to run log
133 lines
4.4 KiB
Bash
Executable File
133 lines
4.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# contract-run.sh — Deterministic contract execution from machine scheduler
|
|
#
|
|
# Takes a contract name, resolves its script, runs it with timeout,
|
|
# logs output to $CONTRACT_RUN_LOG_DIR (default: /var/log/contract-runs/),
|
|
# and alerts on failure.
|
|
#
|
|
# Environment:
|
|
# CONTRACT_RUN_LOG_DIR Override the log directory (default: /var/log/contract-runs)
|
|
#
|
|
# Usage: bash scripts/contract-run.sh <contract-name>
|
|
#
|
|
# Contract names map to scripts as follows:
|
|
# infrastructure-monitoring -> scripts/infra-monitoring.sh
|
|
# proxmox-monitor -> scripts/proxmox-monitor.sh
|
|
# zulip-health -> scripts/zulip-monitor.sh
|
|
# agent-health-check -> scripts/agent-health-check.py
|
|
# litellm-health -> scripts/litellm-health-check.py
|
|
# disk-gc-threat-response -> scripts/disk-gc-scan.py
|
|
# pm2-self-heal -> scripts/pm2-self-heal.sh
|
|
#
|
|
# Exit codes:
|
|
# 0 = contract passed
|
|
# 1 = contract failed (alert sent)
|
|
# 2 = probe failed (script missing, timeout, etc.)
|
|
|
|
set -uo pipefail
|
|
|
|
CONTRACT_NAME="$1"
|
|
SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
LOG_DIR="${CONTRACT_RUN_LOG_DIR:-/var/log/contract-runs}"
|
|
TIMESTAMP=$(date -u '+%Y%m%d-%H%M%S')
|
|
LOG_FILE="${LOG_DIR}/${CONTRACT_NAME}-${TIMESTAMP}.log"
|
|
|
|
# Ensure log directory exists
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Map contract name to script path
|
|
case "$CONTRACT_NAME" in
|
|
infrastructure-monitoring)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/infra-monitoring.sh"
|
|
INTERPRETER="bash"
|
|
;;
|
|
proxmox-monitor)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/proxmox-monitor.sh"
|
|
INTERPRETER="bash"
|
|
;;
|
|
zulip-health)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/zulip-monitor.sh"
|
|
INTERPRETER="bash"
|
|
;;
|
|
agent-health-check)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/agent-health-check.py"
|
|
INTERPRETER="python3"
|
|
;;
|
|
litellm-health)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/litellm-health-check.py"
|
|
INTERPRETER="python3"
|
|
;;
|
|
pm2-self-heal)
|
|
SCRIPT_PATH="${SCRIPTS_DIR}/pm2-self-heal.sh"
|
|
INTERPRETER="bash"
|
|
;;
|
|
*)
|
|
echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE"
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
# Check if script exists
|
|
if [ ! -f "$SCRIPT_PATH" ]; then
|
|
echo "Script not found: $SCRIPT_PATH" | tee -a "$LOG_FILE"
|
|
exit 2
|
|
fi
|
|
|
|
# Run the script with timeout and capture output
|
|
echo "=== Contract: $CONTRACT_NAME ===" | tee "$LOG_FILE"
|
|
echo "Started: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" | tee -a "$LOG_FILE"
|
|
echo "Script: $SCRIPT_PATH" | tee -a "$LOG_FILE"
|
|
echo "" | tee -a "$LOG_FILE"
|
|
|
|
# Use timeout to prevent hangs (10 minutes default)
|
|
TIMEOUT=600
|
|
$INTERPRETER "$SCRIPT_PATH" 2>&1 | tee -a "$LOG_FILE"
|
|
EXIT_CODE=${PIPESTATUS[0]}
|
|
|
|
echo "" | tee -a "$LOG_FILE"
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "✅ VERDICT: PASS" | tee -a "$LOG_FILE"
|
|
exit 0
|
|
else
|
|
echo "🔴 VERDICT: FAIL (exit code $EXIT_CODE)" | tee -a "$LOG_FILE"
|
|
|
|
# Send alert (Zulip DM to user 9 + stream agent-hub topic alerts-infra)
|
|
# Using the same alert path as other monitors
|
|
ALERT_MSG="🔴 Contract $CONTRACT_NAME failed (exit $EXIT_CODE). Log: $LOG_FILE"
|
|
ALERT_SENT=false
|
|
|
|
# Take credentials from environment (ZULIP_API_KEY required)
|
|
ZULIP_API_URL="${ZULIP_API_URL:-https://chat.sysloggh.net/api/v1}"
|
|
ZULIP_API_KEY="${ZULIP_API_KEY:-}"
|
|
ZULIP_USER="${ZULIP_USER:-abiba-bot@chat.sysloggh.net}"
|
|
|
|
if [ -n "$ZULIP_API_KEY" ] && command -v curl &> /dev/null; then
|
|
# DM to user 9
|
|
DM_EXIT=0
|
|
curl -s -X POST "${ZULIP_API_URL}/messages" \
|
|
-u "${ZULIP_USER}:${ZULIP_API_KEY}" \
|
|
-d "type=private" \
|
|
-d "to=9" \
|
|
-d "content=${ALERT_MSG}" > /dev/null 2>&1 || DM_EXIT=$?
|
|
|
|
# Stream agent-hub topic alerts-infra
|
|
STREAM_EXIT=0
|
|
curl -s -X POST "${ZULIP_API_URL}/messages" \
|
|
-u "${ZULIP_USER}:${ZULIP_API_KEY}" \
|
|
-d "type=stream" \
|
|
-d "to=agent-hub" \
|
|
-d "topic=alerts-infra" \
|
|
-d "content=${ALERT_MSG}" > /dev/null 2>&1 || STREAM_EXIT=$?
|
|
|
|
if [ $DM_EXIT -eq 0 ] || [ $STREAM_EXIT -eq 0 ]; then
|
|
ALERT_SENT=true
|
|
else
|
|
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT FAILURE: DM exit=$DM_EXIT, stream exit=$STREAM_EXIT" >> "$LOG_FILE"
|
|
fi
|
|
else
|
|
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') ALERT SKIPPED: no ZULIP_API_KEY or curl" >> "$LOG_FILE"
|
|
fi
|
|
|
|
exit 1
|
|
fi
|