feat(security): commit-time secret guard that FAILS the build on a committed credential #129
@@ -71,6 +71,18 @@ jobs:
|
||||
git fetch origin "${{ gitea.ref }}" --depth=50
|
||||
git checkout "${{ gitea.sha }}"
|
||||
|
||||
- name: Committed-credential scan (secret guard)
|
||||
run: |
|
||||
# Fails the build on a credential-shaped string in the tree. Patterns
|
||||
# live in scripts/secret-patterns.tsv; the only tolerated literal
|
||||
# examples are in scripts/secret-allowlist.tsv, each with a reason.
|
||||
# Do not turn this into a warning: a warning in a stream nobody reads
|
||||
# is how six live credentials sat in this repo for weeks.
|
||||
bash scripts/secret-scan.sh
|
||||
|
||||
- name: Secret guard self-test
|
||||
run: bash tests/test_secret_scan.sh
|
||||
|
||||
- name: Structure + regression + consistency lint
|
||||
run: bash scripts/prose-lint.sh
|
||||
|
||||
|
||||
@@ -51,6 +51,12 @@ Two incidents taught us this:
|
||||
- `/grafana/` nginx route — was reverted Jul 2, must not reappear
|
||||
- `CT 122` or `CT 123` as CT ID labels — don't exist in the cluster
|
||||
- These rules are hardcoded in `scripts/prose-lint.sh`
|
||||
- **Committed-credential guard:** `scripts/secret-scan.sh` FAILS the build on
|
||||
credential-shaped strings (patterns in `scripts/secret-patterns.tsv`, prose
|
||||
included). Tolerated literals are listed one-per-example with a reason in
|
||||
`scripts/secret-allowlist.tsv`; never allowlist a live credential. It runs in
|
||||
the CI lint job, in `scripts/prose-lint.sh`, and via
|
||||
`bash scripts/secret-scan.sh --staged` before committing.
|
||||
|
||||
### Stage 3 — AI Review
|
||||
- Diff is sent to `syslog-auto` model via LiteLLM
|
||||
|
||||
+16
-1
@@ -135,7 +135,22 @@ fi
|
||||
|
||||
echo " Cross-contract: $WARNINGS total warnings across all checks"
|
||||
|
||||
# ── 4. Summary ──
|
||||
# ── 4. Committed-credential scan ──
|
||||
# The 2026-09-17 purge removed six live credentials that had sat in .md prose
|
||||
# and scripts for weeks. This step makes that class of commit FAIL the gate
|
||||
# instead of printing a warning. Patterns: scripts/secret-patterns.tsv.
|
||||
# Only deliberate synthetic examples may be listed in scripts/secret-allowlist.tsv,
|
||||
# each with a reason. Run `bash scripts/secret-scan.sh --staged` before committing.
|
||||
echo ""
|
||||
echo "── 4. Secret scan (committed credentials) ──"
|
||||
if bash scripts/secret-scan.sh; then
|
||||
echo " ✅ No committed credentials"
|
||||
else
|
||||
echo " ❌ COMMITTED CREDENTIAL DETECTED"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
# ── 5. Summary ──
|
||||
echo ""
|
||||
echo "═══════════════════════════════════"
|
||||
if [ $FAILED -eq 1 ]; then
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# secret-allowlist.tsv — exceptions for scripts/secret-scan.sh, every entry with a reason.
|
||||
#
|
||||
# Format: <rule-id|*><TAB><path-glob><TAB><literal-substring><TAB><reason>
|
||||
# Blank lines and lines whose first field starts with '#' are ignored.
|
||||
# A finding is suppressed only when ALL THREE of rule, path and literal match:
|
||||
# * the rule id equals the finding's rule id, or is '*'
|
||||
# * the finding's repo-relative path matches <path-glob> (bash glob)
|
||||
# * the finding's line contains <literal-substring> verbatim
|
||||
# An entry whose reason is empty is a hard error (exit 2) — no silent exceptions.
|
||||
#
|
||||
# RULE: never allowlist a live credential, and never broaden an entry (rule '*',
|
||||
# a wide path glob, or a short generic literal) just to silence a finding.
|
||||
# If the finding is real, remove the credential from the file.
|
||||
#
|
||||
# Entries are one per deliberate synthetic example, so the file reads as an
|
||||
# audit trail of reviewed exceptions rather than a list of things to ignore.
|
||||
# Rule '*' is used only where the same literal is matched by more than one rule.
|
||||
#
|
||||
# ── The 2026-09-17 purge placeholders ─────────────────────────────────────
|
||||
# PR #112 replaced six live credentials with `«vault: <project>/<env> <SECRET>»`
|
||||
# references. Those references are safe by construction (they name where the
|
||||
# secret is read from), but they are listed here explicitly rather than being
|
||||
# filtered by a general "vault" rule, so a new occurrence still needs a
|
||||
# deliberate, reasoned entry.
|
||||
secret-assign litellm-api-keys.prose.md MUMUNI_LITELLM_API_KEY=«vault: agents/production LITELLM_API_KEY» 2026-09-17 purge: replaced the live Mumuni LiteLLM key with its vault reference; no literal credential.
|
||||
secret-assign litellm-api-keys.prose.md MUMUNI_ZULIP_API_KEY=«vault: agents/production ZULIP_API_KEY» 2026-09-17 purge: replaced the live Mumuni Zulip key with its vault reference; no literal credential.
|
||||
* infrastructure-control.prose.md PVEAPIToken=«vault: infrastructure/production PVE_API_TOKEN» 2026-09-17 purge: Proxmox API token is read from the vault; the line only names the vault path.
|
||||
cred-prose infrastructure-control.prose.md Admin credentials: 2026-09-17 purge: the Stirling admin user/password are two `«vault: ...»` references; no literal credential.
|
||||
* scripts/daily-infra-report.py PVEAPIToken=«vault: infrastructure/production PVE_API_TOKEN» 2026-09-17 purge: Proxmox API token is read from the vault; the line only names the vault path.
|
||||
secret-assign stirling-pdf-agent-access.prose.md «vault: infrastructure/production STIRLING_API_KEY» 2026-09-17 purge: Stirling PDF API key is read from the vault; the curl example only names the vault path.
|
||||
bearer-token agent-zero-fix-summary.md «vault: agents/production OPENROUTER_API_KEY» 2026-09-17 purge: OpenRouter key is read from the vault; the example curl only names the vault path.
|
||||
# ── Deliberate synthetic examples in contracts (not from the purge) ───────
|
||||
# These exist to teach the rule they illustrate. They are listed here so the
|
||||
# guard is never taught to skip the words "synthetic"/"example" — a fabricated
|
||||
# example is always an explicit exception, never a pattern-level exemption.
|
||||
* hermes-key-enforcement.prose.md sk-synthetic-external-example Rule 15 illustration of a hardcoded external key that is tolerated; fabricated, never a live key.
|
||||
* hermes-key-enforcement.prose.md sk-synthetic-example-12345 Rule 15 illustration of a forbidden hardcoded key; fabricated, never a live key.
|
||||
openai-key hermes-key-enforcement.prose.md sk-synthetic-litellm- Fabricated key name inside a `grep 'LITELLM_API_KEY=...'` example; not a live key.
|
||||
secret-assign hermes-key-enforcement.prose.md sk-NEW_KEY Placeholder standing for the rotated key in an `infisical secrets set` command; not a literal key.
|
||||
openrouter-key agent-zero-openrouter-key.prose.md sk-or-v1-synthetic Synthetic key prefix in the contract's example response; the real key is read from the vault.
|
||||
openai-key litellm-api-keys.prose.md sk-synthetic-tanko-example Fabricated key name in migration history prose; not a live key.
|
||||
openai-key litellm-self-heal.prose.md sk-syslog-local-master-key Deprecated local LiteLLM master key name documented as no-live-usage; kept for history, not a usable credential.
|
||||
# ── Redacted evidence, not a credential ──────────────────────────────────
|
||||
secret-assign docs/probe-drift-round2-evidence.md =sk-... Probe evidence records redacted key trailers (`sk-...x6uw`); the usable part of the key is not present.
|
||||
# ── tests/test_secret_scan.sh fixtures ───────────────────────────────────
|
||||
# The self-test plants these fabricated values into a TEMP tree, whose path no
|
||||
# entry here covers, so each still fails the guard when planted (see the test's
|
||||
# "... fails the guard" cases). They are listed only so the repo-wide scan of
|
||||
# the test file itself stays quiet.
|
||||
* tests/test_secret_scan.sh sk-or-v1-00000000000000000000000000000000000000000000000000000000deadbeef Self-test fixture: fabricated OpenRouter-shaped key written to a temp tree; the guard must fail on it there.
|
||||
bearer-token tests/test_secret_scan.sh Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbb Self-test fixture: fabricated Bearer token written to a temp tree; the guard must fail on it there.
|
||||
proxmox-token tests/test_secret_scan.sh PVEAPIToken=root@pam!monitor=11111111-2222-3333-4444-555555555555 Self-test fixture: fabricated Proxmox token written to a temp tree; the guard must fail on it there.
|
||||
private-key tests/test_secret_scan.sh -----BEGIN OPENSSH PRIVATE KEY----- Self-test fixture: fabricated PEM banner written to a temp tree; the guard must fail on it there.
|
||||
cred-prose tests/test_secret_scan.sh Admin credentials: Self-test fixture: fabricated prose credential line written to a temp tree; the guard must fail on it there.
|
||||
secret-assign tests/test_secret_scan.sh DB_PASSWORD=correct-horse-battery-staple Self-test fixture: fabricated password assignment written to a temp tree; the guard must fail on it there.
|
||||
|
Can't render this file because it contains an unexpected character in line 23 and column 25.
|
@@ -0,0 +1,22 @@
|
||||
# secret-patterns.tsv — checked-in pattern list for scripts/secret-scan.sh
|
||||
#
|
||||
# Format: <rule-id><TAB><POSIX ERE><TAB><description><TAB><check>
|
||||
# Blank lines and lines whose first field starts with '#' are ignored.
|
||||
# <check> is optional; the only value today is "value", which tells the scanner
|
||||
# to run the matched value through its inert-value classifier (see
|
||||
# value_is_inert in secret-scan.sh) so bare identifiers, env refs and dotted
|
||||
# code access are not reported as credentials. Omit the column to report every
|
||||
# regex hit.
|
||||
# Matching is case-insensitive, so `API_KEY` and `api_key` both count.
|
||||
#
|
||||
# Add a rule here, never inline in secret-scan.sh: this file is the single
|
||||
# auditable list of what the guard considers credential-shaped.
|
||||
openai-key \bsk-[A-Za-z0-9_-]{16,} OpenAI/LiteLLM-style "sk-" secret key (also hyphenated sk-proj- keys)
|
||||
openrouter-key \bsk-or-v1-[A-Za-z0-9_-]{8,} OpenRouter API key
|
||||
stripe-live-key \bsk_live_[A-Za-z0-9]{8,} Stripe live secret key
|
||||
proxmox-token PVEAPIToken=[^[:space:]"']+ Proxmox API token literal
|
||||
bearer-token bearer[[:space:]]+["']?(«.{3,}»|[A-Za-z0-9_./+=-]{20,}) literal Bearer token (http header or prose)
|
||||
auth-header authorization:[[:space:]]+["']?(«.{3,}»|[A-Za-z0-9_./+=-]{20,}) Authorization header carrying a raw literal value
|
||||
private-key -----BEGIN [A-Z ]*PRIVATE KEY----- PEM private key block
|
||||
cred-prose credentials?[[:space:]]*[:=][[:space:]]*[^[:space:]] prose credential line carrying a value
|
||||
secret-assign (api[_-]?key|apikey|passwd|password|secret|token)s?["']?[[:space:]]*[:=][[:space:]]*["']?(«.{3,}»|[A-Za-z0-9_./+=-]{8,}) credential assignment carrying a literal value value
|
||||
|
Can't render this file because it contains an unexpected character in line 5 and column 48.
|
Executable
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env bash
|
||||
# secret-scan.sh — commit-time secret guard. FAILS (exit 1) on a credential-shaped
|
||||
# string, so a build cannot go green with a credential committed to it.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/secret-scan.sh # scan the whole git-tracked tree (default)
|
||||
# scripts/secret-scan.sh --tree
|
||||
# scripts/secret-scan.sh --path DIR # scan an arbitrary directory (git not required)
|
||||
# scripts/secret-scan.sh --staged # scan added lines in the index (pre-commit)
|
||||
# scripts/secret-scan.sh --diff REF # scan added lines since REF (e.g. origin/master)
|
||||
# --quiet only print the verdict and findings, no per-mode banner
|
||||
#
|
||||
# Exit codes: 0 clean, 1 credential found, 2 usage/config error.
|
||||
#
|
||||
# Patterns live in scripts/secret-patterns.tsv
|
||||
# Exceptions live in scripts/secret-allowlist.tsv (every entry carries a reason;
|
||||
# a missing reason is a hard error, so the guard fails closed).
|
||||
#
|
||||
# Dependencies are deliberately bash + coreutils + grep + sed/awk + git. The
|
||||
# Gitea Actions runner executes job steps INSIDE the runner container, which
|
||||
# has no node and no python by default: keep this script free of both.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SELF_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
ROOT=$(cd -- "$SELF_DIR/.." && pwd)
|
||||
PATTERNS_FILE="$SELF_DIR/secret-patterns.tsv"
|
||||
ALLOWLIST_FILE="$SELF_DIR/secret-allowlist.tsv"
|
||||
|
||||
# The guard's own definition files are not scannable content: the pattern list
|
||||
# necessarily contains the pattern text, and the allowlist necessarily contains
|
||||
# the allowed literals. Narrow, exact-path exclusion — not a wildcard.
|
||||
SELF_FILES=(
|
||||
"scripts/secret-scan.sh"
|
||||
"scripts/secret-patterns.tsv"
|
||||
"scripts/secret-allowlist.tsv"
|
||||
)
|
||||
|
||||
MODE="tree"
|
||||
PATH_DIR=""
|
||||
DIFF_REF=""
|
||||
QUIET=0
|
||||
|
||||
usage() {
|
||||
sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
exit 2
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--tree) MODE="tree" ;;
|
||||
--path) MODE="path"; PATH_DIR="${2:-}"; shift ;;
|
||||
--staged) MODE="staged" ;;
|
||||
--diff) MODE="diff"; DIFF_REF="${2:-}"; shift ;;
|
||||
--quiet) QUIET=1 ;;
|
||||
-h|--help) usage ;;
|
||||
*) echo "secret-scan: unknown argument '$1'" >&2; usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ -f "$PATTERNS_FILE" ] || { echo "secret-scan: missing $PATTERNS_FILE" >&2; exit 2; }
|
||||
[ -f "$ALLOWLIST_FILE" ] || { echo "secret-scan: missing $ALLOWLIST_FILE" >&2; exit 2; }
|
||||
if [ "$MODE" = "path" ] && [ -z "$PATH_DIR" ]; then
|
||||
echo "secret-scan: --path needs a directory" >&2; exit 2
|
||||
fi
|
||||
if [ "$MODE" = "diff" ] && [ -z "$DIFF_REF" ]; then
|
||||
echo "secret-scan: --diff needs a base ref" >&2; exit 2
|
||||
fi
|
||||
|
||||
# ── Load patterns ──────────────────────────────────────────────────────────
|
||||
RULE_IDS=()
|
||||
RULE_RES=()
|
||||
RULE_DESCS=()
|
||||
RULE_CHECKS=()
|
||||
COMBINED=""
|
||||
while IFS=$'\t' read -r id re desc check; do
|
||||
case "$id" in ''|'#'*) continue ;; esac
|
||||
[ -n "$re" ] || continue
|
||||
RULE_IDS+=("$id"); RULE_RES+=("$re"); RULE_DESCS+=("$desc"); RULE_CHECKS+=("${check:-}")
|
||||
if [ -z "$COMBINED" ]; then COMBINED="($re)"; else COMBINED="$COMBINED|($re)"; fi
|
||||
done < "$PATTERNS_FILE"
|
||||
if [ "${#RULE_IDS[@]}" -eq 0 ]; then
|
||||
echo "secret-scan: no patterns loaded from $PATTERNS_FILE" >&2; exit 2
|
||||
fi
|
||||
|
||||
# ── Load allowlist (fails closed on a missing reason) ──────────────────────
|
||||
AL_RULES=()
|
||||
AL_GLOBS=()
|
||||
AL_LITS=()
|
||||
AL_REASONS=()
|
||||
AL_LINENO=0
|
||||
while IFS=$'\t' read -r rule glob lit reason; do
|
||||
AL_LINENO=$((AL_LINENO + 1))
|
||||
case "$rule" in ''|'#'*) continue ;; esac
|
||||
if [ -z "$glob" ] || [ -z "$lit" ] || [ -z "$reason" ]; then
|
||||
echo "secret-scan: ❌ $ALLOWLIST_FILE:$AL_LINENO — allowlist entry needs <rule> <path-glob> <literal> <reason>; reason-based exceptions only, refusing to run" >&2
|
||||
exit 2
|
||||
fi
|
||||
AL_RULES+=("$rule"); AL_GLOBS+=("$glob"); AL_LITS+=("$lit"); AL_REASONS+=("$reason")
|
||||
done < "$ALLOWLIST_FILE"
|
||||
|
||||
# nocasematch is toggled only around the regex test; path globs must stay
|
||||
# case-sensitive, so it is never left on.
|
||||
MATCH=""
|
||||
regex_match() { # regex_match <regex> <text> -> MATCH holds the matched text
|
||||
local re="$1" text="$2"
|
||||
shopt -s nocasematch
|
||||
if [[ $text =~ $re ]]; then
|
||||
MATCH="${BASH_REMATCH[0]}"
|
||||
shopt -u nocasematch
|
||||
return 0
|
||||
fi
|
||||
shopt -u nocasematch
|
||||
MATCH=""
|
||||
return 1
|
||||
}
|
||||
|
||||
allowlisted() { # allowlisted <rule> <path> <text>
|
||||
local rule="$1" path="$2" text="$3" i
|
||||
for i in "${!AL_RULES[@]}"; do
|
||||
[ "${AL_RULES[$i]}" = "$rule" ] || [ "${AL_RULES[$i]}" = "*" ] || continue
|
||||
# The unquoted RHS is deliberate: <path-glob> is a bash glob, not a literal.
|
||||
# shellcheck disable=SC2053
|
||||
[[ $path == ${AL_GLOBS[$i]} ]] || continue
|
||||
[[ $text == *"${AL_LITS[$i]}"* ]] || continue
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
mask_value() { # mask_value <text> <match> — never echo a credential to logs.
|
||||
# Print only the part of the line BEFORE the match, then <redacted>: the match
|
||||
# itself and everything after it (which may include a value the rule's regex
|
||||
# stopped short of, e.g. `credentials:` followed by a backticked password) is
|
||||
# never written to stdout.
|
||||
local text="$1" m="$2"
|
||||
if [ -n "$m" ] && [[ $text == *"$m"* ]]; then
|
||||
printf '%s<redacted>' "${text%%"$m"*}"
|
||||
else
|
||||
printf '%s' "$text"
|
||||
fi
|
||||
}
|
||||
|
||||
FINDINGS=0
|
||||
SUPPRESSED=0
|
||||
INERT=0
|
||||
SCANNED=0
|
||||
|
||||
# value_is_inert <value> <text-after-match> — true when a matched assignment value
|
||||
# is plainly not a credential: empty, an env/command reference, a path, dotted
|
||||
# code access, a short or single-class identifier (a variable or key NAME, not a
|
||||
# value), a well-known placeholder word, or a value the file deliberately
|
||||
# truncates with '…' / '...' (a redacted prefix is not a usable credential).
|
||||
# Deliberately does NOT know the words "synthetic" or "example": a fabricated
|
||||
# example must be an explicit allowlist entry.
|
||||
value_is_inert() {
|
||||
local v="$1" rest="$2"
|
||||
case "$rest" in '…'*|'...'*) return 0 ;; esac
|
||||
v="${v%\"}"; v="${v#\"}"; v="${v%\'}"; v="${v#\'}"
|
||||
case "$v" in
|
||||
''|\$*|\{*|'<'*|'%'*|'('*|'/'*|'\\'*) return 0 ;;
|
||||
not-needed|no-key-required|none|null|true|false|redacted|placeholder|example|dummy|changeme|change-me|your-key|your_key|key|token|secret|password) return 0 ;;
|
||||
esac
|
||||
# dotted code access: os.environ.get / process.env.ZULIP_API_KEY / cfg.a
|
||||
if [[ $v =~ ^[a-z_][a-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)+$ ]]; then return 0; fi
|
||||
# bare identifier (no punctuation beyond _): a NAME, not a value. A real
|
||||
# secret in this shape is long and mixes letters with digits.
|
||||
if [[ $v =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
|
||||
[ "${#v}" -lt 20 ] && return 0
|
||||
[[ $v =~ [0-9] ]] || return 0
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
report_finding() { # report_finding <path> <line> <text>
|
||||
local path="$1" line="$2" text="$3" i val
|
||||
for i in "${!RULE_IDS[@]}"; do
|
||||
regex_match "${RULE_RES[$i]}" "$text" || continue
|
||||
SCANNED=$((SCANNED + 1))
|
||||
if [ "${RULE_CHECKS[$i]}" = "value" ]; then
|
||||
val="${MATCH#*[:=]}"
|
||||
val="${val# }"
|
||||
if value_is_inert "$val" "${text#*"$MATCH"}"; then
|
||||
INERT=$((INERT + 1))
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
if allowlisted "${RULE_IDS[$i]}" "$path" "$text"; then
|
||||
SUPPRESSED=$((SUPPRESSED + 1))
|
||||
continue
|
||||
fi
|
||||
FINDINGS=$((FINDINGS + 1))
|
||||
printf ' ❌ %s:%s [%s] %s\n' "$path" "$line" "${RULE_IDS[$i]}" "${RULE_DESCS[$i]}"
|
||||
printf ' | %s\n' "$(mask_value "$text" "$MATCH")"
|
||||
done
|
||||
}
|
||||
|
||||
self_excluded() { # self_excluded <repo-relative-path>
|
||||
local p="$1" s
|
||||
for s in "${SELF_FILES[@]}"; do
|
||||
[ "$p" = "$s" ] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Collect candidate lines and scan them ─────────────────────────────────
|
||||
if [ "$MODE" = "tree" ] || [ "$MODE" = "path" ]; then
|
||||
if [ "$MODE" = "tree" ]; then
|
||||
BASE="$ROOT"
|
||||
git -C "$BASE" rev-parse --git-dir >/dev/null 2>&1 || { echo "secret-scan: --tree needs a git checkout (use --path DIR)" >&2; exit 2; }
|
||||
mapfile -d '' candidate < <(git -C "$BASE" ls-files -z 2>/dev/null)
|
||||
if [ "${#candidate[@]}" -eq 0 ]; then
|
||||
echo "secret-scan: ❌ no tracked files — refusing to report clean" >&2; exit 2
|
||||
fi
|
||||
else
|
||||
BASE=$(cd -- "$PATH_DIR" 2>/dev/null && pwd) || { echo "secret-scan: --path '$PATH_DIR' is not a directory" >&2; exit 2; }
|
||||
mapfile -t candidate < <(cd -- "$BASE" && find . -type f -not -path './.git/*' | sed 's|^\./||')
|
||||
if [ "${#candidate[@]}" -eq 0 ]; then
|
||||
echo "secret-scan: ❌ no files under $BASE — refusing to report clean" >&2; exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
[ "$QUIET" -eq 1 ] || echo "── secret scan ($MODE): ${#candidate[@]} files under $BASE ──"
|
||||
for rel in "${candidate[@]}"; do
|
||||
[ -f "$BASE/$rel" ] || continue
|
||||
self_excluded "$rel" && continue
|
||||
while IFS= read -r hit; do
|
||||
[ -n "$hit" ] || continue
|
||||
report_finding "$rel" "${hit%%:*}" "${hit#*:}"
|
||||
done < <(grep -nEIi -e "$COMBINED" "$BASE/$rel" 2>/dev/null || true)
|
||||
done
|
||||
else
|
||||
# --staged / --diff: only ADDED lines, with the post-change line number.
|
||||
if [ "$MODE" = "staged" ]; then
|
||||
[ "$QUIET" -eq 1 ] || echo "── secret scan: added lines in the index ──"
|
||||
DIFF_TEXT=$(git -C "$ROOT" diff --cached --unified=0 --no-color -- . 2>/dev/null)
|
||||
else
|
||||
[ "$QUIET" -eq 1 ] || echo "── secret scan: added lines since $DIFF_REF ──"
|
||||
DIFF_TEXT=$(git -C "$ROOT" diff --unified=0 --no-color "$DIFF_REF"...HEAD 2>/dev/null \
|
||||
|| git -C "$ROOT" diff --unified=0 --no-color "$DIFF_REF"..HEAD 2>/dev/null)
|
||||
fi
|
||||
if [ -z "$DIFF_TEXT" ]; then
|
||||
[ "$QUIET" -eq 1 ] || echo " (no added lines)"
|
||||
fi
|
||||
while IFS=$'\t' read -r rel line text; do
|
||||
[ -n "$rel" ] || continue
|
||||
self_excluded "$rel" && continue
|
||||
report_finding "$rel" "$line" "$text"
|
||||
done < <(printf '%s\n' "$DIFF_TEXT" | awk '
|
||||
/^\+\+\+ / { f=$2; sub(/^b\//,"",f); next }
|
||||
/^@@ / { if (match($0, /\+[0-9]+/)) ln=substr($0, RSTART+1, RLENGTH-1)+0; next }
|
||||
(/^\+/ && !/^\+\+\+/) { print f "\t" ln "\t" substr($0,2); ln++; next }
|
||||
')
|
||||
fi
|
||||
|
||||
# ── Verdict ────────────────────────────────────────────────────────────────
|
||||
if [ "$FINDINGS" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "❌ SECRET SCAN FAILED — $FINDINGS credential-shaped string(s) in ${MODE} content."
|
||||
echo " Fix: remove the credential and read it from the vault/env."
|
||||
echo " Only a deliberate synthetic example may be added to scripts/secret-allowlist.tsv,"
|
||||
echo " one entry per file/rule/literal, with a reason. Never allowlist a live credential."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ secret scan clean (${MODE}; ${SUPPRESSED} allowlisted exception(s), ${INERT} inert value(s) ignored)"
|
||||
exit 0
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# test_secret_scan.sh — self-test for the commit-time secret guard.
|
||||
#
|
||||
# Run: bash tests/test_secret_scan.sh
|
||||
# Exit: 0 all cases passed, 1 a case failed.
|
||||
#
|
||||
# WHY THIS FILE EXISTS: a scanner that is never observed to fail is not a guard.
|
||||
# Every fixture below is fabricated and pattern-shaped; the test writes it to a
|
||||
# temp tree (a path no allowlist entry covers) and asserts the guard FAILS. The
|
||||
# same fixtures are deliberately listed in scripts/secret-allowlist.tsv, so the
|
||||
# repo-wide tree scan stays quiet while a planted copy still bites — that is the
|
||||
# difference between an explicit, reasoned exception and a guard trained to
|
||||
# ignore a word.
|
||||
#
|
||||
# Only bash + coreutils + grep. No python/node: the Gitea runner executes job
|
||||
# steps inside the runner container, which has neither.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
ROOT=$(cd -- "$HERE/.." && pwd)
|
||||
SCAN="$ROOT/scripts/secret-scan.sh"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
LAST_OUT=""
|
||||
|
||||
ok() { PASS=$((PASS + 1)); echo " ✅ $1"; }
|
||||
bad() { FAIL=$((FAIL + 1)); echo " ❌ $1"; }
|
||||
|
||||
expect_exit() { # expect_exit <want-code> <label> <cmd...>
|
||||
local want="$1" label="$2"; shift 2
|
||||
local rc
|
||||
LAST_OUT=$("$@" 2>&1); rc=$?
|
||||
if [ "$rc" -eq "$want" ]; then ok "$label (exit $rc)"; else
|
||||
bad "$label (wanted exit $want, got $rc)"
|
||||
printf '%s\n' "$LAST_OUT" | sed 's/^/ /' | head -8
|
||||
fi
|
||||
}
|
||||
|
||||
expect_contains() { # expect_contains <label> <needle>
|
||||
if printf '%s' "$LAST_OUT" | grep -qF -- "$2"; then ok "$1"; else
|
||||
bad "$1 (output did not mention: $2)"
|
||||
fi
|
||||
}
|
||||
|
||||
TMPROOT=$(mktemp -d)
|
||||
trap 'rm -rf "$TMPROOT"' EXIT
|
||||
|
||||
echo "── secret-scan self-test ──"
|
||||
|
||||
# ── 1. Guard syntax ───────────────────────────────────────────────────────
|
||||
expect_exit 0 "scanner parses with bash -n" bash -n "$SCAN"
|
||||
|
||||
# ── 2. Guard FAILS on planted, pattern-matching fixtures ──────────────────
|
||||
mkdir -p "$TMPROOT/planted"
|
||||
cat > "$TMPROOT/planted/ops.env" <<'EOF'
|
||||
OPENROUTER_API_KEY=sk-or-v1-00000000000000000000000000000000000000000000000000000000deadbeef
|
||||
EOF
|
||||
expect_exit 1 "planted sk-or-v1 key fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted sk-or-v1 key names the openrouter-key rule" "[openrouter-key]"
|
||||
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/curl.sh" <<'EOF'
|
||||
curl -s -H "Authorization: Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbb" http://example.invalid/
|
||||
EOF
|
||||
expect_exit 1 "planted literal Bearer token fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted Bearer token names the bearer-token rule" "[bearer-token]"
|
||||
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/pve.sh" <<'EOF'
|
||||
AUTH="Authorization: PVEAPIToken=root@pam!monitor=11111111-2222-3333-4444-555555555555"
|
||||
EOF
|
||||
expect_exit 1 "planted Proxmox token fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted Proxmox token names the proxmox-token rule" "[proxmox-token]"
|
||||
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/deploy-key.pem" <<'EOF'
|
||||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
EOF
|
||||
expect_exit 1 "planted PEM private key fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted PEM key names the private-key rule" "[private-key]"
|
||||
|
||||
# Prose is scanned exactly like code — the original exposures were in .md files.
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/handover.md" <<'EOF'
|
||||
- Admin credentials: `admin` / `correct-horse-battery-staple`
|
||||
EOF
|
||||
expect_exit 1 "planted prose credential line fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted prose line names the cred-prose rule" "[cred-prose]"
|
||||
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/config.env" <<'EOF'
|
||||
DB_PASSWORD=correct-horse-battery-staple
|
||||
EOF
|
||||
expect_exit 1 "planted password assignment fails the guard" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
expect_contains "planted password assignment names the secret-assign rule" "[secret-assign]"
|
||||
|
||||
# ── 3. Guard stays QUIET on inert values and on the real tree ─────────────
|
||||
mkdir -p "$TMPROOT/inert"
|
||||
cat > "$TMPROOT/inert/config.yaml" <<'EOF'
|
||||
api_key: not-needed
|
||||
bearer_token=monitor_key
|
||||
api_key: $LITELLM_API_KEY
|
||||
EOF
|
||||
expect_exit 0 "env refs, sentinels and variable names are not credentials" bash "$SCAN" --path "$TMPROOT/inert" --quiet
|
||||
|
||||
expect_exit 0 "current repo tree passes the guard" bash "$SCAN" --tree
|
||||
expect_contains "tree run reports the allowlisted exceptions it applied" "allowlisted exception(s)"
|
||||
|
||||
# ── 4. Allowlist entries are path-explicit, not word-based ────────────────
|
||||
# This exact line is allowlisted in infrastructure-control.prose.md; the same
|
||||
# text at an unlisted path must still fail, proving the exception is per-file
|
||||
# and reviewed, not a blanket "ignore the word vault".
|
||||
rm -f "$TMPROOT/planted/"*
|
||||
cat > "$TMPROOT/planted/unlisted.md" <<'EOF'
|
||||
- Admin credentials: `«vault: infrastructure/production STIRLING_ADMIN_PASSWORD»`
|
||||
EOF
|
||||
expect_exit 1 "allowlisted text at an unlisted path still fails" bash "$SCAN" --path "$TMPROOT/planted" --quiet
|
||||
|
||||
# ── 5. Commit-time mode: the guard blocks a STAGED credential ─────────────
|
||||
# A throwaway git repo with its own copy of the scanner, so this exercises the
|
||||
# real pre-commit path (--staged) without touching this repo's index.
|
||||
mkdir -p "$TMPROOT/repo/scripts"
|
||||
cp "$SCAN" "$TMPROOT/repo/scripts/secret-scan.sh"
|
||||
cp "$ROOT/scripts/secret-patterns.tsv" "$TMPROOT/repo/scripts/secret-patterns.tsv"
|
||||
cp "$ROOT/scripts/secret-allowlist.tsv" "$TMPROOT/repo/scripts/secret-allowlist.tsv"
|
||||
git -C "$TMPROOT/repo" init -q
|
||||
git -C "$TMPROOT/repo" -c user.email=t@example.invalid -c user.name=test commit -q --allow-empty -m base
|
||||
cat > "$TMPROOT/repo/planted.env" <<'EOF'
|
||||
OPENROUTER_API_KEY=sk-or-v1-00000000000000000000000000000000000000000000000000000000deadbeef
|
||||
EOF
|
||||
git -C "$TMPROOT/repo" add planted.env
|
||||
expect_exit 1 "staged credential fails at commit time (--staged)" bash "$TMPROOT/repo/scripts/secret-scan.sh" --staged --quiet
|
||||
expect_contains "staged credential names the openrouter-key rule" "[openrouter-key]"
|
||||
|
||||
# ── 6. Fail closed: an allowlist entry without a reason is a hard error ───
|
||||
mkdir -p "$TMPROOT/scanner" "$TMPROOT/clean"
|
||||
cp "$SCAN" "$TMPROOT/scanner/secret-scan.sh"
|
||||
cp "$ROOT/scripts/secret-patterns.tsv" "$TMPROOT/scanner/secret-patterns.tsv"
|
||||
printf '*\t*.md\twhatever\n' > "$TMPROOT/scanner/secret-allowlist.tsv"
|
||||
echo "placeholder" > "$TMPROOT/clean/ok.md"
|
||||
expect_exit 2 "allowlist entry with no reason fails closed" bash "$TMPROOT/scanner/secret-scan.sh" --path "$TMPROOT/clean" --quiet
|
||||
|
||||
# ── Verdict ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo "❌ secret-scan self-test FAILED — $PASS passed, $FAIL failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ secret-scan self-test passed ($PASS cases)"
|
||||
Reference in New Issue
Block a user