diff --git a/scripts/capture-dsh-token.sh b/scripts/capture-dsh-token.sh index 7a9c1b6..d1ce52c 100755 --- a/scripts/capture-dsh-token.sh +++ b/scripts/capture-dsh-token.sh @@ -14,14 +14,15 @@ # reference the token of the RUNNING process. # # This script: -# 1. reads the LATEST launch token from the running service's journal — it -# NEVER stops or starts dsh-web, +# 1. selects the launch token the RUNNING service actually accepts — it NEVER +# stops or starts dsh-web, # 2. records it in /etc/dsh-web/launch-token, # 3. regenerates the nginx include /etc/dsh-web/nginx-login.conf (the # `proxy_pass ...?token=` line consumed by /dsh-web-login), -# 4. reloads nginx ONLY when the token differs from the token nginx actually -# loaded (tracked in an applied-state stamp written only after a successful -# reload), rolling the include back on failure so the next run retries, +# 4. reloads nginx ONLY when the on-disk include differs from the generated +# one or the applied-state stamp does not match the token (the stamp is +# written only after a successful reload), rolling the include back on +# failure so the next run retries, # 5. removes the legacy unauthenticated :8081 endpoint if it ever reappears. # # Idempotent and safe to run at any time (systemd ExecStartPost or timer). @@ -36,13 +37,21 @@ STAMP_FILE="/etc/dsh-web/nginx-login.conf.applied" SITE_ENABLED="/etc/nginx/sites-enabled/dsh" LEGACY_8081="/etc/nginx/sites-enabled/dsh.token" STASH_DIR="/etc/nginx/sites-available" +LOCK_FILE="/run/capture-dsh-token.lock" +LOGIN_HOST="tankodhs.sysloggh.net" +LOGIN_UPSTREAM="http://127.0.0.1:3080" +TOKEN_WAIT=120 log() { printf 'capture-dsh-token: %s\n' "$*" >&2; } die() { printf 'capture-dsh-token: ERROR: %s\n' "$*" >&2; exit 1; } [ "$(id -u)" -eq 0 ] || die "must run as root" -# ── 0. Remove the legacy unauthenticated :8081 endpoint, if present ───────── +# ── 0. Serialize runs so timer/ExecStartPost/manual runs cannot interleave ── +exec 9>"$LOCK_FILE" +flock -n 9 || { log "another capture-dsh-token run holds $LOCK_FILE; exiting"; exit 0; } + +# ── 1. Remove the legacy unauthenticated :8081 endpoint, if present ───────── # It bypassed Authentik entirely (listened on 0.0.0.0:8081 with no auth_request) # and must never come back. Stash it rather than delete so it is auditable. if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then @@ -58,72 +67,68 @@ if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then log "removed legacy :8081 endpoint -> $STASHED" fi -# ── 1. Read the latest launch token from the RUNNING service ──────────────── -# Scope the journal to the service's CURRENT invocation. While a restarted -# process is still booting (~25s before it prints the banner), the newest token -# in the journal still belongs to the PREVIOUS process; without this filter an -# ExecStartPost run would silently keep the stale token. Never stop/start dsh-web. -INVOCATION="$(systemctl show -p InvocationID --value "$JOURNAL_UNIT" 2>/dev/null || true)" -JOURNAL_ARGS=(-u "$JOURNAL_UNIT") -if [ -n "$INVOCATION" ] && [ "$INVOCATION" != "n/a" ]; then - JOURNAL_ARGS+=("_SYSTEMD_INVOCATION_ID=$INVOCATION") -else - log "WARNING: no invocation id for $JOURNAL_UNIT; using latest journal token" -fi - -extract_token() { - grep -oE 'dsh web: https?://[^[:space:]]+[?&]token=[^[:space:]]+' \ - | tail -n1 | sed -E 's/.*[?&]token=//' || true +# ── 2. Select the token the RUNNING service actually accepts ──────────────── +# Functionally verify each journal candidate against the local dsh-web using the +# public authority, exactly as the /dsh-web-login proxy does. A token from a +# previous invocation is rejected (never 303) and can never be selected, so no +# systemd invocation scoping or newest-overall fallback is needed. Candidates +# are tried newest-first and re-read from the journal each pass until one is +# accepted or the wait elapses. +collect_tokens() { + journalctl -u "$JOURNAL_UNIT" --no-pager -o cat 2>/dev/null \ + | grep -oE 'dsh web: https?://[^[:space:]]+[?&]token=[^[:space:]]+' \ + | sed -E 's/.*[?&]token=//' \ + | grep -E '^[A-Za-z0-9._~+/=:@-]+$' \ + | tac | awk '!seen[$0]++' || true } TOKEN="" -for _ in $(seq 1 60); do - TOKEN="$(journalctl "${JOURNAL_ARGS[@]}" --no-pager -o cat 2>/dev/null | extract_token)" +TRIED=" " +DEADLINE=$((SECONDS + TOKEN_WAIT)) +while [ -z "$TOKEN" ] && [ "$SECONDS" -lt "$DEADLINE" ]; do + for cand in $(collect_tokens); do + case "$TRIED" in *" $cand "*) continue ;; esac + TRIED="$TRIED$cand " + code="$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' \ + -H "Host: $LOGIN_HOST" "$LOGIN_UPSTREAM/?token=$cand" || true)" + if [ "$code" = "303" ]; then + TOKEN="$cand" + break + fi + done [ -n "$TOKEN" ] && break - sleep 1 + sleep 2 done -# Fallback: the current invocation's start banner may have been rotated out of -# the journal; the newest matching line overall is then the best available. if [ -z "$TOKEN" ]; then - log "WARNING: no token for the current invocation; falling back to newest journal token" - TOKEN="$(journalctl -u "$JOURNAL_UNIT" --no-pager -o cat 2>/dev/null | extract_token)" + log "no dsh-web launch token accepted within ${TOKEN_WAIT}s; leaving the include untouched for the next run" + exit 0 fi -[ -n "$TOKEN" ] || die "no launch token found in the $JOURNAL_UNIT journal" -# The token must be safe to embed in a URI and in the nginx config. -printf '%s' "$TOKEN" | grep -qE '^[A-Za-z0-9._~+/=:@-]+$' \ - || die "captured token contains unsupported characters" - -# ── 2. Record the token (atomic, private) ────────────────────────────────── +# ── 3. Record the token (atomic, private) ────────────────────────────────── mkdir -p "$(dirname "$TOKEN_FILE")" if ! printf '%s\n' "$TOKEN" | cmp -s - "$TOKEN_FILE" 2>/dev/null; then printf '%s\n' "$TOKEN" > "$TOKEN_FILE.tmp" chmod 600 "$TOKEN_FILE.tmp" mv "$TOKEN_FILE.tmp" "$TOKEN_FILE" - log "recorded new launch token in $TOKEN_FILE" + log "recorded live launch token in $TOKEN_FILE" fi -# ── 3. Regenerate the nginx login include (reload only when it changes) ──── +# ── 4. Regenerate the nginx login include (reload only when it changes) ──── NEW_INCLUDE="$(mktemp "$INCLUDE_FILE.XXXXXX")" -printf 'proxy_pass http://127.0.0.1:3080/?token=%s;\n' "$TOKEN" > "$NEW_INCLUDE" +printf 'proxy_pass %s/?token=%s;\n' "$LOGIN_UPSTREAM" "$TOKEN" > "$NEW_INCLUDE" chmod 600 "$NEW_INCLUDE" -# The stamp records the token nginx actually loaded. Comparing against it (not -# the on-disk include) means a failed or interrupted reload is retried on the -# next run instead of being mistaken for success. +# The stamp records the token nginx actually loaded. It is written only after a +# successful reload, so the early exit is safe only when both the stamp and the +# on-disk include agree with the live token; anything else falls through to the +# reload path so the include can never silently diverge from what nginx serves. APPLIED="" [ -f "$STAMP_FILE" ] && APPLIED="$(cat "$STAMP_FILE" 2>/dev/null || true)" -if [ "$APPLIED" = "$TOKEN" ]; then - if [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE"; then - rm -f "$NEW_INCLUDE" - log "token unchanged; nginx not reloaded" - exit 0 - fi - mv "$NEW_INCLUDE" "$INCLUDE_FILE" - chmod 600 "$INCLUDE_FILE" - log "include file repaired to match the token nginx already serves" +if [ "$APPLIED" = "$TOKEN" ] && [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE"; then + rm -f "$NEW_INCLUDE" + log "token unchanged; nginx not reloaded" exit 0 fi @@ -165,4 +170,4 @@ chmod 600 "$STAMP_FILE.tmp" mv "$STAMP_FILE.tmp" "$STAMP_FILE" log "token changed; nginx reloaded" -log "login endpoint: https://tankodhs.sysloggh.net/dsh-web-login (Authentik-gated)" +log "login endpoint: https://$LOGIN_HOST/dsh-web-login (Authentik-gated)" diff --git a/zulip-health.prose.md b/zulip-health.prose.md index 349b439..0de8755 100644 --- a/zulip-health.prose.md +++ b/zulip-health.prose.md @@ -292,13 +292,19 @@ proxy_pass http://127.0.0.1:3080/?token=; **Token refresh (non-disruptive):** `/opt/deepseek-harness/capture-dsh-token.sh` (source: -`scripts/capture-dsh-token.sh`) reads the latest launch token from the journal -**of the service's current invocation**, writes `/etc/dsh-web/launch-token` and -regenerates `/etc/dsh-web/nginx-login.conf`, reloading nginx only when the token -differs from the token nginx actually loaded (`nginx -t` guards the reload, and -the applied-state stamp is written only after a successful `nginx -s reload`, so -a failed or interrupted reload is retried on the next run). It **never stops or -starts `dsh-web`**. It is triggered by the `dsh-web.service` drop-in +`scripts/capture-dsh-token.sh`) selects the live launch token by functionally +verifying journal candidates against dsh-web with `Host: tankodhs.sysloggh.net` +and using the first one the running process accepts with `303`; a stale token +from a previous invocation is rejected and can never be selected. It waits up to +120s for a restarted process to accept a token, and if none is accepted it +leaves the include untouched and exits `0` so the timer retries. It writes +`/etc/dsh-web/launch-token` and regenerates `/etc/dsh-web/nginx-login.conf`, +reloading nginx only when the on-disk include differs from the generated one or +the applied-state stamp does not match the token (`nginx -t` guards the reload, +and the stamp is written only after a successful `nginx -s reload`, so a failed +or interrupted reload is retried on the next run). Runs are serialized with +`flock` on `/run/capture-dsh-token.lock`. It **never stops or starts `dsh-web`**. +It is triggered by the `dsh-web.service` drop-in `/etc/systemd/system/dsh-web.service.d/20-token-refresh.conf` (`ExecStartPost=/bin/systemctl --no-block start dsh-web-token.service`) and by `dsh-web-token.timer` every 2 minutes for reconciliation. @@ -312,6 +318,7 @@ Description=Refresh the dsh-web launch token for the nginx login endpoint After=dsh-web.service [Service] Type=oneshot +TimeoutStartSec=180 ExecStart=/opt/deepseek-harness/capture-dsh-token.sh # /etc/systemd/system/dsh-web-token.timer