scripts/capture-dsh-token.sh

@@ -34,6 +34,7 @@ JOURNAL_UNIT="dsh-web.service"
 TOKEN_FILE="/etc/dsh-web/launch-token"
 INCLUDE_FILE="/etc/dsh-web/nginx-login.conf"
 STAMP_FILE="/etc/dsh-web/nginx-login.conf.applied"
+PENDING_FILE="/etc/dsh-web/nginx-reload.pending"
 SITE_ENABLED="/etc/nginx/sites-enabled/dsh"
 LEGACY_8081="/etc/nginx/sites-enabled/dsh.token"
 STASH_DIR="/etc/nginx/sites-available"
@@ -50,6 +51,7 @@ die()  { printf 'capture-dsh-token: ERROR: %s\n' "$*" >&2; exit 1; }
 # ── 0. Serialize runs so timer/ExecStartPost/manual runs cannot interleave ──
 exec 9>"$LOCK_FILE"
 flock -n 9 || { log "another capture-dsh-token run holds $LOCK_FILE; exiting"; exit 0; }
+mkdir -p "$(dirname "$PENDING_FILE")"
 
 # ── 1. Remove the legacy unauthenticated :8081 endpoint, if present ─────────
 # It bypassed Authentik entirely (listened on 0.0.0.0:8081 with no auth_request)
@@ -58,12 +60,14 @@ if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then
   TS="$(date -u +%Y%m%dT%H%M%SZ)"
   STASHED="$STASH_DIR/dsh.token.disabled-$TS"
   mv "$LEGACY_8081" "$STASHED"
+  touch "$PENDING_FILE"
   if ! nginx -t >/dev/null 2>&1; then
-    die "nginx config test failed after disabling $LEGACY_8081 (kept disabled at $STASHED). Fix the nginx config and rerun; the legacy :8081 endpoint will NOT be restored."
+    die "nginx config test failed after disabling $LEGACY_8081 (kept disabled at $STASHED); a pending reload is recorded so running nginx is reloaded once the config is fixed. The legacy :8081 endpoint will NOT be restored."
   fi
   if ! nginx -s reload; then
-    die "nginx reload failed after disabling $LEGACY_8081 (kept disabled at $STASHED). Fix nginx and rerun; the legacy :8081 endpoint will NOT be restored."
+    die "nginx reload failed after disabling $LEGACY_8081 (kept disabled at $STASHED); a pending reload is recorded so running nginx is reloaded on the next run. The legacy :8081 endpoint will NOT be restored."
   fi
+  rm -f "$PENDING_FILE"
   log "removed legacy :8081 endpoint -> $STASHED"
 fi
 
@@ -83,12 +87,9 @@ collect_tokens() {
 }
 
 TOKEN=""
-TRIED=" "
 DEADLINE=$((SECONDS + TOKEN_WAIT))
 while [ -z "$TOKEN" ] && [ "$SECONDS" -lt "$DEADLINE" ]; do
   for cand in $(collect_tokens); do
-    case "$TRIED" in *" $cand "*) continue ;; esac
-    TRIED="$TRIED$cand "
     code="$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' \
       -H "Host: $LOGIN_HOST" "$LOGIN_UPSTREAM/?token=$cand" || true)"
     if [ "$code" = "303" ]; then
@@ -126,7 +127,8 @@ chmod 600 "$NEW_INCLUDE"
 APPLIED=""
 [ -f "$STAMP_FILE" ] && APPLIED="$(cat "$STAMP_FILE" 2>/dev/null || true)"
 
-if [ "$APPLIED" = "$TOKEN" ] && [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE"; then
+if [ "$APPLIED" = "$TOKEN" ] && [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE" \
+  && [ ! -e "$PENDING_FILE" ]; then
   rm -f "$NEW_INCLUDE"
   log "token unchanged; nginx not reloaded"
   exit 0
@@ -158,13 +160,16 @@ if ! nginx -s reload; then
   else
     rm -f "$INCLUDE_FILE"
   fi
-  die "nginx reload failed; previous include restored; will retry next run"
+  touch "$PENDING_FILE"
+  die "nginx reload failed; previous include restored; a pending reload is recorded so the next run retries"
 fi
 
 if [ -n "$RESTORE" ]; then
   rm -f "$RESTORE"
 fi
 
+rm -f "$PENDING_FILE"
+
 printf '%s\n' "$TOKEN" > "$STAMP_FILE.tmp"
 chmod 600 "$STAMP_FILE.tmp"
 mv "$STAMP_FILE.tmp" "$STAMP_FILE"

zulip-health.prose.md

@@ -296,13 +296,18 @@ proxy_pass http://127.0.0.1:3080/?token=<TOKEN>;
 verifying journal candidates against dsh-web with `Host: tankodhs.sysloggh.net`
 and using the first one the running process accepts with `303`; a stale token
 from a previous invocation is rejected and can never be selected. It waits up to
-120s for a restarted process to accept a token, and if none is accepted it
-leaves the include untouched and exits `0` so the timer retries. It writes
+120s for a restarted process to accept a token; every distinct candidate is
+re-probed on each pass, so a token that briefly returns `000` while the service
+is still starting is not disqualified. If none is accepted it leaves the include
+untouched and exits `0` so the timer retries. It writes
 `/etc/dsh-web/launch-token` and regenerates `/etc/dsh-web/nginx-login.conf`,
 reloading nginx only when the on-disk include differs from the generated one or
 the applied-state stamp does not match the token (`nginx -t` guards the reload,
 and the stamp is written only after a successful `nginx -s reload`, so a failed
-or interrupted reload is retried on the next run). Runs are serialized with
+or interrupted reload is retried on the next run). Any failed reload records a
+pending-reload marker under `/etc/dsh-web/` that forces the next run through the
+reload path even when the token is unchanged, so a disabled legacy `:8081` file
+can never leave the running nginx unreloaded. Runs are serialized with
 `flock` on `/run/capture-dsh-token.lock`. It **never stops or starts `dsh-web`**.
 It is triggered by the `dsh-web.service` drop-in
 `/etc/systemd/system/dsh-web.service.d/20-token-refresh.conf`