scripts/capture-dsh-token.sh

@@ -53,6 +53,22 @@ exec 9>"$LOCK_FILE"
 flock -n 9 || { log "another capture-dsh-token run holds $LOCK_FILE; exiting"; exit 0; }
 mkdir -p "$(dirname "$PENDING_FILE")"
 
+# ── 0b. Guarantee the generated include exists before any `nginx -t` ──────
+# The :80 site includes /etc/dsh-web/nginx-login.conf by literal path, so a
+# missing include makes every `nginx -t` fail and can wedge recovery. Seed it
+# from the last known token (or a placeholder); step 4 replaces it.
+if [ ! -f "$INCLUDE_FILE" ]; then
+  SEED="placeholder"
+  if [ -f "$TOKEN_FILE" ]; then
+    SEED="$(cat "$TOKEN_FILE" 2>/dev/null || true)"
+    [ -n "$SEED" ] || SEED="placeholder"
+  fi
+  printf '%s' "$SEED" | grep -qE '^[A-Za-z0-9._~+/=:@-]+$' || SEED="placeholder"
+  printf 'proxy_pass %s/?token=%s;\n' "$LOGIN_UPSTREAM" "$SEED" > "$INCLUDE_FILE"
+  chmod 600 "$INCLUDE_FILE"
+  log "created missing $INCLUDE_FILE"
+fi
+
 # ── 1. Remove the legacy unauthenticated :8081 endpoint, if present ─────────
 # It bypassed Authentik entirely (listened on 0.0.0.0:8081 with no auth_request)
 # and must never come back. Stash it rather than delete so it is auditable.
@@ -60,6 +76,7 @@ if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then
   TS="$(date -u +%Y%m%dT%H%M%SZ)"
   STASHED="$STASH_DIR/dsh.token.disabled-$TS"
   mv "$LEGACY_8081" "$STASHED"
+  chmod 600 "$STASHED" 2>/dev/null || true
   touch "$PENDING_FILE"
   if ! nginx -t >/dev/null 2>&1; then
     die "nginx config test failed after disabling $LEGACY_8081 (kept disabled at $STASHED); a pending reload is recorded so running nginx is reloaded once the config is fixed. The legacy :8081 endpoint will NOT be restored."
@@ -77,13 +94,13 @@ fi
 # answering. Reconcile it before the token wait.
 if [ -e "$PENDING_FILE" ]; then
   if ! nginx -t >/dev/null 2>&1; then
-    die "pending nginx reload recorded but 'nginx -t' fails; fix the config and rerun"
+    log "WARNING: pending nginx reload recorded but 'nginx -t' fails; continuing so the include can be regenerated; will retry next run"
+  elif ! nginx -s reload; then
+    log "WARNING: pending nginx reload recorded but 'nginx -s reload' failed; will retry next run"
+  else
+    rm -f "$PENDING_FILE"
+    log "completed pending nginx reload"
   fi
-  if ! nginx -s reload; then
-    die "pending nginx reload recorded but 'nginx -s reload' failed; will retry next run"
-  fi
-  rm -f "$PENDING_FILE"
-  log "completed pending nginx reload"
 fi
 
 # ── 2. Select the token the RUNNING service actually accepts ────────────────
@@ -118,6 +135,7 @@ done
 
 if [ -z "$TOKEN" ]; then
   log "no dsh-web launch token accepted within ${TOKEN_WAIT}s; leaving the include untouched for the next run"
+  [ -e "$PENDING_FILE" ] && die "pending nginx reload could not be completed; will retry next run"
   exit 0
 fi
 

zulip-health.prose.md

@@ -299,15 +299,18 @@ from a previous invocation is rejected and can never be selected. It waits up to
 120s for a restarted process to accept a token; every distinct candidate is
 re-probed on each pass, so a token that briefly returns `000` while the service
 is still starting is not disqualified. If none is accepted it leaves the include
-untouched and exits `0` so the timer retries. It writes
+untouched and exits so the timer retries (exiting non-zero when a pending reload
+is still outstanding). It writes
 `/etc/dsh-web/launch-token` and regenerates `/etc/dsh-web/nginx-login.conf`,
 reloading nginx only when the on-disk include differs from the generated one or
 the applied-state stamp does not match the token (`nginx -t` guards the reload,
 and the stamp is written only after a successful `nginx -s reload`, so a failed
 or interrupted reload is retried on the next run). Any failed reload records a
-pending-reload marker under `/etc/dsh-web/`; the next run honors it before the
-token wait, reloading nginx and clearing the marker regardless of token state,
-so a disabled legacy `:8081` file can never leave the running nginx unreloaded.
+pending-reload marker under `/etc/dsh-web/`; the next run attempts the reload
+before the token wait, independent of token state, and clears the marker only
+once the reload succeeds, so a disabled legacy `:8081` file can never leave the
+running nginx unreloaded. The generated include is recreated before any
+`nginx -t` if it is missing, so a failed run cannot wedge recovery.
 Runs are serialized with `flock` on `/run/capture-dsh-token.lock`. It **never
 stops or starts `dsh-web`**.
 It is triggered by the `dsh-web.service` drop-in
@@ -394,6 +397,14 @@ fresh cookie. Both verified live 2026-09-11.
 ```bash
 # 5. Cookie survives a dsh-web restart, and the new token mints a new cookie.
 ssh root@192.168.68.15 "pct exec 112 -- systemctl restart dsh-web"
+# dsh-web is Type=simple: restart returns before :3080 is listening. Bounded-poll
+# until the socket answers (any status but 000) before asserting the cookie.
+for i in $(seq 1 60); do
+  UP=$(ssh root@192.168.68.15 "pct exec 112 -- curl -s -o /dev/null -w '%{http_code}' \
+    -H 'Host: tankodhs.sysloggh.net' http://127.0.0.1:3080/")
+  [ "$UP" != "000" ] && break
+  sleep 2
+done
 ssh root@192.168.68.15 "pct exec 112 -- curl -s -b /tmp/dsh.jar -o /dev/null \
   -w '%{http_code}\n' -H 'Host: tankodhs.sysloggh.net' http://127.0.0.1:3080/"
 # Expected: 200 — the pre-restart cookie is still accepted.