no-mistakes(review): simplify dsh token selection and reload state machine

scripts/capture-dsh-token.sh

@@ -14,14 +14,15 @@
 # reference the token of the RUNNING process.
 #
 # This script:
-#   1. reads the LATEST launch token from the running service's journal — it
+#   1. selects the launch token the RUNNING service actually accepts — it NEVER
-#      NEVER stops or starts dsh-web,
+#      stops or starts dsh-web,
 #   2. records it in /etc/dsh-web/launch-token,
 #   3. regenerates the nginx include /etc/dsh-web/nginx-login.conf (the
 #      `proxy_pass ...?token=` line consumed by /dsh-web-login),
-#   4. reloads nginx ONLY when the token differs from the token nginx actually
+#   4. reloads nginx ONLY when the on-disk include differs from the generated
-#      loaded (tracked in an applied-state stamp written only after a successful
+#      one or the applied-state stamp does not match the token (the stamp is
-#      reload), rolling the include back on failure so the next run retries,
+#      written only after a successful reload), rolling the include back on
+#      failure so the next run retries,
 #   5. removes the legacy unauthenticated :8081 endpoint if it ever reappears.
 #
 # Idempotent and safe to run at any time (systemd ExecStartPost or timer).
@@ -36,13 +37,21 @@ STAMP_FILE="/etc/dsh-web/nginx-login.conf.applied"
 SITE_ENABLED="/etc/nginx/sites-enabled/dsh"
 LEGACY_8081="/etc/nginx/sites-enabled/dsh.token"
 STASH_DIR="/etc/nginx/sites-available"
+LOCK_FILE="/run/capture-dsh-token.lock"
+LOGIN_HOST="tankodhs.sysloggh.net"
+LOGIN_UPSTREAM="http://127.0.0.1:3080"
+TOKEN_WAIT=120
 
 log()  { printf 'capture-dsh-token: %s\n' "$*" >&2; }
 die()  { printf 'capture-dsh-token: ERROR: %s\n' "$*" >&2; exit 1; }
 
 [ "$(id -u)" -eq 0 ] || die "must run as root"
 
-# ── 0. Remove the legacy unauthenticated :8081 endpoint, if present ─────────
+# ── 0. Serialize runs so timer/ExecStartPost/manual runs cannot interleave ──
+exec 9>"$LOCK_FILE"
+flock -n 9 || { log "another capture-dsh-token run holds $LOCK_FILE; exiting"; exit 0; }
+
+# ── 1. Remove the legacy unauthenticated :8081 endpoint, if present ─────────
 # It bypassed Authentik entirely (listened on 0.0.0.0:8081 with no auth_request)
 # and must never come back. Stash it rather than delete so it is auditable.
 if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then
@@ -58,72 +67,68 @@ if [ -e "$LEGACY_8081" ] || [ -L "$LEGACY_8081" ]; then
   log "removed legacy :8081 endpoint -> $STASHED"
 fi
 
-# ── 1. Read the latest launch token from the RUNNING service ────────────────
+# ── 2. Select the token the RUNNING service actually accepts ────────────────
-# Scope the journal to the service's CURRENT invocation. While a restarted
+# Functionally verify each journal candidate against the local dsh-web using the
-# process is still booting (~25s before it prints the banner), the newest token
+# public authority, exactly as the /dsh-web-login proxy does. A token from a
-# in the journal still belongs to the PREVIOUS process; without this filter an
+# previous invocation is rejected (never 303) and can never be selected, so no
-# ExecStartPost run would silently keep the stale token. Never stop/start dsh-web.
+# systemd invocation scoping or newest-overall fallback is needed. Candidates
-INVOCATION="$(systemctl show -p InvocationID --value "$JOURNAL_UNIT" 2>/dev/null || true)"
+# are tried newest-first and re-read from the journal each pass until one is
-JOURNAL_ARGS=(-u "$JOURNAL_UNIT")
+# accepted or the wait elapses.
-if [ -n "$INVOCATION" ] && [ "$INVOCATION" != "n/a" ]; then
+collect_tokens() {
-  JOURNAL_ARGS+=("_SYSTEMD_INVOCATION_ID=$INVOCATION")
+  journalctl -u "$JOURNAL_UNIT" --no-pager -o cat 2>/dev/null \
-else
+    | grep -oE 'dsh web: https?://[^[:space:]]+[?&]token=[^[:space:]]+' \
-  log "WARNING: no invocation id for $JOURNAL_UNIT; using latest journal token"
+    | sed -E 's/.*[?&]token=//' \
-fi
+    | grep -E '^[A-Za-z0-9._~+/=:@-]+$' \
-
+    | tac | awk '!seen[$0]++' || true
-extract_token() {
-  grep -oE 'dsh web: https?://[^[:space:]]+[?&]token=[^[:space:]]+' \
-    | tail -n1 | sed -E 's/.*[?&]token=//' || true
 }
 
 TOKEN=""
-for _ in $(seq 1 60); do
+TRIED=" "
-  TOKEN="$(journalctl "${JOURNAL_ARGS[@]}" --no-pager -o cat 2>/dev/null | extract_token)"
+DEADLINE=$((SECONDS + TOKEN_WAIT))
+while [ -z "$TOKEN" ] && [ "$SECONDS" -lt "$DEADLINE" ]; do
+  for cand in $(collect_tokens); do
+    case "$TRIED" in *" $cand "*) continue ;; esac
+    TRIED="$TRIED$cand "
+    code="$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' \
+      -H "Host: $LOGIN_HOST" "$LOGIN_UPSTREAM/?token=$cand" || true)"
+    if [ "$code" = "303" ]; then
+      TOKEN="$cand"
+      break
+    fi
+  done
   [ -n "$TOKEN" ] && break
-  sleep 1
+  sleep 2
 done
 
-# Fallback: the current invocation's start banner may have been rotated out of
-# the journal; the newest matching line overall is then the best available.
 if [ -z "$TOKEN" ]; then
-  log "WARNING: no token for the current invocation; falling back to newest journal token"
+  log "no dsh-web launch token accepted within ${TOKEN_WAIT}s; leaving the include untouched for the next run"
-  TOKEN="$(journalctl -u "$JOURNAL_UNIT" --no-pager -o cat 2>/dev/null | extract_token)"
+  exit 0
 fi
-[ -n "$TOKEN" ] || die "no launch token found in the $JOURNAL_UNIT journal"
 
-# The token must be safe to embed in a URI and in the nginx config.
+# ── 3. Record the token (atomic, private) ──────────────────────────────────
-printf '%s' "$TOKEN" | grep -qE '^[A-Za-z0-9._~+/=:@-]+$' \
-  || die "captured token contains unsupported characters"
-
-# ── 2. Record the token (atomic, private) ──────────────────────────────────
 mkdir -p "$(dirname "$TOKEN_FILE")"
 if ! printf '%s\n' "$TOKEN" | cmp -s - "$TOKEN_FILE" 2>/dev/null; then
   printf '%s\n' "$TOKEN" > "$TOKEN_FILE.tmp"
   chmod 600 "$TOKEN_FILE.tmp"
   mv "$TOKEN_FILE.tmp" "$TOKEN_FILE"
-  log "recorded new launch token in $TOKEN_FILE"
+  log "recorded live launch token in $TOKEN_FILE"
 fi
 
-# ── 3. Regenerate the nginx login include (reload only when it changes) ────
+# ── 4. Regenerate the nginx login include (reload only when it changes) ────
 NEW_INCLUDE="$(mktemp "$INCLUDE_FILE.XXXXXX")"
-printf 'proxy_pass http://127.0.0.1:3080/?token=%s;\n' "$TOKEN" > "$NEW_INCLUDE"
+printf 'proxy_pass %s/?token=%s;\n' "$LOGIN_UPSTREAM" "$TOKEN" > "$NEW_INCLUDE"
 chmod 600 "$NEW_INCLUDE"
 
-# The stamp records the token nginx actually loaded. Comparing against it (not
+# The stamp records the token nginx actually loaded. It is written only after a
-# the on-disk include) means a failed or interrupted reload is retried on the
+# successful reload, so the early exit is safe only when both the stamp and the
-# next run instead of being mistaken for success.
+# on-disk include agree with the live token; anything else falls through to the
+# reload path so the include can never silently diverge from what nginx serves.
 APPLIED=""
 [ -f "$STAMP_FILE" ] && APPLIED="$(cat "$STAMP_FILE" 2>/dev/null || true)"
 
-if [ "$APPLIED" = "$TOKEN" ]; then
+if [ "$APPLIED" = "$TOKEN" ] && [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE"; then
-  if [ -f "$INCLUDE_FILE" ] && cmp -s "$NEW_INCLUDE" "$INCLUDE_FILE"; then
+  rm -f "$NEW_INCLUDE"
-    rm -f "$NEW_INCLUDE"
+  log "token unchanged; nginx not reloaded"
-    log "token unchanged; nginx not reloaded"
-    exit 0
-  fi
-  mv "$NEW_INCLUDE" "$INCLUDE_FILE"
-  chmod 600 "$INCLUDE_FILE"
-  log "include file repaired to match the token nginx already serves"
   exit 0
 fi
 
@@ -165,4 +170,4 @@ chmod 600 "$STAMP_FILE.tmp"
 mv "$STAMP_FILE.tmp" "$STAMP_FILE"
 
 log "token changed; nginx reloaded"
-log "login endpoint: https://tankodhs.sysloggh.net/dsh-web-login (Authentik-gated)"
+log "login endpoint: https://$LOGIN_HOST/dsh-web-login (Authentik-gated)"

zulip-health.prose.md

@@ -292,13 +292,19 @@ proxy_pass http://127.0.0.1:3080/?token=<TOKEN>;
 
 **Token refresh (non-disruptive):**
 `/opt/deepseek-harness/capture-dsh-token.sh` (source:
-`scripts/capture-dsh-token.sh`) reads the latest launch token from the journal
+`scripts/capture-dsh-token.sh`) selects the live launch token by functionally
-**of the service's current invocation**, writes `/etc/dsh-web/launch-token` and
+verifying journal candidates against dsh-web with `Host: tankodhs.sysloggh.net`
-regenerates `/etc/dsh-web/nginx-login.conf`, reloading nginx only when the token
+and using the first one the running process accepts with `303`; a stale token
-differs from the token nginx actually loaded (`nginx -t` guards the reload, and
+from a previous invocation is rejected and can never be selected. It waits up to
-the applied-state stamp is written only after a successful `nginx -s reload`, so
+120s for a restarted process to accept a token, and if none is accepted it
-a failed or interrupted reload is retried on the next run). It **never stops or
+leaves the include untouched and exits `0` so the timer retries. It writes
-starts `dsh-web`**. It is triggered by the `dsh-web.service` drop-in
+`/etc/dsh-web/launch-token` and regenerates `/etc/dsh-web/nginx-login.conf`,
+reloading nginx only when the on-disk include differs from the generated one or
+the applied-state stamp does not match the token (`nginx -t` guards the reload,
+and the stamp is written only after a successful `nginx -s reload`, so a failed
+or interrupted reload is retried on the next run). Runs are serialized with
+`flock` on `/run/capture-dsh-token.lock`. It **never stops or starts `dsh-web`**.
+It is triggered by the `dsh-web.service` drop-in
 `/etc/systemd/system/dsh-web.service.d/20-token-refresh.conf`
 (`ExecStartPost=/bin/systemctl --no-block start dsh-web-token.service`) and by
 `dsh-web-token.timer` every 2 minutes for reconciliation.
@@ -312,6 +318,7 @@ Description=Refresh the dsh-web launch token for the nginx login endpoint
 After=dsh-web.service
 [Service]
 Type=oneshot
+TimeoutStartSec=180
 ExecStart=/opt/deepseek-harness/capture-dsh-token.sh
 
 # /etc/systemd/system/dsh-web-token.timer