From b17c60f997301347bb5e9a015a75809172e003b4 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 24 Jul 2026 19:56:26 +0000 Subject: [PATCH] fix: contract accuracy updates post fleet-wide reboot infrastructure-control.prose.md: - Add hwepve as 6th Proxmox node - Fix Gitea IP: .17 (was .110) - Fix AdGuard IP: .10 on minipve (was .102 on acerpve) - Fix Abiba placement: hwepve (was amdpve) - Fix Mumuni placement: hwepve (was minipve) - Fix Authentik port: add :9000 - Add CT 118 (jdownloader), CT 119 (infisical-vault) - Add last-verified date (2026-07-24) infrastructure-update.prose.md: - Add post-reboot CT sweep procedure - Add Zulip Docker network recovery steps - Add WireGuard tunnel verification scripts/netbird-add-domain.sh: - New script to register domains in Netbird proxy store.db --- infrastructure-control.prose.md | 65 +++++++++++++++++++++------------ infrastructure-update.prose.md | 41 ++++++++++++++++++++- scripts/netbird-add-domain.sh | 65 +++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 26 deletions(-) create mode 100644 scripts/netbird-add-domain.sh diff --git a/infrastructure-control.prose.md b/infrastructure-control.prose.md index 8471017..e7aa436 100644 --- a/infrastructure-control.prose.md +++ b/infrastructure-control.prose.md @@ -3,7 +3,7 @@ kind: pattern name: infrastructure-control description: > Full infrastructure monitoring and control pattern covering the - 5-node Proxmox cluster, 3 Docker ecosystems (22 containers), + 6-node Proxmox cluster, 3 Docker ecosystems (22 containers), NFS storage, and network services. Defines monitors, remediations, and the access matrix for all environments. @@ -12,6 +12,11 @@ description: > never mutate infrastructure based on them without first confirming against the live system. Policy fields are authoritative. See the `verify-before-mutate` skill. + + **Last verified:** 2026-07-24 — corrected Gitea IP (.17 not .110), + AdGuard IP (.10 not .102), AdGuard placement (minipve not acerpve), + Abiba placement (hwepve not amdpve), added hwepve as 6th node, + added dns.sysloggh.net route. See data/learnings.md. --- # Infrastructure Control Pattern @@ -42,8 +47,8 @@ description: > │ │ │ │ │ ┌────┘ ┌────┘ ┌────┘ ┌────┘ ┌────┘ ▼ ▼ ▼ ▼ ▼ - minipve amdpve storepve acerpve ocupve - (.12) (.15) (.6) (.9) (.5) + minipve amdpve storepve acerpve ocupve hwepve + (.12) (.15) (.6) (.9) (.5) (.4) ▼ ┌─────────────────────────────────────────────┐ @@ -95,15 +100,21 @@ description: > ## Section 2: Proxmox Cluster — Monitoring -### Nodes (5) +### Nodes (6) | Node | IP | CPU | RAM | VMs/CTs | Role | |------|----|-----|-----|---------|------| -| minipve | .12 | 16C | 30GB | authentik, gitea, mumuni, syslog-api, jitsi | Auth, git, messaging | -| amdpve | .15 | 32C | 62GB | abiba, kagentz, tanko, tdunna, baggy, scottdenya | Agents, compute | -| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, zulip | Docker, storage, chat | -| acerpve | .9 | 28C | 31GB | llm-gpu, adguard | GPU VMs | +| minipve | .12 | 16C | 30GB | authentik, gitea, syslog-api, infisical-vault, jitsi | Auth, git, messaging | +| amdpve | .15 | 32C | 62GB | kagentz, tanko, tdunna, baggy, scottdenya | Agents, compute | +| storepve | .6 | 28C | 31GB | docker-vm, ra-h-os, PBS, media, jdownloader, zulip | Docker, storage, chat | +| acerpve | .9 | 28C | 31GB | llm-gpu | GPU VMs | | ocupve | .5 | 12C | 14GB | ocu-llm | GPU VMs | +| hwepve | .4 | ? | ? | abiba, (mumuni CT 114 stopped) | Agents (new node) | + +> **Note:** CTs on storepve include jdownloader (CT 118). AdGuard (CT 102) is on +> minipve at .10, not acerpve. Abiba (CT 100) is on hwepve, not amdpve. Mumuni +> (CT 114) is on hwepve (currently stopped), not minipve. Mumuni also has a +> second instance on minipve at .123 — distinguish by CT ID, not hostname. ### Checks (every 5 min) @@ -355,15 +366,17 @@ fine. Services that resolve directly to a LAN IP are NetBird-independent. | Proxmox API | minipve.sysloggh.net:8006 | 192.168.68.12 | LAN IP | No | ✅ | | LiteLLM | litellm.sysloggh.net | 192.168.68.116 | LAN IP | No | ✅ | | Authentik | auth.sysloggh.net:443 | 192.168.68.11 | CNAME → netbird | **Yes** | ⚠️ | -| Gitea | git.sysloggh.net:443 | 192.168.68.110 | CNAME → netbird | **Yes** | ⚠️ | +| Gitea | git.sysloggh.net:443 | 192.168.68.17:3000 | CNAME → netbird | **Yes** | ⚠️ | | Zulip | chat.sysloggh.net:443 | 192.168.68.19 | CNAME → netbird | **Yes** | ⚠️ VERIFY-BEFORE-USE | | Pulse | pulse.sysloggh.net:443 | 192.168.68.7 | CNAME → netbird | **Yes** | ⚠️ | -| DNS UI | dns.sysloggh.net:443 | 192.168.68.102 | CNAME → netbird | **Yes** | ⚠️ | +| DNS UI | dns.sysloggh.net:443 | 192.168.68.10:80 | CNAME → netbird | **Yes** | ⚠️ | | SearXNG | searxng.sysloggh.net:8888 | 192.168.68.7:8888 | LAN IP | No | ✅ | | Firecrawl | firecrawl.sysloggh.net:3002 | 192.168.68.7:3002 | LAN IP | No | ✅ | -**Verified 2026-07-02:** NetBird VPS rebooted after a hang; all CNAME'd +**Verified 2026-07-24:** NetBird VPS rebooted after a hang; all CNAME'd services recovered. LAN-IP-direct paths stayed up throughout the outage. +Also added `dns.sysloggh.net` route (was missing entirely). +See `scripts/netbird-add-domain.sh` for adding new proxy routes. ### 5.2 Checks (every 2 min) @@ -511,8 +524,8 @@ enforced by the `routing-regression.config_url_violations` check in Section | LiteLLM API | `http://192.168.68.116:4000` | `https://litellm.sysloggh.net` | | LiteLLM (nginx) | `http://192.168.68.116` | — | | Grafana | `http://192.168.68.116:3001` | — | -| Authentik | `https://192.168.68.11` | `https://auth.sysloggh.net` | -| Gitea | `http://192.168.68.110:3000` | `https://git.sysloggh.net` | +| Authentik | `https://192.168.68.11:9000` | `https://auth.sysloggh.net` | +| Gitea | `http://192.168.68.17:3000` | `https://git.sysloggh.net` | | Zulip API | `http://192.168.68.19` | `https://chat.sysloggh.net` | | SearXNG | `http://192.168.68.7:8888` | — | | Firecrawl | `http://192.168.68.7:3002` | — | @@ -575,25 +588,28 @@ ssh root@192.168.68.110 "systemctl restart llama-server" | CT | Name | Node | IP | Role | Agent | |----|------|------|----|------|-------| -| 100 | abiba | amdpve | .24 | Pi agent (this host) | ✅ pi | +| CT | Name | Node | IP | Role | Agent | +|----|------|------|----|------|-------| +| 100 | abiba | **hwepve** | .24 | Pi agent | ✅ pi | | 101 | llm-gpu | acerpve | .8 | GPU RTX 3090 | ❌ | -| 102 | adguard | acerpve | — | DNS | ❌ | +| 102 | adguard | **minipve** | **.10** | DNS | ❌ | | 103 | ocu-llm | ocupve | .110 | GPU RTX 5070 | ❌ | | 104 | authentik | minipve | .11 | OIDC | ❌ | -| 105 | kagentz | amdpve | — | Agent Zero | ✅ | +| 105 | kagentz | **hwepve** | — | Agent Zero | ✅ | | 106 | ra-h-os | storepve | .65 | KG bridge | ✅ MCP | | 107 | pbs | storepve | — | Backups | ❌ | | 108 | media | storepve | — | Media | ❌ | | 109 | docker-vm | storepve | .7 | Docker host | ❌ | -| 110 | gitea | minipve | — | Git | ❌ | +| 110 | gitea | minipve | **.17** | Git | ❌ | | 111 | tdunna | amdpve | .129 | Hermes agent | ✅ | | 112 | tanko | amdpve | .122 | Hermes agent | ✅ | -| 113 | baggy | amdpve | ? | Hermes agent | ✅ | -| 114 | mumuni | minipve | .123 | Hermes agent | ✅ | -| 115 | scottdenya | amdpve | — | ? | ❌ | +| 113 | baggy | amdpve | .114 | Hermes agent | ✅ | +| 114 | mumuni | **hwepve** | .123 | Hermes agent (stopped) | ✅ | +| 115 | scottdenya | amdpve | .75 | Denya OneCare | ❌ | | 116 | syslog-api | minipve | .116 | LiteLLM + Grafana | ❌ | -| 117 | zulip | storepve | — | Chat | ❌ | -| 118 | jitsi | minipve | — | Video | ❌ | +| 117 | zulip | storepve | .19 | Chat | ❌ | +| 118 | jdownloader | storepve | — | JDownloader container | ❌ | +| 119 | infisical-vault | minipve | — | Vault | ❌ | ## Appendix C: Docker Compose Files Location @@ -613,7 +629,7 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run. | CT | Name | Node | pct-run | |-----|------|------|---------| -| 100 | abiba | amdpve | `pct-run 100` | +| 100 | abiba | hwepve | `pct-run 100` | | 105 | kagentz | amdpve | `pct-run 105` | | 111 | tdunna | amdpve | `pct-run 111` | | 112 | tanko | amdpve | `pct-run 112` | @@ -627,13 +643,14 @@ Source of truth: `/root/scripts/pct-run.sh` or `prose-contracts/scripts/pct-run. | 107 | proxmox-backup | storepve | `pct-run 107` | | 108 | media | storepve | `pct-run 108` | | 117 | zulip | storepve | `pct-run 117` | -| 102 | adguard | acerpve | `pct-run 102` | +| 102 | adguard | **minipve** | `pct-run 102` | GPU bare-metal hosts (.8 acerpve, .110 ocupve, .15 amdpve) are NOT CTs — use SSH directly: ```bash ssh root@192.168.68.8 # RTX 3090 ssh root@192.168.68.110 # RTX 5070 ssh root@192.168.68.15 # Strix Halo +ssh root@192.168.68.4 # hwepve (abiba, kagentz, mumuni) ``` ## Section 7: Agent Health Check (consolidated — 2026-07-05) diff --git a/infrastructure-update.prose.md b/infrastructure-update.prose.md index 014044b..3ef4eae 100644 --- a/infrastructure-update.prose.md +++ b/infrastructure-update.prose.md @@ -91,17 +91,54 @@ Before ANY update wave: - Firecrawl test: `curl :3002/` - SearXNG test: `curl :8888` -## Wave 4: Proxmox Kernel Reboot (if needed) +## Wave 4: Proxmox Kernel Reboot Only if `[ -f /var/run/reboot-required ]` on any node. | Target | Action | |--------|--------| | Affected PVE node | Verify all CTs/VMs migrated or stopped | -| | `reboot` via PVE API | +| | `reboot` via PVE API (or `systemctl reboot -f` if dbus fails) | | | Wait 120s for node to come back | | | Start any stopped CTs | +### Post-reboot sweep (known gaps) + +After every node reboot, run these checks: + +1. **CT auto-start sweep** — LXC containers sometimes don't start despite + `onboot: 1`. Check every CT on the rebooted node and start any left stopped: + ```bash + pct list | awk '/stopped/{print $1}' | xargs -I{} pct start {} + ``` + Known cases: scottdenya (CT 115 on amdpve), authentik (CT 104 on minipve). + +2. **Zulip recovery** — When docker-vm or storepve reboots, the Zulip main + container loses its Docker network assignment (SIGKILL during storage + outage detaches it from `zulip_default` network). Run: + ```bash + ssh root@192.168.68.19 + docker rm -f zulip-zulip-1 + cd /opt/zulip && docker compose up -d + ``` + The compose restart recreates the container on the correct network. + +3. **docker-vm Docker daemon** — After reboot, Docker can take 3-4 minutes + to become `active`. The docker-proxy for Pulse (port 7655) starts early, + so Pulse is accessible before `docker ps` reports ready. Wait for Docker + before checking other stacks. + +### VPS ↔ docker-vm tunnel + +After any VPS or docker-vm reboot, verify the dedicated WireGuard tunnel: +```bash +ssh root@72.61.0.17 'wg show wg1' | grep "latest handshake" +# If no handshake in >60s: +ssh root@72.61.0.17 'wg-quick up wg1' +``` +The tunnel uses PersistentKeepalive=25 and is systemd-enabled, but should +be verified after a reboot. + ## Rollback Protocol If ANY verification fails: diff --git a/scripts/netbird-add-domain.sh b/scripts/netbird-add-domain.sh new file mode 100644 index 0000000..c3c5211 --- /dev/null +++ b/scripts/netbird-add-domain.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Netbird Reverse Proxy — Add a new domain route +# +# Usage: netbird-add-domain.sh [port] [protocol] +# +# Example: +# netbird-add-domain.sh dns.sysloggh.net 192.168.68.10 80 +# +# This script adds a domain to the Netbird proxy by inserting records +# directly into the management server's SQLite database, then restarting +# the proxy stack. +# +# Prerequisites: SSH root access to 72.61.0.17 +# sqlite3 available on VPS +# +# Requires: The domain must already have a DNS CNAME to netbird.sysloggh.net +# pointing to 72.61.0.17. + +set -euo pipefail + +DOMAIN="${1:?Usage: netbird-add-domain.sh [port] [protocol]}" +BACKEND_IP="${2:?Usage: netbird-add-domain.sh [port] [protocol]}" +PORT="${3:-80}" +PROTOCOL="${4:-http}" + +VPS="root@72.61.0.17" +DB_VOLUME="/var/lib/docker/volumes/root_netbird_data/_data" +DB="$DB_VOLUME/store.db" + +echo "=== Adding Netbird proxy route ===" +echo "Domain: $DOMAIN" +echo "Backend: $BACKEND_IP:$PORT ($PROTOCOL)" +echo "" + +ssh "$VPS" bash << REMOTESCRIPT +set -euo pipefail + +# Generate unique ID using timestamp hash (Netbird format) +ID_SUFFIX=\$(date +%s | md5sum | head -c 16) +SVC_ID="d9\${ID_SUFFIX}ptsnc73\$(date +%s | md5sum | head -c 10)" +TGT_ID=\$(sqlite3 "$DB" "SELECT COALESCE(MAX(id), 100) + 1 FROM targets;") +ACCOUNT_ID="d88av3aptsnc73clmogg" +ZONE_ID="d8adqjaptsnc73fro5g0" + +echo "Service ID: \$SVC_ID" +echo "Target ID: \$TGT_ID" + +# Insert service +sqlite3 "$DB" "INSERT INTO services (id, account_id, name, domain, proxy_cluster, enabled, terminated, pass_host_header, rewrite_redirects, mode, source, port_auto_assigned, private) VALUES (\"\$SVC_ID\", \"\$ACCOUNT_ID\", \"$DOMAIN\", \"$DOMAIN\", \"netbird.sysloggh.net\", 1, 0, 1, 0, \"http\", \"permanent\", 0, 0);" +echo "Service: OK" + +# Insert target +sqlite3 "$DB" "INSERT INTO targets (id, account_id, service_id, host, port, protocol, target_id, target_type, enabled, skip_tls_verify, request_timeout, session_idle_timeout, agent_network, disable_access_log) VALUES (\$TGT_ID, \"\$ACCOUNT_ID\", \"\$SVC_ID\", \"$BACKEND_IP\", $PORT, \"$PROTOCOL\", \"\$ZONE_ID\", \"subnet\", 1, 0, 0, 0, 0, 0);" +echo "Target: OK" + +# Verify +sqlite3 -column "$DB" "SELECT s.name, t.host, t.port, t.protocol FROM services s JOIN targets t ON s.id=t.service_id WHERE s.name=\"$DOMAIN\";" + +echo "" +echo "Restarting proxy stack..." +cd /root && docker compose restart netbird-server 2>/dev/null +sleep 15 +docker compose restart proxy 2>/dev/null +echo "Done. Verify with: curl -sI https://$DOMAIN" +REMOTESCRIPT