Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c243fcddc3 | ||
|
|
50114f32c0 | ||
|
|
4dc59633b3 | ||
|
|
b3193e5e1b | ||
|
|
f1490b656e | ||
|
|
b56501a1cf | ||
|
|
1921937bee |
@@ -584,7 +584,7 @@ contracts:
|
|||||||
verify: curl -sf https://git.sysloggh.net/api/v1/version
|
verify: curl -sf https://git.sysloggh.net/api/v1/version
|
||||||
expect: 200 OK
|
expect: 200 OK
|
||||||
- check: SearXNG reachable
|
- check: SearXNG reachable
|
||||||
verify: curl -sf http://192.168.68.17:8080
|
verify: curl -sf http://192.168.68.7:8888
|
||||||
expect: 200 OK
|
expect: 200 OK
|
||||||
artifact: infrastructure health report
|
artifact: infrastructure health report
|
||||||
receipt:
|
receipt:
|
||||||
|
|||||||
@@ -356,7 +356,7 @@ Postconditions to verify:
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"check": "SearXNG reachable",
|
"check": "SearXNG reachable",
|
||||||
"verify": "curl -sf http://192.168.68.17:8080",
|
"verify": "curl -sf http://192.168.68.7:8888",
|
||||||
"expect": "200 OK"
|
"expect": "200 OK"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ the description:
|
|||||||
|
|
||||||
1. **What system does this contract touch?** Name the hosts, CTs, containers,
|
1. **What system does this contract touch?** Name the hosts, CTs, containers,
|
||||||
and services explicitly. "The inference fleet" is vague. "GPU .8 (RTX 3090,
|
and services explicitly. "The inference fleet" is vague. "GPU .8 (RTX 3090,
|
||||||
qwen), .110 (RTX 5070, gemma), .15 (Strix Halo, ornith), and LiteLLM on CT
|
qwen), .110 (RTX 5070, gemma), .15 (Strix Halo, strix-moe), and LiteLLM on CT
|
||||||
116" is specific.
|
116" is specific.
|
||||||
|
|
||||||
2. **Who runs this contract, and when?** State the agent, the trigger (cron,
|
2. **Who runs this contract, and when?** State the agent, the trigger (cron,
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ done
|
|||||||
| Agent | CT | Node | IP | LiteLLM Alias | Key Source | Platform |
|
| Agent | CT | Node | IP | LiteLLM Alias | Key Source | Platform |
|
||||||
|-------|-----|------|-----|---------------|------------|----------|
|
|-------|-----|------|-----|---------------|------------|----------|
|
||||||
| Tanko | 112 | amdpve | .122 | `tanko` | Infisical vault | Hermes |
|
| Tanko | 112 | amdpve | .122 | `tanko` | Infisical vault | Hermes |
|
||||||
| Mumuni | 114 | hwepve | .123 | `mumuni` | Infisical vault | Hermes |
|
| Mumuni | 100 | hwepve | .24 | `mumuni` | Infisical vault | Hermes |
|
||||||
| Koby | 111 | amdpve | srv1079750 | `koby` | Infisical vault | **Hermes** |
|
| Koby | 111 | amdpve | .129 | `koby` | Infisical vault | **Hermes** |
|
||||||
| Koonimo | 113 | amdpve | .114 | `koonimo` | Infisical vault | Hermes |
|
| Koonimo | 113 | amdpve | .114 | `koonimo` | Infisical vault | Hermes |
|
||||||
| Shumba | — | 192.168.68.119 | N/A | N/A (DeepSeek) | Hermes (RETIRED — CT119 now Infisical vault) |
|
| Shumba | — | 192.168.68.119 | N/A | N/A (DeepSeek) | Hermes (RETIRED — CT119 now Infisical vault) |
|
||||||
|
|
||||||
|
|||||||
@@ -32,10 +32,10 @@ Sub-agent profiles inherit auth from the main config — no separate keys needed
|
|||||||
| Agent | Key Alias | Host | SSH | Sub-Agents |
|
| Agent | Key Alias | Host | SSH | Sub-Agents |
|
||||||
|-------|-----------|------|-----|-----------|
|
|-------|-----------|------|-----|-----------|
|
||||||
| Tanko | `tanko-*` | 192.168.68.122 | jerome@.122 | — |
|
| Tanko | `tanko-*` | 192.168.68.122 | jerome@.122 | — |
|
||||||
| Mumuni | `mumuni` | 192.168.68.123 | root@.123 | 6 profiles ✱ |
|
| Mumuni | `mumuni` | 192.168.68.24 | root@.24 | 6 profiles ✱ |
|
||||||
| Abiba | `abiba-pi` | 192.168.68.24 | local | — |
|
| Abiba | `abiba-pi` | 192.168.68.24 | local | — |
|
||||||
| Koby | `koby` | CT 111 (tdunna) | Zulip | — |
|
| Koby | `koby` | CT 111 (tdunna) | Zulip | — |
|
||||||
| Koonimo | `koonimo` | CT 114 (baggy) | SSH root | — |
|
| Koonimo | `koonimo` | CT 113 (baggy) | SSH root | — |
|
||||||
| Kagenz0 | `kagenz0-*` | ? | Zulip | — |
|
| Kagenz0 | `kagenz0-*` | ? | Zulip | — |
|
||||||
|
|
||||||
> CT hostnames (tdunna, baggy) differ from agent identities (koby, koonimo).
|
> CT hostnames (tdunna, baggy) differ from agent identities (koby, koonimo).
|
||||||
@@ -48,7 +48,7 @@ Sub-agent profiles inherit auth from the main config — no separate keys needed
|
|||||||
| Component | Endpoint | Purpose |
|
| Component | Endpoint | Purpose |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Firecrawl | `http://192.168.68.7:3002/` | Web content extraction |
|
| Firecrawl | `http://192.168.68.7:3002/` | Web content extraction |
|
||||||
| SearXNG | `http://storepve:8888` | Privacy-respecting web search |
|
| SearXNG | `http://192.168.68.7:8888` | Privacy-respecting web search |
|
||||||
| LiteLLM | `http://192.168.68.116/v1` | Unified model gateway (via nginx) |
|
| LiteLLM | `http://192.168.68.116/v1` | Unified model gateway (via nginx) |
|
||||||
| LiteLLM (NetBird) | `https://litellm.sysloggh.net/v1` | Alternative (may have 502 issues) |
|
| LiteLLM (NetBird) | `https://litellm.sysloggh.net/v1` | Alternative (may have 502 issues) |
|
||||||
| RA-H OS MCP | `http://192.168.68.65:3100/mcp` | Knowledge graph bridge |
|
| RA-H OS MCP | `http://192.168.68.65:3100/mcp` | Knowledge graph bridge |
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ connectivity recovery including end-to-end DM validation.
|
|||||||
|
|
||||||
| Host | CT | Proxmox | IP (direct) | Hermes Home | User |
|
| Host | CT | Proxmox | IP (direct) | Hermes Home | User |
|
||||||
|------|-----|---------|-------------|-------------|------|
|
|------|-----|---------|-------------|-------------|------|
|
||||||
| Mumuni | CT114 | — | 192.168.68.123 | /root/.hermes | root |
|
| Mumuni | CT100 | — | 192.168.68.24 | /root/.hermes | root |
|
||||||
| Tanko | CT112 | amdpve | 192.168.68.122 | /home/jerome/.hermes | jerome |
|
| Tanko | CT112 | amdpve | 192.168.68.122 | /home/jerome/.hermes | jerome |
|
||||||
| Koby | CT111 | amdpve | 192.168.68.129 | /root/.hermes | root |
|
| Koby | CT111 | amdpve | 192.168.68.129 | /root/.hermes | root |
|
||||||
| Shumba | — | — | 192.168.68.119 | /home/lucky/.hermes | lucky |
|
| Shumba | — | — | 192.168.68.119 | /home/lucky/.hermes | lucky |
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
kind: function
|
kind: function
|
||||||
name: hermes-zulip-restore
|
name: hermes-zulip-restore
|
||||||
description: >
|
description: >
|
||||||
Restores Zulip connectivity for any Hermes agent (Mumuni CT114, Tanko CT112,
|
Restores Zulip connectivity for any Hermes agent (Mumuni CT100, Tanko CT112,
|
||||||
Koby CT111, Shumba on Lucky's mini PC). Deploys the zulip-platform adapter to the correct bundled plugin
|
Koby CT111, Shumba on Lucky's mini PC). Deploys the zulip-platform adapter to the correct bundled plugin
|
||||||
path, verifies env credentials, restarts the gateway, and confirms Zulip
|
path, verifies env credentials, restarts the gateway, and confirms Zulip
|
||||||
connects. Run this whenever a Hermes agent stops responding on Zulip or after
|
connects. Run this whenever a Hermes agent stops responding on Zulip or after
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ call apply-liteLLM-routing
|
|||||||
|
|
||||||
call apply-agent-compression
|
call apply-agent-compression
|
||||||
agent: mumuni
|
agent: mumuni
|
||||||
host: 192.168.68.123
|
host: 192.168.68.24
|
||||||
config_path: /root/.hermes/config.yaml
|
config_path: /root/.hermes/config.yaml
|
||||||
|
|
||||||
-- Phase 4: Enable llama.cpp prompt caching on GPU hosts
|
-- Phase 4: Enable llama.cpp prompt caching on GPU hosts
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ description: >
|
|||||||
|
|
||||||
### Reachability Matrix
|
### Reachability Matrix
|
||||||
|
|
||||||
| From / To | PVE API | docker-vm (.7) | CT 116 | Tanko (.122) | Mumuni (.123) | Baggy (.114) |
|
| From / To | PVE API | docker-vm (.7) | CT 116 | Tanko (.122) | Mumuni (.24) | Baggy (.114) |
|
||||||
|-----------|---------|----------------|--------|-------------|---------------|----------------|
|
|-----------|---------|----------------|--------|-------------|---------------|----------------|
|
||||||
| **Abiba** (.24) | ✅ :443 | ✅ SSH | ✅ SSH | ✅ SSH jerome | ✅ SSH root | ❌ SSH |
|
| **Abiba** (.24) | ✅ :443 | ✅ SSH | ✅ SSH | ✅ SSH jerome | ✅ SSH root | ❌ SSH |
|
||||||
| **Tanko** (.122) | ❌ | ❌ | ❌ via NetBird | ✅ | ❌ | ❌ |
|
| **Tanko** (.122) | ❌ | ❌ | ❌ via NetBird | ✅ | ❌ | ❌ |
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ After ALL updates (apt + images + restarts), verify every critical service is ba
|
|||||||
| Zulip | `curl -sf https://chat.sysloggh.net/api/v1/server_settings` | 200 OK |
|
| Zulip | `curl -sf https://chat.sysloggh.net/api/v1/server_settings` | 200 OK |
|
||||||
| Gitea | `curl -sf https://git.sysloggh.net/api/v1/version` | 200 OK |
|
| Gitea | `curl -sf https://git.sysloggh.net/api/v1/version` | 200 OK |
|
||||||
| PM2 processes | `pm2 jlist` (CT 100) | all pi-agent processes `online` |
|
| PM2 processes | `pm2 jlist` (CT 100) | all pi-agent processes `online` |
|
||||||
| Hermes gateways | SSH to Mumuni CT 114, Tanko CT 112; `systemctl is-active hermes-gateway` | `active` for each |
|
| Hermes gateways | SSH to Mumuni CT 100, Tanko CT 112; `systemctl is-active hermes-gateway` | `active` for each |
|
||||||
|
|
||||||
Regression check: every service that was GREEN in `health-baseline` must still be GREEN. A service that was already RED (and caused a preflight abort) is excluded — but Phase 0 should have aborted before we got here.
|
Regression check: every service that was GREEN in `health-baseline` must still be GREEN. A service that was already RED (and caused a preflight abort) is excluded — but Phase 0 should have aborted before we got here.
|
||||||
|
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ Before ANY update wave:
|
|||||||
| CT 100 (.24) | Abiba (pi) | `apt update && apt upgrade -y` | 3 min |
|
| CT 100 (.24) | Abiba (pi) | `apt update && apt upgrade -y` | 3 min |
|
||||||
| CT 116 (.116) | syslog-api (LiteLLM host) | `apt update && apt upgrade -y` | 3 min |
|
| CT 116 (.116) | syslog-api (LiteLLM host) | `apt update && apt upgrade -y` | 3 min |
|
||||||
| CT 112 (tanko, amdpve) | Tanko | `apt update && apt upgrade -y` | 3 min |
|
| CT 112 (tanko, amdpve) | Tanko | `apt update && apt upgrade -y` | 3 min |
|
||||||
| CT 114 (mumuni, hwepve) | Mumuni | `apt update && apt upgrade -y` | 3 min |
|
| CT 100 (mumuni/abiba, hwepve) | Mumuni | `apt update && apt upgrade -y` | 3 min |
|
||||||
| VM 101 (.8) | llm-gpu (RTX 3090) | `apt update && apt upgrade -y` | 3 min |
|
| VM 101 (.8) | llm-gpu (RTX 3090) | `apt update && apt upgrade -y` | 3 min |
|
||||||
| VM 103 (.110) | ocu-llm (RTX 5070) | `apt update && apt upgrade -y` | 3 min |
|
| VM 103 (.110) | ocu-llm (RTX 5070) | `apt update && apt upgrade -y` | 3 min |
|
||||||
|
|
||||||
@@ -160,7 +160,7 @@ Before Wave 1, snapshot these files:
|
|||||||
/opt/home_stack/docker-compose.yml (VM 109 .7)
|
/opt/home_stack/docker-compose.yml (VM 109 .7)
|
||||||
/opt/audiobookshelf/docker-compose.yml (VM 109 .7)
|
/opt/audiobookshelf/docker-compose.yml (VM 109 .7)
|
||||||
/root/.pi/agent/extensions/config.yaml (CT 100 .24)
|
/root/.pi/agent/extensions/config.yaml (CT 100 .24)
|
||||||
/etc/systemd/system/ornith-server.service (amdpve .15 — strix-moe)
|
/etc/systemd/system/strix-server.service (amdpve .15 — strix-moe)
|
||||||
/etc/systemd/system/llama-server.service (VM 101 .8, VM 103 .110)
|
/etc/systemd/system/llama-server.service (VM 101 .8, VM 103 .110)
|
||||||
# Hermes agent configs (key enforcement — 2026-07-10)
|
# Hermes agent configs (key enforcement — 2026-07-10)
|
||||||
/root/.hermes/config.yaml (Mumuni CT 114, Tanko CT 112, etc.)
|
/root/.hermes/config.yaml (Mumuni CT 114, Tanko CT 112, etc.)
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ AGENTS = {
|
|||||||
GPU_HOSTS = {
|
GPU_HOSTS = {
|
||||||
"gpu-rtx3090 (.8)": {"host": "192.168.68.8", "port": 8080, "service": "llama-server"},
|
"gpu-rtx3090 (.8)": {"host": "192.168.68.8", "port": 8080, "service": "llama-server"},
|
||||||
"gpu-rtx5070 (.110)": {"host": "192.168.68.110", "port": 8080, "service": "llama-server"},
|
"gpu-rtx5070 (.110)": {"host": "192.168.68.110", "port": 8080, "service": "llama-server"},
|
||||||
"gpu-strixhalo (.15)": {"host": "192.168.68.15", "port": 8080, "service": "ornith-server"},
|
"gpu-strixhalo (.15)": {"host": "192.168.68.15", "port": 8080, "service": "strix-server"},
|
||||||
}
|
}
|
||||||
|
|
||||||
FAIL = []
|
FAIL = []
|
||||||
@@ -57,6 +57,16 @@ FAIL = []
|
|||||||
INFISICAL_TOKEN = os.environ.get("INFISICAL_TOKEN")
|
INFISICAL_TOKEN = os.environ.get("INFISICAL_TOKEN")
|
||||||
INFISICAL_API_URL = os.environ.get("INFISICAL_API_URL", "https://vault.sysloggh.net")
|
INFISICAL_API_URL = os.environ.get("INFISICAL_API_URL", "https://vault.sysloggh.net")
|
||||||
|
|
||||||
|
# Fallback: if no env token, read the shared vault token file
|
||||||
|
if not INFISICAL_TOKEN:
|
||||||
|
_token_path = os.path.expanduser("~/.infisical-token")
|
||||||
|
if os.path.isfile(_token_path):
|
||||||
|
try:
|
||||||
|
with open(_token_path) as _f:
|
||||||
|
INFISICAL_TOKEN = _f.read().strip()
|
||||||
|
except (OSError, UnicodeDecodeError):
|
||||||
|
pass
|
||||||
|
|
||||||
# ── Helpers ──────────────────────────────────────────────────────────
|
# ── Helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def ssh(host, cmd, user="root"):
|
def ssh(host, cmd, user="root"):
|
||||||
|
|||||||
@@ -313,8 +313,8 @@ def collect():
|
|||||||
"updated_at": tanko_data.get("updated_at"),
|
"updated_at": tanko_data.get("updated_at"),
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mumuni (CT 114, IP 192.168.68.123)
|
# Mumuni (CT 100, IP 192.168.68.24)
|
||||||
mumuni_state = ssh("192.168.68.123", "cat ~/.hermes/gateway_state.json 2>/dev/null")
|
mumuni_state = ssh("192.168.68.24", "cat ~/.hermes/gateway_state.json 2>/dev/null")
|
||||||
mumuni_data = {}
|
mumuni_data = {}
|
||||||
try:
|
try:
|
||||||
mumuni_data = json.loads(mumuni_state) if mumuni_state else {}
|
mumuni_data = json.loads(mumuni_state) if mumuni_state else {}
|
||||||
@@ -322,7 +322,7 @@ def collect():
|
|||||||
mumuni_data = {}
|
mumuni_data = {}
|
||||||
mumuni_platforms = mumuni_data.get("platforms", {})
|
mumuni_platforms = mumuni_data.get("platforms", {})
|
||||||
report["agents"]["mumuni"] = {
|
report["agents"]["mumuni"] = {
|
||||||
"platform": "hermes", "ct": 114, "ip": "192.168.68.123",
|
"platform": "hermes", "ct": 114, "ip": "192.168.68.24",
|
||||||
"gateway_state": mumuni_data.get("gateway_state", "unknown"),
|
"gateway_state": mumuni_data.get("gateway_state", "unknown"),
|
||||||
"telegram_state": mumuni_platforms.get("telegram", {}).get("state", "unknown"),
|
"telegram_state": mumuni_platforms.get("telegram", {}).get("state", "unknown"),
|
||||||
"zulip_state": mumuni_platforms.get("zulip", {}).get("state", "not_installed"),
|
"zulip_state": mumuni_platforms.get("zulip", {}).get("state", "not_installed"),
|
||||||
@@ -330,7 +330,7 @@ def collect():
|
|||||||
"hermes_version": "",
|
"hermes_version": "",
|
||||||
}
|
}
|
||||||
# Get Hermes version
|
# Get Hermes version
|
||||||
ver = ssh("192.168.68.123", "hermes --version 2>/dev/null | head -1")
|
ver = ssh("192.168.68.24", "hermes --version 2>/dev/null | head -1")
|
||||||
if ver:
|
if ver:
|
||||||
report["agents"]["mumuni"]["hermes_version"] = ver.split("·")[0].replace("Hermes Agent ","").strip()
|
report["agents"]["mumuni"]["hermes_version"] = ver.split("·")[0].replace("Hermes Agent ","").strip()
|
||||||
|
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Platform B: Hermes (Mumuni) ──
|
# ── Platform B: Hermes (Mumuni) ──
|
||||||
MUMUNI_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.123 \
|
MUMUNI_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.24 \
|
||||||
"cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}")
|
"cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}")
|
||||||
MUMUNI_ZULIP=$(echo "$MUMUNI_STATE" | python3 -c "
|
MUMUNI_ZULIP=$(echo "$MUMUNI_STATE" | python3 -c "
|
||||||
import sys,json
|
import sys,json
|
||||||
|
|||||||
Reference in New Issue
Block a user