SyslogSolution/prose-contracts
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: multi-engine search stack + visibility check #132

Merged
abiba-bot merged 2 commits from fix/search-stack-multi-engine-20260925 into master 2026-09-25 01:31:56 +00:00
Conversation 0 Commits 2 Files Changed 3
+358
3 changed files with 358 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/contract-run.sh
+4
View File
@@ -65,6 +65,10 @@ case "$CONTRACT_NAME" in
SCRIPT_PATH="${SCRIPTS_DIR}/disk-gc-scan.py"
INTERPRETER="python3"
;;
search-stack-visibility)
SCRIPT_PATH="${SCRIPTS_DIR}/search-stack-check.py"
INTERPRETER="python3"
;;
*)
echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE"
# Send alert for unknown contract
scripts/search-stack-check.py
Executable
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
"""Search-stack visibility check.
The fleet shares one SearXNG instance (search) plus one extraction service
(Firecrawl). A broken search stack used to fail silently: one engine answered
and nobody could tell that the other engines had stopped contributing, or that
an enabled engine was returning nothing at all without reporting an error.
This check makes those failures visible and non-zero:
* runs two fixed queries against SearXNG; FAILS when fewer than two engines
contribute to a query, printing the contributing engines and every
``unresponsive_engines`` entry;
* FAILS when a known page cannot be extracted to non-empty markdown through
Firecrawl;
* reports every *silent zero* engine explicitly -- an engine that is enabled,
is eligible for the query category, is not listed in
``unresponsive_engines``, and still contributed no results.
Exit code 0 = healthy, 1 = degraded, 2 = the check could not run at all.
Environment overrides (all optional):
SEARXNG_URL default http://192.168.68.7:8888
FIRECRAWL_URL default http://192.168.68.7:3002
SEARCH_CHECK_QUERIES comma-separated fixed queries
SEARCH_CHECK_MIN_ENGINES default 2
SEARCH_CHECK_TIMEOUT per-request timeout in seconds, default 25
SEARCH_CHECK_EXTRACT_URL page used for the extraction leg
SEARCH_CHECK_ENGINES comma-separated engine names the stack is expected to
run; a silent zero is reported for any of them that is
enabled but contributes nothing with no error
"""
from __future__ import annotations
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://192.168.68.7:8888").rstrip("/")
FIRECRAWL_URL = os.environ.get("FIRECRAWL_URL", "http://192.168.68.7:3002").rstrip("/")
QUERIES = [
q.strip()
for q in os.environ.get(
"SEARCH_CHECK_QUERIES", "proxmox backup server,python asyncio tutorial"
).split(",")
if q.strip()
]
MIN_ENGINES = int(os.environ.get("SEARCH_CHECK_MIN_ENGINES", "2"))
TIMEOUT = float(os.environ.get("SEARCH_CHECK_TIMEOUT", "25"))
EXTRACT_URL = os.environ.get(
"SEARCH_CHECK_EXTRACT_URL", "https://en.wikipedia.org/wiki/Proxmox_Virtual_Environment"
)
# The general web-search engines this stack intentionally runs. A general query
# is expected to draw on these; an enabled one that returns nothing without an
# error is the silent-zero failure this check exists to expose. Specialised
# engines (images, videos, translate, currency, arxiv, npm, ...) are excluded on
# purpose -- contributing nothing to a general query is correct for them.
DEFAULT_EXPECTED_ENGINES = [
"bing",
"brave",
"google cse",
"yandex",
"duckduckgo",
]
EXPECTED_ENGINES = [
e.strip()
for e in os.environ.get(
"SEARCH_CHECK_ENGINES", ",".join(DEFAULT_EXPECTED_ENGINES)
).split(",")
if e.strip()
]
def _get_json(url: str) -> dict:
req = urllib.request.Request(url, headers={"User-Agent": "search-stack-check/1.0"})
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
return json.loads(resp.read().decode("utf-8", "replace"))
def _post_json(url: str, payload: dict) -> dict:
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
url,
data=data,
headers={
"Content-Type": "application/json",
"User-Agent": "search-stack-check/1.0",
},
)
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
return json.loads(resp.read().decode("utf-8", "replace"))
def enabled_expected_engines() -> set[str]:
"""Expected engines that SearXNG reports as actually enabled."""
cfg = _get_json(f"{SEARXNG_URL}/config")
enabled = {e["name"] for e in cfg.get("engines", []) if e.get("enabled")}
return {name for name in EXPECTED_ENGINES if name in enabled}
def unresponsive_names(pairs: list) -> dict[str, str]:
"""``unresponsive_engines`` is a list of [name, reason] pairs (or strings)."""
out: dict[str, str] = {}
for item in pairs or []:
if isinstance(item, (list, tuple)) and len(item) >= 2:
out[str(item[0])] = str(item[1])
elif isinstance(item, str):
out[item] = "unresponsive"
return out
def main() -> int:
failures: list[str] = []
print(f"Search stack check -- {SEARXNG_URL}")
print(f"Queries: {QUERIES!r} min contributing engines: {MIN_ENGINES}")
print("=" * 72)
try:
eligible = enabled_expected_engines()
except Exception as exc: # noqa: BLE001 - report, do not traceback
print(f"FAIL: could not read /config from SearXNG: {exc!r}")
return 2
print(f"Expected engines, enabled ({len(eligible)}): {sorted(eligible)}")
missing = sorted(set(EXPECTED_ENGINES) - eligible)
if missing:
print(f"Expected engines NOT enabled: {missing}")
failures.append(f"expected engines not enabled in SearXNG: {missing}")
contributed: dict[str, int] = {name: 0 for name in eligible}
silent_zero_all: dict[str, list[str]] = {}
for query in QUERIES:
url = f"{SEARXNG_URL}/search?" + urllib.parse.urlencode(
{"q": query, "format": "json"}
)
print("-" * 72)
print(f"QUERY: {query!r}")
try:
data = _get_json(url)
except Exception as exc: # noqa: BLE001
print(f" FAIL: query request failed: {exc!r}")
failures.append(f"query {query!r} request failed: {exc!r}")
continue
results = data.get("results", [])
engines: dict[str, int] = {}
for r in results:
name = r.get("engine", "?")
engines[name] = engines.get(name, 0) + 1
unresponsive = unresponsive_names(data.get("unresponsive_engines", []))
print(f" results: {len(results)}")
print(f" contributing engines: {engines or '(none)'}")
print(f" unresponsive_engines: {unresponsive or '(none)'}")
for name in engines:
contributed[name] = contributed.get(name, 0) + engines[name]
if len(engines) < MIN_ENGINES:
msg = (
f"query {query!r} had only {len(engines)} contributing engine(s) "
f"({sorted(engines)}); need >= {MIN_ENGINES}"
)
print(f" FAIL: {msg}")
failures.append(msg)
silent = sorted(
n for n in eligible if n not in engines and n not in unresponsive
)
if silent:
silent_zero_all[query] = silent
print(
" SILENT ZERO (enabled, no error, no results -- reported, "
f"not fatal): {silent}"
)
print("=" * 72)
print("Engine contribution across all queries:")
for name in sorted(contributed):
status = "ZERO" if contributed[name] == 0 else "ok"
print(f" {name:<24} {contributed[name]:>4} {status}")
if silent_zero_all:
print("-" * 72)
print("SILENT-ZERO ENGINES REPORTED (no error raised, no results returned):")
for query, names in silent_zero_all.items():
print(f" {query!r}: {names}")
print(" NOTE: a silent zero is REPORTED, not counted as a failure. These")
print(" engines are expected to answer a general query, but contributing")
print(" nothing to one query can be legitimate (result de-duplication, or")
print(" an engine that only fires on certain query shapes). Only the")
print(f" <{MIN_ENGINES}-contributing-engine floor and the extraction leg fail the run.")
print("-" * 72)
print(f"EXTRACTION: scraping {EXTRACT_URL} via {FIRECRAWL_URL}/v1/scrape")
try:
payload = _post_json(
f"{FIRECRAWL_URL}/v1/scrape",
{"url": EXTRACT_URL, "formats": ["markdown"]},
)
markdown = ((payload.get("data") or {}).get("markdown") or "").strip()
if not markdown:
msg = "extraction returned empty markdown"
print(f" FAIL: {msg}")
failures.append(msg)
else:
print(f" ok: {len(markdown)} chars of markdown returned")
print(f" first line: {markdown.splitlines()[0][:120]!r}")
except Exception as exc: # noqa: BLE001
msg = f"extraction request failed: {exc!r}"
print(f" FAIL: {msg}")
failures.append(msg)
print("=" * 72)
if failures:
print("VERDICT: FAIL")
for f in failures:
print(f" - {f}")
return 1
print("VERDICT: PASS -- multiple engines contributing, extraction healthy")
return 0
if __name__ == "__main__":
sys.exit(main())
search-stack-visibility.prose.md
+124
View File
@@ -0,0 +1,124 @@
---
kind: function
name: search-stack-visibility
description: >
Makes the shared search stack observable. Every agent reaches one SearXNG
instance (http://192.168.68.7:8888) and one extraction service (Firecrawl,
http://192.168.68.7:3002). Before this check the stack could degrade to a
single engine, or an enabled engine could return nothing at all, without any
error surfacing anywhere.
This contract runs scripts/search-stack-check.py, which:
* runs two fixed queries against SearXNG and FAILS when fewer than two
engines contribute, printing the contributing engines and every
unresponsive_engines entry;
* checks extraction by scraping a known page through Firecrawl and FAILS
when the returned markdown is empty or the request fails;
* reports every silent-zero engine explicitly (enabled, not in
unresponsive_engines, contributed no results).
Multi-engine state (2026-09-25): bing, google cse, brave and yandex
contribute on every query. duckduckgo is NOT working: the house egress IP
and the VPS fallback egress are both flagged by DuckDuckGo and it reports
CAPTCHA. It is left enabled as best-effort coverage so that a recovery shows
up as a contribution.
google cse is a third party's public search-engine id hardcoded in the
SearXNG build. Quota and availability are outside our control.
SCHEDULED: /etc/cron.d/contract-runner on CT 100 (abiba), hourly at :15,
via scripts/contract-run.sh search-stack-visibility. Logs land in
/var/log/contract-runs/. A failure also raises a firstmate inbox note.
version: 1.1.0
---
## Purpose
The fleet has exactly one search endpoint and one extraction endpoint. If
either degrades, every agent silently loses capability at the same moment.
The failure mode this contract exists to close is *silent* degradation: a
query that still returns a page of results while all but one engine have
stopped contributing, or an enabled engine that answers with zero results and
raises no error.
## Execution model
The contract is a host-scheduled check, not an agent workflow. It is driven by
`scripts/contract-run.sh search-stack-visibility` from
`/etc/cron.d/contract-runner` on CT 100. `contract-run.sh` resolves the
mapping to `scripts/search-stack-check.py`, runs it under a timeout, writes a
timestamped log to `/var/log/contract-runs/`, and on non-zero exit raises a
firstmate inbox note through `bin/fm-inbox.sh`.
## What passing looks like
```
$ bash scripts/contract-run.sh search-stack-visibility
Expected engines, enabled (5): ['bing', 'brave', 'duckduckgo', 'google cse', 'yandex']
queries: 'proxmox backup server' -> contributing: bing, brave, google cse, yandex
unresponsive: duckduckgo=CAPTCHA
'python asyncio tutorial' -> contributing: bing, brave, google cse, yandex
EXTRACTION: 71016 chars of markdown returned
VERDICT: PASS -- multiple engines contributing, extraction healthy
```
## What failing looks like
* A query whose results come from fewer than `SEARCH_CHECK_MIN_ENGINES`
engines (default 2) fails and names the engines that did contribute.
* An extraction request that errors or returns empty markdown fails.
## Silent zeros are reported, not fatal
An enabled, expected engine that contributed nothing **without reporting an
error** is printed under `SILENT-ZERO ENGINES REPORTED`, and each occurrence is
annotated `reported, not fatal`. This is deliberate:
* a general query can legitimately draw zero results from an engine that only
fires on certain query shapes, and results are de-duplicated across engines,
so a zero does not by itself prove the engine is broken;
* the run therefore fails only on the two conditions that do prove loss of
capability -- fewer than two contributing engines, and a broken extraction
leg.
A run can consequently print `VERDICT: PASS` while still listing a silent
zero. That is the intended relationship: the zero is *visible*, not *fatal*.
An engine that fails with an error (for example DuckDuckGo returning CAPTCHA)
appears in `unresponsive_engines` instead.
## Google coverage is third-party, not ours
The free Google-derived results come from the SearXNG build's built-in
`google cse` engine. It uses **a third party's public search-engine id
hardcoded in the build** (`google_cse.py`, `CX = "partner-pub-8993..."`,
blackle.com), not a key or id we own. Its quota and availability are outside
our control and it can be rate-limited or withdrawn without notice. No engine
in this build accepts our own Google Custom Search key; using our own free key
would require a small wrapper service, which is deliberately **not** built.
## Configuration
Environment overrides (see the script docstring for the full list):
| Variable | Default | Meaning |
| --- | --- | --- |
| `SEARXNG_URL` | `http://192.168.68.7:8888` | SearXNG base URL |
| `FIRECRAWL_URL` | `http://192.168.68.7:3002` | Firecrawl base URL |
| `SEARCH_CHECK_QUERIES` | `proxmox backup server,python asyncio tutorial` | fixed queries |
| `SEARCH_CHECK_MIN_ENGINES` | `2` | minimum contributing engines per query |
| `SEARCH_CHECK_ENGINES` | `bing,brave,google cse,yandex,duckduckgo` | engines a silent zero is reported for |
| `SEARCH_CHECK_EXTRACT_URL` | Wikipedia Proxmox article | page used for the extraction leg |
## Known residual risk
DuckDuckGo is **not** working. The house egress IP is CAPTCHA'd by
DuckDuckGo, and a forward proxy on the VPS (`10.10.10.1:3128`, WireGuard) was
built as a second egress -- but DuckDuckGo has since flagged the VPS address
too (HTTP 202 with challenge markers), so DuckDuckGo now reports CAPTCHA on
both paths. It is left enabled as best-effort coverage: if DuckDuckGo
unflags either address it will show up as a contribution, and until then it is
visible in `unresponsive_engines` every run. It is never a required engine.
The VPS forward proxy remains a real service (`/opt/fwd-proxy`,
`restart: unless-stopped`, healthy healthcheck, Docker enabled at boot) so the
second egress path is available for any engine that benefits from it in future.