scripts/contract-run.sh

@@ -65,6 +65,10 @@ case "$CONTRACT_NAME" in
         SCRIPT_PATH="${SCRIPTS_DIR}/disk-gc-scan.py"
         INTERPRETER="python3"
         ;;
+    search-stack-visibility)
+        SCRIPT_PATH="${SCRIPTS_DIR}/search-stack-check.py"
+        INTERPRETER="python3"
+        ;;
     *)
         echo "Unknown contract: $CONTRACT_NAME" | tee -a "$LOG_FILE"
         # Send alert for unknown contract

scripts/search-stack-check.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Search-stack visibility check.
+
+The fleet shares one SearXNG instance (search) plus one extraction service
+(Firecrawl).  A broken search stack used to fail silently: one engine answered
+and nobody could tell that the other engines had stopped contributing, or that
+an enabled engine was returning nothing at all without reporting an error.
+
+This check makes those failures visible and non-zero:
+
+  * runs two fixed queries against SearXNG; FAILS when fewer than two engines
+    contribute to a query, printing the contributing engines and every
+    ``unresponsive_engines`` entry;
+  * FAILS when a known page cannot be extracted to non-empty markdown through
+    Firecrawl;
+  * reports every *silent zero* engine explicitly -- an engine that is enabled,
+    is eligible for the query category, is not listed in
+    ``unresponsive_engines``, and still contributed no results.
+
+Exit code 0 = healthy, 1 = degraded, 2 = the check could not run at all.
+
+Environment overrides (all optional):
+  SEARXNG_URL         default http://192.168.68.7:8888
+  FIRECRAWL_URL       default http://192.168.68.7:3002
+  SEARCH_CHECK_QUERIES  comma-separated fixed queries
+  SEARCH_CHECK_MIN_ENGINES  default 2
+  SEARCH_CHECK_TIMEOUT  per-request timeout in seconds, default 25
+  SEARCH_CHECK_EXTRACT_URL  page used for the extraction leg
+  SEARCH_CHECK_ENGINES  comma-separated engine names the stack is expected to
+                        run; a silent zero is reported for any of them that is
+                        enabled but contributes nothing with no error
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://192.168.68.7:8888").rstrip("/")
+FIRECRAWL_URL = os.environ.get("FIRECRAWL_URL", "http://192.168.68.7:3002").rstrip("/")
+QUERIES = [
+    q.strip()
+    for q in os.environ.get(
+        "SEARCH_CHECK_QUERIES", "proxmox backup server,python asyncio tutorial"
+    ).split(",")
+    if q.strip()
+]
+MIN_ENGINES = int(os.environ.get("SEARCH_CHECK_MIN_ENGINES", "2"))
+TIMEOUT = float(os.environ.get("SEARCH_CHECK_TIMEOUT", "25"))
+EXTRACT_URL = os.environ.get(
+    "SEARCH_CHECK_EXTRACT_URL", "https://en.wikipedia.org/wiki/Proxmox_Virtual_Environment"
+)
+
+# The general web-search engines this stack intentionally runs.  A general query
+# is expected to draw on these; an enabled one that returns nothing without an
+# error is the silent-zero failure this check exists to expose.  Specialised
+# engines (images, videos, translate, currency, arxiv, npm, ...) are excluded on
+# purpose -- contributing nothing to a general query is correct for them.
+DEFAULT_EXPECTED_ENGINES = [
+    "bing",
+    "brave",
+    "google cse",
+    "yandex",
+    "duckduckgo",
+]
+EXPECTED_ENGINES = [
+    e.strip()
+    for e in os.environ.get(
+        "SEARCH_CHECK_ENGINES", ",".join(DEFAULT_EXPECTED_ENGINES)
+    ).split(",")
+    if e.strip()
+]
+
+
+def _get_json(url: str) -> dict:
+    req = urllib.request.Request(url, headers={"User-Agent": "search-stack-check/1.0"})
+    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+        return json.loads(resp.read().decode("utf-8", "replace"))
+
+
+def _post_json(url: str, payload: dict) -> dict:
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        url,
+        data=data,
+        headers={
+            "Content-Type": "application/json",
+            "User-Agent": "search-stack-check/1.0",
+        },
+    )
+    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+        return json.loads(resp.read().decode("utf-8", "replace"))
+
+
+def enabled_expected_engines() -> set[str]:
+    """Expected engines that SearXNG reports as actually enabled."""
+    cfg = _get_json(f"{SEARXNG_URL}/config")
+    enabled = {e["name"] for e in cfg.get("engines", []) if e.get("enabled")}
+    return {name for name in EXPECTED_ENGINES if name in enabled}
+
+
+def unresponsive_names(pairs: list) -> dict[str, str]:
+    """``unresponsive_engines`` is a list of [name, reason] pairs (or strings)."""
+    out: dict[str, str] = {}
+    for item in pairs or []:
+        if isinstance(item, (list, tuple)) and len(item) >= 2:
+            out[str(item[0])] = str(item[1])
+        elif isinstance(item, str):
+            out[item] = "unresponsive"
+    return out
+
+
+def main() -> int:
+    failures: list[str] = []
+    print(f"Search stack check -- {SEARXNG_URL}")
+    print(f"Queries: {QUERIES!r}  min contributing engines: {MIN_ENGINES}")
+    print("=" * 72)
+
+    try:
+        eligible = enabled_expected_engines()
+    except Exception as exc:  # noqa: BLE001 - report, do not traceback
+        print(f"FAIL: could not read /config from SearXNG: {exc!r}")
+        return 2
+    print(f"Expected engines, enabled ({len(eligible)}): {sorted(eligible)}")
+    missing = sorted(set(EXPECTED_ENGINES) - eligible)
+    if missing:
+        print(f"Expected engines NOT enabled: {missing}")
+        failures.append(f"expected engines not enabled in SearXNG: {missing}")
+
+    contributed: dict[str, int] = {name: 0 for name in eligible}
+    silent_zero_all: dict[str, list[str]] = {}
+
+    for query in QUERIES:
+        url = f"{SEARXNG_URL}/search?" + urllib.parse.urlencode(
+            {"q": query, "format": "json"}
+        )
+        print("-" * 72)
+        print(f"QUERY: {query!r}")
+        try:
+            data = _get_json(url)
+        except Exception as exc:  # noqa: BLE001
+            print(f"  FAIL: query request failed: {exc!r}")
+            failures.append(f"query {query!r} request failed: {exc!r}")
+            continue
+
+        results = data.get("results", [])
+        engines: dict[str, int] = {}
+        for r in results:
+            name = r.get("engine", "?")
+            engines[name] = engines.get(name, 0) + 1
+        unresponsive = unresponsive_names(data.get("unresponsive_engines", []))
+
+        print(f"  results: {len(results)}")
+        print(f"  contributing engines: {engines or '(none)'}")
+        print(f"  unresponsive_engines: {unresponsive or '(none)'}")
+
+        for name in engines:
+            contributed[name] = contributed.get(name, 0) + engines[name]
+
+        if len(engines) < MIN_ENGINES:
+            msg = (
+                f"query {query!r} had only {len(engines)} contributing engine(s) "
+                f"({sorted(engines)}); need >= {MIN_ENGINES}"
+            )
+            print(f"  FAIL: {msg}")
+            failures.append(msg)
+
+        silent = sorted(
+            n for n in eligible if n not in engines and n not in unresponsive
+        )
+        if silent:
+            silent_zero_all[query] = silent
+            print(
+                "  SILENT ZERO (enabled, no error, no results -- reported, "
+                f"not fatal): {silent}"
+            )
+
+    print("=" * 72)
+    print("Engine contribution across all queries:")
+    for name in sorted(contributed):
+        status = "ZERO" if contributed[name] == 0 else "ok"
+        print(f"  {name:<24} {contributed[name]:>4}  {status}")
+
+    if silent_zero_all:
+        print("-" * 72)
+        print("SILENT-ZERO ENGINES REPORTED (no error raised, no results returned):")
+        for query, names in silent_zero_all.items():
+            print(f"  {query!r}: {names}")
+        print("  NOTE: a silent zero is REPORTED, not counted as a failure.  These")
+        print("  engines are expected to answer a general query, but contributing")
+        print("  nothing to one query can be legitimate (result de-duplication, or")
+        print("  an engine that only fires on certain query shapes).  Only the")
+        print(f"  <{MIN_ENGINES}-contributing-engine floor and the extraction leg fail the run.")
+
+    print("-" * 72)
+    print(f"EXTRACTION: scraping {EXTRACT_URL} via {FIRECRAWL_URL}/v1/scrape")
+    try:
+        payload = _post_json(
+            f"{FIRECRAWL_URL}/v1/scrape",
+            {"url": EXTRACT_URL, "formats": ["markdown"]},
+        )
+        markdown = ((payload.get("data") or {}).get("markdown") or "").strip()
+        if not markdown:
+            msg = "extraction returned empty markdown"
+            print(f"  FAIL: {msg}")
+            failures.append(msg)
+        else:
+            print(f"  ok: {len(markdown)} chars of markdown returned")
+            print(f"  first line: {markdown.splitlines()[0][:120]!r}")
+    except Exception as exc:  # noqa: BLE001
+        msg = f"extraction request failed: {exc!r}"
+        print(f"  FAIL: {msg}")
+        failures.append(msg)
+
+    print("=" * 72)
+    if failures:
+        print("VERDICT: FAIL")
+        for f in failures:
+            print(f"  - {f}")
+        return 1
+    print("VERDICT: PASS -- multiple engines contributing, extraction healthy")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

search-stack-visibility.prose.md

@@ -0,0 +1,124 @@
+---
+kind: function
+name: search-stack-visibility
+description: >
+  Makes the shared search stack observable. Every agent reaches one SearXNG
+  instance (http://192.168.68.7:8888) and one extraction service (Firecrawl,
+  http://192.168.68.7:3002). Before this check the stack could degrade to a
+  single engine, or an enabled engine could return nothing at all, without any
+  error surfacing anywhere.
+
+  This contract runs scripts/search-stack-check.py, which:
+    * runs two fixed queries against SearXNG and FAILS when fewer than two
+      engines contribute, printing the contributing engines and every
+      unresponsive_engines entry;
+    * checks extraction by scraping a known page through Firecrawl and FAILS
+      when the returned markdown is empty or the request fails;
+    * reports every silent-zero engine explicitly (enabled, not in
+      unresponsive_engines, contributed no results).
+
+  Multi-engine state (2026-09-25): bing, google cse, brave and yandex
+  contribute on every query. duckduckgo is NOT working: the house egress IP
+  and the VPS fallback egress are both flagged by DuckDuckGo and it reports
+  CAPTCHA. It is left enabled as best-effort coverage so that a recovery shows
+  up as a contribution.
+
+  google cse is a third party's public search-engine id hardcoded in the
+  SearXNG build. Quota and availability are outside our control.
+
+  SCHEDULED: /etc/cron.d/contract-runner on CT 100 (abiba), hourly at :15,
+  via scripts/contract-run.sh search-stack-visibility. Logs land in
+  /var/log/contract-runs/. A failure also raises a firstmate inbox note.
+version: 1.1.0
+---
+
+## Purpose
+
+The fleet has exactly one search endpoint and one extraction endpoint. If
+either degrades, every agent silently loses capability at the same moment.
+The failure mode this contract exists to close is *silent* degradation: a
+query that still returns a page of results while all but one engine have
+stopped contributing, or an enabled engine that answers with zero results and
+raises no error.
+
+## Execution model
+
+The contract is a host-scheduled check, not an agent workflow. It is driven by
+`scripts/contract-run.sh search-stack-visibility` from
+`/etc/cron.d/contract-runner` on CT 100. `contract-run.sh` resolves the
+mapping to `scripts/search-stack-check.py`, runs it under a timeout, writes a
+timestamped log to `/var/log/contract-runs/`, and on non-zero exit raises a
+firstmate inbox note through `bin/fm-inbox.sh`.
+
+## What passing looks like
+
+```
+$ bash scripts/contract-run.sh search-stack-visibility
+Expected engines, enabled (5): ['bing', 'brave', 'duckduckgo', 'google cse', 'yandex']
+queries: 'proxmox backup server' -> contributing: bing, brave, google cse, yandex
+                                    unresponsive: duckduckgo=CAPTCHA
+         'python asyncio tutorial' -> contributing: bing, brave, google cse, yandex
+EXTRACTION: 71016 chars of markdown returned
+VERDICT: PASS -- multiple engines contributing, extraction healthy
+```
+
+## What failing looks like
+
+* A query whose results come from fewer than `SEARCH_CHECK_MIN_ENGINES`
+  engines (default 2) fails and names the engines that did contribute.
+* An extraction request that errors or returns empty markdown fails.
+
+## Silent zeros are reported, not fatal
+
+An enabled, expected engine that contributed nothing **without reporting an
+error** is printed under `SILENT-ZERO ENGINES REPORTED`, and each occurrence is
+annotated `reported, not fatal`.  This is deliberate:
+
+* a general query can legitimately draw zero results from an engine that only
+  fires on certain query shapes, and results are de-duplicated across engines,
+  so a zero does not by itself prove the engine is broken;
+* the run therefore fails only on the two conditions that do prove loss of
+  capability -- fewer than two contributing engines, and a broken extraction
+  leg.
+
+A run can consequently print `VERDICT: PASS` while still listing a silent
+zero.  That is the intended relationship: the zero is *visible*, not *fatal*.
+An engine that fails with an error (for example DuckDuckGo returning CAPTCHA)
+appears in `unresponsive_engines` instead.
+
+## Google coverage is third-party, not ours
+
+The free Google-derived results come from the SearXNG build's built-in
+`google cse` engine.  It uses **a third party's public search-engine id
+hardcoded in the build** (`google_cse.py`, `CX = "partner-pub-8993..."`,
+blackle.com), not a key or id we own.  Its quota and availability are outside
+our control and it can be rate-limited or withdrawn without notice.  No engine
+in this build accepts our own Google Custom Search key; using our own free key
+would require a small wrapper service, which is deliberately **not** built.
+
+## Configuration
+
+Environment overrides (see the script docstring for the full list):
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `SEARXNG_URL` | `http://192.168.68.7:8888` | SearXNG base URL |
+| `FIRECRAWL_URL` | `http://192.168.68.7:3002` | Firecrawl base URL |
+| `SEARCH_CHECK_QUERIES` | `proxmox backup server,python asyncio tutorial` | fixed queries |
+| `SEARCH_CHECK_MIN_ENGINES` | `2` | minimum contributing engines per query |
+| `SEARCH_CHECK_ENGINES` | `bing,brave,google cse,yandex,duckduckgo` | engines a silent zero is reported for |
+| `SEARCH_CHECK_EXTRACT_URL` | Wikipedia Proxmox article | page used for the extraction leg |
+
+## Known residual risk
+
+DuckDuckGo is **not** working.  The house egress IP is CAPTCHA'd by
+DuckDuckGo, and a forward proxy on the VPS (`10.10.10.1:3128`, WireGuard) was
+built as a second egress -- but DuckDuckGo has since flagged the VPS address
+too (HTTP 202 with challenge markers), so DuckDuckGo now reports CAPTCHA on
+both paths.  It is left enabled as best-effort coverage: if DuckDuckGo
+unflags either address it will show up as a contribution, and until then it is
+visible in `unresponsive_engines` every run.  It is never a required engine.
+
+The VPS forward proxy remains a real service (`/opt/fwd-proxy`,
+`restart: unless-stopped`, healthy healthcheck, Docker enabled at boot) so the
+second egress path is available for any engine that benefits from it in future.