From 9edc258245dca0a016eb9f6d1d450d494cd14af0 Mon Sep 17 00:00:00 2001 From: Mumuni Date: Fri, 25 Sep 2026 15:50:16 +0000 Subject: [PATCH] =?UTF-8?q?fix(ci):=20make=20the=20validate=20job=20actual?= =?UTF-8?q?ly=20able=20to=20run=20=E2=80=94=20it=20never=20had?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci.yml has been invalid YAML since introduction: the 'Config validation' and old inline checks dedented out of their run:| block scalar, so Gitea could never parse the workflow — CI never ran on any PR despite CI_STATUS.md claiming 'Active'. The old 'No secrets check' also always passed (|| echo swallows the grep hit) and never scanned *.yml — where six embedded credentials were living. - validation logic moved to ci_check.py (testable locally: python3 ci_check.py all) - secrets check now FAILS on embedded http-basic URLs and long api_keys, across .py/.ts/.yaml/.yml/.cjs/.sh, with placeholder allowlist - added workflow-YAML parse gate so this class of breakage can't recur - py_compile steps no longer swallow errors with '|| echo skipped' - removed ci.yml's duplicate deploy job: deploy.yml is the sole deploy pipeline (rc tags → Tanko canary only; stable → all agents). The ci.yml copy would have deployed Mumuni on rc tags too, breaking canary policy, and never ran anyway. - CI_STATUS.md rewritten with the real state + caveats (history still contains the old creds — rotation is a server-side task) --- .gitea/workflows/ci.yml | 33 +++---------- CI_STATUS.md | 27 ++++++++++- ci_check.py | 105 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 26 deletions(-) create mode 100644 ci_check.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 483cfbe..41a33ee 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -21,35 +21,18 @@ jobs: - name: Python syntax check run: | - python3 -m py_compile plugins/platforms/zulip/adapter.py 2>/dev/null && echo "✅ adapter.py OK" || echo "⚠️ adapter.py check skipped" - python3 -m py_compile agent-zero-zulip/src/agent_zero_zulip/adapter.py 2>/dev/null && echo "✅ a2a adapter OK" || echo "⚠️ a2a check skipped" + python3 -m py_compile plugins/platforms/zulip/adapter.py && echo "✅ adapter.py OK" + python3 -m py_compile agent-zero-zulip/src/agent_zero_zulip/adapter.py && echo "✅ a2a adapter OK" + + - name: Workflow YAML parse check + run: python3 ci_check.py workflows - name: Config validation - run: | - python3 -c " -import yaml -cfg = yaml.safe_load(open('config.yaml.example')) -assert 'zulip' in cfg, 'Missing zulip config' -print('✅ config.yaml.example valid') -" 2>/dev/null || echo "⚠️ Config check skipped (no pyyaml)" + run: python3 ci_check.py config - name: No secrets check - run: | - ! grep -r "api_key.*[A-Za-z0-9]\{20,\}" --include="*.py" --include="*.ts" --include="*.yaml" . 2>/dev/null || echo "⚠️ Possible API key found!" + run: python3 ci_check.py secrets - name: Deploy script syntax - run: bash -n scripts/deploy.sh && echo "✅ deploy.sh syntax OK" || echo "⚠️ deploy.sh syntax issue" + run: bash -n scripts/deploy.sh && echo "✅ deploy.sh syntax OK" - deploy: - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - needs: [validate] - steps: - - uses: actions/checkout@v4 - - run: echo "🚀 Deploy tag $(echo $GITHUB_REF_NAME)" - - - name: Deploy to Tanko (canary) - run: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@192.168.68.122 "cd /root && bash -s" < scripts/deploy.sh --ct=tanko --mode=native "$GITHUB_REF_NAME" 2>&1 || echo "⚠️ Tanko deploy skipped" - - - name: Deploy to Mumuni - run: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@192.168.68.123 "cd /root && bash -s" < scripts/deploy.sh --ct=mumuni --mode=native "$GITHUB_REF_NAME" 2>&1 || echo "⚠️ Mumuni deploy skipped" diff --git a/CI_STATUS.md b/CI_STATUS.md index 3d709d0..1926152 100644 --- a/CI_STATUS.md +++ b/CI_STATUS.md @@ -1,2 +1,27 @@ # CI Pipeline Status -Status: Active + +**Last verified: 2026-09-25** — see PR "main security + truncate + CI repair". + +## Reality check (before that PR) +`ci.yml` was **invalid YAML** — the inline `run: |` blocks for the config and +secrets checks dedented out of their block scalar, so Gitea Actions could never +parse the workflow. CI never ran on any PR, and this file's "Active" status was +fiction. The old "No secrets check" also swallowed hits with `|| echo` (always +green) and never scanned `.yml` files — which is where six embedded +credentials were living. + +## Current pipeline +| Trigger | Job | Checks | +|---|---|---| +| PR → main, push main, tag `v*` | `validate` | adapter + a2a `py_compile`; workflow YAML parse; `config.yaml.example`; secret scan; `bash -n scripts/deploy.sh` | +| tag `v*` | `deploy` | Tanko canary + Mumuni via `scripts/deploy.sh` (native mode) | + +All validation logic lives in **`ci_check.py`** — run `python3 ci_check.py all` +locally before pushing; it fails the check on real hits instead of echoing +warnings. + +## Known caveats +- The removed credentials still exist in git history (pre-July commits) — + rotating `abiba-bot`'s password is a **server-side** task, out of repo scope. +- Runner availability (`zulip-runner` on CT 116) is not verifiable from agents + — the first green run after merge is the proof. diff --git a/ci_check.py b/ci_check.py new file mode 100644 index 0000000..2b6ee2a --- /dev/null +++ b/ci_check.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""CI validation checks for zulip-platform-plugins. + +The inline validation steps this script replaced were never valid YAML in the +first place (the `run: |` blocks dedented out of their block scalar), so the +whole `validate` job silently never ran. Keeping the logic in a real file +means it is testable locally — run `python3 ci_check.py all` before pushing. + +Usage: python3 ci_check.py {workflows|config|secrets|all} +""" +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +CODE_GLOBS = ("*.py", "*.ts", "*.yaml", "*.yml", "*.cjs", "*.sh") +# scheme://user:pass@host — embedded HTTP Basic credentials +CRED_RE = re.compile(r"://[^/ \"']+:[^/ \"'@]+@") +API_KEY_RE = re.compile(r"""api_key["']?\s*[:=]\s*["']?[A-Za-z0-9]{20,}""") +# Allowlist: placeholder patterns, not real secrets +PLACEHOLDER = ("${", "{{", "user:pass", "USER:TOKEN", "user:token", "example") + + +def iter_code_files(): + for path in sorted(ROOT.rglob("*")): + if not path.is_file(): + continue + if any(part in {".git", "__pycache__", "node_modules"} for part in path.parts): + continue + if path.name == Path(__file__).name: # this file documents the patterns + continue + if any(path.match(glob) for glob in CODE_GLOBS): + yield path + + +def check_workflows() -> bool: + try: + import yaml + except ModuleNotFoundError: + print("⚠️ pyyaml not installed — workflow parse check skipped") + return True + ok = True + for f in sorted((ROOT / ".gitea" / "workflows").glob("*.yml")): + try: + doc = yaml.safe_load(f.read_text()) + assert isinstance(doc, dict) and "jobs" in doc, "missing 'jobs' mapping" + print(f"✅ {f.relative_to(ROOT)} valid — jobs: {list(doc['jobs'])}") + except Exception as e: + print(f"❌ {f.relative_to(ROOT)}: {e}") + ok = False + return ok + + +def check_config() -> bool: + try: + import yaml + except ModuleNotFoundError: + print("⚠️ pyyaml not installed — config check skipped") + return True + try: + cfg = yaml.safe_load((ROOT / "config.yaml.example").read_text()) + assert isinstance(cfg, dict) and "zulip" in cfg, "missing 'zulip' section" + print("✅ config.yaml.example valid") + return True + except Exception as e: + print(f"❌ config.yaml.example: {e}") + return False + + +def check_secrets() -> bool: + hits = [] + for path in iter_code_files(): + try: + text = path.read_text(errors="ignore") + except OSError: + continue + for lineno, line in enumerate(text.splitlines(), 1): + for rx in (CRED_RE, API_KEY_RE): + if rx.search(line) and not any(p in line for p in PLACEHOLDER): + hits.append(f"{path.relative_to(ROOT)}:{lineno}: {line.strip()[:100]}") + break + if hits: + print("❌ Embedded credentials detected:") + for h in hits: + print(f" {h}") + return False + print("✅ No embedded credentials in tracked code/workflow files") + return True + + +CHECKS = {"workflows": check_workflows, "config": check_config, "secrets": check_secrets} + + +def main() -> int: + args = sys.argv[1:] or ["all"] + names = list(CHECKS) if "all" in args else args + failed = [n for n in names if n not in CHECKS or not CHECKS[n]()] + if failed: + print(f"❌ CI checks failed: {', '.join(failed)}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())