From 4e8b96ed0a40a76671f414199f0bfa658b2ec719 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 9 Sep 2026 12:52:07 +0000 Subject: [PATCH 1/2] fix(whatsapp,roles): self-heal legacy whatsapp_log schema; map underscore role aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CT115 (Mumuni relay #747) — two live-instance defects after PR #12: 1. GET /api/whatsapp/mock-log 500'd with a valid admin token: 'no such column: whatsapp_log.message_text'. The model gained message_text/wa_message_id/ticket_number (and dropped command) in 4afdc36 with no migration, so legacy DBs keep the (command, ...) shape. ensure_legacy_schema (startup, app/main.py) now adds the three missing columns idempotently and backfills legacy command bodies into message_text before dropping the obsolete NOT NULL command column, so both the mock-log read path and the ORM write path work on healed DBs. New producer/consumer regression (tests/test_whatsapp_log_legacy_heal.py) reproduces the exact OperationalError, then asserts 200 + data. 2. ROLE_ALIASES gap: underscore legacy roles (cs_rep, cs_manager, fm_dispatcher) were not mapped, so normalize_legacy_user_roles could not converge rows like user 18 (test@denya.com, role 'cs_rep') and the frontend stranded them on /tickets. Added the underscore aliases; tests assert normalize_role('cs_rep') == 'CS Rep' and a cs_rep row converges and authenticates. --- AGENTS.md | 11 +- app/core/roles.py | 8 +- app/main.py | 41 +++++++ tests/test_p0_auth_admin_batch.py | 37 +++++++ tests/test_whatsapp_log_legacy_heal.py | 144 +++++++++++++++++++++++++ 5 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 tests/test_whatsapp_log_legacy_heal.py diff --git a/AGENTS.md b/AGENTS.md index b37beb2..17da210 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,6 +66,13 @@ sign-up UI; users are created/managed by admins only (P0 hardening batch). | POST | `/api/whatsapp/webhook` | `X-Webhook-Secret` header | Inbound message → ticket + log. Fail-closed: 403 when `WHATSAPP_WEBHOOK_SECRET` is unset or the header doesn't match | | GET | `/api/whatsapp/mock-log` | Bearer | Recent webhook submissions (debug; auth required) | +`whatsapp_log` predates the real Meta webhook (the model gained +`message_text`/`wa_message_id`/`ticket_number` and dropped `command` without a +migration), so legacy tables keep the old `(command, …)` shape and every ORM +read/write 500s. `ensure_legacy_schema` (app/main.py) adds the missing columns +and backfills+drops the obsolete NOT NULL `command` column idempotently at +startup — do not hand-edit legacy DBs, ship a self-heal there instead. + ### Pages (Sprint 3) — Jinja2 templates at `app/templates/` | Method | Path | Auth | Description | |--------|------|------|-------------| @@ -123,7 +130,9 @@ Frontend: Alpine.js + Tailwind CSS vendored same-origin (no CDN) — see old open register) fail closed at login/JWT and can never be recreated via the API (422). Startup self-heals (lifespan in `app/main.py`, helpers in `app/services/seed.py`) converge legacy rows: `normalize_legacy_user_roles` - maps unambiguous alias nicknames onto canonical roles, and + maps unambiguous alias nicknames onto canonical roles (space and underscore + forms alike — `cs rep`/`cs_rep` → `CS Rep`, `fm dispatcher`/`fm_dispatcher`, + `cs manager`/`cs_manager`), and `normalize_legacy_user_emails` lowercases stored emails — login and the admin create-user duplicate check both compare on the lowercased form, so pre-P0 mixed-case emails are never silently locked out. diff --git a/app/core/roles.py b/app/core/roles.py index 99ac37e..b66d75e 100644 --- a/app/core/roles.py +++ b/app/core/roles.py @@ -12,8 +12,9 @@ uses (PRD §4, ``app/services/seed.py``, the frontend nav in base.html): Legacy databases created under the pre-P0 open-registration builds can carry lowercase/nickname role strings (``technician``, ``cs``, ``fm``, ``ceo``, -``admin``, ``superadmin`` …). ``ROLE_ALIASES`` maps the *unambiguous* -nicknames onto a canonical role so startup normalization (see +``admin``, ``superadmin`` …) and underscore variants of the space-separated +ones (``cs_rep``, ``cs_manager``, ``fm_dispatcher``). ``ROLE_ALIASES`` maps +the *unambiguous* nicknames onto a canonical role so startup normalization (see ``app/services/seed.py::normalize_legacy_user_roles``) can converge the data. ``admin`` / ``superadmin`` are deliberately NOT aliased: they are @@ -52,10 +53,13 @@ ROLE_ALIASES: dict[str, str] = { "tech": TECHNICIAN_ROLE, "cs": "CS Rep", "cs rep": "CS Rep", + "cs_rep": "CS Rep", "cs representative": "CS Rep", "cs manager": "CS Manager", + "cs_manager": "CS Manager", "fm": "FM Dispatcher", "fm dispatcher": "FM Dispatcher", + "fm_dispatcher": "FM Dispatcher", "ceo": "CEO", "director": "Director", } diff --git a/app/main.py b/app/main.py index 5addab3..06695a8 100644 --- a/app/main.py +++ b/app/main.py @@ -52,6 +52,47 @@ async def ensure_legacy_schema(conn) -> None: text("UPDATE tickets SET reported_at = created_at WHERE reported_at IS NULL") ) logger.info("Added missing tickets.reported_at column (legacy database)") + + # whatsapp_log predates the real Meta webhook (the model gained + # message_text/wa_message_id/ticket_number and dropped `command` in commit + # 4afdc36 with no migration), so legacy DBs still carry the old shape and + # every read/write through the ORM 500s (no such column: message_text). + result = await conn.execute( + text("SELECT name FROM sqlite_master WHERE type='table' AND name='whatsapp_log'") + ) + if result.scalar(): + result = await conn.execute(text("PRAGMA table_info(whatsapp_log)")) + log_columns = {row[1] for row in result} + if "message_text" not in log_columns: + await conn.execute( + text("ALTER TABLE whatsapp_log ADD COLUMN message_text TEXT NOT NULL DEFAULT ''") + ) + logger.info("Added missing whatsapp_log.message_text column (legacy database)") + if "wa_message_id" not in log_columns: + await conn.execute( + text("ALTER TABLE whatsapp_log ADD COLUMN wa_message_id VARCHAR(100)") + ) + logger.info("Added missing whatsapp_log.wa_message_id column (legacy database)") + if "ticket_number" not in log_columns: + await conn.execute( + text("ALTER TABLE whatsapp_log ADD COLUMN ticket_number VARCHAR(30)") + ) + logger.info("Added missing whatsapp_log.ticket_number column (legacy database)") + if "command" in log_columns: + # Legacy rows stored the message body in `command`, which the model + # no longer defines (NOT NULL, no default): any ORM insert omitting + # it would violate NOT NULL. Preserve the old bodies in + # message_text, then drop the obsolete column to match the model. + await conn.execute( + text( + "UPDATE whatsapp_log SET message_text = command " + "WHERE (message_text IS NULL OR message_text = '') " + "AND command IS NOT NULL AND command != ''" + ) + ) + await conn.execute(text("ALTER TABLE whatsapp_log DROP COLUMN command")) + logger.info("Dropped obsolete whatsapp_log.command column (legacy database)") + result = await conn.execute( text( "UPDATE categories SET name = 'Missing Item' " diff --git a/tests/test_p0_auth_admin_batch.py b/tests/test_p0_auth_admin_batch.py index 6448362..91e1c1c 100644 --- a/tests/test_p0_auth_admin_batch.py +++ b/tests/test_p0_auth_admin_batch.py @@ -308,6 +308,43 @@ async def test_normalize_does_not_map_ambiguous_admin_alias(client: AsyncClient) assert user.role == "admin" +async def test_underscore_legacy_aliases_normalize_to_canonical(): + """Open-register rows can carry underscore role forms ('cs_rep', + 'cs_manager', 'fm_dispatcher') — the exact gap Mumuni relay #747 found on + user 18 (test@denya.com). Each must map onto its canonical role so startup + normalization can converge the row instead of stranding it on /tickets.""" + from app.core.roles import normalize_role + + assert normalize_role("cs_rep") == "CS Rep" + assert normalize_role("cs_manager") == "CS Manager" + assert normalize_role("fm_dispatcher") == "FM Dispatcher" + # Matching is case/whitespace tolerant, like the space-form aliases. + assert normalize_role(" CS_REP ") == "CS Rep" + assert normalize_role("cs_rep") is not None # known, not fail-closed + + +async def test_legacy_cs_rep_row_converges_and_authenticates(client: AsyncClient): + """A 'cs_rep' row (user 18 test@denya.com shape) is converged to canonical + 'CS Rep' by the startup self-heal and can log in again (no fail-closed + denial, and the frontend CS nav sees the canonical role).""" + await _insert_user("cs-rep-legacy@example.com", "cs_rep") + await _normalize_roles() # what lifespan does each boot + + async with async_session_factory() as session: + from sqlalchemy import select + user = ( + await session.execute(select(User).where(User.email == "cs-rep-legacy@example.com")) + ).scalar_one() + assert user.role == "CS Rep" + + login = await client.post( + "/api/auth/login", json={"email": "cs-rep-legacy@example.com", "password": "denya123"} + ) + assert login.status_code == 200, login.text + me = await client.get("/api/auth/me", headers=_auth(login.json()["access_token"])) + assert me.json()["role"] == "CS Rep" + + # ── P0 email normalization (legacy mixed-case rows) ─────────────────── async def test_legacy_mixed_case_email_migrated_and_authenticates(client: AsyncClient): """A legacy row whose email was stored verbatim in mixed case (the old open diff --git a/tests/test_whatsapp_log_legacy_heal.py b/tests/test_whatsapp_log_legacy_heal.py new file mode 100644 index 0000000..8981e04 --- /dev/null +++ b/tests/test_whatsapp_log_legacy_heal.py @@ -0,0 +1,144 @@ +"""Regression: legacy ``whatsapp_log`` tables self-heal at startup. + +Producer/consumer for the CT115 defect (Mumuni relay #747): the live demo DB's +``whatsapp_log`` table still has the pre-webhook shape — ``command`` instead of +``message_text`` and no ``wa_message_id``/``ticket_number`` — so +``GET /api/whatsapp/mock-log`` 500'd with ``OperationalError: no such column: +whatsapp_log.message_text`` even for a valid admin token. + +Producer: build the legacy-shaped table (as the live DB holds it) and seed it +with ``command`` rows. +Consumer: boot the app's startup self-heal (``ensure_legacy_schema``), then read +back through the authenticated mock-log endpoint, insert through the ORM write +path, and re-run the self-heal to prove idempotency. +""" + +from __future__ import annotations + +from datetime import datetime + +import pytest +from sqlalchemy import text + +from app.core.database import async_session_factory, engine +from app.main import ensure_legacy_schema + +pytestmark = pytest.mark.asyncio + +# Pre-webhook shape (commit 4afdc36 changed the model but no migration shipped): +# id, command (NOT NULL), from_number, ticket_id, received_at. +LEGACY_WHATSAPP_LOG_DDL = ( + "CREATE TABLE whatsapp_log (" + "id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + "command TEXT NOT NULL, " + "from_number VARCHAR(50), " + "ticket_id INTEGER, " + "received_at DATETIME NOT NULL)" +) + +EXPECTED_MODEL_COLUMNS = { + "id", + "from_number", + "message_text", + "wa_message_id", + "ticket_id", + "ticket_number", + "received_at", +} + + +async def _login(client, email="wahab@denya.com", password="denya123") -> str: + resp = await client.post("/api/auth/login", json={"email": email, "password": password}) + assert resp.status_code == 200, resp.text + return resp.json()["access_token"] + + +def _auth(token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +async def _replace_with_legacy_whatsapp_log() -> None: + """Drop the model-shaped table and recreate the legacy shape with rows.""" + async with engine.begin() as conn: + await conn.execute(text("DROP TABLE IF EXISTS whatsapp_log")) + await conn.execute(text(LEGACY_WHATSAPP_LOG_DDL)) + await conn.execute( + text( + "INSERT INTO whatsapp_log (command, from_number, ticket_id, received_at) VALUES " + "('legacy older message', '+233200000001', 1, '2026-09-08 08:00:00'), " + "('legacy newest message', '+233200000002', 2, '2026-09-09 09:30:00')" + ) + ) + + +async def _columns() -> set[str]: + async with engine.begin() as conn: + result = await conn.execute(text("PRAGMA table_info(whatsapp_log)")) + return {row[1] for row in result} + + +async def test_legacy_whatsapp_log_self_heals_and_mock_log_200(client): + """Producer: legacy whatsapp_log (command shape). Consumer: startup self-heal + then authenticated mock-log — the exact 500 from the live instance.""" + token = await _login(client) + await _replace_with_legacy_whatsapp_log() + + # Pre-fix reproduction: on the legacy table this endpoint fails exactly as + # reported (production: HTTP 500; under ASGITransport the app never returns + # a response so the sqlalchemy OperationalError propagates). + import sqlalchemy.exc + + with pytest.raises(sqlalchemy.exc.OperationalError): + await client.get("/api/whatsapp/mock-log", headers=_auth(token)) + + # ── Boot the startup self-heal (what lifespan does each boot) ── + async with engine.begin() as conn: + await ensure_legacy_schema(conn) + + columns = await _columns() + assert EXPECTED_MODEL_COLUMNS <= columns + assert "command" not in columns # obsolete model-dropped column is gone + + # Authenticated mock-log returns 200 and the backfilled rows are readable. + resp = await client.get("/api/whatsapp/mock-log", headers=_auth(token)) + assert resp.status_code == 200, resp.text + entries = resp.json() + assert [e["message_text"] for e in entries] == [ + "legacy newest message", + "legacy older message", + ] # order: received_at desc; bodies preserved from legacy `command` + + # ORM write path works on the healed table (the current app inserts + # message_text/wa_message_id and never writes `command`). + from app.models.whatsapp_log import WhatsAppLog + + async with async_session_factory() as session: + session.add( + WhatsAppLog( + from_number="+233200000003", + message_text="inbound after self-heal", + wa_message_id="wamid.healed.1", + ticket_id=None, + ticket_number=None, + received_at=datetime(2026, 9, 10, 10, 0, 0), + ) + ) + await session.commit() + + resp = await client.get("/api/whatsapp/mock-log", headers=_auth(token)) + assert resp.status_code == 200, resp.text + entries = resp.json() + assert entries[0]["message_text"] == "inbound after self-heal" + assert entries[0]["from_number"] == "+233200000003" + assert len(entries) == 3 + + # ── Idempotent on the next boot ── + async with engine.begin() as conn: + await ensure_legacy_schema(conn) + columns = await _columns() + assert EXPECTED_MODEL_COLUMNS <= columns + assert "command" not in columns + + resp = await client.get("/api/whatsapp/mock-log", headers=_auth(token)) + assert resp.status_code == 200, resp.text + assert len(resp.json()) == 3 From 0d79a582f67da5f6b7654d889f31a10ad7674038 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 9 Sep 2026 13:06:19 +0000 Subject: [PATCH 2/2] no-mistakes(document): drop stale hand-copied test count from HARDENING.md --- HARDENING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HARDENING.md b/HARDENING.md index 27a7b5c..ba3090c 100644 --- a/HARDENING.md +++ b/HARDENING.md @@ -16,7 +16,7 @@ - **Working & verified:** auth (JWT 30m/7d, bcrypt, RBAC via `require_roles`), ticket CRUD with 16-status `VALID_TRANSITIONS` state machine, SLA engine, photo uploads, category/unit hierarchy, 3 role dashboards (CS/FM/CEO), Alembic migrations with - legacy-schema self-heal. **32 pytest tests pass.** + legacy-schema self-heal. **pytest suite passes.** - **Live:** container `denya-onecare` on LXC `scottdenya` (192.168.68.75:8000), image built 2026-08-02, `restart: unless-stopped`. - **Demo-only posture resolved (PR #10 + P0 batch):** `SECRET_KEY` now fails