From 0dd3c5868d85d0c9c719cc9bca850d443096287d Mon Sep 17 00:00:00 2001 From: Jerome Date: Sat, 15 Aug 2026 23:56:44 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20Zulip=20attachment=20handling=20?= =?UTF-8?q?=E2=80=94=20download=20user=20uploads=20for=20vision=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapter stripped all HTML tags via _strip_html() before the gateway could see image references, and always sent MessageType.TEXT. Uploaded files (PNG, PDF, audio, etc.) arrived as or markdown-style references to /user_uploads/... paths that were never downloaded locally — so vision_analyze had no file to read. Changes: - Add _extract_inline_media(): finds /user_uploads/... references in message text, downloads them via httpx, caches locally - Add _extract_event_attachments(): handles message.attachments from Zulip UI file uploads (structured event data) - Add _cache_attachment(): routes image/audio/document bytes to Hermes media storage via cache_image_from_bytes / cache_audio_from_bytes / cache_document_from_bytes - Update _route_message() to extract media and set MessageType.PHOTO, VOICE, or DOCUMENT when attachments are present (was always TEXT) - Pass media_urls and media_types to MessageEvent so the gateway delivery pipeline can attach local file paths to the conversation Ported from hermes-zulip-plugin (deprecated) and adapted for the httpx-based platform adapter. --- plugins/platforms/zulip/adapter.py | 131 ++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/plugins/platforms/zulip/adapter.py b/plugins/platforms/zulip/adapter.py index a9c6877..125a807 100644 --- a/plugins/platforms/zulip/adapter.py +++ b/plugins/platforms/zulip/adapter.py @@ -43,6 +43,7 @@ import time import uuid from collections import deque from datetime import datetime, timezone +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple try: @@ -881,6 +882,116 @@ class ZulipAdapter(BasePlatformAdapter): reply_to_topic=topic, ) + # ------------------------------------------------------------------ + # Media / attachment extraction (vision support) + # ------------------------------------------------------------------ + + async def _extract_inline_media( + self, content: str, media_urls: List[str], media_types: List[str], + ) -> None: + """Download Zulip ``/user_uploads/...`` attachments referenced in a + message so vision/transcription tools can read them locally. + + Zulip renders uploaded files as ```` + in HTML. After ``_strip_html()`` the markdown-style references + survive in the plain text. This method finds those references, + downloads the files, and caches them into Hermes media storage. + """ + for rel in re.findall(r"\]\((/user_uploads/[^)]+)\)", content): + dl_url = f"{self._site}{rel}" + try: + resp = await self._http_client.get( + dl_url, + headers={"Authorization": self._auth_header}, + timeout=30.0, + ) + if resp.status_code >= 400: + logger.debug( + "[%s] Inline media %s -> %d", + self.name, rel, resp.status_code, + ) + continue + data = resp.content + mime = ( + resp.headers.get("content-type", "application/octet-stream") + .split(";")[0] + .strip() + ) + except Exception: + logger.debug("[%s] Failed to download inline media %s", self.name, rel) + continue + fname = rel.rsplit("/", 1)[-1] or "file" + await self._cache_attachment(data, fname, mime, media_urls, media_types) + + async def _extract_event_attachments( + self, message: Dict[str, Any], media_urls: List[str], media_types: List[str], + ) -> None: + """Download files from Zulip's ``message.attachments`` field (UI file + uploads that Zulip sends as structured event data rather than inline + markdown links).""" + atts = message.get("attachments") or [] + for att in atts: + path_id = att.get("path_id") + fname = att.get("name") or "file" + if not path_id: + continue + dl_url = f"{self._site}/api/v1/user_uploads/{path_id}" + try: + resp = await self._http_client.get( + dl_url, + headers={"Authorization": self._auth_header}, + timeout=30.0, + ) + if resp.status_code >= 400: + logger.debug( + "[%s] Event attachment %s -> %d", + self.name, fname, resp.status_code, + ) + continue + data = resp.content + mime = ( + resp.headers.get("content-type", "application/octet-stream") + .split(";")[0] + .strip() + ) + except Exception: + logger.debug("[%s] Failed to download event attachment %s", self.name, fname) + continue + await self._cache_attachment(data, fname, mime, media_urls, media_types) + + async def _cache_attachment( + self, data: bytes, fname: str, mime: str, + media_urls: List[str], media_types: List[str], + ) -> None: + """Cache a downloaded attachment into Hermes media storage so the + vision_analyze / ocr tools can access it via a local file path.""" + try: + from gateway.platforms.base import ( + cache_image_from_bytes, + cache_audio_from_bytes, + cache_document_from_bytes, + ) + except ImportError: + logger.warning("[%s] cache_*_from_bytes not available in gateway", self.name) + return + ext = Path(fname).suffix + try: + if mime.startswith("image/"): + media_urls.append(cache_image_from_bytes(data, ext or ".png")) + media_types.append(mime) + elif mime.startswith("audio/"): + media_urls.append(cache_audio_from_bytes(data, ext or ".ogg")) + media_types.append(mime) + else: + media_urls.append(cache_document_from_bytes(data, fname)) + media_types.append(mime) + except Exception as exc: + logger.warning("[%s] Failed to cache attachment %s: %s", self.name, fname, exc) + + # ------------------------------------------------------------------ + # Message routing + # ------------------------------------------------------------------ + async def _route_message( self, text: str, @@ -911,8 +1022,22 @@ class ZulipAdapter(BasePlatformAdapter): chat_id_alt=str(reply_to_id) if reply_to_id else None, ) - # Derive message type - message_type = MessageType.TEXT + # Gen 5: Extract media/attachments so vision tools can read them + media_urls: List[str] = [] + media_types: List[str] = [] + await self._extract_inline_media(text, media_urls, media_types) + await self._extract_event_attachments(raw, media_urls, media_types) + + # Derive message type — upgrade to PHOTO/VOICe/DOCUMENT when media present + if media_types: + if any(m.startswith("image/") for m in media_types): + message_type = MessageType.PHOTO + elif any(m.startswith("audio/") for m in media_types): + message_type = MessageType.VOICE + else: + message_type = MessageType.DOCUMENT + else: + message_type = MessageType.TEXT message_event = MessageEvent( text=text, @@ -921,6 +1046,8 @@ class ZulipAdapter(BasePlatformAdapter): message_id=msg_id, raw_message=raw, timestamp=_parse_zulip_timestamp(timestamp), + media_urls=media_urls or None, + media_types=media_types or None, ) # Store reply metadata so send() can respond to the right thread