diff --git a/plugins/platforms/zulip/adapter.py b/plugins/platforms/zulip/adapter.py index a9c6877..125a807 100644 --- a/plugins/platforms/zulip/adapter.py +++ b/plugins/platforms/zulip/adapter.py @@ -43,6 +43,7 @@ import time import uuid from collections import deque from datetime import datetime, timezone +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple try: @@ -881,6 +882,116 @@ class ZulipAdapter(BasePlatformAdapter): reply_to_topic=topic, ) + # ------------------------------------------------------------------ + # Media / attachment extraction (vision support) + # ------------------------------------------------------------------ + + async def _extract_inline_media( + self, content: str, media_urls: List[str], media_types: List[str], + ) -> None: + """Download Zulip ``/user_uploads/...`` attachments referenced in a + message so vision/transcription tools can read them locally. + + Zulip renders uploaded files as ```` + in HTML. After ``_strip_html()`` the markdown-style references + survive in the plain text. This method finds those references, + downloads the files, and caches them into Hermes media storage. + """ + for rel in re.findall(r"\]\((/user_uploads/[^)]+)\)", content): + dl_url = f"{self._site}{rel}" + try: + resp = await self._http_client.get( + dl_url, + headers={"Authorization": self._auth_header}, + timeout=30.0, + ) + if resp.status_code >= 400: + logger.debug( + "[%s] Inline media %s -> %d", + self.name, rel, resp.status_code, + ) + continue + data = resp.content + mime = ( + resp.headers.get("content-type", "application/octet-stream") + .split(";")[0] + .strip() + ) + except Exception: + logger.debug("[%s] Failed to download inline media %s", self.name, rel) + continue + fname = rel.rsplit("/", 1)[-1] or "file" + await self._cache_attachment(data, fname, mime, media_urls, media_types) + + async def _extract_event_attachments( + self, message: Dict[str, Any], media_urls: List[str], media_types: List[str], + ) -> None: + """Download files from Zulip's ``message.attachments`` field (UI file + uploads that Zulip sends as structured event data rather than inline + markdown links).""" + atts = message.get("attachments") or [] + for att in atts: + path_id = att.get("path_id") + fname = att.get("name") or "file" + if not path_id: + continue + dl_url = f"{self._site}/api/v1/user_uploads/{path_id}" + try: + resp = await self._http_client.get( + dl_url, + headers={"Authorization": self._auth_header}, + timeout=30.0, + ) + if resp.status_code >= 400: + logger.debug( + "[%s] Event attachment %s -> %d", + self.name, fname, resp.status_code, + ) + continue + data = resp.content + mime = ( + resp.headers.get("content-type", "application/octet-stream") + .split(";")[0] + .strip() + ) + except Exception: + logger.debug("[%s] Failed to download event attachment %s", self.name, fname) + continue + await self._cache_attachment(data, fname, mime, media_urls, media_types) + + async def _cache_attachment( + self, data: bytes, fname: str, mime: str, + media_urls: List[str], media_types: List[str], + ) -> None: + """Cache a downloaded attachment into Hermes media storage so the + vision_analyze / ocr tools can access it via a local file path.""" + try: + from gateway.platforms.base import ( + cache_image_from_bytes, + cache_audio_from_bytes, + cache_document_from_bytes, + ) + except ImportError: + logger.warning("[%s] cache_*_from_bytes not available in gateway", self.name) + return + ext = Path(fname).suffix + try: + if mime.startswith("image/"): + media_urls.append(cache_image_from_bytes(data, ext or ".png")) + media_types.append(mime) + elif mime.startswith("audio/"): + media_urls.append(cache_audio_from_bytes(data, ext or ".ogg")) + media_types.append(mime) + else: + media_urls.append(cache_document_from_bytes(data, fname)) + media_types.append(mime) + except Exception as exc: + logger.warning("[%s] Failed to cache attachment %s: %s", self.name, fname, exc) + + # ------------------------------------------------------------------ + # Message routing + # ------------------------------------------------------------------ + async def _route_message( self, text: str, @@ -911,8 +1022,22 @@ class ZulipAdapter(BasePlatformAdapter): chat_id_alt=str(reply_to_id) if reply_to_id else None, ) - # Derive message type - message_type = MessageType.TEXT + # Gen 5: Extract media/attachments so vision tools can read them + media_urls: List[str] = [] + media_types: List[str] = [] + await self._extract_inline_media(text, media_urls, media_types) + await self._extract_event_attachments(raw, media_urls, media_types) + + # Derive message type — upgrade to PHOTO/VOICe/DOCUMENT when media present + if media_types: + if any(m.startswith("image/") for m in media_types): + message_type = MessageType.PHOTO + elif any(m.startswith("audio/") for m in media_types): + message_type = MessageType.VOICE + else: + message_type = MessageType.DOCUMENT + else: + message_type = MessageType.TEXT message_event = MessageEvent( text=text, @@ -921,6 +1046,8 @@ class ZulipAdapter(BasePlatformAdapter): message_id=msg_id, raw_message=raw, timestamp=_parse_zulip_timestamp(timestamp), + media_urls=media_urls or None, + media_types=media_types or None, ) # Store reply metadata so send() can respond to the right thread