fix: Zulip attachment handling — download user uploads for vision tools
The adapter stripped all HTML tags via _strip_html() before the gateway could see image references, and always sent MessageType.TEXT. Uploaded files (PNG, PDF, audio, etc.) arrived as <img> or markdown-style references to /user_uploads/... paths that were never downloaded locally — so vision_analyze had no file to read. Changes: - Add _extract_inline_media(): finds /user_uploads/... references in message text, downloads them via httpx, caches locally - Add _extract_event_attachments(): handles message.attachments from Zulip UI file uploads (structured event data) - Add _cache_attachment(): routes image/audio/document bytes to Hermes media storage via cache_image_from_bytes / cache_audio_from_bytes / cache_document_from_bytes - Update _route_message() to extract media and set MessageType.PHOTO, VOICE, or DOCUMENT when attachments are present (was always TEXT) - Pass media_urls and media_types to MessageEvent so the gateway delivery pipeline can attach local file paths to the conversation Ported from hermes-zulip-plugin (deprecated) and adapted for the httpx-based platform adapter.
This commit is contained in:
Jerome
@@ -43,6 +43,7 @@ import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
@@ -881,6 +882,116 @@ class ZulipAdapter(BasePlatformAdapter):
|
||||
reply_to_topic=topic,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Media / attachment extraction (vision support)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _extract_inline_media(
|
||||
self, content: str, media_urls: List[str], media_types: List[str],
|
||||
) -> None:
|
||||
"""Download Zulip ``/user_uploads/...`` attachments referenced in a
|
||||
message so vision/transcription tools can read them locally.
|
||||
|
||||
Zulip renders uploaded files as ``<img src="/user_uploads/...">``
|
||||
in HTML. After ``_strip_html()`` the markdown-style references
|
||||
survive in the plain text. This method finds those references,
|
||||
downloads the files, and caches them into Hermes media storage.
|
||||
"""
|
||||
for rel in re.findall(r"\]\((/user_uploads/[^)]+)\)", content):
|
||||
dl_url = f"{self._site}{rel}"
|
||||
try:
|
||||
resp = await self._http_client.get(
|
||||
dl_url,
|
||||
headers={"Authorization": self._auth_header},
|
||||
timeout=30.0,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
logger.debug(
|
||||
"[%s] Inline media %s -> %d",
|
||||
self.name, rel, resp.status_code,
|
||||
)
|
||||
continue
|
||||
data = resp.content
|
||||
mime = (
|
||||
resp.headers.get("content-type", "application/octet-stream")
|
||||
.split(";")[0]
|
||||
.strip()
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("[%s] Failed to download inline media %s", self.name, rel)
|
||||
continue
|
||||
fname = rel.rsplit("/", 1)[-1] or "file"
|
||||
await self._cache_attachment(data, fname, mime, media_urls, media_types)
|
||||
|
||||
async def _extract_event_attachments(
|
||||
self, message: Dict[str, Any], media_urls: List[str], media_types: List[str],
|
||||
) -> None:
|
||||
"""Download files from Zulip's ``message.attachments`` field (UI file
|
||||
uploads that Zulip sends as structured event data rather than inline
|
||||
markdown links)."""
|
||||
atts = message.get("attachments") or []
|
||||
for att in atts:
|
||||
path_id = att.get("path_id")
|
||||
fname = att.get("name") or "file"
|
||||
if not path_id:
|
||||
continue
|
||||
dl_url = f"{self._site}/api/v1/user_uploads/{path_id}"
|
||||
try:
|
||||
resp = await self._http_client.get(
|
||||
dl_url,
|
||||
headers={"Authorization": self._auth_header},
|
||||
timeout=30.0,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
logger.debug(
|
||||
"[%s] Event attachment %s -> %d",
|
||||
self.name, fname, resp.status_code,
|
||||
)
|
||||
continue
|
||||
data = resp.content
|
||||
mime = (
|
||||
resp.headers.get("content-type", "application/octet-stream")
|
||||
.split(";")[0]
|
||||
.strip()
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("[%s] Failed to download event attachment %s", self.name, fname)
|
||||
continue
|
||||
await self._cache_attachment(data, fname, mime, media_urls, media_types)
|
||||
|
||||
async def _cache_attachment(
|
||||
self, data: bytes, fname: str, mime: str,
|
||||
media_urls: List[str], media_types: List[str],
|
||||
) -> None:
|
||||
"""Cache a downloaded attachment into Hermes media storage so the
|
||||
vision_analyze / ocr tools can access it via a local file path."""
|
||||
try:
|
||||
from gateway.platforms.base import (
|
||||
cache_image_from_bytes,
|
||||
cache_audio_from_bytes,
|
||||
cache_document_from_bytes,
|
||||
)
|
||||
except ImportError:
|
||||
logger.warning("[%s] cache_*_from_bytes not available in gateway", self.name)
|
||||
return
|
||||
ext = Path(fname).suffix
|
||||
try:
|
||||
if mime.startswith("image/"):
|
||||
media_urls.append(cache_image_from_bytes(data, ext or ".png"))
|
||||
media_types.append(mime)
|
||||
elif mime.startswith("audio/"):
|
||||
media_urls.append(cache_audio_from_bytes(data, ext or ".ogg"))
|
||||
media_types.append(mime)
|
||||
else:
|
||||
media_urls.append(cache_document_from_bytes(data, fname))
|
||||
media_types.append(mime)
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] Failed to cache attachment %s: %s", self.name, fname, exc)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Message routing
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _route_message(
|
||||
self,
|
||||
text: str,
|
||||
@@ -911,8 +1022,22 @@ class ZulipAdapter(BasePlatformAdapter):
|
||||
chat_id_alt=str(reply_to_id) if reply_to_id else None,
|
||||
)
|
||||
|
||||
# Derive message type
|
||||
message_type = MessageType.TEXT
|
||||
# Gen 5: Extract media/attachments so vision tools can read them
|
||||
media_urls: List[str] = []
|
||||
media_types: List[str] = []
|
||||
await self._extract_inline_media(text, media_urls, media_types)
|
||||
await self._extract_event_attachments(raw, media_urls, media_types)
|
||||
|
||||
# Derive message type — upgrade to PHOTO/VOICe/DOCUMENT when media present
|
||||
if media_types:
|
||||
if any(m.startswith("image/") for m in media_types):
|
||||
message_type = MessageType.PHOTO
|
||||
elif any(m.startswith("audio/") for m in media_types):
|
||||
message_type = MessageType.VOICE
|
||||
else:
|
||||
message_type = MessageType.DOCUMENT
|
||||
else:
|
||||
message_type = MessageType.TEXT
|
||||
|
||||
message_event = MessageEvent(
|
||||
text=text,
|
||||
@@ -921,6 +1046,8 @@ class ZulipAdapter(BasePlatformAdapter):
|
||||
message_id=msg_id,
|
||||
raw_message=raw,
|
||||
timestamp=_parse_zulip_timestamp(timestamp),
|
||||
media_urls=media_urls or None,
|
||||
media_types=media_types or None,
|
||||
)
|
||||
|
||||
# Store reply metadata so send() can respond to the right thread
|
||||
|
||||
Reference in New Issue
Block a user