feat: image attachments via multimodal messages for vision models

- Backend: ChatRequest.attachments; image files read as base64 data URLs (8MB cap) and passed to run/run_stream as image_urls; last user message converted to multimodal parts only on the LLM-bound copy (history stays text); vision whitelist (kimi-k3, gpt-4o, claude, etc.), non-vision models keep text/OCR path
- Frontend: send attachments metadata with chat body (also fixes attachment context never being sent — body now uses fullText); user bubble shows image thumbnails
This commit is contained in:
2026-07-26 23:48:29 +08:00
parent fe37f8be5c
commit 8d9ddd3aa0
3 changed files with 136 additions and 12 deletions

View File

@@ -132,6 +132,65 @@ async def _sse_stream(gen: AsyncGenerator[dict, None]) -> AsyncGenerator[str, No
yield f"event: {event_type}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
# 支持图像理解的模型名特征(小写包含即视为支持)
_VISION_MODEL_HINTS = (
"vision", "gpt-4o", "claude", "kimi-k3", "kimi-latest",
"qwen-vl", "glm-4v", "gemini", "pixtral",
)
# 单张图片进入多模态的大小上限(base64 前)
_IMAGE_MAX_BYTES = 8 * 1024 * 1024
_IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"}
def _model_supports_vision(model: str) -> bool:
m = (model or "").lower()
return any(h in m for h in _VISION_MODEL_HINTS)
def _image_data_urls(attachments: Optional[List["ChatAttachment"]], model: str) -> List[str]:
"""将图片附件读为 base64 data URL(仅当模型支持视觉时)。
非图片附件或不支持视觉的模型返回空列表,调用方仍走文本路径(file_read/OCR)。
"""
if not attachments or not _model_supports_vision(model):
return []
import base64
from app.services.builtin_tools import _resolve_path_under_workspace
urls: List[str] = []
for att in attachments:
ct = (att.content_type or "").lower()
ext = (att.filename or att.relative_path).rsplit(".", 1)[-1].lower() if "." in (att.filename or att.relative_path) else ""
if not (att.is_image or ct.startswith("image/") or ext in _IMAGE_EXTS):
continue
path, err = _resolve_path_under_workspace(att.relative_path)
if err or path is None or not path.is_file():
logger.warning("图片附件路径无效: %s (%s)", att.relative_path, err)
continue
try:
size = path.stat().st_size
if size <= 0 or size > _IMAGE_MAX_BYTES:
logger.warning("图片附件大小超限,跳过: %s (%d bytes)", att.relative_path, size)
continue
raw = path.read_bytes()
except OSError as e:
logger.warning("读取图片附件失败: %s (%s)", att.relative_path, e)
continue
mime = ct if ct.startswith("image/") else f"image/{'jpeg' if ext in ('jpg', 'jpeg') else (ext or 'png')}"
urls.append(f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}")
return urls
class ChatAttachment(BaseModel):
"""聊天附件(图片可走多模态)"""
relative_path: str
filename: Optional[str] = None
content_type: Optional[str] = None
is_image: bool = False
class ChatRequest(BaseModel):
message: str
session_id: Optional[str] = None
@@ -143,6 +202,7 @@ class ChatRequest(BaseModel):
streamlined: bool = Field(default=False, description="启用工具结果流式美化")
prompt_sections_enabled: bool = Field(default=True, description="启用系统提示词分层装配")
system_prompt_override: Optional[str] = Field(default=None, description="覆盖 Agent 的 System Prompt")
attachments: Optional[List[ChatAttachment]] = Field(default=None, description="消息附件(图片将走多模态)")
class ChatResponse(BaseModel):
@@ -392,7 +452,7 @@ async def chat_bare(
on_message = _make_message_saver(db, agent_id=None, user_id=current_user.id, workspace_id=workspace_id)
context = AgentContext(session_id=req.session_id)
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
result = await runtime.run(req.message)
result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))
# 流式美化:为 steps 生成累计摘要
streamlined_summary = None
@@ -462,7 +522,7 @@ async def chat_bare_stream(
context = AgentContext(session_id=req.session_id)
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
return StreamingResponse(
_sse_stream(runtime.run_stream(req.message)),
_sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
@@ -550,7 +610,7 @@ async def chat_with_agent(
on_message = _make_message_saver(db, agent_id=agent_id, user_id=current_user.id, workspace_id=workspace_id)
context = AgentContext(session_id=req.session_id)
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
result = await runtime.run(req.message)
result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))
# 流式美化:为 steps 生成累计摘要
streamlined_summary = None
@@ -650,7 +710,7 @@ async def chat_with_agent_stream(
context = AgentContext(session_id=req.session_id)
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
return StreamingResponse(
_sse_stream(runtime.run_stream(req.message)),
_sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
@@ -924,10 +984,11 @@ def _resolve_model_config(db: Session, model_config_id: Optional[str], user_id:
).first()
if not mc:
raise HTTPException(status_code=404, detail="模型配置不存在")
from app.services.encryption_service import EncryptionService
return {
"provider": mc.provider,
"model": mc.model_name,
"api_key": mc.api_key,
"api_key": EncryptionService.decrypt(mc.api_key),
"base_url": mc.base_url,
}
@@ -944,6 +1005,9 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]]
llm_kwargs["api_key"] = mc["api_key"]
if mc.get("base_url"):
llm_kwargs["base_url"] = mc["base_url"]
# Kimi K3 等推理模型强制要求 temperature=1
if mc.get("provider") == "moonshot":
llm_kwargs["temperature"] = 1.0
return llm_kwargs
"""从工作流节点列表中查找第一个 agent 类型或 llm 类型的节点配置。"""
if not nodes:
@@ -955,6 +1019,17 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]]
return {}
def _find_agent_node_config(nodes: list) -> Dict[str, Any]:
"""从工作流节点列表中提取 Agent/LLM 节点的配置。"""
if not nodes:
return {}
for node in nodes:
typ = node.get("type", "")
if typ in ("agent", "llm", "template"):
return node.get("data") or {}
return {}
def _build_memory_config_from_node(agent_node_cfg: dict) -> AgentMemoryConfig:
"""从 Agent 工作流节点配置中提取记忆配置。"""
from app.core.compaction_config import CompactionConfig