From 8d9ddd3aa05070042a69a97521b67ca50df1e035 Mon Sep 17 00:00:00 2001 From: renjianbo <263303411@qq.com> Date: Sun, 26 Jul 2026 23:48:29 +0800 Subject: [PATCH] feat: image attachments via multimodal messages for vision models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Backend: ChatRequest.attachments; image files read as base64 data URLs (8MB cap) and passed to run/run_stream as image_urls; last user message converted to multimodal parts only on the LLM-bound copy (history stays text); vision whitelist (kimi-k3, gpt-4o, claude, etc.), non-vision models keep text/OCR path - Frontend: send attachments metadata with chat body (also fixes attachment context never being sent — body now uses fullText); user bubble shows image thumbnails --- backend/app/agent_runtime/core.py | 34 ++++++++++++- backend/app/api/agent_chat.py | 85 +++++++++++++++++++++++++++++-- frontend/src/views/AgentChat.vue | 29 +++++++++-- 3 files changed, 136 insertions(+), 12 deletions(-) diff --git a/backend/app/agent_runtime/core.py b/backend/app/agent_runtime/core.py index a0a8d6d..923d4c4 100644 --- a/backend/app/agent_runtime/core.py +++ b/backend/app/agent_runtime/core.py @@ -175,6 +175,9 @@ class AgentRuntime: self._streamlined_transformer = create_streamlined_transformer(enabled=True) logger.info("工具结果流式美化已启用") + # 本轮用户消息附带的图片（多模态），run()/run_stream() 调用时注入 + self._turn_image_urls: List[str] = [] + # 系统提示词分层装配 (P2 — 参考 Claude Code systemPromptSections.ts) self._prompt_composer: Optional[PromptComposer] = None self._prompt_sections_enabled = self.config.prompt_sections.enabled @@ -369,12 +372,15 @@ class AgentRuntime: except Exception: pass # 日志记录失败不影响主流程 - async def run(self, user_input: str) -> AgentResult: + async def run(self, user_input: str, image_urls: Optional[List[str]] = None) -> AgentResult: """ 执行 Agent 单轮对话。 流程：加载记忆 → 追加用户消息 → ReAct 循环 → 保存记忆 → 返回结果。 + image_urls: 本轮用户消息附带的图片（data URL 或可访问的 http URL）， + 将以多模态 parts 形式挂到最后一条 user 消息上（不写入上下文历史）。 """ + self._turn_image_urls = image_urls or [] max_iter = max(1, self.config.llm.max_iterations) self.context.iteration = 0 self.context.tool_calls_made = 0 @@ -487,6 +493,7 @@ class AgentRuntime: # 裁剪过长历史 messages = self.memory.trim_messages(self.context.messages) + messages = self._with_image_parts(messages) # 预算检查：LLM 调用次数（在调用 LLM 之前检查，避免浪费额度） budget = self.config.budget @@ -806,13 +813,15 @@ class AgentRuntime: self._attach_token_usage(result) return result - async def run_stream(self, user_input: str) -> AsyncGenerator[dict, None]: + async def run_stream(self, user_input: str, image_urls: Optional[List[str]] = None) -> AsyncGenerator[dict, None]: """ 流式执行 Agent 单轮对话（支持 streamlined 模式）。 + image_urls: 本轮用户消息附带的图片（多模态 parts 挂到最后一条 user 消息）。 与 run() 逻辑相同，但在每个关键步骤 yield SSE 事件。 当 streamlined=True 时，工具调用会被折叠为累计摘要。 """ + self._turn_image_urls = image_urls or [] if self._streamlined_transformer: self._streamlined_transformer.reset() async for event in self._run_stream_impl(user_input): @@ -964,6 +973,7 @@ class AgentRuntime: ) messages = self.memory.trim_messages(self.context.messages) + messages = self._with_image_parts(messages) # 预算检查：LLM 调用次数（在调用 LLM 之前检查，避免浪费额度） budget = self.config.budget @@ -1361,6 +1371,26 @@ class AgentRuntime: "token_usage": token_usage_truncated, } + def _with_image_parts(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """若本轮带图片，将最后一条 user 消息改为多模态 parts（文本 + image_url）。 + + 只在发给 LLM 的 messages 副本上修改，不污染 context 历史（历史里仍是纯文本， + 避免图片 base64 反复进入后续轮次与持久化）。 + """ + if not self._turn_image_urls: + return messages + msgs = [dict(m) for m in messages] + for i in range(len(msgs) - 1, -1, -1): + if msgs[i].get("role") == "user": + content = msgs[i].get("content") + text = content if isinstance(content, str) else "" + parts: List[Dict[str, Any]] = [{"type": "text", "text": text}] + for url in self._turn_image_urls: + parts.append({"type": "image_url", "image_url": {"url": url}}) + msgs[i]["content"] = parts + break + return msgs + async def _compose_system_prompt(self, query: str = "") -> str: """使用分层装配构建完整系统提示词。 diff --git a/backend/app/api/agent_chat.py b/backend/app/api/agent_chat.py index bd66e72..e3aa01e 100644 --- a/backend/app/api/agent_chat.py +++ b/backend/app/api/agent_chat.py @@ -132,6 +132,65 @@ async def _sse_stream(gen: AsyncGenerator[dict, None]) -> AsyncGenerator[str, No yield f"event: {event_type}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n" +# 支持图像理解的模型名特征（小写包含即视为支持） +_VISION_MODEL_HINTS = ( + "vision", "gpt-4o", "claude", "kimi-k3", "kimi-latest", + "qwen-vl", "glm-4v", "gemini", "pixtral", +) + +# 单张图片进入多模态的大小上限（base64 前） +_IMAGE_MAX_BYTES = 8 * 1024 * 1024 + +_IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"} + + +def _model_supports_vision(model: str) -> bool: + m = (model or "").lower() + return any(h in m for h in _VISION_MODEL_HINTS) + + +def _image_data_urls(attachments: Optional[List["ChatAttachment"]], model: str) -> List[str]: + """将图片附件读为 base64 data URL（仅当模型支持视觉时）。 + + 非图片附件或不支持视觉的模型返回空列表，调用方仍走文本路径（file_read/OCR）。 + """ + if not attachments or not _model_supports_vision(model): + return [] + import base64 + from app.services.builtin_tools import _resolve_path_under_workspace + + urls: List[str] = [] + for att in attachments: + ct = (att.content_type or "").lower() + ext = (att.filename or att.relative_path).rsplit(".", 1)[-1].lower() if "." in (att.filename or att.relative_path) else "" + if not (att.is_image or ct.startswith("image/") or ext in _IMAGE_EXTS): + continue + path, err = _resolve_path_under_workspace(att.relative_path) + if err or path is None or not path.is_file(): + logger.warning("图片附件路径无效: %s (%s)", att.relative_path, err) + continue + try: + size = path.stat().st_size + if size <= 0 or size > _IMAGE_MAX_BYTES: + logger.warning("图片附件大小超限，跳过: %s (%d bytes)", att.relative_path, size) + continue + raw = path.read_bytes() + except OSError as e: + logger.warning("读取图片附件失败: %s (%s)", att.relative_path, e) + continue + mime = ct if ct.startswith("image/") else f"image/{'jpeg' if ext in ('jpg', 'jpeg') else (ext or 'png')}" + urls.append(f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}") + return urls + + +class ChatAttachment(BaseModel): + """聊天附件（图片可走多模态）""" + relative_path: str + filename: Optional[str] = None + content_type: Optional[str] = None + is_image: bool = False + + class ChatRequest(BaseModel): message: str session_id: Optional[str] = None @@ -143,6 +202,7 @@ class ChatRequest(BaseModel): streamlined: bool = Field(default=False, description="启用工具结果流式美化") prompt_sections_enabled: bool = Field(default=True, description="启用系统提示词分层装配") system_prompt_override: Optional[str] = Field(default=None, description="覆盖 Agent 的 System Prompt") + attachments: Optional[List[ChatAttachment]] = Field(default=None, description="消息附件（图片将走多模态）") class ChatResponse(BaseModel): @@ -392,7 +452,7 @@ async def chat_bare( on_message = _make_message_saver(db, agent_id=None, user_id=current_user.id, workspace_id=workspace_id) context = AgentContext(session_id=req.session_id) runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined) - result = await runtime.run(req.message) + result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", ""))) # 流式美化：为 steps 生成累计摘要 streamlined_summary = None @@ -462,7 +522,7 @@ async def chat_bare_stream( context = AgentContext(session_id=req.session_id) runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined) return StreamingResponse( - _sse_stream(runtime.run_stream(req.message)), + _sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -550,7 +610,7 @@ async def chat_with_agent( on_message = _make_message_saver(db, agent_id=agent_id, user_id=current_user.id, workspace_id=workspace_id) context = AgentContext(session_id=req.session_id) runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined) - result = await runtime.run(req.message) + result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", ""))) # 流式美化：为 steps 生成累计摘要 streamlined_summary = None @@ -650,7 +710,7 @@ async def chat_with_agent_stream( context = AgentContext(session_id=req.session_id) runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined) return StreamingResponse( - _sse_stream(runtime.run_stream(req.message)), + _sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -924,10 +984,11 @@ def _resolve_model_config(db: Session, model_config_id: Optional[str], user_id: ).first() if not mc: raise HTTPException(status_code=404, detail="模型配置不存在") + from app.services.encryption_service import EncryptionService return { "provider": mc.provider, "model": mc.model_name, - "api_key": mc.api_key, + "api_key": EncryptionService.decrypt(mc.api_key), "base_url": mc.base_url, } @@ -944,6 +1005,9 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]] llm_kwargs["api_key"] = mc["api_key"] if mc.get("base_url"): llm_kwargs["base_url"] = mc["base_url"] + # Kimi K3 等推理模型强制要求 temperature=1 + if mc.get("provider") == "moonshot": + llm_kwargs["temperature"] = 1.0 return llm_kwargs """从工作流节点列表中查找第一个 agent 类型或 llm 类型的节点配置。""" if not nodes: @@ -955,6 +1019,17 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]] return {} +def _find_agent_node_config(nodes: list) -> Dict[str, Any]: + """从工作流节点列表中提取 Agent/LLM 节点的配置。""" + if not nodes: + return {} + for node in nodes: + typ = node.get("type", "") + if typ in ("agent", "llm", "template"): + return node.get("data") or {} + return {} + + def _build_memory_config_from_node(agent_node_cfg: dict) -> AgentMemoryConfig: """从 Agent 工作流节点配置中提取记忆配置。""" from app.core.compaction_config import CompactionConfig diff --git a/frontend/src/views/AgentChat.vue b/frontend/src/views/AgentChat.vue index 082cb45..ec589e5 100644 --- a/frontend/src/views/AgentChat.vue +++ b/frontend/src/views/AgentChat.vue @@ -200,6 +200,7 @@ :iterations="item.msg.iterations" :tool-calls-made="item.msg.tool_calls_made" :token-usage="item.msg.token_usage" + :attachments="item.msg.attachments" >