feat: image attachments via multimodal messages for vision models
- Backend: ChatRequest.attachments; image files read as base64 data URLs (8MB cap) and passed to run/run_stream as image_urls; last user message converted to multimodal parts only on the LLM-bound copy (history stays text); vision whitelist (kimi-k3, gpt-4o, claude, etc.), non-vision models keep text/OCR path - Frontend: send attachments metadata with chat body (also fixes attachment context never being sent — body now uses fullText); user bubble shows image thumbnails
This commit is contained in:
@@ -132,6 +132,65 @@ async def _sse_stream(gen: AsyncGenerator[dict, None]) -> AsyncGenerator[str, No
|
||||
yield f"event: {event_type}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
|
||||
|
||||
# 支持图像理解的模型名特征(小写包含即视为支持)
|
||||
_VISION_MODEL_HINTS = (
|
||||
"vision", "gpt-4o", "claude", "kimi-k3", "kimi-latest",
|
||||
"qwen-vl", "glm-4v", "gemini", "pixtral",
|
||||
)
|
||||
|
||||
# 单张图片进入多模态的大小上限(base64 前)
|
||||
_IMAGE_MAX_BYTES = 8 * 1024 * 1024
|
||||
|
||||
_IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"}
|
||||
|
||||
|
||||
def _model_supports_vision(model: str) -> bool:
|
||||
m = (model or "").lower()
|
||||
return any(h in m for h in _VISION_MODEL_HINTS)
|
||||
|
||||
|
||||
def _image_data_urls(attachments: Optional[List["ChatAttachment"]], model: str) -> List[str]:
|
||||
"""将图片附件读为 base64 data URL(仅当模型支持视觉时)。
|
||||
|
||||
非图片附件或不支持视觉的模型返回空列表,调用方仍走文本路径(file_read/OCR)。
|
||||
"""
|
||||
if not attachments or not _model_supports_vision(model):
|
||||
return []
|
||||
import base64
|
||||
from app.services.builtin_tools import _resolve_path_under_workspace
|
||||
|
||||
urls: List[str] = []
|
||||
for att in attachments:
|
||||
ct = (att.content_type or "").lower()
|
||||
ext = (att.filename or att.relative_path).rsplit(".", 1)[-1].lower() if "." in (att.filename or att.relative_path) else ""
|
||||
if not (att.is_image or ct.startswith("image/") or ext in _IMAGE_EXTS):
|
||||
continue
|
||||
path, err = _resolve_path_under_workspace(att.relative_path)
|
||||
if err or path is None or not path.is_file():
|
||||
logger.warning("图片附件路径无效: %s (%s)", att.relative_path, err)
|
||||
continue
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
if size <= 0 or size > _IMAGE_MAX_BYTES:
|
||||
logger.warning("图片附件大小超限,跳过: %s (%d bytes)", att.relative_path, size)
|
||||
continue
|
||||
raw = path.read_bytes()
|
||||
except OSError as e:
|
||||
logger.warning("读取图片附件失败: %s (%s)", att.relative_path, e)
|
||||
continue
|
||||
mime = ct if ct.startswith("image/") else f"image/{'jpeg' if ext in ('jpg', 'jpeg') else (ext or 'png')}"
|
||||
urls.append(f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}")
|
||||
return urls
|
||||
|
||||
|
||||
class ChatAttachment(BaseModel):
|
||||
"""聊天附件(图片可走多模态)"""
|
||||
relative_path: str
|
||||
filename: Optional[str] = None
|
||||
content_type: Optional[str] = None
|
||||
is_image: bool = False
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
message: str
|
||||
session_id: Optional[str] = None
|
||||
@@ -143,6 +202,7 @@ class ChatRequest(BaseModel):
|
||||
streamlined: bool = Field(default=False, description="启用工具结果流式美化")
|
||||
prompt_sections_enabled: bool = Field(default=True, description="启用系统提示词分层装配")
|
||||
system_prompt_override: Optional[str] = Field(default=None, description="覆盖 Agent 的 System Prompt")
|
||||
attachments: Optional[List[ChatAttachment]] = Field(default=None, description="消息附件(图片将走多模态)")
|
||||
|
||||
|
||||
class ChatResponse(BaseModel):
|
||||
@@ -392,7 +452,7 @@ async def chat_bare(
|
||||
on_message = _make_message_saver(db, agent_id=None, user_id=current_user.id, workspace_id=workspace_id)
|
||||
context = AgentContext(session_id=req.session_id)
|
||||
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
|
||||
result = await runtime.run(req.message)
|
||||
result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))
|
||||
|
||||
# 流式美化:为 steps 生成累计摘要
|
||||
streamlined_summary = None
|
||||
@@ -462,7 +522,7 @@ async def chat_bare_stream(
|
||||
context = AgentContext(session_id=req.session_id)
|
||||
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
|
||||
return StreamingResponse(
|
||||
_sse_stream(runtime.run_stream(req.message)),
|
||||
_sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
@@ -550,7 +610,7 @@ async def chat_with_agent(
|
||||
on_message = _make_message_saver(db, agent_id=agent_id, user_id=current_user.id, workspace_id=workspace_id)
|
||||
context = AgentContext(session_id=req.session_id)
|
||||
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
|
||||
result = await runtime.run(req.message)
|
||||
result = await runtime.run(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))
|
||||
|
||||
# 流式美化:为 steps 生成累计摘要
|
||||
streamlined_summary = None
|
||||
@@ -650,7 +710,7 @@ async def chat_with_agent_stream(
|
||||
context = AgentContext(session_id=req.session_id)
|
||||
runtime = AgentRuntime(config=config, context=context, on_llm_call=on_llm_call, on_message=on_message, streamlined=req.streamlined)
|
||||
return StreamingResponse(
|
||||
_sse_stream(runtime.run_stream(req.message)),
|
||||
_sse_stream(runtime.run_stream(req.message, image_urls=_image_data_urls(req.attachments, llm_kwargs.get("model", "")))),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
@@ -924,10 +984,11 @@ def _resolve_model_config(db: Session, model_config_id: Optional[str], user_id:
|
||||
).first()
|
||||
if not mc:
|
||||
raise HTTPException(status_code=404, detail="模型配置不存在")
|
||||
from app.services.encryption_service import EncryptionService
|
||||
return {
|
||||
"provider": mc.provider,
|
||||
"model": mc.model_name,
|
||||
"api_key": mc.api_key,
|
||||
"api_key": EncryptionService.decrypt(mc.api_key),
|
||||
"base_url": mc.base_url,
|
||||
}
|
||||
|
||||
@@ -944,6 +1005,9 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]]
|
||||
llm_kwargs["api_key"] = mc["api_key"]
|
||||
if mc.get("base_url"):
|
||||
llm_kwargs["base_url"] = mc["base_url"]
|
||||
# Kimi K3 等推理模型强制要求 temperature=1
|
||||
if mc.get("provider") == "moonshot":
|
||||
llm_kwargs["temperature"] = 1.0
|
||||
return llm_kwargs
|
||||
"""从工作流节点列表中查找第一个 agent 类型或 llm 类型的节点配置。"""
|
||||
if not nodes:
|
||||
@@ -955,6 +1019,17 @@ def _apply_model_config(llm_kwargs: Dict[str, Any], mc: Optional[Dict[str, Any]]
|
||||
return {}
|
||||
|
||||
|
||||
def _find_agent_node_config(nodes: list) -> Dict[str, Any]:
|
||||
"""从工作流节点列表中提取 Agent/LLM 节点的配置。"""
|
||||
if not nodes:
|
||||
return {}
|
||||
for node in nodes:
|
||||
typ = node.get("type", "")
|
||||
if typ in ("agent", "llm", "template"):
|
||||
return node.get("data") or {}
|
||||
return {}
|
||||
|
||||
|
||||
def _build_memory_config_from_node(agent_node_cfg: dict) -> AgentMemoryConfig:
|
||||
"""从 Agent 工作流节点配置中提取记忆配置。"""
|
||||
from app.core.compaction_config import CompactionConfig
|
||||
|
||||
Reference in New Issue
Block a user