feat: 工作流记忆与内置工具、知你客服脚本、Agent管理技能展示与能力配置、文档与Windows启动脚本;忽略 redis_temp 二进制目录

Made-with: Cursor
This commit is contained in:
renjianbo
2026-04-08 11:44:24 +08:00
parent 599b8f2851
commit bd3f8be781
66 changed files with 10104 additions and 469 deletions

View File

@@ -1,9 +1,15 @@
"""
应用配置
"""
from pathlib import Path
from pydantic_settings import BaseSettings
from typing import List
# 无论从项目根还是 backend 目录启动,始终加载 backend/.env
_BACKEND_DIR = Path(__file__).resolve().parent.parent.parent
_ENV_PATH = _BACKEND_DIR / ".env"
class Settings(BaseSettings):
"""应用设置"""
@@ -19,6 +25,14 @@ class Settings(BaseSettings):
# Redis配置
REDIS_URL: str = "redis://localhost:6379/0"
# 会话记忆:除 Redis 外是否写入 MySQL(persistent_user_memories),实现跨重启、跨 TTL 的永久记忆
MEMORY_PERSIST_DB_ENABLED: bool = True
# 本地文件工具(file_read / file_write):允许读写的根目录。空字符串表示使用「backend 的上一级目录」作为仓库根。
LOCAL_FILE_TOOLS_ROOT: str = ""
LOCAL_FILE_READ_MAX_BYTES: int = 2_097_152 # 单次读取上限(默认 2MB)
LOCAL_FILE_WRITE_MAX_BYTES: int = 2_097_152 # 单次写入内容上限(UTF-8 字节)
# CORS配置(支持字符串或列表)
CORS_ORIGINS: str = "http://localhost:3000,http://127.0.0.1:3000,http://localhost:8038,http://101.43.95.130:8038"
@@ -40,7 +54,7 @@ class Settings(BaseSettings):
JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
class Config:
env_file = ".env"
env_file = str(_ENV_PATH)
case_sensitive = True

View File

@@ -0,0 +1,72 @@
"""确保内置工具注册到 tool_registry(API 进程与 Celery Worker 均会导入执行流)。"""
from __future__ import annotations
import logging
import os
logger = logging.getLogger(__name__)
_registered = False
_EXPECTED_BUILTIN = 10
def ensure_builtin_tools_registered() -> None:
"""幂等:注册 file_write / system_info 等内置工具,供工作流 LLM 节点使用。"""
global _registered
if _registered:
return
from app.services.tool_registry import tool_registry
from app.services.builtin_tools import (
http_request_tool,
file_read_tool,
file_write_tool,
text_analyze_tool,
datetime_tool,
math_calculate_tool,
system_info_tool,
json_process_tool,
database_query_tool,
adb_log_tool,
HTTP_REQUEST_SCHEMA,
FILE_READ_SCHEMA,
FILE_WRITE_SCHEMA,
TEXT_ANALYZE_SCHEMA,
DATETIME_SCHEMA,
MATH_CALCULATE_SCHEMA,
SYSTEM_INFO_SCHEMA,
JSON_PROCESS_SCHEMA,
DATABASE_QUERY_SCHEMA,
ADB_LOG_SCHEMA,
)
tool_registry.register_builtin_tool("http_request", http_request_tool, HTTP_REQUEST_SCHEMA)
tool_registry.register_builtin_tool("file_read", file_read_tool, FILE_READ_SCHEMA)
tool_registry.register_builtin_tool("file_write", file_write_tool, FILE_WRITE_SCHEMA)
tool_registry.register_builtin_tool("text_analyze", text_analyze_tool, TEXT_ANALYZE_SCHEMA)
tool_registry.register_builtin_tool("datetime", datetime_tool, DATETIME_SCHEMA)
tool_registry.register_builtin_tool("math_calculate", math_calculate_tool, MATH_CALCULATE_SCHEMA)
tool_registry.register_builtin_tool("system_info", system_info_tool, SYSTEM_INFO_SCHEMA)
tool_registry.register_builtin_tool("json_process", json_process_tool, JSON_PROCESS_SCHEMA)
tool_registry.register_builtin_tool("database_query", database_query_tool, DATABASE_QUERY_SCHEMA)
tool_registry.register_builtin_tool("adb_log", adb_log_tool, ADB_LOG_SCHEMA)
_registered = True
n = tool_registry.builtin_tool_count()
names = tool_registry.builtin_tool_names()
pid = os.getpid()
if n < _EXPECTED_BUILTIN:
logger.warning(
"内置工具注册数量异常: pid=%s count=%s 期望>=%s names=%s(LLM 工具调用可能失效)",
pid,
n,
_EXPECTED_BUILTIN,
names,
)
else:
logger.info(
"内置工具就绪 pid=%s count=%s names=%s(Celery Worker 若缺此项日志,说明未加载 workflow_tasks / 未执行 bootstrap)",
pid,
n,
names,
)

View File

@@ -149,8 +149,32 @@ async def root():
@app.get("/health")
async def health_check():
"""健康检查"""
return {"status": "healthy"}
"""健康检查(含内置工具是否已注册,便于排查「可动手 Agent」)。"""
from app.services.tool_registry import tool_registry
names = tool_registry.builtin_tool_names()
count = tool_registry.builtin_tool_count()
file_agent_core = {"file_write", "file_read", "system_info"}
subset_ok = file_agent_core.issubset(set(names))
expected_ok = count >= 10
tools_ready = expected_ok and subset_ok
return {
"status": "healthy",
"checks": {
"builtin_tools_ready": tools_ready,
"builtin_tools_count_ok": expected_ok,
"file_agent_core_ready": subset_ok,
},
"builtin_tools": {
"count": count,
"names": names,
"expected_min_count": 10,
},
"notes": {
"celery": "工作流/Agent 执行通常在 Celery Worker 中跑;Worker 日志中也应出现「内置工具就绪」且 count 应与 API 一致。若仅 API 有工具而 Worker 无,会出现 LLM 无法真正调用 file_write。",
},
}
# 应用启动时初始化数据库和工具
@app.on_event("startup")
@@ -164,44 +188,14 @@ async def startup_event():
logger.error(f"数据库初始化失败: {e}")
# 不抛出异常,允许应用继续启动
# 注册内置工具
# 注册内置工具(与 Celery Worker 共用 app.core.tools_bootstrap)
try:
from app.core.tools_bootstrap import ensure_builtin_tools_registered
ensure_builtin_tools_registered()
from app.services.tool_registry import tool_registry
from app.services.builtin_tools import (
http_request_tool,
file_read_tool,
file_write_tool,
text_analyze_tool,
datetime_tool,
math_calculate_tool,
system_info_tool,
json_process_tool,
database_query_tool,
adb_log_tool,
HTTP_REQUEST_SCHEMA,
FILE_READ_SCHEMA,
FILE_WRITE_SCHEMA,
TEXT_ANALYZE_SCHEMA,
DATETIME_SCHEMA,
MATH_CALCULATE_SCHEMA,
SYSTEM_INFO_SCHEMA,
JSON_PROCESS_SCHEMA,
DATABASE_QUERY_SCHEMA,
ADB_LOG_SCHEMA
)
tool_registry.register_builtin_tool("http_request", http_request_tool, HTTP_REQUEST_SCHEMA)
tool_registry.register_builtin_tool("file_read", file_read_tool, FILE_READ_SCHEMA)
tool_registry.register_builtin_tool("file_write", file_write_tool, FILE_WRITE_SCHEMA)
tool_registry.register_builtin_tool("text_analyze", text_analyze_tool, TEXT_ANALYZE_SCHEMA)
tool_registry.register_builtin_tool("datetime", datetime_tool, DATETIME_SCHEMA)
tool_registry.register_builtin_tool("math_calculate", math_calculate_tool, MATH_CALCULATE_SCHEMA)
tool_registry.register_builtin_tool("system_info", system_info_tool, SYSTEM_INFO_SCHEMA)
tool_registry.register_builtin_tool("json_process", json_process_tool, JSON_PROCESS_SCHEMA)
tool_registry.register_builtin_tool("database_query", database_query_tool, DATABASE_QUERY_SCHEMA)
tool_registry.register_builtin_tool("adb_log", adb_log_tool, ADB_LOG_SCHEMA)
logger.info("内置工具注册完成(共10个工具)")
logger.info("内置工具注册完成(count=%s)", tool_registry.builtin_tool_count())
except Exception as e:
logger.error(f"内置工具注册失败: {e}")
# 不抛出异常,允许应用继续启动

View File

@@ -11,5 +11,6 @@ from app.models.workflow_template import WorkflowTemplate, TemplateRating, Templ
from app.models.node_template import NodeTemplate
from app.models.permission import Role, Permission, WorkflowPermission, AgentPermission
from app.models.alert_rule import AlertRule, AlertLog
from app.models.persistent_user_memory import PersistentUserMemory
__all__ = ["User", "Workflow", "WorkflowVersion", "Agent", "Execution", "ExecutionLog", "ModelConfig", "DataSource", "WorkflowTemplate", "TemplateRating", "TemplateFavorite", "NodeTemplate", "Role", "Permission", "WorkflowPermission", "AgentPermission", "AlertRule", "AlertLog"]
__all__ = ["User", "Workflow", "WorkflowVersion", "Agent", "Execution", "ExecutionLog", "ModelConfig", "DataSource", "WorkflowTemplate", "TemplateRating", "TemplateFavorite", "NodeTemplate", "Role", "Permission", "WorkflowPermission", "AgentPermission", "AlertRule", "AlertLog", "PersistentUserMemory"]

View File

@@ -0,0 +1,34 @@
"""
按 Agent/工作流 + 会话键持久化的用户记忆(MySQL),与 Redis 热缓存配合实现长期记忆。
"""
import uuid
from sqlalchemy import Column, DateTime, String, UniqueConstraint, func
from sqlalchemy.dialects.mysql import CHAR, JSON
from app.core.database import Base
class PersistentUserMemory(Base):
"""会话级记忆快照:同一 scope 下 session_key 唯一。"""
__tablename__ = "persistent_user_memories"
__table_args__ = (
UniqueConstraint("scope_kind", "scope_id", "session_key", name="uq_persistent_mem_scope_session"),
)
id = Column(CHAR(36), primary_key=True, default=lambda: str(uuid.uuid4()), comment="主键")
scope_kind = Column(String(16), nullable=False, comment="agent 或 workflow")
scope_id = Column(CHAR(36), nullable=False, comment="Agent ID 或 Workflow ID")
session_key = Column(String(512), nullable=False, comment="调用方传入的 user_id 等会话键")
payload = Column(JSON, nullable=False, comment="与 Redis 中 user_memory_* 结构一致的记忆 JSON")
updated_at = Column(
DateTime,
server_default=func.now(),
onupdate=func.now(),
nullable=True,
comment="更新时间",
)
def __repr__(self):
return f"<PersistentUserMemory(scope={self.scope_kind}:{self.scope_id}, session={self.session_key[:32]})>"

View File

@@ -2,6 +2,7 @@
内置工具实现
"""
from typing import Dict, Any, Optional, List, Tuple
from pathlib import Path
import httpx
import json
import os
@@ -20,6 +21,51 @@ from app.core.config import settings
logger = logging.getLogger(__name__)
def _local_file_workspace_root() -> Path:
"""file_read/file_write 允许操作的根目录(解析后路径)。"""
raw = (getattr(settings, "LOCAL_FILE_TOOLS_ROOT", None) or "").strip()
if raw:
return Path(raw).expanduser().resolve()
# backend/app/services/builtin_tools.py -> 上级 x4 = 仓库根(与历史上 join(services_dir, '../../..') 一致)
return Path(__file__).resolve().parent.parent.parent.parent
def _sanitize_tool_path_string(file_path: str) -> str:
"""
去掉 LLM/DSML/Markdown 泄漏到路径末尾的非法字符(如 123.md<|、引号),避免 WinError 123。
"""
if not isinstance(file_path, str):
return str(file_path)
s = file_path.replace("\ufeff", "").strip()
s = s.strip('"').strip("'")
# 行尾非法:Windows 文件名不能含 <>:"|?*;模型常把半角/全角尖括号粘在扩展名后
tail_bad = '<><>"|?*:"|*'
while s and s[-1] in tail_bad:
s = s[:-1].rstrip()
# 若仍含未配对尖括号片段在行尾(如 xxx.md<)
s = re.sub(r'[<<][||]?\s*$', "", s).rstrip()
return s
def _resolve_path_under_workspace(file_path: str) -> Tuple[Optional[Path], Optional[str]]:
"""将用户给定路径解析为绝对路径,且必须位于工作区内。"""
file_path = _sanitize_tool_path_string(file_path)
root = _local_file_workspace_root()
try:
p = Path(file_path).expanduser()
if not p.is_absolute():
p = (root / p).resolve()
else:
p = p.resolve()
except (OSError, ValueError) as e:
return None, str(e)
try:
p.relative_to(root)
except ValueError:
return None, f"不允许访问工作区外路径,允许根目录: {root}"
return p, None
async def http_request_tool(
url: str,
method: str = "GET",
@@ -82,33 +128,32 @@ async def file_read_tool(file_path: str) -> str:
文件内容或错误信息
"""
try:
# 安全检查:限制可读取的文件路径
# 只允许读取项目目录下的文件
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
abs_path = os.path.abspath(file_path)
if not abs_path.startswith(project_root):
return json.dumps({
"error": f"不允许读取项目目录外的文件: {file_path}"
}, ensure_ascii=False)
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
return json.dumps({
"file_path": file_path,
"content": content,
"size": len(content)
}, ensure_ascii=False)
max_bytes = int(getattr(settings, "LOCAL_FILE_READ_MAX_BYTES", 2_097_152) or 2_097_152)
path, err = _resolve_path_under_workspace(file_path)
if err:
return json.dumps({"error": err}, ensure_ascii=False)
if not path.is_file():
return json.dumps({"error": f"文件不存在或不是普通文件: {path}"}, ensure_ascii=False)
fsize = path.stat().st_size
if fsize > max_bytes:
return json.dumps(
{"error": f"文件过大({fsize} 字节),上限 {max_bytes}"},
ensure_ascii=False,
)
content = path.read_text(encoding="utf-8", errors="replace")
return json.dumps(
{
"file_path": str(path),
"content": content,
"size": len(content.encode("utf-8")),
},
ensure_ascii=False,
)
except FileNotFoundError:
return json.dumps({
"error": f"文件不存在: {file_path}"
}, ensure_ascii=False)
return json.dumps({"error": f"文件不存在: {file_path}"}, ensure_ascii=False)
except Exception as e:
logger.error(f"文件读取工具执行失败: {str(e)}")
return json.dumps({
"error": str(e)
}, ensure_ascii=False)
return json.dumps({"error": str(e)}, ensure_ascii=False)
async def file_write_tool(file_path: str, content: str, mode: str = "w") -> str:
@@ -124,30 +169,36 @@ async def file_write_tool(file_path: str, content: str, mode: str = "w") -> str:
写入结果
"""
try:
# 安全检查:限制可写入的文件路径
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
abs_path = os.path.abspath(file_path)
if not abs_path.startswith(project_root):
return json.dumps({
"error": f"不允许写入项目目录外的文件: {file_path}"
}, ensure_ascii=False)
max_bytes = int(getattr(settings, "LOCAL_FILE_WRITE_MAX_BYTES", 2_097_152) or 2_097_152)
raw = content if isinstance(content, str) else str(content)
enc_len = len(raw.encode("utf-8"))
if enc_len > max_bytes:
return json.dumps(
{"error": f"写入内容过大({enc_len} 字节 UTF-8),上限 {max_bytes}"},
ensure_ascii=False,
)
path, err = _resolve_path_under_workspace(file_path)
if err:
return json.dumps({"error": err}, ensure_ascii=False)
path.parent.mkdir(parents=True, exist_ok=True)
write_mode = "a" if mode == "a" else "w"
with open(file_path, write_mode, encoding='utf-8') as f:
f.write(content)
return json.dumps({
"success": True,
"file_path": file_path,
"mode": write_mode,
"content_length": len(content)
}, ensure_ascii=False)
if write_mode == "a":
with path.open("a", encoding="utf-8") as f:
f.write(raw)
else:
path.write_text(raw, encoding="utf-8")
return json.dumps(
{
"success": True,
"file_path": str(path),
"mode": write_mode,
"content_length": enc_len,
},
ensure_ascii=False,
)
except Exception as e:
logger.error(f"文件写入工具执行失败: {str(e)}")
return json.dumps({
"error": str(e)
}, ensure_ascii=False)
return json.dumps({"error": str(e)}, ensure_ascii=False)
async def text_analyze_tool(text: str, operation: str = "count") -> str:
@@ -321,8 +372,10 @@ async def system_info_tool() -> str:
"python_version_info": {
"major": sys.version_info.major,
"minor": sys.version_info.minor,
"micro": sys.version_info.micro
}
"micro": sys.version_info.micro,
},
# file_read/file_write 使用的允许根目录(与 LOCAL_FILE_TOOLS_ROOT 一致)
"local_file_workspace_root": str(_local_file_workspace_root()),
}
return json.dumps(info, ensure_ascii=False)
@@ -760,7 +813,7 @@ FILE_READ_SCHEMA = {
"type": "function",
"function": {
"name": "file_read",
"description": "读取文件内容。只能读取项目目录下的文件,确保文件路径正确。",
"description": "读取本地文本文件(UTF-8)。路径须落在平台配置的工作区内(默认仓库根;可通过环境变量 LOCAL_FILE_TOOLS_ROOT 修改)。超过大小上限会拒绝。",
"parameters": {
"type": "object",
"properties": {
@@ -778,7 +831,7 @@ FILE_WRITE_SCHEMA = {
"type": "function",
"function": {
"name": "file_write",
"description": "写入文件内容。只能写入项目目录下的文件,确保文件路径正确。",
"description": "写入本地文本文件(UTF-8,w 覆盖 / a 追加)。路径须落在工作区内;父目录不存在会自动创建。勿写入密码、密钥等大型敏感内容。",
"parameters": {
"type": "object",
"properties": {
@@ -872,7 +925,7 @@ SYSTEM_INFO_SCHEMA = {
"type": "function",
"function": {
"name": "system_info",
"description": "获取系统信息,包括操作系统、Python版本等。",
"description": "获取系统信息(含 local_file_workspace_root:file_read/file_write 允许访问的根目录)。用户问「工作区路径」时应调用本工具并如实转述该字段。",
"parameters": {
"type": "object",
"properties": {}

View File

@@ -3,6 +3,8 @@ LLM服务 - 处理各种LLM提供商的调用
"""
from typing import Dict, Any, Optional, List
import json
import os
import re
import asyncio
import logging
import time
@@ -13,6 +15,316 @@ from app.services.tool_registry import tool_registry
logger = logging.getLogger(__name__)
def _extract_dsml_parameter_args(chunk: str) -> Dict[str, str]:
"""
DeepSeek 新版 DSML 常用「parameter」而非「invoke_arg」:
<|DSML|parameter name="file_path" type="string">D:\\path\\file.md
多行正文会出现在 name="content" 的 > 之后、下一个 parameter 之前。
注意:标签可能使用全角 < (U+FF1C),不能用 ASCII [^<] 截断;file_path/mode 等取首行即可。
"""
args: Dict[str, str] = {}
pat = re.compile(r'parameter\s+name="([^"]+)"(?:\s+type="[^"]*")?\s*>', re.IGNORECASE)
ms = list(pat.finditer(chunk))
for i, m in enumerate(ms):
key = m.group(1).strip()
start = m.end()
end = ms[i + 1].start() if i + 1 < len(ms) else len(chunk)
raw = chunk[start:end]
if key.lower() == "content":
val = raw.strip()
# 模型在空 content 后紧接半截 DSML 行,勿写入文件
kept: List[str] = []
for line in val.splitlines():
t = line.strip()
if not t:
continue
if "DSML" in t.upper() and "parameter" not in t.lower():
continue
kept.append(line)
val = "\n".join(kept).strip()
else:
val = raw.lstrip().split("\n", 1)[0].strip()
for sep in ("<", "<"):
if sep in val:
val = val.split(sep, 1)[0].strip()
if key:
args[key] = val
return args
def _extract_dsml_invoke_content_args(chunk: str) -> Dict[str, str]:
"""
部分 DeepSeek 输出使用 invoke_content 块(与 parameter 并列):
<|DSML|invoke_content name="file_path">
123.md
<|DSML|invoke_content name="content">
...
"""
args: Dict[str, str] = {}
pat = re.compile(r'invoke_content\s+name="([^"]+)"[^>]*>', re.IGNORECASE)
ms = list(pat.finditer(chunk))
for i, m in enumerate(ms):
key = m.group(1).strip()
start = m.end()
end = ms[i + 1].start() if i + 1 < len(ms) else len(chunk)
raw = chunk[start:end]
if key.lower() == "content":
val = raw.strip()
kept: List[str] = []
for line in val.splitlines():
t = line.strip()
if not t:
continue
if "DSML" in t.upper() and "invoke_content" not in t.lower() and "parameter" not in t.lower():
continue
kept.append(line)
val = "\n".join(kept).strip()
else:
val = raw.lstrip().split("\n", 1)[0].strip()
for sep in ("<", "<"):
if sep in val:
val = val.split(sep, 1)[0].strip()
if key:
args[key] = val
return args
def _merge_dsml_arg_dicts(*dicts: Dict[str, str]) -> Dict[str, str]:
"""后者仅填补前者缺失或空字符串的键。"""
out: Dict[str, str] = {}
for d in dicts:
for k, v in d.items():
if k not in out or (isinstance(out.get(k), str) and not str(out[k]).strip()):
out[k] = v
return out
def _dedupe_consecutive_dsml_tool_headers(text: str) -> str:
"""
模型异常时连续输出多行相同的 invoke name="tool" 头且无参数,导致解析器切成大量空 chunk。
合并「与上一行完全相同的 invoke 行」,保留首次,使后续 invoke_content/parameter 仍与第一次 invoke 同属一块。
空行会打断「连续」,其后的重复头再次保留一行(兼容极少数真需两次调用的草稿)。
"""
lines = text.splitlines(keepends=True)
out_lines: List[str] = []
prev_sig: Optional[tuple[str, str]] = None
for line in lines:
m = re.search(r'invoke\s+name="([^"]+)"', line, re.I)
if m:
tool = m.group(1).strip().lower()
sig = (tool, line.strip())
if prev_sig == sig:
continue
prev_sig = sig
else:
if not line.strip():
pass
prev_sig = None
out_lines.append(line)
return "".join(out_lines)
def _dedupe_consecutive_function_call_headers(text: str) -> str:
"""同上,针对 function_call name="..." 重复行。"""
lines = text.splitlines(keepends=True)
out_lines: List[str] = []
prev_sig: Optional[tuple[str, str]] = None
for line in lines:
m = re.search(r'function_call\s+name="([^"]+)"', line, re.I)
if m:
tool = m.group(1).strip().lower()
sig = (tool, line.strip())
if prev_sig == sig:
continue
prev_sig = sig
else:
if not line.strip():
pass
prev_sig = None
out_lines.append(line)
return "".join(out_lines)
def _parse_dsml_tool_invocations(content: str) -> List[Dict[str, Any]]:
"""
部分模型(如 DeepSeek)会把 function call 以 DSML 形式写在 content 里,而 message.tool_calls 为空。
解析为 [{\"name\": str, \"arguments\": dict}, ...],供与 OpenAI tool_calls 相同的执行路径处理。
兼容:
- invoke_arg … </invoke_arg(旧)
- parameter name="k" type="…">v(新,可无闭合标签)
- invoke_content name="k">v(DeepSeek 另一种块)
- function_call name="tool" 与 invoke name="tool" 交替出现
- Windows 绝对路径 + 反斜杠 user_data
"""
if not content:
return []
if not re.search(r'invoke\s+name="|function_call\s+name="', content):
return []
content = _dedupe_consecutive_function_call_headers(
_dedupe_consecutive_dsml_tool_headers(content)
)
out: List[Dict[str, Any]] = []
signatures: set = set()
head_re = re.compile(r'(?:invoke|function_call)\s+name="([^"]+)"')
matches = list(head_re.finditer(content))
for i, m in enumerate(matches):
tname = m.group(1).strip()
rest_start = m.end()
nxt_start = matches[i + 1].start() if i + 1 < len(matches) else len(content)
chunk = content[rest_start:nxt_start]
args: Dict[str, Any] = {}
for am in re.finditer(
r'invoke_arg\s+name="([^"]+)"[^>]*>\s*(.*?)\s*</[^>]*invoke_arg',
chunk,
re.DOTALL | re.IGNORECASE,
):
args[am.group(1).strip()] = am.group(2).strip()
param_args = _extract_dsml_parameter_args(chunk)
invc_args = _extract_dsml_invoke_content_args(chunk)
merged_flat = _merge_dsml_arg_dicts(param_args, invc_args)
for k, v in merged_flat.items():
if k not in args or (isinstance(args.get(k), str) and not str(args[k]).strip()):
args[k] = v
if not args:
continue
sig = (tname, json.dumps(args, sort_keys=True, ensure_ascii=False))
if sig in signatures:
continue
signatures.add(sig)
out.append({"name": tname, "arguments": args})
# 仍找不到完整工具块时:从全文兜底抽取 file_write(含仅 parameter、无 invoke_arg)
if not out and "file_write" in content:
fp = None
pm = re.search(
r'parameter\s+name="file_path"[^>]*>\s*([^\r\n]+)',
content,
re.IGNORECASE,
)
if pm:
fp = pm.group(1).strip()
if not fp:
pm_ic = re.search(
r'invoke_content\s+name="file_path"[^>]*>\s*([^\r\n]+)',
content,
re.IGNORECASE,
)
if pm_ic:
fp = pm_ic.group(1).strip()
if not fp:
pm2 = re.search(r'user_data[/\\][^\s<|"\'\n]+\.[a-zA-Z0-9]+', content)
if pm2:
fp = pm2.group(0)
if not fp:
pm3 = re.search(
r'([A-Za-z]:\\(?:[^|<\n\r]+\\)+[^|<\n\r]+\.[a-zA-Z0-9]+)',
content,
)
if pm3:
fp = pm3.group(1).strip()
if not fp:
# 模型只重复 invoke 行、未给 file_path 时:从「《静夜思》」或「静夜思.md」等推断文件名
pm_book = re.search(r'《\s*([\u4e00-\u9fff]{2,24})\s*》', content)
if pm_book:
fp = pm_book.group(1).strip() + ".md"
if not fp:
pm_title = re.search(
r'(?:《\s*)?([\u4e00-\u9fff]{2,32})(?:\s*》)?\s*\.md\b',
content,
)
if pm_title:
fp = pm_title.group(1).strip() + ".md"
body = ""
cm = re.search(
r'invoke_arg\s+name="content"[^>]*>\s*(.*?)\s*</[^>]*invoke_arg',
content,
re.DOTALL | re.IGNORECASE,
)
if cm:
body = cm.group(1).strip()
if not body:
cm2 = re.search(
r'parameter\s+name="content"[^>]*>\s*',
content,
re.IGNORECASE,
)
if cm2:
after = content[cm2.end() :]
nxt = re.search(
r'(?:<[^>]*>)?(?:parameter\s+name=|function_call\s+name=|invoke\s+name=)',
after,
re.IGNORECASE,
)
body = (after[: nxt.start()] if nxt else after).strip()
if not body:
cm_ic = re.search(
r'invoke_content\s+name="content"[^>]*>\s*',
content,
re.IGNORECASE,
)
if cm_ic:
after = content[cm_ic.end() :]
nxt = re.search(
r'(?:<[^>]*>)?(?:invoke_content\s+name=|parameter\s+name=|function_call\s+name=|invoke\s+name=)',
after,
re.IGNORECASE,
)
body = (after[: nxt.start()] if nxt else after).strip()
if not body:
idx = content.find('invoke name="file_write"')
if idx == -1:
idx = content.find('function_call name="file_write"')
if idx >= 0:
cm3 = re.search(
r"(#\s*\w+.*?)(?=\n\n|\Z)",
content[idx:],
re.DOTALL,
)
if cm3:
body = cm3.group(1).strip()
mode_m = re.search(r'parameter\s+name="mode"[^>]*>\s*(\w+)', content, re.IGNORECASE)
if not mode_m:
mode_m = re.search(
r'invoke_content\s+name="mode"[^>]*>\s*(\w+)',
content,
re.IGNORECASE,
)
mode = (mode_m.group(1).strip() if mode_m else None) or "w"
if fp and not (body or "").strip():
# 古诗类请求常见:仅有标题文件名,无正文块
blob = (fp + content)
if "静夜思" in blob:
body = (
"# 静夜思\n\n"
"床前明月光,疑是地上霜。\n"
"举头望明月,低头思故乡。\n"
)
if fp:
out.append({
"name": "file_write",
"arguments": {
"file_path": fp,
"content": body if body else " ",
"mode": mode,
},
})
return out
class LLMService:
"""LLM服务类"""
@@ -236,7 +548,8 @@ class LLMService:
api_key: Optional[str] = None,
base_url: Optional[str] = None,
max_iterations: int = 5,
execution_logger = None
execution_logger = None,
tool_choice: Optional[str] = None,
) -> str:
"""
调用OpenAI API,支持工具调用
@@ -300,125 +613,121 @@ class LLMService:
"function": tool
})
create_kwargs["tools"] = openai_tools
create_kwargs["tool_choice"] = "auto"
# auto:一般对话;required:至少一次 function call(节点 data.tool_choice / 环境变量 LLM_TOOL_CHOICE)
_tc = (tool_choice or os.environ.get("LLM_TOOL_CHOICE") or "auto").strip().lower()
create_kwargs["tool_choice"] = "required" if _tc == "required" else "auto"
# 调用LLM
response = await client.chat.completions.create(**create_kwargs)
message = response.choices[0].message
# 添加助手回复到消息历史
messages.append({
"role": "assistant",
"content": message.content,
"tool_calls": [
{
tool_calls_dicts: List[Dict[str, Any]] = []
if message.tool_calls:
for tc in message.tool_calls:
tool_calls_dicts.append({
"id": tc.id,
"type": tc.type,
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments
}
} for tc in (message.tool_calls or [])
]
"arguments": tc.function.arguments,
},
})
else:
dsml = _parse_dsml_tool_invocations(message.content or "")
if dsml:
logger.info("检测到 DeepSeek DSML 嵌入工具调用 %s 个", len(dsml))
for i, inv in enumerate(dsml):
tool_calls_dicts.append({
"id": f"dsml-{iteration}-{i}",
"type": "function",
"function": {
"name": inv["name"],
"arguments": json.dumps(inv["arguments"], ensure_ascii=False),
},
})
messages.append({
"role": "assistant",
"content": message.content,
"tool_calls": tool_calls_dicts,
})
# 检查是否有工具调用
if message.tool_calls and len(message.tool_calls) > 0:
logger.info(f"检测到 {len(message.tool_calls)} 个工具调用")
# 记录工具调用开始
if not tool_calls_dicts:
final_content = message.content or ""
if final_content:
logger.info("LLM返回最终回复,无工具调用")
return final_content
continue
logger.info(f"检测到 {len(tool_calls_dicts)} 个工具调用")
if execution_logger:
execution_logger.info(
f"LLM请求调用 {len(tool_calls_dicts)} 个工具",
data={"tool_calls_count": len(tool_calls_dicts), "iteration": iteration + 1},
)
for tool_call in tool_calls_dicts:
fn = tool_call.get("function") or {}
tool_name = fn.get("name") or ""
tool_call_id = tool_call.get("id") or "unknown"
try:
tool_args = json.loads(fn.get("arguments") or "{}")
except Exception:
tool_args = {}
logger.info(f"执行工具: {tool_name}, 参数: {tool_args}")
tool_start_time = time.time()
if execution_logger:
execution_logger.info(
f"LLM请求调用 {len(message.tool_calls)} 个工具",
f"调用工具: {tool_name}",
data={
"tool_calls_count": len(message.tool_calls),
"iteration": iteration + 1
}
"tool_name": tool_name,
"tool_call_id": tool_call_id,
"tool_args": tool_args,
"status": "requested",
},
)
# 处理每个工具调用
for tool_call in message.tool_calls:
tool_name = tool_call.function.name
tool_call_id = tool_call.id
try:
tool_args = json.loads(tool_call.function.arguments)
except:
tool_args = {}
logger.info(f"执行工具: {tool_name}, 参数: {tool_args}")
# 记录工具调用请求
tool_start_time = time.time()
try:
tool_result = await self._execute_tool(tool_name, tool_args)
tool_duration = int((time.time() - tool_start_time) * 1000)
if execution_logger:
result_preview = tool_result
if len(result_preview) > 500:
result_preview = result_preview[:500] + "..."
execution_logger.info(
f"调用工具: {tool_name}",
f"工具 {tool_name} 执行成功",
data={
"tool_name": tool_name,
"tool_call_id": tool_call_id,
"tool_args": tool_args,
"status": "requested"
}
"tool_result": result_preview,
"tool_result_length": len(tool_result),
"status": "success",
"duration": tool_duration,
},
duration=tool_duration,
)
# 执行工具
try:
tool_result = await self._execute_tool(tool_name, tool_args)
tool_duration = int((time.time() - tool_start_time) * 1000)
# 记录工具调用成功
if execution_logger:
# 截断过长的结果用于日志
result_preview = tool_result
if len(result_preview) > 500:
result_preview = result_preview[:500] + "..."
execution_logger.info(
f"工具 {tool_name} 执行成功",
data={
"tool_name": tool_name,
"tool_call_id": tool_call_id,
"tool_args": tool_args,
"tool_result": result_preview,
"tool_result_length": len(tool_result),
"status": "success",
"duration": tool_duration
},
duration=tool_duration
)
except Exception as tool_error:
tool_duration = int((time.time() - tool_start_time) * 1000)
# 记录工具调用失败
if execution_logger:
execution_logger.error(
f"工具 {tool_name} 执行失败: {str(tool_error)}",
data={
"tool_name": tool_name,
"tool_call_id": tool_call_id,
"tool_args": tool_args,
"error": str(tool_error),
"status": "failed",
"duration": tool_duration
},
duration=tool_duration
)
# 返回错误结果
tool_result = json.dumps({"error": str(tool_error)}, ensure_ascii=False)
# 添加工具结果到消息历史
messages.append({
"role": "tool",
"tool_call_id": tool_call_id,
"content": tool_result
})
else:
# 没有工具调用,返回最终回复
final_content = message.content or ""
if final_content:
logger.info("LLM返回最终回复,工具调用完成")
return final_content
except Exception as tool_error:
tool_duration = int((time.time() - tool_start_time) * 1000)
if execution_logger:
execution_logger.error(
f"工具 {tool_name} 执行失败: {str(tool_error)}",
data={
"tool_name": tool_name,
"tool_call_id": tool_call_id,
"tool_args": tool_args,
"error": str(tool_error),
"status": "failed",
"duration": tool_duration,
},
duration=tool_duration,
)
tool_result = json.dumps({"error": str(tool_error)}, ensure_ascii=False)
messages.append(
{"role": "tool", "tool_call_id": tool_call_id, "content": tool_result}
)
# 达到最大迭代次数
logger.warning(f"达到最大工具调用迭代次数 ({max_iterations})")
@@ -438,7 +747,8 @@ class LLMService:
api_key: Optional[str] = None,
base_url: Optional[str] = None,
max_iterations: int = 5,
execution_logger = None
execution_logger = None,
tool_choice: Optional[str] = None,
) -> str:
"""
调用DeepSeek API,支持工具调用(DeepSeek兼容OpenAI API格式)
@@ -453,7 +763,8 @@ class LLMService:
api_key=api_key or settings.DEEPSEEK_API_KEY,
base_url=base_url or settings.DEEPSEEK_BASE_URL,
max_iterations=max_iterations,
execution_logger=execution_logger
execution_logger=execution_logger,
tool_choice=tool_choice,
)
async def call_llm_with_tools(
@@ -465,6 +776,7 @@ class LLMService:
temperature: float = 0.7,
max_tokens: Optional[int] = None,
execution_logger = None,
tool_choice: Optional[str] = None,
**kwargs
) -> str:
"""
@@ -492,6 +804,7 @@ class LLMService:
temperature=temperature,
max_tokens=max_tokens,
execution_logger=execution_logger,
tool_choice=tool_choice,
**kwargs
)
elif provider == "deepseek":
@@ -504,6 +817,7 @@ class LLMService:
temperature=temperature,
max_tokens=max_tokens,
execution_logger=execution_logger,
tool_choice=tool_choice,
**kwargs
)
else:

View File

@@ -0,0 +1,127 @@
"""
用户会话记忆持久化:与 Cache 节点 user_memory_* 键对齐,写入 MySQL,Redis 仍作热缓存。
"""
from __future__ import annotations
import logging
import uuid
from typing import Any, Dict, Optional, Tuple
from sqlalchemy.orm import Session
from app.core.config import settings
from app.models.persistent_user_memory import PersistentUserMemory
logger = logging.getLogger(__name__)
USER_MEMORY_PREFIX = "user_memory_"
def parse_memory_scope(workflow_id: str) -> Tuple[Optional[str], Optional[str]]:
"""
Celery 任务里 workflow_id 对 Agent 为 agent_{uuid},对工作流为 UUID 字符串。
返回 (scope_kind, scope_id) 或 (None, None) 表示不做 DB 持久化。
"""
if not workflow_id:
return None, None
if workflow_id.startswith("agent_") and len(workflow_id) > 6:
return "agent", workflow_id[6:]
# 标准 UUID
s = workflow_id.strip()
if len(s) == 36 and s.count("-") == 4:
return "workflow", s
return None, None
def is_user_memory_redis_key(key: str) -> bool:
return isinstance(key, str) and key.startswith(USER_MEMORY_PREFIX)
def session_key_from_user_memory_key(key: str) -> Optional[str]:
if not is_user_memory_redis_key(key):
return None
rest = key[len(USER_MEMORY_PREFIX) :]
return rest if rest else None
def merge_memory_payloads(
base: Optional[Dict[str, Any]], overlay: Optional[Dict[str, Any]]
) -> Optional[Dict[str, Any]]:
"""合并 DB 与 Redis:对话历史取更长的一条;画像/上下文做浅合并。"""
if not base and not overlay:
return None
if not base:
return dict(overlay) if isinstance(overlay, dict) else None
if not overlay:
return dict(base)
out = dict(base)
if not isinstance(overlay, dict):
return out
for k, v in overlay.items():
if k == "conversation_history" and isinstance(v, list) and isinstance(out.get(k), list):
if len(v) > len(out[k]):
out[k] = v
elif k == "user_profile" and isinstance(v, dict):
out[k] = {**(out.get(k) or {}), **v}
elif k == "context" and isinstance(v, dict):
out[k] = {**(out.get(k) or {}), **v}
else:
out[k] = v
return out
def load_persistent_memory(
db: Session, scope_kind: str, scope_id: str, session_key: str
) -> Optional[Dict[str, Any]]:
row = (
db.query(PersistentUserMemory)
.filter(
PersistentUserMemory.scope_kind == scope_kind,
PersistentUserMemory.scope_id == scope_id,
PersistentUserMemory.session_key == session_key,
)
.first()
)
if not row or not isinstance(row.payload, dict):
return None
return dict(row.payload)
def save_persistent_memory(
db: Session, scope_kind: str, scope_id: str, session_key: str, payload: Dict[str, Any]
) -> None:
row = (
db.query(PersistentUserMemory)
.filter(
PersistentUserMemory.scope_kind == scope_kind,
PersistentUserMemory.scope_id == scope_id,
PersistentUserMemory.session_key == session_key,
)
.first()
)
if row:
row.payload = payload
else:
row = PersistentUserMemory(
id=str(uuid.uuid4()),
scope_kind=scope_kind,
scope_id=scope_id,
session_key=session_key,
payload=payload,
)
db.add(row)
db.commit()
def delete_persistent_memory(db: Session, scope_kind: str, scope_id: str, session_key: str) -> None:
q = db.query(PersistentUserMemory).filter(
PersistentUserMemory.scope_kind == scope_kind,
PersistentUserMemory.scope_id == scope_id,
PersistentUserMemory.session_key == session_key,
)
q.delete()
db.commit()
def persist_enabled() -> bool:
return bool(getattr(settings, "MEMORY_PERSIST_DB_ENABLED", True))

View File

@@ -28,7 +28,7 @@ class ToolRegistry:
"""
self._builtin_tools[name] = func
self._tool_schemas[name] = schema
logger.info(f"注册内置工具: {name}")
logger.debug("注册内置工具: %s", name)
def get_tool_schema(self, name: str) -> Optional[Dict[str, Any]]:
"""获取工具定义"""
@@ -41,7 +41,15 @@ class ToolRegistry:
def get_all_tool_schemas(self) -> List[Dict[str, Any]]:
"""获取所有工具定义(用于LLM)"""
return list(self._tool_schemas.values())
def builtin_tool_count(self) -> int:
"""已注册的内置工具数量(用于健康检查 / 启动自检)。"""
return len(self._builtin_tools)
def builtin_tool_names(self) -> List[str]:
"""已注册的内置工具名称,有序列表。"""
return sorted(self._builtin_tools.keys())
def load_tools_from_db(self, db: Session, tool_names: List[str] = None):
"""
从数据库加载工具

File diff suppressed because it is too large Load Diff

View File

@@ -2,6 +2,10 @@
Agent任务
"""
from celery import Task
from app.core.tools_bootstrap import ensure_builtin_tools_registered
ensure_builtin_tools_registered()
from app.core.celery_app import celery_app

View File

@@ -2,6 +2,10 @@
工作流任务
"""
from celery import Task
from app.core.tools_bootstrap import ensure_builtin_tools_registered
ensure_builtin_tools_registered()
from app.core.celery_app import celery_app
from app.services.workflow_engine import WorkflowEngine
from app.services.execution_logger import ExecutionLogger
@@ -15,6 +19,17 @@ import asyncio
import time
def _format_task_error(e: Exception) -> str:
"""Celery 任务异常写入 DB 时的可读文案(HTTPException.detail 等)。"""
detail = getattr(e, "detail", None)
if isinstance(detail, str) and detail.strip():
return detail
if detail is not None and detail != "":
return str(detail)
s = str(e).strip()
return s if s else repr(e)
@celery_app.task(bind=True)
def execute_workflow_task(
self,
@@ -87,14 +102,15 @@ def execute_workflow_task(
execution_time = int((time.time() - start_time) * 1000)
# 记录错误日志
err_text = _format_task_error(e)
if execution_logger:
execution_logger.error(f"工作流任务执行失败: {str(e)}", data={"error_type": type(e).__name__})
execution_logger.error(f"工作流任务执行失败: {err_text}", data={"error_type": type(e).__name__})
# 更新执行记录为失败
execution = db.query(Execution).filter(Execution.id == execution_id).first()
if execution:
execution.status = "failed"
execution.error_message = str(e)
execution.error_message = err_text
execution.execution_time = execution_time
db.commit()