625 lines
24 KiB
Python
625 lines
24 KiB
Python
|
"""
|
|||
|
|
知识闭环端到端测试
|
|||
|
|
|
|||
|
|
验证 执行日志 → 知识提取 → 知识存储 → 检索 → 注入 Prompt 的完整闭环。
|
|||
|
|
"""
|
|||
|
|
import pytest
|
|||
|
|
from unittest.mock import patch, MagicMock
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.unit
|
|||
|
|
@pytest.mark.knowledge
|
|||
|
|
class TestKnowledgeExtractionPipeline:
|
|||
|
|
"""知识提取管道：日志 → LLM 提取 → KnowledgeEntry 存入"""
|
|||
|
|
|
|||
|
|
def test_agent_execution_log_has_extraction_flag(self, db_session):
|
|||
|
|
"""AgentExecutionLog 模型包含 knowledge_extracted 字段"""
|
|||
|
|
from app.models.agent_execution_log import AgentExecutionLog
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
log = AgentExecutionLog(
|
|||
|
|
id=str(uuid.uuid4()),
|
|||
|
|
agent_name="test_agent",
|
|||
|
|
input_text="如何优化数据库查询性能？",
|
|||
|
|
output_text="可以通过建立索引、优化SQL语句、使用连接池等方式提升性能。",
|
|||
|
|
success=True,
|
|||
|
|
iterations_used=3,
|
|||
|
|
tool_calls_made=5,
|
|||
|
|
)
|
|||
|
|
db_session.add(log)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
assert log.knowledge_extracted is False
|
|||
|
|
assert log.success is True
|
|||
|
|
assert log.agent_name == "test_agent"
|
|||
|
|
|
|||
|
|
def test_extract_from_execution_with_llm(self, db_session):
|
|||
|
|
"""知识提取器从执行日志中提取知识（mock LLM）"""
|
|||
|
|
from app.services.knowledge_extractor import KnowledgeExtractor
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
log_data = {
|
|||
|
|
"input_text": "Nginx 502 Bad Gateway 错误应该如何排查和修复？请给出完整步骤",
|
|||
|
|
"output_text": "先检查后端服务是否正常运行，再查看 Nginx 错误日志定位 upstream 连接问题，确认超时配置和缓冲区大小是否合理。",
|
|||
|
|
"success": True,
|
|||
|
|
"tool_chain": [{"name": "web_search", "result": "ok"}],
|
|||
|
|
"iterations_used": 3,
|
|||
|
|
"tool_calls_made": 2,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
mock_llm_response = json.dumps({
|
|||
|
|
"title": "Nginx 502 错误排查方法",
|
|||
|
|
"category": "best_practice",
|
|||
|
|
"tags": ["nginx", "502", "排查"],
|
|||
|
|
"situation": "Nginx 返回 502 Bad Gateway",
|
|||
|
|
"solution": "1.检查后端服务状态 2.查看error.log 3.调整超时配置",
|
|||
|
|
"caveats": "注意区分 502 和 504 的不同排查路径",
|
|||
|
|
"confidence": 0.85,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
extractor = KnowledgeExtractor(llm_model="test-model")
|
|||
|
|
with patch.object(extractor, "_sync_llm_call", return_value=mock_llm_response):
|
|||
|
|
result = extractor.extract_from_execution(log_data)
|
|||
|
|
|
|||
|
|
assert result is not None
|
|||
|
|
assert result["title"] == "Nginx 502 错误排查方法"
|
|||
|
|
assert result["category"] == "best_practice"
|
|||
|
|
assert result["confidence"] == 0.85
|
|||
|
|
assert len(result["tags"]) == 3
|
|||
|
|
assert "检查后端服务状态" in result["solution"]
|
|||
|
|
|
|||
|
|
def test_extract_skips_low_confidence(self, db_session):
|
|||
|
|
"""低置信度 (<0.3) 的知识应被跳过"""
|
|||
|
|
from app.services.knowledge_extractor import KnowledgeExtractor
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
log_data = {
|
|||
|
|
"input_text": "测试问题" * 10,
|
|||
|
|
"output_text": "测试回答" * 20,
|
|||
|
|
"success": True,
|
|||
|
|
"tool_chain": [],
|
|||
|
|
"iterations_used": 1,
|
|||
|
|
"tool_calls_made": 0,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
mock_llm_response = json.dumps({
|
|||
|
|
"title": "低质量知识",
|
|||
|
|
"category": "insight",
|
|||
|
|
"tags": [],
|
|||
|
|
"situation": "测试",
|
|||
|
|
"solution": "测试",
|
|||
|
|
"caveats": "",
|
|||
|
|
"confidence": 0.15,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
extractor = KnowledgeExtractor(llm_model="test-model")
|
|||
|
|
with patch.object(extractor, "_sync_llm_call", return_value=mock_llm_response):
|
|||
|
|
result = extractor.extract_from_execution(log_data)
|
|||
|
|
|
|||
|
|
assert result is None
|
|||
|
|
|
|||
|
|
def test_extract_skips_marked_skip(self, db_session):
|
|||
|
|
"""LLM 标记 skip 时应跳过"""
|
|||
|
|
from app.services.knowledge_extractor import KnowledgeExtractor
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
log_data = {
|
|||
|
|
"input_text": "你好" * 10,
|
|||
|
|
"output_text": "你好！有什么可以帮助你的吗？" * 10,
|
|||
|
|
"success": True,
|
|||
|
|
"tool_chain": [],
|
|||
|
|
"iterations_used": 1,
|
|||
|
|
"tool_calls_made": 0,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
mock_llm_response = json.dumps({"skip": True, "reason": "对话过于简单"})
|
|||
|
|
|
|||
|
|
extractor = KnowledgeExtractor(llm_model="test-model")
|
|||
|
|
with patch.object(extractor, "_sync_llm_call", return_value=mock_llm_response):
|
|||
|
|
result = extractor.extract_from_execution(log_data)
|
|||
|
|
|
|||
|
|
assert result is None
|
|||
|
|
|
|||
|
|
def test_extract_skips_short_content(self, db_session):
|
|||
|
|
"""输入/输出太短时应跳过（不调用 LLM）"""
|
|||
|
|
from app.services.knowledge_extractor import KnowledgeExtractor
|
|||
|
|
|
|||
|
|
log_data = {
|
|||
|
|
"input_text": "hi",
|
|||
|
|
"output_text": "hello",
|
|||
|
|
"success": True,
|
|||
|
|
"tool_chain": [],
|
|||
|
|
"iterations_used": 1,
|
|||
|
|
"tool_calls_made": 0,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
extractor = KnowledgeExtractor()
|
|||
|
|
result = extractor.extract_from_execution(log_data)
|
|||
|
|
assert result is None
|
|||
|
|
|
|||
|
|
def test_knowledge_entry_created_from_extraction(self, db_session):
|
|||
|
|
"""提取的知识应正确存入 KnowledgeEntry"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="MySQL 死锁重试策略",
|
|||
|
|
category="bug_fix",
|
|||
|
|
tags=["mysql", "deadlock", "retry"],
|
|||
|
|
situation="高并发下 MySQL 发生死锁",
|
|||
|
|
solution="捕获 DeadlockError 后休眠随机毫秒数再重试，最多3次",
|
|||
|
|
caveats="幂等操作才可安全重试",
|
|||
|
|
source_execution_ids=[str(uuid.uuid4())],
|
|||
|
|
source_agent_name="db_admin_agent",
|
|||
|
|
embedding_text="MySQL 死锁重试策略 高并发下 MySQL 发生死锁 捕获 DeadlockError 后休眠随机毫秒数再重试",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.9,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
assert entry.id is not None
|
|||
|
|
assert entry.category == "bug_fix"
|
|||
|
|
assert entry.confidence == 0.9
|
|||
|
|
assert entry.retrieval_count == 0
|
|||
|
|
assert entry.success_rate is None
|
|||
|
|
assert entry.is_active is True
|
|||
|
|
|
|||
|
|
def test_pipeline_dedup_by_title(self, db_session):
|
|||
|
|
"""管道去重：同名标题的知识条目不重复创建"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
|
|||
|
|
title = "唯一标题去重测试"
|
|||
|
|
entry1 = KnowledgeEntry(title=title, category="insight", extracted_by="llm_auto", confidence=0.8)
|
|||
|
|
db_session.add(entry1)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
existing = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.title == title
|
|||
|
|
).first()
|
|||
|
|
assert existing is not None
|
|||
|
|
assert existing.id == entry1.id
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.unit
|
|||
|
|
@pytest.mark.knowledge
|
|||
|
|
class TestKnowledgeRetrievalPipeline:
|
|||
|
|
"""知识检索管道：关键词搜索 → 排序 → 格式化注入"""
|
|||
|
|
|
|||
|
|
def test_query_by_keyword(self, db_session):
|
|||
|
|
"""关键词搜索应能通过 LIKE 查询找到匹配条目"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
from sqlalchemy import or_
|
|||
|
|
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="Redis 缓存穿透防护",
|
|||
|
|
category="best_practice",
|
|||
|
|
situation="大量请求查询不存在的缓存key导致DB压力",
|
|||
|
|
solution="使用布隆过滤器或缓存空值（短TTL）",
|
|||
|
|
caveats="布隆过滤器有误判率",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.9,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 模拟 retriever 的关键词 LIKE 查询
|
|||
|
|
keywords = ["Redis", "缓存穿透"]
|
|||
|
|
conditions = []
|
|||
|
|
for kw in keywords:
|
|||
|
|
like_pat = f"%{kw}%"
|
|||
|
|
conditions.append(KnowledgeEntry.title.like(like_pat))
|
|||
|
|
conditions.append(KnowledgeEntry.situation.like(like_pat))
|
|||
|
|
conditions.append(KnowledgeEntry.solution.like(like_pat))
|
|||
|
|
|
|||
|
|
results = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.is_active == True,
|
|||
|
|
or_(*conditions),
|
|||
|
|
).all()
|
|||
|
|
|
|||
|
|
assert len(results) >= 1
|
|||
|
|
assert results[0].title == "Redis 缓存穿透防护"
|
|||
|
|
|
|||
|
|
def test_retrieval_increments_count(self, db_session):
|
|||
|
|
"""检索后 retrieval_count 应 +1"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="测试检索计数",
|
|||
|
|
category="insight",
|
|||
|
|
situation="测试场景",
|
|||
|
|
solution="测试方案",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.8,
|
|||
|
|
retrieval_count=0,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
entry_id = entry.id
|
|||
|
|
|
|||
|
|
# 模拟检索计数更新
|
|||
|
|
entry.retrieval_count = (entry.retrieval_count or 0) + 1
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
updated = db_session.query(KnowledgeEntry).filter(KnowledgeEntry.id == entry_id).first()
|
|||
|
|
assert updated is not None
|
|||
|
|
assert updated.retrieval_count == 1
|
|||
|
|
|
|||
|
|
def test_retrieve_respects_top_k(self, db_session):
|
|||
|
|
"""检索应限制返回 top_k 条"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
from sqlalchemy import or_
|
|||
|
|
|
|||
|
|
for i in range(5):
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title=f"共享关键词条目{i}",
|
|||
|
|
category="insight",
|
|||
|
|
situation=f"场景{i}",
|
|||
|
|
solution=f"方案{i}",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.5 + i * 0.1,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 模拟 top_k 限制
|
|||
|
|
top_k = 3
|
|||
|
|
conditions = []
|
|||
|
|
for kw in ["共享关键词条目"]:
|
|||
|
|
like_pat = f"%{kw}%"
|
|||
|
|
conditions.append(KnowledgeEntry.title.like(like_pat))
|
|||
|
|
|
|||
|
|
results = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.is_active == True,
|
|||
|
|
or_(*conditions),
|
|||
|
|
).limit(top_k).all()
|
|||
|
|
|
|||
|
|
assert len(results) <= top_k
|
|||
|
|
|
|||
|
|
def test_retrieve_filter_by_category(self, db_session):
|
|||
|
|
"""应按类别过滤检索结果"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
|
|||
|
|
entry_bug = KnowledgeEntry(title="公共关键词Bug条目", category="bug_fix",
|
|||
|
|
situation="bug", solution="fix",
|
|||
|
|
extracted_by="llm_auto", confidence=0.8)
|
|||
|
|
entry_bp = KnowledgeEntry(title="公共关键词最佳实践", category="best_practice",
|
|||
|
|
situation="practice", solution="do",
|
|||
|
|
extracted_by="llm_auto", confidence=0.9)
|
|||
|
|
db_session.add_all([entry_bug, entry_bp])
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 模拟类别过滤
|
|||
|
|
results = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.is_active == True,
|
|||
|
|
KnowledgeEntry.category == "bug_fix",
|
|||
|
|
).all()
|
|||
|
|
|
|||
|
|
for r in results:
|
|||
|
|
assert r.category == "bug_fix"
|
|||
|
|
|
|||
|
|
def test_format_for_prompt(self, db_session):
|
|||
|
|
"""知识条目格式化应为 Markdown 格式"""
|
|||
|
|
from app.services.knowledge_retriever import KnowledgeRetriever
|
|||
|
|
|
|||
|
|
entries = [{
|
|||
|
|
"id": "test-1",
|
|||
|
|
"title": "测试知识",
|
|||
|
|
"category": "best_practice",
|
|||
|
|
"tags": ["test"],
|
|||
|
|
"situation": "测试场景",
|
|||
|
|
"solution": "执行以下步骤\n1. 步骤A\n2. 步骤B",
|
|||
|
|
"caveats": "注意安全",
|
|||
|
|
"confidence": 0.9,
|
|||
|
|
}]
|
|||
|
|
|
|||
|
|
retriever = KnowledgeRetriever()
|
|||
|
|
formatted = retriever.format_for_prompt(entries)
|
|||
|
|
|
|||
|
|
assert "相关知识库经验" in formatted
|
|||
|
|
assert "测试知识" in formatted
|
|||
|
|
assert "置信度: 90%" in formatted
|
|||
|
|
assert "步骤A" in formatted
|
|||
|
|
assert "注意安全" in formatted
|
|||
|
|
|
|||
|
|
def test_inject_knowledge_appends_to_prompt(self, db_session):
|
|||
|
|
"""inject_knowledge 应检索并追加知识到 system prompt"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
from app.services.knowledge_retriever import KnowledgeRetriever
|
|||
|
|
|
|||
|
|
# 存入知识条目
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="注入测试条目",
|
|||
|
|
category="insight",
|
|||
|
|
situation="注入测试",
|
|||
|
|
solution="应出现在结果中",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.9,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 使用 mock retrieve 返回已知条目
|
|||
|
|
mock_entries = [{
|
|||
|
|
"id": str(entry.id),
|
|||
|
|
"title": "注入测试条目",
|
|||
|
|
"category": "insight",
|
|||
|
|
"tags": [],
|
|||
|
|
"situation": "注入测试",
|
|||
|
|
"solution": "应出现在结果中",
|
|||
|
|
"caveats": "",
|
|||
|
|
"confidence": 0.9,
|
|||
|
|
}]
|
|||
|
|
|
|||
|
|
retriever = KnowledgeRetriever()
|
|||
|
|
with patch.object(retriever, "retrieve", return_value=mock_entries):
|
|||
|
|
system_prompt = "你是一个有用的助手。"
|
|||
|
|
result = retriever.inject_knowledge(system_prompt, "注入测试")
|
|||
|
|
|
|||
|
|
assert result.startswith(system_prompt)
|
|||
|
|
assert "相关知识库经验" in result
|
|||
|
|
assert "注入测试条目" in result
|
|||
|
|
|
|||
|
|
def test_empty_retrieval_returns_original_prompt(self, db_session):
|
|||
|
|
"""无匹配时 inject_knowledge 应返回原始的 system prompt"""
|
|||
|
|
from app.services.knowledge_retriever import KnowledgeRetriever
|
|||
|
|
|
|||
|
|
retriever = KnowledgeRetriever()
|
|||
|
|
with patch.object(retriever, "retrieve", return_value=[]):
|
|||
|
|
system_prompt = "原始系统提示"
|
|||
|
|
result = retriever.inject_knowledge(system_prompt, "不存在的关键词")
|
|||
|
|
|
|||
|
|
assert result == system_prompt
|
|||
|
|
|
|||
|
|
def test_knowledge_with_tags_stored(self, db_session):
|
|||
|
|
"""知识标签应正确存储"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
|
|||
|
|
tags = ["python", "async", "性能优化"]
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="Python 异步性能优化",
|
|||
|
|
category="optimization",
|
|||
|
|
tags=tags,
|
|||
|
|
situation="高并发异步任务",
|
|||
|
|
solution="使用 asyncio.gather 并发执行",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.85,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
refreshed = db_session.query(KnowledgeEntry).filter(KnowledgeEntry.id == entry.id).first()
|
|||
|
|
assert refreshed is not None
|
|||
|
|
assert "python" in refreshed.tags
|
|||
|
|
assert len(refreshed.tags) == 3
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.unit
|
|||
|
|
@pytest.mark.knowledge
|
|||
|
|
class TestKnowledgeLoopEndToEnd:
|
|||
|
|
"""知识闭环端到端：执行→提取→检索→注入→验证"""
|
|||
|
|
|
|||
|
|
def test_full_loop_log_to_retrieval(self, db_session):
|
|||
|
|
"""端到端：模拟执行日志 → 提取知识 → 检索 → 注入 Prompt"""
|
|||
|
|
from app.models.agent_execution_log import AgentExecutionLog
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
from app.services.knowledge_extractor import KnowledgeExtractor
|
|||
|
|
from app.services.knowledge_retriever import KnowledgeRetriever
|
|||
|
|
import json
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
# Step 1: 创建执行日志
|
|||
|
|
log = AgentExecutionLog(
|
|||
|
|
id=str(uuid.uuid4()),
|
|||
|
|
agent_name="devops_agent",
|
|||
|
|
input_text="生产环境 Redis 连接池耗尽导致服务不可用，如何预防？",
|
|||
|
|
output_text=(
|
|||
|
|
"1. 设置合理的 max_connections 上限\n"
|
|||
|
|
"2. 配置连接空闲超时和最大等待时间\n"
|
|||
|
|
"3. 添加连接池监控告警（可用连接数 < 20% 时告警）\n"
|
|||
|
|
"4. 使用连接池预热避免冷启动雪崩"
|
|||
|
|
),
|
|||
|
|
success=True,
|
|||
|
|
iterations_used=4,
|
|||
|
|
tool_calls_made=3,
|
|||
|
|
)
|
|||
|
|
db_session.add(log)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# Step 2: 模拟知识提取（mock LLM）
|
|||
|
|
mock_result = json.dumps({
|
|||
|
|
"title": "Redis 连接池耗尽预防策略",
|
|||
|
|
"category": "best_practice",
|
|||
|
|
"tags": ["redis", "连接池", "高可用"],
|
|||
|
|
"situation": "生产环境 Redis 连接池耗尽",
|
|||
|
|
"solution": "设置max_connections上限+空闲超时+监控告警+连接池预热",
|
|||
|
|
"caveats": "max_connections 需要根据实例规格合理设置",
|
|||
|
|
"confidence": 0.92,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
extractor = KnowledgeExtractor()
|
|||
|
|
log_data = {
|
|||
|
|
"input_text": log.input_text,
|
|||
|
|
"output_text": log.output_text,
|
|||
|
|
"success": log.success,
|
|||
|
|
"tool_chain": [],
|
|||
|
|
"iterations_used": log.iterations_used,
|
|||
|
|
"tool_calls_made": log.tool_calls_made,
|
|||
|
|
}
|
|||
|
|
with patch.object(extractor, "_sync_llm_call", return_value=mock_result):
|
|||
|
|
knowledge = extractor.extract_from_execution(log_data)
|
|||
|
|
|
|||
|
|
assert knowledge is not None
|
|||
|
|
assert knowledge["category"] == "best_practice"
|
|||
|
|
assert knowledge["confidence"] == 0.92
|
|||
|
|
|
|||
|
|
# Step 3: 创建 KnowledgeEntry
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title=knowledge["title"],
|
|||
|
|
category=knowledge["category"],
|
|||
|
|
tags=knowledge["tags"],
|
|||
|
|
situation=knowledge["situation"],
|
|||
|
|
solution=knowledge["solution"],
|
|||
|
|
caveats=knowledge["caveats"],
|
|||
|
|
source_execution_ids=[str(log.id)],
|
|||
|
|
source_agent_name=log.agent_name,
|
|||
|
|
embedding_text=f"{knowledge['title']} {knowledge['situation']} {knowledge['solution']}",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=knowledge["confidence"],
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# Step 4: 通过 DB 查询验证知识已存储
|
|||
|
|
stored = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.title == "Redis 连接池耗尽预防策略"
|
|||
|
|
).first()
|
|||
|
|
assert stored is not None
|
|||
|
|
assert stored.category == "best_practice"
|
|||
|
|
assert stored.source_execution_ids == [str(log.id)]
|
|||
|
|
assert stored.source_agent_name == "devops_agent"
|
|||
|
|
|
|||
|
|
# Step 5: 格式化 Prompt（不用真实的 retriever，直接测试 format_for_prompt）
|
|||
|
|
retriever = KnowledgeRetriever()
|
|||
|
|
mock_entries = [{
|
|||
|
|
"id": str(stored.id),
|
|||
|
|
"title": stored.title,
|
|||
|
|
"category": stored.category,
|
|||
|
|
"tags": stored.tags,
|
|||
|
|
"situation": stored.situation,
|
|||
|
|
"solution": stored.solution,
|
|||
|
|
"caveats": stored.caveats,
|
|||
|
|
"confidence": stored.confidence,
|
|||
|
|
}]
|
|||
|
|
enriched = retriever.format_for_prompt(mock_entries)
|
|||
|
|
|
|||
|
|
assert "Redis 连接池耗尽预防策略" in enriched
|
|||
|
|
assert "相关知识库经验" in enriched
|
|||
|
|
|
|||
|
|
# Step 6: 验证格式化后的内容可注入 system prompt
|
|||
|
|
system_prompt = "你是一个运维助手。"
|
|||
|
|
final_with_mock = system_prompt + enriched
|
|||
|
|
assert system_prompt in final_with_mock
|
|||
|
|
assert "相关知识库经验" in final_with_mock
|
|||
|
|
assert "Redis 连接池耗尽预防策略" in final_with_mock
|
|||
|
|
|
|||
|
|
def test_multiple_executions_accumulate_knowledge(self, db_session):
|
|||
|
|
"""多次执行应累积多条知识"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
entries_data = [
|
|||
|
|
("Nginx配置优化", "optimization", 0.9),
|
|||
|
|
("MySQL慢查询优化", "optimization", 0.85),
|
|||
|
|
("Docker内存泄漏处理", "bug_fix", 0.8),
|
|||
|
|
("API限流策略", "best_practice", 0.95),
|
|||
|
|
("日志收集最佳实践", "best_practice", 0.88),
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for title, cat, conf in entries_data:
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title=title, category=cat, extracted_by="llm_auto",
|
|||
|
|
confidence=conf, embedding_text=title,
|
|||
|
|
source_execution_ids=[str(uuid.uuid4())],
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
all_entries = db_session.query(KnowledgeEntry).all()
|
|||
|
|
assert len(all_entries) >= 5
|
|||
|
|
|
|||
|
|
optimizations = [e for e in all_entries if e.category == "optimization"]
|
|||
|
|
assert len(optimizations) >= 2
|
|||
|
|
|
|||
|
|
best_practices = [e for e in all_entries if e.category == "best_practice"]
|
|||
|
|
assert len(best_practices) >= 2
|
|||
|
|
|
|||
|
|
def test_knowledge_entry_has_all_required_fields(self, db_session):
|
|||
|
|
"""KnowledgeEntry 应包含所有闭环所需的字段"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
entry = KnowledgeEntry(
|
|||
|
|
title="完整字段测试",
|
|||
|
|
category="insight",
|
|||
|
|
tags=["a", "b", "c"],
|
|||
|
|
situation="场景描述",
|
|||
|
|
solution="解决方案",
|
|||
|
|
caveats="注意事项",
|
|||
|
|
source_execution_ids=[str(uuid.uuid4()), str(uuid.uuid4())],
|
|||
|
|
source_agent_name="source_agent",
|
|||
|
|
source_model="deepseek-chat",
|
|||
|
|
embedding_text="embedding text",
|
|||
|
|
extracted_by="llm_auto",
|
|||
|
|
confidence=0.75,
|
|||
|
|
retrieval_count=5,
|
|||
|
|
success_rate=0.8,
|
|||
|
|
is_active=True,
|
|||
|
|
)
|
|||
|
|
db_session.add(entry)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
refreshed = db_session.query(KnowledgeEntry).filter(KnowledgeEntry.id == entry.id).first()
|
|||
|
|
assert refreshed is not None
|
|||
|
|
assert refreshed.title == "完整字段测试"
|
|||
|
|
assert refreshed.confidence == 0.75
|
|||
|
|
assert refreshed.retrieval_count == 5
|
|||
|
|
assert refreshed.success_rate == 0.8
|
|||
|
|
assert refreshed.is_active is True
|
|||
|
|
assert len(refreshed.source_execution_ids) == 2
|
|||
|
|
assert len(refreshed.tags) == 3
|
|||
|
|
|
|||
|
|
def test_deactivated_knowledge_not_retrieved(self, db_session):
|
|||
|
|
"""is_active=False 的条目不应被检索到"""
|
|||
|
|
from app.models.knowledge_entry import KnowledgeEntry
|
|||
|
|
|
|||
|
|
inactive = KnowledgeEntry(
|
|||
|
|
title="已停用知识点", category="insight",
|
|||
|
|
situation="不应出现", solution="不应出现",
|
|||
|
|
extracted_by="llm_auto", confidence=0.9,
|
|||
|
|
is_active=False,
|
|||
|
|
)
|
|||
|
|
db_session.add(inactive)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 查询活跃条目不应包含已停用的
|
|||
|
|
active_results = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.is_active == True,
|
|||
|
|
KnowledgeEntry.title == "已停用知识点",
|
|||
|
|
).all()
|
|||
|
|
assert len(active_results) == 0
|
|||
|
|
|
|||
|
|
# 但停用条目仍在数据库中
|
|||
|
|
all_results = db_session.query(KnowledgeEntry).filter(
|
|||
|
|
KnowledgeEntry.title == "已停用知识点",
|
|||
|
|
).all()
|
|||
|
|
assert len(all_results) == 1
|
|||
|
|
assert all_results[0].is_active is False
|
|||
|
|
|
|||
|
|
def test_extraction_pipeline_marks_log_processed(self, db_session):
|
|||
|
|
"""提取管道处理后应标记日志为已提取"""
|
|||
|
|
from app.models.agent_execution_log import AgentExecutionLog
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
log = AgentExecutionLog(
|
|||
|
|
id=str(uuid.uuid4()),
|
|||
|
|
agent_name="test_agent",
|
|||
|
|
input_text="生产环境数据库连接超时如何排查？",
|
|||
|
|
output_text="逐步排查：1.检查网络连通性 2.检查连接池配置 3.检查数据库负载 4.检查慢查询",
|
|||
|
|
success=True,
|
|||
|
|
iterations_used=2,
|
|||
|
|
tool_calls_made=4,
|
|||
|
|
)
|
|||
|
|
db_session.add(log)
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
# 模拟提取后的标记
|
|||
|
|
log.knowledge_extracted = True
|
|||
|
|
db_session.commit()
|
|||
|
|
|
|||
|
|
refreshed = db_session.query(AgentExecutionLog).filter(
|
|||
|
|
AgentExecutionLog.id == log.id
|
|||
|
|
).first()
|
|||
|
|
assert refreshed is not None
|
|||
|
|
assert refreshed.knowledge_extracted is True
|