renjianbo
f2e65a8fbb
- Add company module (3-tier org, CEO planning, parallel departments) - Add company orchestrator, knowledge extractor, presets, scheduler - Add company API endpoints, models, and frontend views - Add 今天吃啥 PWA app (69 dishes, real images, offline support) - Add team_projects output directory structure - Add unified manage.ps1 for service lifecycle - Add Windows startup guide v1.0 - Add TTS troubleshooting doc - Update frontend (AgentChat UX overhaul, new views) - Update backend (voice engine fix, multi-tenant, RBAC) - Remove deprecated startup scripts and old docs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
253 lines
8.7 KiB
Python
253 lines
8.7 KiB
Python
"""
|
||
Project Supervisor — 监控所有公司项目的执行状态，检测僵尸项目并自动恢复。
|
||
|
||
僵尸项目定义：
|
||
- status="in_progress" 且 updated_at 超过 ZOMBIE_TIMEOUT_SECONDS 未更新
|
||
- SSE 连接断开后，Agent 仍在后台运行但结果无法写回
|
||
|
||
职责:
|
||
1. 周期性扫描（默认60秒）
|
||
2. 检测僵尸项目并标记为 "zombie"
|
||
3. 提供恢复接口，允许用户重试僵尸项目
|
||
4. 发送告警通知
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
from datetime import datetime, timedelta
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.models.company import Company, CompanyProject
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 默认僵尸超时：15 分钟（900 秒）
|
||
ZOMBIE_TIMEOUT_SECONDS = 900
|
||
# 扫描间隔：60 秒
|
||
SCAN_INTERVAL_SECONDS = 60
|
||
|
||
|
||
class ProjectSupervisor:
|
||
"""项目监管器 — 检测并处理僵尸项目。"""
|
||
|
||
def __init__(self, db: Session):
|
||
self.db = db
|
||
self._last_scan: Optional[datetime] = None
|
||
self._zombie_cache: Dict[str, datetime] = {} # project_id -> detected_at
|
||
|
||
def scan_zombie_projects(self, timeout_seconds: int = ZOMBIE_TIMEOUT_SECONDS) -> List[CompanyProject]:
|
||
"""扫描所有僵尸项目。
|
||
|
||
Args:
|
||
timeout_seconds: 超过此秒数未更新的 in_progress 项目视为僵尸
|
||
|
||
Returns:
|
||
检测到的僵尸项目列表
|
||
"""
|
||
cutoff = datetime.now() - timedelta(seconds=timeout_seconds)
|
||
|
||
zombies = (
|
||
self.db.query(CompanyProject)
|
||
.filter(
|
||
CompanyProject.status == "in_progress",
|
||
CompanyProject.updated_at < cutoff,
|
||
)
|
||
.all()
|
||
)
|
||
|
||
self._last_scan = datetime.now()
|
||
return zombies
|
||
|
||
def mark_as_zombie(self, project_id: str, reason: str = "") -> bool:
|
||
"""将项目标记为僵尸状态。
|
||
|
||
Args:
|
||
project_id: 项目ID
|
||
reason: 标记原因
|
||
|
||
Returns:
|
||
是否成功
|
||
"""
|
||
project = self.db.query(CompanyProject).filter(CompanyProject.id == project_id).first()
|
||
if not project:
|
||
return False
|
||
|
||
old_status = project.status
|
||
project.status = "zombie"
|
||
# 将原因存入 description 字段
|
||
suffix = f"\n[ZOMBIE] {reason}" if reason else "\n[ZOMBIE] SSE disconnected or execution stalled"
|
||
if suffix not in (project.description or ""):
|
||
project.description = (project.description or "") + suffix
|
||
project.updated_at = datetime.now()
|
||
self.db.commit()
|
||
|
||
logger.warning(
|
||
"Project %s marked as zombie (was: %s). Reason: %s",
|
||
project_id[:8], old_status, reason or "timeout"
|
||
)
|
||
self._zombie_cache[project_id] = datetime.now()
|
||
return True
|
||
|
||
def auto_detect_and_mark(self, timeout_seconds: int = ZOMBIE_TIMEOUT_SECONDS) -> Dict[str, Any]:
|
||
"""自动检测并标记僵尸项目。
|
||
|
||
Returns:
|
||
扫描结果摘要
|
||
"""
|
||
zombies = self.scan_zombie_projects(timeout_seconds)
|
||
result = {
|
||
"scanned_at": datetime.now().isoformat(),
|
||
"zombie_count": len(zombies),
|
||
"marked": 0,
|
||
"projects": [],
|
||
}
|
||
|
||
for p in zombies:
|
||
idle_minutes = round((datetime.now() - p.updated_at).total_seconds() / 60, 1)
|
||
reason = f"Project stuck in_progress for {idle_minutes} minutes without updates (last update: {p.updated_at.isoformat()})"
|
||
if self.mark_as_zombie(p.id, reason):
|
||
result["marked"] += 1
|
||
result["projects"].append({
|
||
"project_id": p.id,
|
||
"name": p.name,
|
||
"company_id": p.company_id,
|
||
"idle_minutes": idle_minutes,
|
||
"previous_status": "in_progress",
|
||
})
|
||
|
||
if result["marked"] > 0:
|
||
logger.warning(
|
||
"Supervisor scan: found %d zombies, marked %d",
|
||
result["zombie_count"], result["marked"]
|
||
)
|
||
|
||
return result
|
||
|
||
def recover_project(self, project_id: str) -> Dict[str, Any]:
|
||
"""尝试恢复僵尸项目（将其重置为 failed 以便重试）。
|
||
|
||
Args:
|
||
project_id: 项目ID
|
||
|
||
Returns:
|
||
恢复结果
|
||
"""
|
||
project = self.db.query(CompanyProject).filter(CompanyProject.id == project_id).first()
|
||
if not project:
|
||
return {"success": False, "error": "Project not found"}
|
||
|
||
if project.status != "zombie":
|
||
return {"success": False, "error": f"Project is not zombie (current: {project.status})"}
|
||
|
||
old_status = project.status
|
||
project.status = "failed"
|
||
project.description = (project.description or "") + f"\n[RECOVERED] Manually recovered from zombie state at {datetime.now().isoformat()}"
|
||
project.updated_at = datetime.now()
|
||
project.completed_at = datetime.now()
|
||
self.db.commit()
|
||
|
||
logger.info("Project %s recovered from zombie to failed (ready for retry)", project_id[:8])
|
||
|
||
if project_id in self._zombie_cache:
|
||
del self._zombie_cache[project_id]
|
||
|
||
return {
|
||
"success": True,
|
||
"project_id": project_id,
|
||
"previous_status": old_status,
|
||
"current_status": "failed",
|
||
"message": "Project recovered. You can now re-execute it.",
|
||
}
|
||
|
||
def get_zombie_summary(self) -> Dict[str, Any]:
|
||
"""获取所有僵尸项目的摘要信息。
|
||
|
||
Returns:
|
||
{zombie_count, projects: [{project_id, name, company_id, idle_minutes, detected_at}]}
|
||
"""
|
||
zombies = (
|
||
self.db.query(CompanyProject)
|
||
.filter(CompanyProject.status == "zombie")
|
||
.all()
|
||
)
|
||
|
||
projects = []
|
||
for p in zombies:
|
||
idle_seconds = (datetime.now() - p.updated_at).total_seconds()
|
||
projects.append({
|
||
"project_id": p.id,
|
||
"name": p.name,
|
||
"company_id": p.company_id,
|
||
"status": p.status,
|
||
"idle_minutes": round(idle_seconds / 60, 1),
|
||
"last_updated": p.updated_at.isoformat() if p.updated_at else None,
|
||
"created_at": p.created_at.isoformat() if p.created_at else None,
|
||
})
|
||
|
||
return {
|
||
"zombie_count": len(zombies),
|
||
"projects": projects,
|
||
"scanned_at": (self._last_scan.isoformat() if self._last_scan else None),
|
||
}
|
||
|
||
def get_all_in_progress(self) -> List[Dict[str, Any]]:
|
||
"""获取所有 in_progress 项目（含正常和即将成为僵尸的）。
|
||
|
||
Returns:
|
||
项目列表，包含 idle_minutes 字段
|
||
"""
|
||
projects = (
|
||
self.db.query(CompanyProject)
|
||
.filter(CompanyProject.status == "in_progress")
|
||
.all()
|
||
)
|
||
|
||
result = []
|
||
for p in projects:
|
||
idle_seconds = (datetime.now() - p.updated_at).total_seconds()
|
||
result.append({
|
||
"project_id": p.id,
|
||
"name": p.name,
|
||
"company_id": p.company_id,
|
||
"status": p.status,
|
||
"idle_minutes": round(idle_seconds / 60, 1),
|
||
"is_at_risk": idle_seconds > ZOMBIE_TIMEOUT_SECONDS * 0.5, # 50% 阈值 = 预警
|
||
"will_become_zombie_in_minutes": max(0, round((ZOMBIE_TIMEOUT_SECONDS - idle_seconds) / 60, 1)),
|
||
"last_updated": p.updated_at.isoformat() if p.updated_at else None,
|
||
})
|
||
|
||
return result
|
||
|
||
|
||
async def run_supervisor_loop(db_session_factory) -> None:
|
||
"""监管器主循环 — 集成到现有调度器中。
|
||
|
||
Args:
|
||
db_session_factory: 返回新 Session 的可调用对象
|
||
"""
|
||
logger.info("Project Supervisor loop started (scan interval: %ds, zombie timeout: %ds)",
|
||
SCAN_INTERVAL_SECONDS, ZOMBIE_TIMEOUT_SECONDS)
|
||
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(SCAN_INTERVAL_SECONDS)
|
||
db = db_session_factory()
|
||
try:
|
||
supervisor = ProjectSupervisor(db)
|
||
result = supervisor.auto_detect_and_mark()
|
||
if result["zombie_count"] > 0:
|
||
logger.warning(
|
||
"Supervisor: %d zombie projects detected, %d marked",
|
||
result["zombie_count"], result["marked"]
|
||
)
|
||
finally:
|
||
db.close()
|
||
except asyncio.CancelledError:
|
||
logger.info("Project Supervisor loop cancelled")
|
||
break
|
||
except Exception as e:
|
||
logger.error("Project Supervisor loop error: %s", e, exc_info=True)
|