""" Project Supervisor — 监控所有公司项目的执行状态，检测僵尸项目并自动恢复。 僵尸项目定义： - status="in_progress" 且 updated_at 超过 ZOMBIE_TIMEOUT_SECONDS 未更新 - SSE 连接断开后，Agent 仍在后台运行但结果无法写回 职责: 1. 周期性扫描（默认60秒） 2. 检测僵尸项目并标记为 "zombie" 3. 提供恢复接口，允许用户重试僵尸项目 4. 发送告警通知 """ from __future__ import annotations import asyncio import logging from datetime import datetime, timedelta from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from app.models.company import Company, CompanyProject logger = logging.getLogger(__name__) # 默认僵尸超时：15 分钟（900 秒） ZOMBIE_TIMEOUT_SECONDS = 900 # 扫描间隔：60 秒 SCAN_INTERVAL_SECONDS = 60 class ProjectSupervisor: """项目监管器 — 检测并处理僵尸项目。""" def __init__(self, db: Session): self.db = db self._last_scan: Optional[datetime] = None self._zombie_cache: Dict[str, datetime] = {} # project_id -> detected_at def scan_zombie_projects(self, timeout_seconds: int = ZOMBIE_TIMEOUT_SECONDS) -> List[CompanyProject]: """扫描所有僵尸项目。 Args: timeout_seconds: 超过此秒数未更新的 in_progress 项目视为僵尸 Returns: 检测到的僵尸项目列表 """ cutoff = datetime.now() - timedelta(seconds=timeout_seconds) zombies = ( self.db.query(CompanyProject) .filter( CompanyProject.status == "in_progress", CompanyProject.updated_at < cutoff, ) .all() ) self._last_scan = datetime.now() return zombies def mark_as_zombie(self, project_id: str, reason: str = "") -> bool: """将项目标记为僵尸状态。 Args: project_id: 项目ID reason: 标记原因 Returns: 是否成功 """ project = self.db.query(CompanyProject).filter(CompanyProject.id == project_id).first() if not project: return False old_status = project.status project.status = "zombie" # 将原因存入 description 字段 suffix = f"\n[ZOMBIE] {reason}" if reason else "\n[ZOMBIE] SSE disconnected or execution stalled" if suffix not in (project.description or ""): project.description = (project.description or "") + suffix project.updated_at = datetime.now() self.db.commit() logger.warning( "Project %s marked as zombie (was: %s). Reason: %s", project_id[:8], old_status, reason or "timeout" ) self._zombie_cache[project_id] = datetime.now() return True def auto_detect_and_mark(self, timeout_seconds: int = ZOMBIE_TIMEOUT_SECONDS) -> Dict[str, Any]: """自动检测并标记僵尸项目。 Returns: 扫描结果摘要 """ zombies = self.scan_zombie_projects(timeout_seconds) result = { "scanned_at": datetime.now().isoformat(), "zombie_count": len(zombies), "marked": 0, "projects": [], } for p in zombies: idle_minutes = round((datetime.now() - p.updated_at).total_seconds() / 60, 1) reason = f"Project stuck in_progress for {idle_minutes} minutes without updates (last update: {p.updated_at.isoformat()})" if self.mark_as_zombie(p.id, reason): result["marked"] += 1 result["projects"].append({ "project_id": p.id, "name": p.name, "company_id": p.company_id, "idle_minutes": idle_minutes, "previous_status": "in_progress", }) if result["marked"] > 0: logger.warning( "Supervisor scan: found %d zombies, marked %d", result["zombie_count"], result["marked"] ) return result def recover_project(self, project_id: str) -> Dict[str, Any]: """尝试恢复僵尸项目（将其重置为 failed 以便重试）。 Args: project_id: 项目ID Returns: 恢复结果 """ project = self.db.query(CompanyProject).filter(CompanyProject.id == project_id).first() if not project: return {"success": False, "error": "Project not found"} if project.status != "zombie": return {"success": False, "error": f"Project is not zombie (current: {project.status})"} old_status = project.status project.status = "failed" project.description = (project.description or "") + f"\n[RECOVERED] Manually recovered from zombie state at {datetime.now().isoformat()}" project.updated_at = datetime.now() project.completed_at = datetime.now() self.db.commit() logger.info("Project %s recovered from zombie to failed (ready for retry)", project_id[:8]) if project_id in self._zombie_cache: del self._zombie_cache[project_id] return { "success": True, "project_id": project_id, "previous_status": old_status, "current_status": "failed", "message": "Project recovered. You can now re-execute it.", } def get_zombie_summary(self) -> Dict[str, Any]: """获取所有僵尸项目的摘要信息。 Returns: {zombie_count, projects: [{project_id, name, company_id, idle_minutes, detected_at}]} """ zombies = ( self.db.query(CompanyProject) .filter(CompanyProject.status == "zombie") .all() ) projects = [] for p in zombies: idle_seconds = (datetime.now() - p.updated_at).total_seconds() projects.append({ "project_id": p.id, "name": p.name, "company_id": p.company_id, "status": p.status, "idle_minutes": round(idle_seconds / 60, 1), "last_updated": p.updated_at.isoformat() if p.updated_at else None, "created_at": p.created_at.isoformat() if p.created_at else None, }) return { "zombie_count": len(zombies), "projects": projects, "scanned_at": (self._last_scan.isoformat() if self._last_scan else None), } def get_all_in_progress(self) -> List[Dict[str, Any]]: """获取所有 in_progress 项目（含正常和即将成为僵尸的）。 Returns: 项目列表，包含 idle_minutes 字段 """ projects = ( self.db.query(CompanyProject) .filter(CompanyProject.status == "in_progress") .all() ) result = [] for p in projects: idle_seconds = (datetime.now() - p.updated_at).total_seconds() result.append({ "project_id": p.id, "name": p.name, "company_id": p.company_id, "status": p.status, "idle_minutes": round(idle_seconds / 60, 1), "is_at_risk": idle_seconds > ZOMBIE_TIMEOUT_SECONDS * 0.5, # 50% 阈值 = 预警 "will_become_zombie_in_minutes": max(0, round((ZOMBIE_TIMEOUT_SECONDS - idle_seconds) / 60, 1)), "last_updated": p.updated_at.isoformat() if p.updated_at else None, }) return result async def run_supervisor_loop(db_session_factory) -> None: """监管器主循环 — 集成到现有调度器中。 Args: db_session_factory: 返回新 Session 的可调用对象 """ logger.info("Project Supervisor loop started (scan interval: %ds, zombie timeout: %ds)", SCAN_INTERVAL_SECONDS, ZOMBIE_TIMEOUT_SECONDS) while True: try: await asyncio.sleep(SCAN_INTERVAL_SECONDS) db = db_session_factory() try: supervisor = ProjectSupervisor(db) result = supervisor.auto_detect_and_mark() if result["zombie_count"] > 0: logger.warning( "Supervisor: %d zombie projects detected, %d marked", result["zombie_count"], result["marked"] ) finally: db.close() except asyncio.CancelledError: logger.info("Project Supervisor loop cancelled") break except Exception as e: logger.error("Project Supervisor loop error: %s", e, exc_info=True)