Enhanced GraphEngine Pause Handling (#28196)

This commit: 

1. Convert `pause_reason` to `pause_reasons` in `GraphExecution` and relevant classes. Change the field from a scalar value to a list that can contain multiple `PauseReason` objects, ensuring all pause events are properly captured.
2. Introduce a new `WorkflowPauseReason` model to record reasons associated with a specific `WorkflowPause`.

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

api/core/workflow/graph_engine/domain/graph_execution.py

@@ -42,7 +42,7 @@ class GraphExecutionState(BaseModel):
     completed: bool = Field(default=False)
     aborted: bool = Field(default=False)
     paused: bool = Field(default=False)
-    pause_reason: PauseReason | None = Field(default=None)
+    pause_reasons: list[PauseReason] = Field(default_factory=list)
     error: GraphExecutionErrorState | None = Field(default=None)
     exceptions_count: int = Field(default=0)
     node_executions: list[NodeExecutionState] = Field(default_factory=list[NodeExecutionState])
@@ -107,7 +107,7 @@ class GraphExecution:
     completed: bool = False
     aborted: bool = False
     paused: bool = False
-    pause_reason: PauseReason | None = None
+    pause_reasons: list[PauseReason] = field(default_factory=list)
     error: Exception | None = None
     node_executions: dict[str, NodeExecution] = field(default_factory=dict[str, NodeExecution])
     exceptions_count: int = 0
@@ -137,10 +137,8 @@ class GraphExecution:
             raise RuntimeError("Cannot pause execution that has completed")
         if self.aborted:
             raise RuntimeError("Cannot pause execution that has been aborted")
-        if self.paused:
-            return
         self.paused = True
-        self.pause_reason = reason
+        self.pause_reasons.append(reason)
 
     def fail(self, error: Exception) -> None:
         """Mark the graph execution as failed."""
@@ -195,7 +193,7 @@ class GraphExecution:
             completed=self.completed,
             aborted=self.aborted,
             paused=self.paused,
-            pause_reason=self.pause_reason,
+            pause_reasons=self.pause_reasons,
             error=_serialize_error(self.error),
             exceptions_count=self.exceptions_count,
             node_executions=node_states,
@@ -221,7 +219,7 @@ class GraphExecution:
         self.completed = state.completed
         self.aborted = state.aborted
         self.paused = state.paused
-        self.pause_reason = state.pause_reason
+        self.pause_reasons = state.pause_reasons
         self.error = _deserialize_error(state.error)
         self.exceptions_count = state.exceptions_count
         self.node_executions = {

api/core/workflow/graph_engine/event_management/event_manager.py

@@ -110,7 +110,13 @@ class EventManager:
         """
         with self._lock.write_lock():
             self._events.append(event)
-            self._notify_layers(event)
+
+        # NOTE: `_notify_layers` is intentionally called outside the critical section
+        # to minimize lock contention and avoid blocking other readers or writers.
+        #
+        # The public `notify_layers` method also does not use a write lock,
+        # so protecting `_notify_layers` with a lock here is unnecessary.
+        self._notify_layers(event)
 
     def _get_new_events(self, start_index: int) -> list[GraphEngineEvent]:
         """

api/core/workflow/graph_engine/graph_engine.py

@@ -232,7 +232,7 @@ class GraphEngine:
                 self._graph_execution.start()
             else:
                 self._graph_execution.paused = False
-                self._graph_execution.pause_reason = None
+                self._graph_execution.pause_reasons = []
 
             start_event = GraphRunStartedEvent()
             self._event_manager.notify_layers(start_event)
@@ -246,11 +246,11 @@ class GraphEngine:
 
             # Handle completion
             if self._graph_execution.is_paused:
-                pause_reason = self._graph_execution.pause_reason
-                assert pause_reason is not None, "pause_reason should not be None when execution is paused."
+                pause_reasons = self._graph_execution.pause_reasons
+                assert pause_reasons, "pause_reasons should not be empty when execution is paused."
                 # Ensure we have a valid PauseReason for the event
                 paused_event = GraphRunPausedEvent(
-                    reason=pause_reason,
+                    reasons=pause_reasons,
                     outputs=self._graph_runtime_state.outputs,
                 )
                 self._event_manager.notify_layers(paused_event)