Enhanced GraphEngine Pause Handling (#28196)

This commit: 

1. Convert `pause_reason` to `pause_reasons` in `GraphExecution` and relevant classes. Change the field from a scalar value to a list that can contain multiple `PauseReason` objects, ensuring all pause events are properly captured.
2. Introduce a new `WorkflowPauseReason` model to record reasons associated with a specific `WorkflowPause`.

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py

@@ -31,7 +31,7 @@ class TestDataFactory:
 
     @staticmethod
     def create_graph_run_paused_event(outputs: dict[str, object] | None = None) -> GraphRunPausedEvent:
-        return GraphRunPausedEvent(reason=SchedulingPause(message="test pause"), outputs=outputs or {})
+        return GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")], outputs=outputs or {})
 
     @staticmethod
     def create_graph_run_started_event() -> GraphRunStartedEvent:
@@ -255,15 +255,17 @@ class TestPauseStatePersistenceLayer:
         layer.on_event(event)
 
         mock_factory.assert_called_once_with(session_factory)
-        mock_repo.create_workflow_pause.assert_called_once_with(
-            workflow_run_id="run-123",
-            state_owner_user_id="owner-123",
-            state=mock_repo.create_workflow_pause.call_args.kwargs["state"],
-        )
-        serialized_state = mock_repo.create_workflow_pause.call_args.kwargs["state"]
+        assert mock_repo.create_workflow_pause.call_count == 1
+        call_kwargs = mock_repo.create_workflow_pause.call_args.kwargs
+        assert call_kwargs["workflow_run_id"] == "run-123"
+        assert call_kwargs["state_owner_user_id"] == "owner-123"
+        serialized_state = call_kwargs["state"]
         resumption_context = WorkflowResumptionContext.loads(serialized_state)
         assert resumption_context.serialized_graph_runtime_state == expected_state
         assert resumption_context.get_generate_entity().model_dump() == generate_entity.model_dump()
+        pause_reasons = call_kwargs["pause_reasons"]
+
+        assert isinstance(pause_reasons, list)
 
     def test_on_event_ignores_non_paused_events(self, monkeypatch: pytest.MonkeyPatch):
         session_factory = Mock(name="session_factory")

api/tests/unit_tests/core/workflow/entities/test_private_workflow_pause.py

@@ -19,38 +19,18 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model.resumed_at = None
 
         # Create entity
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         # Verify initialization
         assert entity._pause_model is mock_pause_model
         assert entity._cached_state is None
 
-    def test_from_models_classmethod(self):
-        """Test from_models class method."""
-        # Create mock models
-        mock_pause_model = MagicMock(spec=WorkflowPauseModel)
-        mock_pause_model.id = "pause-123"
-        mock_pause_model.workflow_run_id = "execution-456"
-
-        # Create entity using from_models
-        entity = _PrivateWorkflowPauseEntity.from_models(
-            workflow_pause_model=mock_pause_model,
-        )
-
-        # Verify entity creation
-        assert isinstance(entity, _PrivateWorkflowPauseEntity)
-        assert entity._pause_model is mock_pause_model
-
     def test_id_property(self):
         """Test id property returns pause model ID."""
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.id = "pause-123"
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         assert entity.id == "pause-123"
 
@@ -59,9 +39,7 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.workflow_run_id = "execution-456"
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         assert entity.workflow_execution_id == "execution-456"
 
@@ -72,9 +50,7 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.resumed_at = resumed_at
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         assert entity.resumed_at == resumed_at
 
@@ -83,9 +59,7 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.resumed_at = None
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         assert entity.resumed_at is None
 
@@ -98,9 +72,7 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.state_object_key = "test-state-key"
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         # First call should load from storage
         result = entity.get_state()
@@ -118,9 +90,7 @@ class TestPrivateWorkflowPauseEntity:
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
         mock_pause_model.state_object_key = "test-state-key"
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         # First call
         result1 = entity.get_state()
@@ -139,9 +109,7 @@ class TestPrivateWorkflowPauseEntity:
 
         mock_pause_model = MagicMock(spec=WorkflowPauseModel)
 
-        entity = _PrivateWorkflowPauseEntity(
-            pause_model=mock_pause_model,
-        )
+        entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
         # Pre-cache data
         entity._cached_state = state_data
@@ -162,9 +130,7 @@ class TestPrivateWorkflowPauseEntity:
 
             mock_pause_model = MagicMock(spec=WorkflowPauseModel)
 
-            entity = _PrivateWorkflowPauseEntity(
-                pause_model=mock_pause_model,
-            )
+            entity = _PrivateWorkflowPauseEntity(pause_model=mock_pause_model, reason_models=[], human_input_form=[])
 
             result = entity.get_state()
 

api/tests/unit_tests/core/workflow/graph/test_graph_validation.py

@@ -8,12 +8,13 @@ from typing import Any
 import pytest
 
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
+from core.workflow.entities import GraphInitParams
 from core.workflow.enums import ErrorStrategy, NodeExecutionType, NodeType
 from core.workflow.graph import Graph
 from core.workflow.graph.validation import GraphValidationError
 from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig
 from core.workflow.nodes.base.node import Node
+from core.workflow.runtime import GraphRuntimeState, VariablePool
 from core.workflow.system_variable import SystemVariable
 from models.enums import UserFrom
 

api/tests/unit_tests/core/workflow/graph_engine/test_command_system.py

@@ -178,8 +178,7 @@ def test_pause_command():
     assert any(isinstance(e, GraphRunStartedEvent) for e in events)
     pause_events = [e for e in events if isinstance(e, GraphRunPausedEvent)]
     assert len(pause_events) == 1
-    assert pause_events[0].reason == SchedulingPause(message="User requested pause")
+    assert pause_events[0].reasons == [SchedulingPause(message="User requested pause")]
 
     graph_execution = engine.graph_runtime_state.graph_execution
-    assert graph_execution.paused
-    assert graph_execution.pause_reason == SchedulingPause(message="User requested pause")
+    assert graph_execution.pause_reasons == [SchedulingPause(message="User requested pause")]