feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

api/core/workflow/graph_events/__init__.py

@@ -0,0 +1,72 @@
+# Agent events
+from .agent import NodeRunAgentLogEvent
+
+# Base events
+from .base import (
+    BaseGraphEvent,
+    GraphEngineEvent,
+    GraphNodeEventBase,
+)
+
+# Graph events
+from .graph import (
+    GraphRunAbortedEvent,
+    GraphRunFailedEvent,
+    GraphRunPartialSucceededEvent,
+    GraphRunStartedEvent,
+    GraphRunSucceededEvent,
+)
+
+# Iteration events
+from .iteration import (
+    NodeRunIterationFailedEvent,
+    NodeRunIterationNextEvent,
+    NodeRunIterationStartedEvent,
+    NodeRunIterationSucceededEvent,
+)
+
+# Loop events
+from .loop import (
+    NodeRunLoopFailedEvent,
+    NodeRunLoopNextEvent,
+    NodeRunLoopStartedEvent,
+    NodeRunLoopSucceededEvent,
+)
+
+# Node events
+from .node import (
+    NodeRunExceptionEvent,
+    NodeRunFailedEvent,
+    NodeRunRetrieverResourceEvent,
+    NodeRunRetryEvent,
+    NodeRunStartedEvent,
+    NodeRunStreamChunkEvent,
+    NodeRunSucceededEvent,
+)
+
+__all__ = [
+    "BaseGraphEvent",
+    "GraphEngineEvent",
+    "GraphNodeEventBase",
+    "GraphRunAbortedEvent",
+    "GraphRunFailedEvent",
+    "GraphRunPartialSucceededEvent",
+    "GraphRunStartedEvent",
+    "GraphRunSucceededEvent",
+    "NodeRunAgentLogEvent",
+    "NodeRunExceptionEvent",
+    "NodeRunFailedEvent",
+    "NodeRunIterationFailedEvent",
+    "NodeRunIterationNextEvent",
+    "NodeRunIterationStartedEvent",
+    "NodeRunIterationSucceededEvent",
+    "NodeRunLoopFailedEvent",
+    "NodeRunLoopNextEvent",
+    "NodeRunLoopStartedEvent",
+    "NodeRunLoopSucceededEvent",
+    "NodeRunRetrieverResourceEvent",
+    "NodeRunRetryEvent",
+    "NodeRunStartedEvent",
+    "NodeRunStreamChunkEvent",
+    "NodeRunSucceededEvent",
+]

api/core/workflow/graph_events/agent.py

@@ -0,0 +1,17 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import Field
+
+from .base import GraphAgentNodeEventBase
+
+
+class NodeRunAgentLogEvent(GraphAgentNodeEventBase):
+    message_id: str = Field(..., description="message id")
+    label: str = Field(..., description="label")
+    node_execution_id: str = Field(..., description="node execution id")
+    parent_id: str | None = Field(..., description="parent id")
+    error: str | None = Field(..., description="error")
+    status: str = Field(..., description="status")
+    data: Mapping[str, Any] = Field(..., description="data")
+    metadata: Mapping[str, object] = Field(default_factory=dict)

api/core/workflow/graph_events/base.py

@@ -0,0 +1,31 @@
+from pydantic import BaseModel, Field
+
+from core.workflow.enums import NodeType
+from core.workflow.node_events import NodeRunResult
+
+
+class GraphEngineEvent(BaseModel):
+    pass
+
+
+class BaseGraphEvent(GraphEngineEvent):
+    pass
+
+
+class GraphNodeEventBase(GraphEngineEvent):
+    id: str = Field(..., description="node execution id")
+    node_id: str
+    node_type: NodeType
+
+    in_iteration_id: str | None = None
+    """iteration id if node is in iteration"""
+    in_loop_id: str | None = None
+    """loop id if node is in loop"""
+
+    # The version of the node, or "1" if not specified.
+    node_version: str = "1"
+    node_run_result: NodeRunResult = Field(default_factory=NodeRunResult)
+
+
+class GraphAgentNodeEventBase(GraphNodeEventBase):
+    pass

api/core/workflow/graph_events/graph.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+
+from core.workflow.graph_events import BaseGraphEvent
+
+
+class GraphRunStartedEvent(BaseGraphEvent):
+    pass
+
+
+class GraphRunSucceededEvent(BaseGraphEvent):
+    outputs: dict[str, object] = Field(default_factory=dict)
+
+
+class GraphRunFailedEvent(BaseGraphEvent):
+    error: str = Field(..., description="failed reason")
+    exceptions_count: int = Field(description="exception count", default=0)
+
+
+class GraphRunPartialSucceededEvent(BaseGraphEvent):
+    exceptions_count: int = Field(..., description="exception count")
+    outputs: dict[str, object] = Field(default_factory=dict)
+
+
+class GraphRunAbortedEvent(BaseGraphEvent):
+    """Event emitted when a graph run is aborted by user command."""
+
+    reason: str | None = Field(default=None, description="reason for abort")
+    outputs: dict[str, object] = Field(default_factory=dict, description="partial outputs if any")

api/core/workflow/graph_events/iteration.py

@@ -0,0 +1,40 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from .base import GraphNodeEventBase
+
+
+class NodeRunIterationStartedEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    predecessor_node_id: str | None = None
+
+
+class NodeRunIterationNextEvent(GraphNodeEventBase):
+    node_title: str
+    index: int = Field(..., description="index")
+    pre_iteration_output: Any = None
+
+
+class NodeRunIterationSucceededEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+
+
+class NodeRunIterationFailedEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+    error: str = Field(..., description="failed reason")

api/core/workflow/graph_events/loop.py

@@ -0,0 +1,40 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from .base import GraphNodeEventBase
+
+
+class NodeRunLoopStartedEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    predecessor_node_id: str | None = None
+
+
+class NodeRunLoopNextEvent(GraphNodeEventBase):
+    node_title: str
+    index: int = Field(..., description="index")
+    pre_loop_output: Any = None
+
+
+class NodeRunLoopSucceededEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+
+
+class NodeRunLoopFailedEvent(GraphNodeEventBase):
+    node_title: str
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+    error: str = Field(..., description="failed reason")

api/core/workflow/graph_events/node.py

@@ -0,0 +1,53 @@
+from collections.abc import Sequence
+from datetime import datetime
+
+from pydantic import Field
+
+from core.rag.entities.citation_metadata import RetrievalSourceMetadata
+from core.workflow.entities import AgentNodeStrategyInit
+
+from .base import GraphNodeEventBase
+
+
+class NodeRunStartedEvent(GraphNodeEventBase):
+    node_title: str
+    predecessor_node_id: str | None = None
+    agent_strategy: AgentNodeStrategyInit | None = None
+    start_at: datetime = Field(..., description="node start time")
+
+    # FIXME(-LAN-): only for ToolNode
+    provider_type: str = ""
+    provider_id: str = ""
+
+
+class NodeRunStreamChunkEvent(GraphNodeEventBase):
+    # Spec-compliant fields
+    selector: Sequence[str] = Field(
+        ..., description="selector identifying the output location (e.g., ['nodeA', 'text'])"
+    )
+    chunk: str = Field(..., description="the actual chunk content")
+    is_final: bool = Field(default=False, description="indicates if this is the last chunk")
+
+
+class NodeRunRetrieverResourceEvent(GraphNodeEventBase):
+    retriever_resources: Sequence[RetrievalSourceMetadata] = Field(..., description="retriever resources")
+    context: str = Field(..., description="context")
+
+
+class NodeRunSucceededEvent(GraphNodeEventBase):
+    start_at: datetime = Field(..., description="node start time")
+
+
+class NodeRunFailedEvent(GraphNodeEventBase):
+    error: str = Field(..., description="error")
+    start_at: datetime = Field(..., description="node start time")
+
+
+class NodeRunExceptionEvent(GraphNodeEventBase):
+    error: str = Field(..., description="error")
+    start_at: datetime = Field(..., description="node start time")
+
+
+class NodeRunRetryEvent(NodeRunStartedEvent):
+    error: str = Field(..., description="error")
+    retry_index: int = Field(..., description="which retry attempt is about to be performed")