feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

api/core/workflow/node_events/__init__.py

@@ -0,0 +1,40 @@
+from .agent import AgentLogEvent
+from .base import NodeEventBase, NodeRunResult
+from .iteration import (
+    IterationFailedEvent,
+    IterationNextEvent,
+    IterationStartedEvent,
+    IterationSucceededEvent,
+)
+from .loop import (
+    LoopFailedEvent,
+    LoopNextEvent,
+    LoopStartedEvent,
+    LoopSucceededEvent,
+)
+from .node import (
+    ModelInvokeCompletedEvent,
+    RunRetrieverResourceEvent,
+    RunRetryEvent,
+    StreamChunkEvent,
+    StreamCompletedEvent,
+)
+
+__all__ = [
+    "AgentLogEvent",
+    "IterationFailedEvent",
+    "IterationNextEvent",
+    "IterationStartedEvent",
+    "IterationSucceededEvent",
+    "LoopFailedEvent",
+    "LoopNextEvent",
+    "LoopStartedEvent",
+    "LoopSucceededEvent",
+    "ModelInvokeCompletedEvent",
+    "NodeEventBase",
+    "NodeRunResult",
+    "RunRetrieverResourceEvent",
+    "RunRetryEvent",
+    "StreamChunkEvent",
+    "StreamCompletedEvent",
+]

api/core/workflow/node_events/agent.py

@@ -0,0 +1,18 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import Field
+
+from .base import NodeEventBase
+
+
+class AgentLogEvent(NodeEventBase):
+    message_id: str = Field(..., description="id")
+    label: str = Field(..., description="label")
+    node_execution_id: str = Field(..., description="node execution id")
+    parent_id: str | None = Field(..., description="parent id")
+    error: str | None = Field(..., description="error")
+    status: str = Field(..., description="status")
+    data: Mapping[str, Any] = Field(..., description="data")
+    metadata: Mapping[str, Any] = Field(default_factory=dict, description="metadata")
+    node_id: str = Field(..., description="node id")

api/core/workflow/node_events/base.py

@@ -0,0 +1,40 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from core.model_runtime.entities.llm_entities import LLMUsage
+from core.workflow.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
+
+
+class NodeEventBase(BaseModel):
+    """Base class for all node events"""
+
+    pass
+
+
+def _default_metadata():
+    v: Mapping[WorkflowNodeExecutionMetadataKey, Any] = {}
+    return v
+
+
+class NodeRunResult(BaseModel):
+    """
+    Node Run Result.
+    """
+
+    status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.PENDING
+
+    inputs: Mapping[str, Any] = Field(default_factory=dict)
+    process_data: Mapping[str, Any] = Field(default_factory=dict)
+    outputs: Mapping[str, Any] = Field(default_factory=dict)
+    metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] = Field(default_factory=_default_metadata)
+    llm_usage: LLMUsage = Field(default_factory=LLMUsage.empty_usage)
+
+    edge_source_handle: str = "source"  # source handle id of node with multiple branches
+
+    error: str = ""
+    error_type: str = ""
+
+    # single step node run retry
+    retry_index: int = 0

api/core/workflow/node_events/iteration.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from .base import NodeEventBase
+
+
+class IterationStartedEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    predecessor_node_id: str | None = None
+
+
+class IterationNextEvent(NodeEventBase):
+    index: int = Field(..., description="index")
+    pre_iteration_output: Any = None
+
+
+class IterationSucceededEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+
+
+class IterationFailedEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+    error: str = Field(..., description="failed reason")

api/core/workflow/node_events/loop.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from .base import NodeEventBase
+
+
+class LoopStartedEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    predecessor_node_id: str | None = None
+
+
+class LoopNextEvent(NodeEventBase):
+    index: int = Field(..., description="index")
+    pre_loop_output: Any = None
+
+
+class LoopSucceededEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+
+
+class LoopFailedEvent(NodeEventBase):
+    start_at: datetime = Field(..., description="start at")
+    inputs: Mapping[str, object] = Field(default_factory=dict)
+    outputs: Mapping[str, object] = Field(default_factory=dict)
+    metadata: Mapping[str, object] = Field(default_factory=dict)
+    steps: int = 0
+    error: str = Field(..., description="failed reason")

api/core/workflow/node_events/node.py

@@ -0,0 +1,41 @@
+from collections.abc import Sequence
+from datetime import datetime
+
+from pydantic import Field
+
+from core.model_runtime.entities.llm_entities import LLMUsage
+from core.rag.entities.citation_metadata import RetrievalSourceMetadata
+from core.workflow.node_events import NodeRunResult
+
+from .base import NodeEventBase
+
+
+class RunRetrieverResourceEvent(NodeEventBase):
+    retriever_resources: Sequence[RetrievalSourceMetadata] = Field(..., description="retriever resources")
+    context: str = Field(..., description="context")
+
+
+class ModelInvokeCompletedEvent(NodeEventBase):
+    text: str
+    usage: LLMUsage
+    finish_reason: str | None = None
+    reasoning_content: str | None = None
+
+
+class RunRetryEvent(NodeEventBase):
+    error: str = Field(..., description="error")
+    retry_index: int = Field(..., description="Retry attempt number")
+    start_at: datetime = Field(..., description="Retry start time")
+
+
+class StreamChunkEvent(NodeEventBase):
+    # Spec-compliant fields
+    selector: Sequence[str] = Field(
+        ..., description="selector identifying the output location (e.g., ['nodeA', 'text'])"
+    )
+    chunk: str = Field(..., description="the actual chunk content")
+    is_final: bool = Field(default=False, description="indicates if this is the last chunk")
+
+
+class StreamCompletedEvent(NodeEventBase):
+    node_run_result: NodeRunResult = Field(..., description="run result")