feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/rag-pipeline/hooks/index.ts

@@ -0,0 +1,9 @@
+export * from './use-available-nodes-meta-data'
+export * from './use-pipeline-refresh-draft'
+export * from './use-nodes-sync-draft'
+export * from './use-pipeline-run'
+export * from './use-pipeline-start-run'
+export * from './use-pipeline-init'
+export * from './use-get-run-and-trace-url'
+export * from './use-DSL'
+export * from './use-input-field-panel'

web/app/components/rag-pipeline/hooks/use-DSL.ts

@@ -0,0 +1,83 @@
+import {
+  useCallback,
+  useState,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  DSL_EXPORT_CHECK,
+} from '@/app/components/workflow/constants'
+import { useNodesSyncDraft } from './use-nodes-sync-draft'
+import { useEventEmitterContextContext } from '@/context/event-emitter'
+import { fetchWorkflowDraft } from '@/service/workflow'
+import { useToastContext } from '@/app/components/base/toast'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+import { useExportPipelineDSL } from '@/service/use-pipeline'
+
+export const useDSL = () => {
+  const { t } = useTranslation()
+  const { notify } = useToastContext()
+  const { eventEmitter } = useEventEmitterContextContext()
+  const [exporting, setExporting] = useState(false)
+  const { doSyncWorkflowDraft } = useNodesSyncDraft()
+  const workflowStore = useWorkflowStore()
+  const { mutateAsync: exportPipelineConfig } = useExportPipelineDSL()
+
+  const handleExportDSL = useCallback(async (include = false) => {
+    const { pipelineId, knowledgeName } = workflowStore.getState()
+    if (!pipelineId)
+      return
+
+    if (exporting)
+      return
+
+    try {
+      setExporting(true)
+      await doSyncWorkflowDraft()
+      const { data } = await exportPipelineConfig({
+        pipelineId,
+        include,
+      })
+      const a = document.createElement('a')
+      const file = new Blob([data], { type: 'application/yaml' })
+      const url = URL.createObjectURL(file)
+      a.href = url
+      a.download = `${knowledgeName}.pipeline`
+      a.click()
+      URL.revokeObjectURL(url)
+    }
+    catch {
+      notify({ type: 'error', message: t('app.exportFailed') })
+    }
+    finally {
+      setExporting(false)
+    }
+  }, [notify, t, doSyncWorkflowDraft, exporting, exportPipelineConfig, workflowStore])
+
+  const exportCheck = useCallback(async () => {
+    const { pipelineId } = workflowStore.getState()
+    if (!pipelineId)
+      return
+    try {
+      const workflowDraft = await fetchWorkflowDraft(`/rag/pipelines/${pipelineId}/workflows/draft`)
+      const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret')
+      if (list.length === 0) {
+        handleExportDSL()
+        return
+      }
+      eventEmitter?.emit({
+        type: DSL_EXPORT_CHECK,
+        payload: {
+          data: list,
+        },
+      } as any)
+    }
+    catch {
+      notify({ type: 'error', message: t('app.exportFailed') })
+    }
+  }, [eventEmitter, handleExportDSL, notify, t, workflowStore])
+
+  return {
+    exportCheck,
+    handleExportDSL,
+  }
+}

web/app/components/rag-pipeline/hooks/use-available-nodes-meta-data.ts

@@ -0,0 +1,71 @@
+import { useMemo } from 'react'
+import { useTranslation } from 'react-i18next'
+import { useGetLanguage } from '@/context/i18n'
+import knowledgeBaseDefault from '@/app/components/workflow/nodes/knowledge-base/default'
+import dataSourceDefault from '@/app/components/workflow/nodes/data-source/default'
+import dataSourceEmptyDefault from '@/app/components/workflow/nodes/data-source-empty/default'
+import { WORKFLOW_COMMON_NODES } from '@/app/components/workflow/constants/node'
+import type { AvailableNodesMetaData } from '@/app/components/workflow/hooks-store/store'
+import { BlockEnum } from '@/app/components/workflow/types'
+
+export const useAvailableNodesMetaData = () => {
+  const { t } = useTranslation()
+  const language = useGetLanguage()
+
+  const mergedNodesMetaData = useMemo(() => [
+    ...WORKFLOW_COMMON_NODES,
+    {
+      ...dataSourceDefault,
+      defaultValue: {
+        ...dataSourceDefault.defaultValue,
+        _dataSourceStartToAdd: true,
+      },
+    },
+    knowledgeBaseDefault,
+    dataSourceEmptyDefault,
+  ], [])
+
+  const helpLinkUri = useMemo(() => {
+    if (language === 'zh_Hans')
+      return 'https://docs.dify.ai/zh-hans/guides/knowledge-base/knowledge-pipeline/knowledge-pipeline-orchestration#%E6%AD%A5%E9%AA%A4%E4%B8%80%EF%BC%9A%E6%95%B0%E6%8D%AE%E6%BA%90%E9%85%8D%E7%BD%AE'
+    if (language === 'ja_JP')
+      return 'https://docs.dify.ai/ja-jp/guides/knowledge-base/knowledge-pipeline/knowledge-pipeline-orchestration#%E3%82%B9%E3%83%86%E3%83%83%E3%83%971%EF%BC%9A%E3%83%87%E3%83%BC%E3%82%BF%E3%82%BD%E3%83%BC%E3%82%B9%E3%81%AE%E8%A8%AD%E5%AE%9A'
+
+    return 'https://docs.dify.ai/en/guides/knowledge-base/knowledge-pipeline/knowledge-pipeline-orchestration#step-1%3A-data-source'
+  }, [language])
+
+  const availableNodesMetaData = useMemo(() => mergedNodesMetaData.map((node) => {
+    const { metaData } = node
+    const title = t(`workflow.blocks.${metaData.type}`)
+    const description = t(`workflow.blocksAbout.${metaData.type}`)
+    return {
+      ...node,
+      metaData: {
+        ...metaData,
+        title,
+        description,
+        helpLinkUri,
+      },
+      defaultValue: {
+        ...node.defaultValue,
+        type: metaData.type,
+        title,
+      },
+    }
+  }), [mergedNodesMetaData, t])
+
+  const availableNodesMetaDataMap = useMemo(() => availableNodesMetaData.reduce((acc, node) => {
+    acc![node.metaData.type] = node
+    return acc
+  }, {} as AvailableNodesMetaData['nodesMap']), [availableNodesMetaData])
+
+  return useMemo(() => {
+    return {
+      nodes: availableNodesMetaData,
+      nodesMap: {
+        ...availableNodesMetaDataMap,
+        [BlockEnum.VariableAssigner]: availableNodesMetaDataMap?.[BlockEnum.VariableAggregator],
+      },
+    }
+  }, [availableNodesMetaData, availableNodesMetaDataMap])
+}

web/app/components/rag-pipeline/hooks/use-configs-map.ts

@@ -0,0 +1,24 @@
+import { useMemo } from 'react'
+import { useStore } from '@/app/components/workflow/store'
+import { FlowType } from '@/types/common'
+import { Resolution, TransferMethod } from '@/types/app'
+
+export const useConfigsMap = () => {
+  const pipelineId = useStore(s => s.pipelineId)
+  const fileUploadConfig = useStore(s => s.fileUploadConfig)
+  return useMemo(() => {
+    return {
+      flowId: pipelineId!,
+      flowType: FlowType.ragPipeline,
+      fileSettings: {
+        image: {
+          enabled: false,
+          detail: Resolution.high,
+          number_limits: 3,
+          transfer_methods: [TransferMethod.local_file, TransferMethod.remote_url],
+        },
+        fileUploadConfig,
+      },
+    }
+  }, [pipelineId])
+}

web/app/components/rag-pipeline/hooks/use-get-run-and-trace-url.ts

@@ -0,0 +1,18 @@
+import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+
+export const useGetRunAndTraceUrl = () => {
+  const workflowStore = useWorkflowStore()
+  const getWorkflowRunAndTraceUrl = useCallback((runId: string) => {
+    const { pipelineId } = workflowStore.getState()
+
+    return {
+      runUrl: `/rag/pipelines/${pipelineId}/workflow-runs/${runId}`,
+      traceUrl: `/rag/pipelines/${pipelineId}/workflow-runs/${runId}/node-executions`,
+    }
+  }, [workflowStore])
+
+  return {
+    getWorkflowRunAndTraceUrl,
+  }
+}

web/app/components/rag-pipeline/hooks/use-input-field-panel.ts

@@ -0,0 +1,54 @@
+import { useCallback, useMemo } from 'react'
+import { useStore, useWorkflowStore } from '@/app/components/workflow/store'
+import type { InputFieldEditorProps } from '../components/panel/input-field/editor'
+
+export const useInputFieldPanel = () => {
+  const workflowStore = useWorkflowStore()
+  const showInputFieldPreviewPanel = useStore(state => state.showInputFieldPreviewPanel)
+  const inputFieldEditPanelProps = useStore(state => state.inputFieldEditPanelProps)
+
+  const isPreviewing = useMemo(() => {
+    return showInputFieldPreviewPanel
+  }, [showInputFieldPreviewPanel])
+
+  const isEditing = useMemo(() => {
+    return !!inputFieldEditPanelProps
+  }, [inputFieldEditPanelProps])
+
+  const closeAllInputFieldPanels = useCallback(() => {
+    const {
+      setShowInputFieldPanel,
+      setShowInputFieldPreviewPanel,
+      setInputFieldEditPanelProps,
+    } = workflowStore.getState()
+
+    setShowInputFieldPanel?.(false)
+    setShowInputFieldPreviewPanel?.(false)
+    setInputFieldEditPanelProps?.(null)
+  }, [workflowStore])
+
+  const toggleInputFieldPreviewPanel = useCallback(() => {
+    const {
+      showInputFieldPreviewPanel,
+      setShowInputFieldPreviewPanel,
+    } = workflowStore.getState()
+
+    setShowInputFieldPreviewPanel?.(!showInputFieldPreviewPanel)
+  }, [workflowStore])
+
+  const toggleInputFieldEditPanel = useCallback((editContent: InputFieldEditorProps | null) => {
+    const {
+      setInputFieldEditPanelProps,
+    } = workflowStore.getState()
+
+    setInputFieldEditPanelProps?.(editContent)
+  }, [workflowStore])
+
+  return {
+    closeAllInputFieldPanels,
+    toggleInputFieldPreviewPanel,
+    toggleInputFieldEditPanel,
+    isPreviewing,
+    isEditing,
+  }
+}

web/app/components/rag-pipeline/hooks/use-input-fields.ts

@@ -0,0 +1,54 @@
+import { useMemo } from 'react'
+import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
+import { type RAGPipelineVariables, VAR_TYPE_MAP } from '@/models/pipeline'
+import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
+
+export const useInitialData = (variables: RAGPipelineVariables, lastRunInputData?: Record<string, any>) => {
+  const initialData = useMemo(() => {
+    return variables.reduce((acc, item) => {
+      const type = VAR_TYPE_MAP[item.type]
+      const variableName = item.variable
+      const defaultValue = lastRunInputData?.[variableName] || item.default_value
+      if ([BaseFieldType.textInput, BaseFieldType.paragraph, BaseFieldType.select].includes(type))
+        acc[variableName] = defaultValue ?? ''
+      if (type === BaseFieldType.numberInput)
+        acc[variableName] = defaultValue ?? 0
+      if (type === BaseFieldType.checkbox)
+        acc[variableName] = defaultValue ?? false
+      if ([BaseFieldType.file, BaseFieldType.fileList].includes(type))
+        acc[variableName] = defaultValue ?? []
+      return acc
+    }, {} as Record<string, any>)
+  }, [lastRunInputData, variables])
+
+  return initialData
+}
+
+export const useConfigurations = (variables: RAGPipelineVariables) => {
+  const configurations = useMemo(() => {
+    const configurations: BaseConfiguration[] = []
+    variables.forEach((item) => {
+      configurations.push({
+        type: VAR_TYPE_MAP[item.type],
+        variable: item.variable,
+        label: item.label,
+        required: item.required,
+        maxLength: item.max_length,
+        options: item.options?.map(option => ({
+          label: option,
+          value: option,
+        })),
+        showConditions: [],
+        placeholder: item.placeholder,
+        tooltip: item.tooltips,
+        unit: item.unit,
+        allowedFileTypes: item.allowed_file_types,
+        allowedFileExtensions: item.allowed_file_extensions,
+        allowedFileUploadMethods: item.allowed_file_upload_methods,
+      })
+    })
+    return configurations
+  }, [variables])
+
+  return configurations
+}

web/app/components/rag-pipeline/hooks/use-inspect-vars-crud.ts

@@ -0,0 +1,13 @@
+import { useInspectVarsCrudCommon } from '../../workflow/hooks/use-inspect-vars-crud-common'
+import { useConfigsMap } from './use-configs-map'
+
+export const useInspectVarsCrud = () => {
+  const configsMap = useConfigsMap()
+  const apis = useInspectVarsCrudCommon({
+    ...configsMap,
+  })
+
+  return {
+    ...apis,
+  }
+}

web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts

@@ -0,0 +1,128 @@
+import { useCallback } from 'react'
+import produce from 'immer'
+import { useStoreApi } from 'reactflow'
+import {
+  useWorkflowStore,
+} from '@/app/components/workflow/store'
+import {
+  useNodesReadOnly,
+} from '@/app/components/workflow/hooks/use-workflow'
+import { API_PREFIX } from '@/config'
+import { syncWorkflowDraft } from '@/service/workflow'
+import { usePipelineRefreshDraft } from '.'
+
+export const useNodesSyncDraft = () => {
+  const store = useStoreApi()
+  const workflowStore = useWorkflowStore()
+  const { getNodesReadOnly } = useNodesReadOnly()
+  const { handleRefreshWorkflowDraft } = usePipelineRefreshDraft()
+
+  const getPostParams = useCallback(() => {
+    const {
+      getNodes,
+      edges,
+      transform,
+    } = store.getState()
+    const nodesOriginal = getNodes()
+    const nodes = nodesOriginal.filter(node => !node.data._isTempNode)
+    const [x, y, zoom] = transform
+    const {
+      pipelineId,
+      environmentVariables,
+      syncWorkflowDraftHash,
+      ragPipelineVariables,
+    } = workflowStore.getState()
+
+    if (pipelineId && !!nodes.length) {
+      const producedNodes = produce(nodes, (draft) => {
+        draft.forEach((node) => {
+          Object.keys(node.data).forEach((key) => {
+            if (key.startsWith('_'))
+              delete node.data[key]
+          })
+        })
+      })
+      const producedEdges = produce(edges, (draft) => {
+        draft.forEach((edge) => {
+          Object.keys(edge.data).forEach((key) => {
+            if (key.startsWith('_'))
+              delete edge.data[key]
+          })
+        })
+      })
+      return {
+        url: `/rag/pipelines/${pipelineId}/workflows/draft`,
+        params: {
+          graph: {
+            nodes: producedNodes,
+            edges: producedEdges,
+            viewport: {
+              x,
+              y,
+              zoom,
+            },
+          },
+          environment_variables: environmentVariables,
+          rag_pipeline_variables: ragPipelineVariables,
+          hash: syncWorkflowDraftHash,
+        },
+      }
+    }
+  }, [store, workflowStore])
+
+  const syncWorkflowDraftWhenPageClose = useCallback(() => {
+    if (getNodesReadOnly())
+      return
+    const postParams = getPostParams()
+
+    if (postParams) {
+      navigator.sendBeacon(
+        `${API_PREFIX}${postParams.url}?_token=${localStorage.getItem('console_token')}`,
+        JSON.stringify(postParams.params),
+      )
+    }
+  }, [getPostParams, getNodesReadOnly])
+
+  const doSyncWorkflowDraft = useCallback(async (
+    notRefreshWhenSyncError?: boolean,
+    callback?: {
+      onSuccess?: () => void
+      onError?: () => void
+      onSettled?: () => void
+    },
+  ) => {
+    if (getNodesReadOnly())
+      return
+
+    const postParams = getPostParams()
+    if (postParams) {
+      const {
+        setSyncWorkflowDraftHash,
+        setDraftUpdatedAt,
+      } = workflowStore.getState()
+      try {
+        const res = await syncWorkflowDraft(postParams)
+        setSyncWorkflowDraftHash(res.hash)
+        setDraftUpdatedAt(res.updated_at)
+        callback?.onSuccess && callback.onSuccess()
+      }
+      catch (error: any) {
+        if (error && error.json && !error.bodyUsed) {
+          error.json().then((err: any) => {
+            if (err.code === 'draft_workflow_not_sync' && !notRefreshWhenSyncError)
+              handleRefreshWorkflowDraft()
+          })
+        }
+        callback?.onError && callback.onError()
+      }
+      finally {
+        callback?.onSettled && callback.onSettled()
+      }
+    }
+  }, [getPostParams, getNodesReadOnly, workflowStore, handleRefreshWorkflowDraft])
+
+  return {
+    doSyncWorkflowDraft,
+    syncWorkflowDraftWhenPageClose,
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline-config.ts

@@ -0,0 +1,73 @@
+import { useCallback } from 'react'
+import {
+  useStore,
+  useWorkflowStore,
+} from '@/app/components/workflow/store'
+import { useWorkflowConfig } from '@/service/use-workflow'
+import type { FetchWorkflowDraftResponse } from '@/types/workflow'
+import { useDataSourceList } from '@/service/use-pipeline'
+import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
+import { basePath } from '@/utils/var'
+import type { FileUploadConfigResponse } from '@/models/common'
+
+export const usePipelineConfig = () => {
+  const pipelineId = useStore(s => s.pipelineId)
+  const workflowStore = useWorkflowStore()
+
+  const handleUpdateWorkflowConfig = useCallback((config: Record<string, any>) => {
+    const { setWorkflowConfig } = workflowStore.getState()
+
+    setWorkflowConfig(config)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/draft/config` : '',
+    handleUpdateWorkflowConfig,
+  )
+
+  const handleUpdateNodesDefaultConfigs = useCallback((nodesDefaultConfigs: Record<string, any> | Record<string, any>[]) => {
+    const { setNodesDefaultConfigs } = workflowStore.getState()
+    let res: Record<string, any> = {}
+    if (Array.isArray(nodesDefaultConfigs)) {
+      nodesDefaultConfigs.forEach((item) => {
+        res[item.type] = item.config
+      })
+    }
+    else {
+      res = nodesDefaultConfigs as Record<string, any>
+    }
+
+    setNodesDefaultConfigs!(res)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/default-workflow-block-configs` : '',
+    handleUpdateNodesDefaultConfigs,
+  )
+
+  const handleUpdatePublishedAt = useCallback((publishedWorkflow: FetchWorkflowDraftResponse) => {
+    const { setPublishedAt } = workflowStore.getState()
+
+    setPublishedAt(publishedWorkflow?.created_at)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipelines/${pipelineId}/workflows/publish` : '',
+    handleUpdatePublishedAt,
+  )
+
+  const handleUpdateDataSourceList = useCallback((dataSourceList: DataSourceItem[]) => {
+    dataSourceList.forEach((item) => {
+      const icon = item.declaration.identity.icon
+      if (typeof icon == 'string' && !icon.includes(basePath))
+        item.declaration.identity.icon = `${basePath}${icon}`
+    })
+    const { setDataSourceList } = workflowStore.getState()
+    setDataSourceList!(dataSourceList)
+  }, [workflowStore])
+
+  const handleUpdateWorkflowFileUploadConfig = useCallback((config: FileUploadConfigResponse) => {
+    const { setFileUploadConfig } = workflowStore.getState()
+    setFileUploadConfig(config)
+  }, [workflowStore])
+  useWorkflowConfig('/files/upload', handleUpdateWorkflowFileUploadConfig)
+
+  useDataSourceList(!!pipelineId, handleUpdateDataSourceList)
+}

web/app/components/rag-pipeline/hooks/use-pipeline-init.ts

@@ -0,0 +1,92 @@
+import {
+  useCallback,
+  useEffect,
+  useState,
+} from 'react'
+import {
+  useWorkflowStore,
+} from '@/app/components/workflow/store'
+import { usePipelineTemplate } from './use-pipeline-template'
+import {
+  fetchWorkflowDraft,
+  syncWorkflowDraft,
+} from '@/service/workflow'
+import type { FetchWorkflowDraftResponse } from '@/types/workflow'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { usePipelineConfig } from './use-pipeline-config'
+
+export const usePipelineInit = () => {
+  const workflowStore = useWorkflowStore()
+  const {
+    nodes: nodesTemplate,
+    edges: edgesTemplate,
+  } = usePipelineTemplate()
+  const [data, setData] = useState<FetchWorkflowDraftResponse>()
+  const [isLoading, setIsLoading] = useState(true)
+  const datasetId = useDatasetDetailContextWithSelector(s => s.dataset)?.pipeline_id
+  const knowledgeName = useDatasetDetailContextWithSelector(s => s.dataset)?.name
+  const knowledgeIcon = useDatasetDetailContextWithSelector(s => s.dataset)?.icon_info
+
+  useEffect(() => {
+    workflowStore.setState({ pipelineId: datasetId, knowledgeName, knowledgeIcon })
+  }, [datasetId, workflowStore, knowledgeName, knowledgeIcon])
+
+  usePipelineConfig()
+
+  const handleGetInitialWorkflowData = useCallback(async () => {
+    const {
+      setEnvSecrets,
+      setEnvironmentVariables,
+      setSyncWorkflowDraftHash,
+      setDraftUpdatedAt,
+      setToolPublished,
+      setRagPipelineVariables,
+    } = workflowStore.getState()
+    try {
+      const res = await fetchWorkflowDraft(`/rag/pipelines/${datasetId}/workflows/draft`)
+      setData(res)
+      setDraftUpdatedAt(res.updated_at)
+      setToolPublished(res.tool_published)
+      setEnvSecrets((res.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
+        acc[env.id] = env.value
+        return acc
+      }, {} as Record<string, string>))
+      setEnvironmentVariables(res.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
+      setSyncWorkflowDraftHash(res.hash)
+      setRagPipelineVariables?.(res.rag_pipeline_variables || [])
+      setIsLoading(false)
+    }
+    catch (error: any) {
+      if (error && error.json && !error.bodyUsed && datasetId) {
+        error.json().then((err: any) => {
+          if (err.code === 'draft_workflow_not_exist') {
+            workflowStore.setState({ notInitialWorkflow: true })
+            syncWorkflowDraft({
+              url: `/rag/pipelines/${datasetId}/workflows/draft`,
+              params: {
+                graph: {
+                  nodes: nodesTemplate,
+                  edges: edgesTemplate,
+                },
+                environment_variables: [],
+              },
+            }).then((res) => {
+              const { setDraftUpdatedAt } = workflowStore.getState()
+              setDraftUpdatedAt(res.updated_at)
+              handleGetInitialWorkflowData()
+            })
+          }
+        })
+      }
+    }
+  }, [nodesTemplate, edgesTemplate, workflowStore, datasetId])
+
+  useEffect(() => {
+    handleGetInitialWorkflowData()
+  }, [])
+
+  return {
+    data,
+    isLoading,
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline-refresh-draft.ts

@@ -0,0 +1,43 @@
+import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+import { fetchWorkflowDraft } from '@/service/workflow'
+import type { WorkflowDataUpdater } from '@/app/components/workflow/types'
+import { useWorkflowUpdate } from '@/app/components/workflow/hooks'
+import { processNodesWithoutDataSource } from '../utils'
+
+export const usePipelineRefreshDraft = () => {
+  const workflowStore = useWorkflowStore()
+  const { handleUpdateWorkflowCanvas } = useWorkflowUpdate()
+
+  const handleRefreshWorkflowDraft = useCallback(() => {
+    const {
+      pipelineId,
+      setSyncWorkflowDraftHash,
+      setIsSyncingWorkflowDraft,
+      setEnvironmentVariables,
+      setEnvSecrets,
+    } = workflowStore.getState()
+    setIsSyncingWorkflowDraft(true)
+    fetchWorkflowDraft(`/rag/pipelines/${pipelineId}/workflows/draft`).then((response) => {
+      const {
+        nodes: processedNodes,
+        viewport,
+      } = processNodesWithoutDataSource(response.graph.nodes, response.graph.viewport)
+      handleUpdateWorkflowCanvas({
+        ...response.graph,
+        nodes: processedNodes,
+        viewport,
+      } as WorkflowDataUpdater)
+      setSyncWorkflowDraftHash(response.hash)
+      setEnvSecrets((response.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
+        acc[env.id] = env.value
+        return acc
+      }, {} as Record<string, string>))
+      setEnvironmentVariables(response.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
+    }).finally(() => setIsSyncingWorkflowDraft(false))
+  }, [handleUpdateWorkflowCanvas, workflowStore])
+
+  return {
+    handleRefreshWorkflowDraft,
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline-run.ts

@@ -0,0 +1,316 @@
+import { useCallback } from 'react'
+import {
+  useReactFlow,
+  useStoreApi,
+} from 'reactflow'
+import produce from 'immer'
+import { useStore, useWorkflowStore } from '@/app/components/workflow/store'
+import { WorkflowRunningStatus } from '@/app/components/workflow/types'
+import { useWorkflowUpdate } from '@/app/components/workflow/hooks/use-workflow-interactions'
+import { useWorkflowRunEvent } from '@/app/components/workflow/hooks/use-workflow-run-event/use-workflow-run-event'
+import type { IOtherOptions } from '@/service/base'
+import { ssePost } from '@/service/base'
+import { stopWorkflowRun } from '@/service/workflow'
+import type { VersionHistory } from '@/types/workflow'
+import { useNodesSyncDraft } from './use-nodes-sync-draft'
+import { useSetWorkflowVarsWithValue } from '@/app/components/workflow/hooks/use-fetch-workflow-inspect-vars'
+import { useInvalidAllLastRun } from '@/service/use-workflow'
+import { FlowType } from '@/types/common'
+
+export const usePipelineRun = () => {
+  const store = useStoreApi()
+  const workflowStore = useWorkflowStore()
+  const reactflow = useReactFlow()
+  const { doSyncWorkflowDraft } = useNodesSyncDraft()
+  const { handleUpdateWorkflowCanvas } = useWorkflowUpdate()
+
+  const {
+    handleWorkflowStarted,
+    handleWorkflowFinished,
+    handleWorkflowFailed,
+    handleWorkflowNodeStarted,
+    handleWorkflowNodeFinished,
+    handleWorkflowNodeIterationStarted,
+    handleWorkflowNodeIterationNext,
+    handleWorkflowNodeIterationFinished,
+    handleWorkflowNodeLoopStarted,
+    handleWorkflowNodeLoopNext,
+    handleWorkflowNodeLoopFinished,
+    handleWorkflowNodeRetry,
+    handleWorkflowAgentLog,
+    handleWorkflowTextChunk,
+    handleWorkflowTextReplace,
+  } = useWorkflowRunEvent()
+
+  const handleBackupDraft = useCallback(() => {
+    const {
+      getNodes,
+      edges,
+    } = store.getState()
+    const { getViewport } = reactflow
+    const {
+      backupDraft,
+      setBackupDraft,
+      environmentVariables,
+    } = workflowStore.getState()
+
+    if (!backupDraft) {
+      setBackupDraft({
+        nodes: getNodes(),
+        edges,
+        viewport: getViewport(),
+        environmentVariables,
+      })
+      doSyncWorkflowDraft()
+    }
+  }, [reactflow, workflowStore, store, doSyncWorkflowDraft])
+
+  const handleLoadBackupDraft = useCallback(() => {
+    const {
+      backupDraft,
+      setBackupDraft,
+      setEnvironmentVariables,
+    } = workflowStore.getState()
+
+    if (backupDraft) {
+      const {
+        nodes,
+        edges,
+        viewport,
+        environmentVariables,
+      } = backupDraft
+      handleUpdateWorkflowCanvas({
+        nodes,
+        edges,
+        viewport,
+      })
+      setEnvironmentVariables(environmentVariables)
+      setBackupDraft(undefined)
+    }
+  }, [handleUpdateWorkflowCanvas, workflowStore])
+
+  const pipelineId = useStore(s => s.pipelineId)
+  const invalidAllLastRun = useInvalidAllLastRun(FlowType.ragPipeline, pipelineId)
+  const { fetchInspectVars } = useSetWorkflowVarsWithValue({
+    flowType: FlowType.ragPipeline,
+    flowId: pipelineId!,
+  })
+
+  const handleRun = useCallback(async (
+    params: any,
+    callback?: IOtherOptions,
+  ) => {
+    const {
+      getNodes,
+      setNodes,
+    } = store.getState()
+    const newNodes = produce(getNodes(), (draft) => {
+      draft.forEach((node) => {
+        node.data.selected = false
+        node.data._runningStatus = undefined
+      })
+    })
+    setNodes(newNodes)
+    await doSyncWorkflowDraft()
+
+    const {
+      onWorkflowStarted,
+      onWorkflowFinished,
+      onNodeStarted,
+      onNodeFinished,
+      onIterationStart,
+      onIterationNext,
+      onIterationFinish,
+      onLoopStart,
+      onLoopNext,
+      onLoopFinish,
+      onNodeRetry,
+      onAgentLog,
+      onError,
+      ...restCallback
+    } = callback || {}
+    const { pipelineId } = workflowStore.getState()
+    workflowStore.setState({ historyWorkflowData: undefined })
+    const workflowContainer = document.getElementById('workflow-container')
+
+    const {
+      clientWidth,
+      clientHeight,
+    } = workflowContainer!
+
+    const url = `/rag/pipelines/${pipelineId}/workflows/draft/run`
+
+    const {
+      setWorkflowRunningData,
+    } = workflowStore.getState()
+    setWorkflowRunningData({
+      result: {
+        status: WorkflowRunningStatus.Running,
+      },
+      tracing: [],
+      resultText: '',
+    })
+
+    ssePost(
+      url,
+      {
+        body: params,
+      },
+      {
+        onWorkflowStarted: (params) => {
+          handleWorkflowStarted(params)
+
+          if (onWorkflowStarted)
+            onWorkflowStarted(params)
+        },
+        onWorkflowFinished: (params) => {
+          handleWorkflowFinished(params)
+          fetchInspectVars({})
+          invalidAllLastRun()
+
+          if (onWorkflowFinished)
+            onWorkflowFinished(params)
+        },
+        onError: (params) => {
+          handleWorkflowFailed()
+
+          if (onError)
+            onError(params)
+        },
+        onNodeStarted: (params) => {
+          handleWorkflowNodeStarted(
+            params,
+            {
+              clientWidth,
+              clientHeight,
+            },
+          )
+
+          if (onNodeStarted)
+            onNodeStarted(params)
+        },
+        onNodeFinished: (params) => {
+          handleWorkflowNodeFinished(params)
+
+          if (onNodeFinished)
+            onNodeFinished(params)
+        },
+        onIterationStart: (params) => {
+          handleWorkflowNodeIterationStarted(
+            params,
+            {
+              clientWidth,
+              clientHeight,
+            },
+          )
+
+          if (onIterationStart)
+            onIterationStart(params)
+        },
+        onIterationNext: (params) => {
+          handleWorkflowNodeIterationNext(params)
+
+          if (onIterationNext)
+            onIterationNext(params)
+        },
+        onIterationFinish: (params) => {
+          handleWorkflowNodeIterationFinished(params)
+
+          if (onIterationFinish)
+            onIterationFinish(params)
+        },
+        onLoopStart: (params) => {
+          handleWorkflowNodeLoopStarted(
+            params,
+            {
+              clientWidth,
+              clientHeight,
+            },
+          )
+
+          if (onLoopStart)
+            onLoopStart(params)
+        },
+        onLoopNext: (params) => {
+          handleWorkflowNodeLoopNext(params)
+
+          if (onLoopNext)
+            onLoopNext(params)
+        },
+        onLoopFinish: (params) => {
+          handleWorkflowNodeLoopFinished(params)
+
+          if (onLoopFinish)
+            onLoopFinish(params)
+        },
+        onNodeRetry: (params) => {
+          handleWorkflowNodeRetry(params)
+
+          if (onNodeRetry)
+            onNodeRetry(params)
+        },
+        onAgentLog: (params) => {
+          handleWorkflowAgentLog(params)
+
+          if (onAgentLog)
+            onAgentLog(params)
+        },
+        onTextChunk: (params) => {
+          handleWorkflowTextChunk(params)
+        },
+        onTextReplace: (params) => {
+          handleWorkflowTextReplace(params)
+        },
+        ...restCallback,
+      },
+    )
+  }, [
+    store,
+    workflowStore,
+    doSyncWorkflowDraft,
+    handleWorkflowStarted,
+    handleWorkflowFinished,
+    handleWorkflowFailed,
+    handleWorkflowNodeStarted,
+    handleWorkflowNodeFinished,
+    handleWorkflowNodeIterationStarted,
+    handleWorkflowNodeIterationNext,
+    handleWorkflowNodeIterationFinished,
+    handleWorkflowNodeLoopStarted,
+    handleWorkflowNodeLoopNext,
+    handleWorkflowNodeLoopFinished,
+    handleWorkflowNodeRetry,
+    handleWorkflowTextChunk,
+    handleWorkflowTextReplace,
+    handleWorkflowAgentLog,
+  ],
+  )
+
+  const handleStopRun = useCallback((taskId: string) => {
+    const { pipelineId } = workflowStore.getState()
+
+    stopWorkflowRun(`/rag/pipelines/${pipelineId}/workflow-runs/tasks/${taskId}/stop`)
+  }, [workflowStore])
+
+  const handleRestoreFromPublishedWorkflow = useCallback((publishedWorkflow: VersionHistory) => {
+    const nodes = publishedWorkflow.graph.nodes.map(node => ({ ...node, selected: false, data: { ...node.data, selected: false } }))
+    const edges = publishedWorkflow.graph.edges
+    const viewport = publishedWorkflow.graph.viewport!
+    handleUpdateWorkflowCanvas({
+      nodes,
+      edges,
+      viewport,
+    })
+
+    workflowStore.getState().setEnvironmentVariables(publishedWorkflow.environment_variables || [])
+    workflowStore.getState().setRagPipelineVariables?.(publishedWorkflow.rag_pipeline_variables || [])
+  }, [handleUpdateWorkflowCanvas, workflowStore])
+
+  return {
+    handleBackupDraft,
+    handleLoadBackupDraft,
+    handleRun,
+    handleStopRun,
+    handleRestoreFromPublishedWorkflow,
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline-start-run.tsx

@@ -0,0 +1,64 @@
+import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+import {
+  WorkflowRunningStatus,
+} from '@/app/components/workflow/types'
+import { useWorkflowInteractions } from '@/app/components/workflow/hooks'
+import {
+  useInputFieldPanel,
+  useNodesSyncDraft,
+} from '.'
+
+export const usePipelineStartRun = () => {
+  const workflowStore = useWorkflowStore()
+  const { handleCancelDebugAndPreviewPanel } = useWorkflowInteractions()
+  const { doSyncWorkflowDraft } = useNodesSyncDraft()
+  const { closeAllInputFieldPanels } = useInputFieldPanel()
+
+  const handleWorkflowStartRunInWorkflow = useCallback(async () => {
+    const {
+      workflowRunningData,
+    } = workflowStore.getState()
+
+    if (workflowRunningData?.result.status === WorkflowRunningStatus.Running)
+      return
+
+    const {
+      isPreparingDataSource,
+      setIsPreparingDataSource,
+      showDebugAndPreviewPanel,
+      setShowEnvPanel,
+      setShowDebugAndPreviewPanel,
+    } = workflowStore.getState()
+
+    if (!isPreparingDataSource && workflowRunningData) {
+      workflowStore.setState({
+        isPreparingDataSource: true,
+        workflowRunningData: undefined,
+      })
+      return
+    }
+
+    setShowEnvPanel(false)
+    closeAllInputFieldPanels()
+
+    if (showDebugAndPreviewPanel) {
+      setIsPreparingDataSource?.(false)
+      handleCancelDebugAndPreviewPanel()
+      return
+    }
+
+    await doSyncWorkflowDraft()
+    setIsPreparingDataSource?.(true)
+    setShowDebugAndPreviewPanel(true)
+  }, [workflowStore, handleCancelDebugAndPreviewPanel, doSyncWorkflowDraft])
+
+  const handleStartWorkflowRun = useCallback(() => {
+    handleWorkflowStartRunInWorkflow()
+  }, [handleWorkflowStartRunInWorkflow])
+
+  return {
+    handleStartWorkflowRun,
+    handleWorkflowStartRunInWorkflow,
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline-template.ts

@@ -0,0 +1,30 @@
+import { useTranslation } from 'react-i18next'
+import { generateNewNode } from '@/app/components/workflow/utils'
+import {
+  START_INITIAL_POSITION,
+} from '@/app/components/workflow/constants'
+import type { KnowledgeBaseNodeType } from '@/app/components/workflow/nodes/knowledge-base/types'
+import knowledgeBaseDefault from '@/app/components/workflow/nodes/knowledge-base/default'
+
+export const usePipelineTemplate = () => {
+  const { t } = useTranslation()
+
+  const { newNode: knowledgeBaseNode } = generateNewNode({
+    id: 'knowledgeBase',
+    data: {
+      ...knowledgeBaseDefault.defaultValue as KnowledgeBaseNodeType,
+      type: knowledgeBaseDefault.metaData.type,
+      title: t(`workflow.blocks.${knowledgeBaseDefault.metaData.type}`),
+      selected: true,
+    },
+    position: {
+      x: START_INITIAL_POSITION.x + 500,
+      y: START_INITIAL_POSITION.y,
+    },
+  })
+
+  return {
+    nodes: [knowledgeBaseNode],
+    edges: [],
+  }
+}

web/app/components/rag-pipeline/hooks/use-pipeline.tsx

@@ -0,0 +1,115 @@
+import { useCallback } from 'react'
+import { getOutgoers, useStoreApi } from 'reactflow'
+import { BlockEnum, type Node, type ValueSelector } from '../../workflow/types'
+import { uniqBy } from 'lodash-es'
+import { findUsedVarNodes, updateNodeVars } from '../../workflow/nodes/_base/components/variable/utils'
+import type { DataSourceNodeType } from '../../workflow/nodes/data-source/types'
+
+export const usePipeline = () => {
+  const store = useStoreApi()
+
+  const getAllDatasourceNodes = useCallback(() => {
+    const {
+      getNodes,
+    } = store.getState()
+    const nodes = getNodes() as Node<DataSourceNodeType>[]
+    const datasourceNodes = nodes.filter(node => node.data.type === BlockEnum.DataSource)
+
+    return datasourceNodes
+  }, [store])
+
+  const getAllNodesInSameBranch = useCallback((nodeId: string) => {
+    const {
+      getNodes,
+      edges,
+    } = store.getState()
+    const nodes = getNodes()
+    const list: Node[] = []
+
+    const traverse = (root: Node, callback: (node: Node) => void) => {
+      if (root) {
+        const outgoers = getOutgoers(root, nodes, edges)
+
+        if (outgoers.length) {
+          outgoers.forEach((node) => {
+            callback(node)
+            traverse(node, callback)
+          })
+        }
+      }
+    }
+
+    if (nodeId === 'shared') {
+      const allDatasourceNodes = getAllDatasourceNodes()
+
+      if (allDatasourceNodes.length === 0)
+        return []
+
+      list.push(...allDatasourceNodes)
+
+      allDatasourceNodes.forEach((node) => {
+        traverse(node, (childNode) => {
+          list.push(childNode)
+        })
+      })
+    }
+    else {
+      const currentNode = nodes.find(node => node.id === nodeId)!
+
+      if (!currentNode)
+        return []
+
+      list.push(currentNode)
+
+      traverse(currentNode, (node) => {
+        list.push(node)
+      })
+    }
+
+    return uniqBy(list, 'id')
+  }, [getAllDatasourceNodes, store])
+
+  const isVarUsedInNodes = useCallback((varSelector: ValueSelector) => {
+    const nodeId = varSelector[1] // Assuming the first element is always 'VARIABLE_PREFIX'(rag)
+    const afterNodes = getAllNodesInSameBranch(nodeId)
+    const effectNodes = findUsedVarNodes(varSelector, afterNodes)
+    return effectNodes.length > 0
+  }, [getAllNodesInSameBranch])
+
+  const handleInputVarRename = useCallback((nodeId: string, oldValeSelector: ValueSelector, newVarSelector: ValueSelector) => {
+    const { getNodes, setNodes } = store.getState()
+    const afterNodes = getAllNodesInSameBranch(nodeId)
+    const effectNodes = findUsedVarNodes(oldValeSelector, afterNodes)
+    if (effectNodes.length > 0) {
+      const newNodes = getNodes().map((node) => {
+        if (effectNodes.find(n => n.id === node.id))
+          return updateNodeVars(node, oldValeSelector, newVarSelector)
+
+        return node
+      })
+      setNodes(newNodes)
+    }
+  }, [getAllNodesInSameBranch, store])
+
+  const removeUsedVarInNodes = useCallback((varSelector: ValueSelector) => {
+    const nodeId = varSelector[1] // Assuming the first element is always 'VARIABLE_PREFIX'(rag)
+    const { getNodes, setNodes } = store.getState()
+    const afterNodes = getAllNodesInSameBranch(nodeId)
+    const effectNodes = findUsedVarNodes(varSelector, afterNodes)
+    if (effectNodes.length > 0) {
+      const newNodes = getNodes().map((node) => {
+        if (effectNodes.find(n => n.id === node.id))
+          return updateNodeVars(node, varSelector, [])
+
+        return node
+      })
+      setNodes(newNodes)
+    }
+  }, [getAllNodesInSameBranch, store])
+
+  return {
+    handleInputVarRename,
+    isVarUsedInNodes,
+    removeUsedVarInNodes,
+  }
+}