feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/service/base.ts
+++ b/web/service/base.ts
@@ -23,8 +23,13 @@ import type {
 } from '@/types/workflow'
 import { removeAccessToken } from '@/app/components/share/utils'
 import type { FetchOptionType, ResponseError } from './fetch'
-import { ContentType, base, baseOptions, getAccessToken } from './fetch'
+import { ContentType, base, getAccessToken, getBaseOptions } from './fetch'
 import { asyncRunSafe } from '@/utils'
+import type {
+  DataSourceNodeCompletedResponse,
+  DataSourceNodeErrorResponse,
+  DataSourceNodeProcessingResponse,
+} from '@/types/pipeline'
 const TIME_OUT = 100000

 export type IOnDataMoreInfo = {
@@ -63,6 +68,10 @@ export type IOnLoopNext = (workflowStarted: LoopNextResponse) => void
 export type IOnLoopFinished = (workflowFinished: LoopFinishedResponse) => void
 export type IOnAgentLog = (agentLog: AgentLogResponse) => void

+export type IOnDataSourceNodeProcessing = (dataSourceNodeProcessing: DataSourceNodeProcessingResponse) => void
+export type IOnDataSourceNodeCompleted = (dataSourceNodeCompleted: DataSourceNodeCompletedResponse) => void
+export type IOnDataSourceNodeError = (dataSourceNodeError: DataSourceNodeErrorResponse) => void
+
 export type IOtherOptions = {
  isPublicAPI?: boolean
  isMarketplaceAPI?: boolean
@@ -97,6 +106,11 @@ export type IOtherOptions = {
  onLoopNext?: IOnLoopNext
  onLoopFinish?: IOnLoopFinished
  onAgentLog?: IOnAgentLog
+
+  // Pipeline data source node run
+  onDataSourceNodeProcessing?: IOnDataSourceNodeProcessing
+  onDataSourceNodeCompleted?: IOnDataSourceNodeCompleted
+  onDataSourceNodeError?: IOnDataSourceNodeError
 }

 function unicodeToChar(text: string) {
@@ -152,6 +166,9 @@ const handleStream = (
  onTTSEnd?: IOnTTSEnd,
  onTextReplace?: IOnTextReplace,
  onAgentLog?: IOnAgentLog,
+  onDataSourceNodeProcessing?: IOnDataSourceNodeProcessing,
+  onDataSourceNodeCompleted?: IOnDataSourceNodeCompleted,
+  onDataSourceNodeError?: IOnDataSourceNodeError,
 ) => {
  if (!response.ok)
    throw new Error('Network response was not ok')
@@ -270,6 +287,18 @@ const handleStream = (
            else if (bufferObj.event === 'tts_message_end') {
              onTTSEnd?.(bufferObj.message_id, bufferObj.audio)
            }
+            else if (bufferObj.event === 'datasource_processing') {
+              onDataSourceNodeProcessing?.(bufferObj as DataSourceNodeProcessingResponse)
+            }
+            else if (bufferObj.event === 'datasource_completed') {
+              onDataSourceNodeCompleted?.(bufferObj as DataSourceNodeCompletedResponse)
+            }
+            else if (bufferObj.event === 'datasource_error') {
+              onDataSourceNodeError?.(bufferObj as DataSourceNodeErrorResponse)
+            }
+            else {
+              console.warn(`Unknown event: ${bufferObj.event}`, bufferObj)
+            }
          }
        })
        buffer = lines[lines.length - 1]
@@ -363,11 +392,15 @@ export const ssePost = async (
    onLoopStart,
    onLoopNext,
    onLoopFinish,
+    onDataSourceNodeProcessing,
+    onDataSourceNodeCompleted,
+    onDataSourceNodeError,
  } = otherOptions
  const abortController = new AbortController()

  const token = localStorage.getItem('console_token')

+  const baseOptions = getBaseOptions()
  const options = Object.assign({}, baseOptions, {
    method: 'POST',
    signal: abortController.signal,
@@ -432,39 +465,44 @@ export const ssePost = async (
        }
        return
      }
-      return handleStream(res, (str: string, isFirstMessage: boolean, moreInfo: IOnDataMoreInfo) => {
-        if (moreInfo.errorMessage) {
-          onError?.(moreInfo.errorMessage, moreInfo.errorCode)
-          // TypeError: Cannot assign to read only property ... will happen in page leave, so it should be ignored.
-          if (moreInfo.errorMessage !== 'AbortError: The user aborted a request.' && !moreInfo.errorMessage.includes('TypeError: Cannot assign to read only property'))
-            Toast.notify({ type: 'error', message: moreInfo.errorMessage })
-          return
-        }
-        onData?.(str, isFirstMessage, moreInfo)
-      },
-      onCompleted,
-      onThought,
-      onMessageEnd,
-      onMessageReplace,
-      onFile,
-      onWorkflowStarted,
-      onWorkflowFinished,
-      onNodeStarted,
-      onNodeFinished,
-      onIterationStart,
-      onIterationNext,
-      onIterationFinish,
-      onLoopStart,
-      onLoopNext,
-      onLoopFinish,
-      onNodeRetry,
-      onParallelBranchStarted,
-      onParallelBranchFinished,
-      onTextChunk,
-      onTTSChunk,
-      onTTSEnd,
-      onTextReplace,
-      onAgentLog,
+      return handleStream(
+        res,
+        (str: string, isFirstMessage: boolean, moreInfo: IOnDataMoreInfo) => {
+          if (moreInfo.errorMessage) {
+            onError?.(moreInfo.errorMessage, moreInfo.errorCode)
+            // TypeError: Cannot assign to read only property ... will happen in page leave, so it should be ignored.
+            if (moreInfo.errorMessage !== 'AbortError: The user aborted a request.' && !moreInfo.errorMessage.includes('TypeError: Cannot assign to read only property'))
+              Toast.notify({ type: 'error', message: moreInfo.errorMessage })
+            return
+          }
+          onData?.(str, isFirstMessage, moreInfo)
+        },
+        onCompleted,
+        onThought,
+        onMessageEnd,
+        onMessageReplace,
+        onFile,
+        onWorkflowStarted,
+        onWorkflowFinished,
+        onNodeStarted,
+        onNodeFinished,
+        onIterationStart,
+        onIterationNext,
+        onIterationFinish,
+        onLoopStart,
+        onLoopNext,
+        onLoopFinish,
+        onNodeRetry,
+        onParallelBranchStarted,
+        onParallelBranchFinished,
+        onTextChunk,
+        onTTSChunk,
+        onTTSEnd,
+        onTextReplace,
+        onAgentLog,
+        onDataSourceNodeProcessing,
+        onDataSourceNodeCompleted,
+        onDataSourceNodeError,
      )
    }).catch((e) => {
      if (e.toString() !== 'AbortError: The user aborted a request.' && !e.toString().includes('TypeError: Cannot assign to read only property'))
@@ -482,18 +520,14 @@ export const request = async<T>(url: string, options = {}, otherOptions?: IOther
      return resp
    const errResp: Response = err as any
    if (errResp.status === 401) {
-      if(/\/login/.test(url)) {
-        const clonedResponse = errResp.clone()
-        const bodyJson = await clonedResponse.json() as Promise<ResponseError>
-        return bodyJson
-      }
-
      const [parseErr, errRespData] = await asyncRunSafe<ResponseError>(errResp.json())
      const loginUrl = `${globalThis.location.origin}${basePath}/signin`
      if (parseErr) {
        globalThis.location.href = loginUrl
        return Promise.reject(err)
      }
+      if (/\/login/.test(url))
+        return Promise.reject(errRespData)
      // special code
      const { code, message } = errRespData
      // webapp sso
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -57,7 +57,7 @@ export const fetchDatasetDetail: Fetcher<DataSet, string> = (datasetId: string)
 export const updateDatasetSetting: Fetcher<DataSet, {
  datasetId: string
  body: Partial<Pick<DataSet,
-    'name' | 'description' | 'permission' | 'partial_member_list' | 'indexing_technique' | 'retrieval_model' | 'embedding_model' | 'embedding_model_provider'
+    'name' | 'description' | 'permission' | 'partial_member_list' | 'indexing_technique' | 'retrieval_model' | 'embedding_model' | 'embedding_model_provider' | 'icon_info' | 'doc_form'
  >>
 }> = ({ datasetId, body }) => {
  return patch<DataSet>(`/datasets/${datasetId}`, { body })
@@ -183,8 +183,12 @@ export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, In
  return post<FileIndexingEstimateResponse>('/datasets/indexing-estimate', { body })
 }

-export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string }> = ({ workspaceID, pageID, pageType }) => {
-  return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`)
+export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string; credentialID: string; }> = ({ workspaceID, pageID, pageType, credentialID }) => {
+  return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`, {
+    params: {
+      credential_id: credentialID,
+    },
+  })
 }

 export const fetchApiKeysList: Fetcher<ApiKeysListResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
@@ -199,10 +203,6 @@ export const createApikey: Fetcher<CreateApiKeyResponse, { url: string; body: Re
  return post<CreateApiKeyResponse>(url, body)
 }

-export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> = (url) => {
-  return get<{ api_base_url: string }>(url)
-}
-
 export const fetchDataSources = () => {
  return get<CommonResponse>('api-key-auth/data-source')
 }
@@ -272,7 +272,7 @@ export const checkWatercrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId
  })
 }

-type FileTypesRes = {
+export type FileTypesRes = {
  allowed_extensions: string[]
 }

--- a/web/service/fetch.ts
+++ b/web/service/fetch.ts
@@ -2,7 +2,7 @@ import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Hooks } fro
 import ky from 'ky'
 import type { IOtherOptions } from './base'
 import Toast from '@/app/components/base/toast'
-import { API_PREFIX, MARKETPLACE_API_PREFIX, PUBLIC_API_PREFIX } from '@/config'
+import { API_PREFIX, APP_VERSION, MARKETPLACE_API_PREFIX, PUBLIC_API_PREFIX } from '@/config'
 import { getInitialTokenV2, isTokenV1 } from '@/app/components/share/utils'
 import { getProcessedSystemVariablesFromUrlParams } from '@/app/components/base/chat/utils'

@@ -111,7 +111,7 @@ const baseClient = ky.create({
  timeout: TIME_OUT,
 })

-export const baseOptions: RequestInit = {
+export const getBaseOptions = (): RequestInit => ({
  method: 'GET',
  mode: 'cors',
  credentials: 'include', // always send cookies、HTTP Basic authentication.
@@ -119,9 +119,10 @@ export const baseOptions: RequestInit = {
    'Content-Type': ContentType.json,
  }),
  redirect: 'follow',
-}
+})

 async function base<T>(url: string, options: FetchOptionType = {}, otherOptions: IOtherOptions = {}): Promise<T> {
+  const baseOptions = getBaseOptions()
  const { params, body, headers, ...init } = Object.assign({}, baseOptions, options)
  const {
    isPublicAPI = false,
@@ -151,6 +152,10 @@ async function base<T>(url: string, options: FetchOptionType = {}, otherOptions:
  if (deleteContentType)
    (headers as any).delete('Content-Type')

+  // ! For Marketplace API, help to filter tags added in new version
+  if (isMarketplaceAPI)
+    (headers as any).set('X-Dify-Version', APP_VERSION)
+
  const client = baseClient.extend({
    hooks: {
      ...baseHooks,
--- a/web/service/knowledge/use-create-dataset.ts
+++ b/web/service/knowledge/use-create-dataset.ts
@@ -3,11 +3,30 @@ import type { MutationOptions } from '@tanstack/react-query'
 import { useMutation } from '@tanstack/react-query'
 import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from '../datasets'
 import type { IndexingType } from '@/app/components/datasets/create/step-two'
-import type { ChunkingMode, CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
+import type {
+  ChunkingMode,
+  CrawlOptions,
+  CrawlResultItem,
+  CreateDatasetReq,
+  CreateDatasetResponse,
+  CreateDocumentReq,
+  CustomFile,
+  DataSourceType,
+  FileIndexingEstimateResponse,
+  IndexingEstimateParams,
+  NotionInfo,
+  ProcessRule,
+  ProcessRuleResponse,
+  createDocumentResponse,
+} from '@/models/datasets'
 import type { DataSourceProvider, NotionPage } from '@/models/common'
+import { post } from '../base'
+
+const NAME_SPACE = 'knowledge/create-dataset'

 export const getNotionInfo = (
  notionPages: NotionPage[],
+  credentialId: string,
 ) => {
  const workspacesMap = groupBy(notionPages, 'workspace_id')
  const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
@@ -18,6 +37,7 @@ export const getNotionInfo = (
  })
  return workspaces.map((workspace) => {
    return {
+      credential_id: credentialId,
      workspace_id: workspace.workspaceId,
      pages: workspace.pages.map((page) => {
        const { page_id, page_name, page_icon, type } = page
@@ -101,6 +121,7 @@ export const useFetchFileIndexingEstimateForFile = (
 type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
  dataSourceType: DataSourceType.NOTION
  notionPages: NotionPage[]
+  credential_id: string
 }

 const getFileIndexingEstimateParamsForNotion = ({
@@ -111,11 +132,12 @@ const getFileIndexingEstimateParamsForNotion = ({
  indexingTechnique,
  processRule,
  dataset_id,
+  credential_id,
 }: GetFileIndexingEstimateParamsOptionNotion): IndexingEstimateParams => {
  return {
    info_list: {
      data_source_type: dataSourceType,
-      notion_info_list: getNotionInfo(notionPages),
+      notion_info_list: getNotionInfo(notionPages, credential_id),
    },
    indexing_technique: indexingTechnique,
    process_rule: processRule,
@@ -221,3 +243,27 @@ export const useFetchDefaultProcessRule = (
    ...mutationOptions,
  })
 }
+
+export const useCreatePipelineDataset = (
+  mutationOptions: MutationOptions<CreateDatasetResponse, Error> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'create-pipeline-empty-dataset'],
+    mutationFn: () => {
+      return post<CreateDatasetResponse>('/rag/pipeline/empty-dataset')
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useCreatePipelineDatasetFromCustomized = (
+  mutationOptions: MutationOptions<CreateDatasetResponse, Error, CreateDatasetReq> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'create-pipeline-dataset'],
+    mutationFn: (req: CreateDatasetReq) => {
+      return post<CreateDatasetResponse>('/rag/pipeline/dataset', { body: req })
+    },
+    ...mutationOptions,
+  })
+}
--- a/web/service/knowledge/use-dataset.ts
+++ b/web/service/knowledge/use-dataset.ts
@@ -0,0 +1,99 @@
+import type { MutationOptions } from '@tanstack/react-query'
+import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query'
+import type {
+  DataSet,
+  DataSetListResponse,
+  DatasetListRequest,
+  IndexingStatusBatchRequest,
+  IndexingStatusBatchResponse,
+  ProcessRuleResponse,
+  RelatedAppResponse,
+} from '@/models/datasets'
+import { get, post } from '../base'
+import { useInvalid } from '../use-base'
+import qs from 'qs'
+import type { CommonResponse } from '@/models/common'
+
+const NAME_SPACE = 'dataset'
+
+const DatasetListKey = [NAME_SPACE, 'list']
+
+export const useDatasetList = (params: DatasetListRequest) => {
+  const { initialPage, tag_ids, limit, include_all, keyword } = params
+  return useInfiniteQuery({
+    queryKey: [...DatasetListKey, initialPage, tag_ids, limit, include_all, keyword],
+    queryFn: ({ pageParam = 1 }) => {
+      const urlParams = qs.stringify({
+        tag_ids,
+        limit,
+        include_all,
+        keyword,
+        page: pageParam,
+      }, { indices: false })
+      return get<DataSetListResponse>(`/datasets?${urlParams}`)
+    },
+    getNextPageParam: lastPage => lastPage.has_more ? lastPage.page + 1 : null,
+    initialPageParam: initialPage,
+  })
+}
+
+export const useInvalidDatasetList = () => {
+  return useInvalid([...DatasetListKey])
+}
+
+export const datasetDetailQueryKeyPrefix = [NAME_SPACE, 'detail']
+
+export const useDatasetDetail = (datasetId: string) => {
+  return useQuery({
+    queryKey: [...datasetDetailQueryKeyPrefix, datasetId],
+    queryFn: () => get<DataSet>(`/datasets/${datasetId}`),
+    enabled: !!datasetId,
+  })
+}
+
+export const useDatasetRelatedApps = (datasetId: string) => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'related-apps', datasetId],
+    queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`),
+  })
+}
+
+export const useIndexingStatusBatch = (
+  params: IndexingStatusBatchRequest,
+  mutationOptions: MutationOptions<IndexingStatusBatchResponse, Error> = {},
+) => {
+  const { datasetId, batchId } = params
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'indexing-status-batch', datasetId, batchId],
+    mutationFn: () => get<IndexingStatusBatchResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-status`),
+    ...mutationOptions,
+  })
+}
+
+export const useProcessRule = (documentId: string) => {
+  return useQuery<ProcessRuleResponse>({
+    queryKey: [NAME_SPACE, 'process-rule', documentId],
+    queryFn: () => get<ProcessRuleResponse>('/datasets/process-rule', { params: { document_id: documentId } }),
+  })
+}
+
+export const useDatasetApiBaseUrl = () => {
+  return useQuery<{ api_base_url: string }>({
+    queryKey: [NAME_SPACE, 'api-base-info'],
+    queryFn: () => get<{ api_base_url: string }>('/datasets/api-base-info'),
+  })
+}
+
+export const useEnableDatasetServiceApi = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'enable-api'],
+    mutationFn: (datasetId: string) => post<CommonResponse>(`/datasets/${datasetId}/api-keys/enable`),
+  })
+}
+
+export const useDisableDatasetServiceApi = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'disable-api'],
+    mutationFn: (datasetId: string) => post<CommonResponse>(`/datasets/${datasetId}/api-keys/disable`),
+  })
+}
--- a/web/service/knowledge/use-document.ts
+++ b/web/service/knowledge/use-document.ts
@@ -128,7 +128,7 @@ export const useDocumentMetadata = (payload: {
  })
 }

-export const useInvalidDocumentDetailKey = () => {
+export const useInvalidDocumentDetail = () => {
  return useInvalid(useDocumentDetailKey)
 }

--- a/web/service/knowledge/use-import.ts
+++ b/web/service/knowledge/use-import.ts
@@ -0,0 +1,42 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query'
+import { get } from '../base'
+import type { DataSourceNotionWorkspace } from '@/models/common'
+
+type PreImportNotionPagesParams = {
+  datasetId: string
+  credentialId: string
+}
+
+const PRE_IMPORT_NOTION_PAGES_QUERY_KEY = 'notion-pre-import-pages'
+
+export const usePreImportNotionPages = ({
+  datasetId,
+  credentialId,
+}: PreImportNotionPagesParams) => {
+  return useQuery({
+    queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
+    queryFn: async () => {
+      return get<{ notion_info: DataSourceNotionWorkspace[] }>('/notion/pre-import/pages', {
+        params: {
+          dataset_id: datasetId,
+          credential_id: credentialId,
+        },
+      })
+    },
+    retry: 0,
+  })
+}
+
+export const useInvalidPreImportNotionPages = () => {
+  const queryClient = useQueryClient()
+  return ({
+    datasetId,
+    credentialId,
+  }: PreImportNotionPagesParams) => {
+    queryClient.invalidateQueries(
+      {
+        queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
+      },
+    )
+  }
+}
--- a/web/service/log.ts
+++ b/web/service/log.ts
@@ -67,8 +67,8 @@ export const fetchWorkflowLogs: Fetcher<WorkflowLogsResponse, { url: string; par
  return get<WorkflowLogsResponse>(url, { params })
 }

-export const fetchRunDetail = ({ appID, runID }: { appID: string; runID: string }) => {
-  return get<WorkflowRunDetailResponse>(`/apps/${appID}/workflow-runs/${runID}`)
+export const fetchRunDetail = (url: string) => {
+  return get<WorkflowRunDetailResponse>(url)
 }

 export const fetchTracingList: Fetcher<NodeTracingListResponse, { url: string }> = ({ url }) => {
--- a/web/service/use-common.ts
+++ b/web/service/use-common.ts
@@ -1,10 +1,12 @@
 import { get, post } from './base'
 import type {
  FileUploadConfigResponse,
+  Member,
  StructuredOutputRulesRequestBody,
  StructuredOutputRulesResponse,
 } from '@/models/common'
 import { useMutation, useQuery } from '@tanstack/react-query'
+import type { FileTypesRes } from './datasets'

 const NAME_SPACE = 'common'

@@ -58,3 +60,49 @@ export const useMailRegister = () => {
    },
  })
 }
+
+export const useFileSupportTypes = () => {
+  return useQuery<FileTypesRes>({
+    queryKey: [NAME_SPACE, 'file-types'],
+    queryFn: () => get<FileTypesRes>('/files/support-type'),
+  })
+}
+
+type MemberResponse = {
+  accounts: Member[] | null
+}
+
+export const useMembers = () => {
+  return useQuery<MemberResponse>({
+    queryKey: [NAME_SPACE, 'members'],
+    queryFn: (params: Record<string, any>) => get<MemberResponse>('/workspaces/current/members', {
+      params,
+    }),
+  })
+}
+
+type FilePreviewResponse = {
+  content: string
+}
+
+export const useFilePreview = (fileID: string) => {
+  return useQuery<FilePreviewResponse>({
+    queryKey: [NAME_SPACE, 'file-preview', fileID],
+    queryFn: () => get<FilePreviewResponse>(`/files/${fileID}/preview`),
+    enabled: !!fileID,
+  })
+}
+
+export type SchemaTypeDefinition = {
+  name: string
+  schema: {
+    properties: Record<string, any>
+  }
+}
+
+export const useSchemaTypeDefinitions = () => {
+  return useQuery<SchemaTypeDefinition[]>({
+    queryKey: [NAME_SPACE, 'schema-type-definitions'],
+    queryFn: () => get<SchemaTypeDefinition[]>('/spec/schema-definitions'),
+  })
+}
--- a/web/service/use-datasource.ts
+++ b/web/service/use-datasource.ts
@@ -0,0 +1,79 @@
+import {
+  useMutation,
+  useQuery,
+} from '@tanstack/react-query'
+import { get } from './base'
+import { useInvalid } from './use-base'
+import type {
+  DataSourceAuth,
+  DataSourceCredential,
+} from '@/app/components/header/account-setting/data-source-page-new/types'
+
+const NAME_SPACE = 'data-source-auth'
+
+export const useGetDataSourceListAuth = () => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'list'],
+    queryFn: () => get<{ result: DataSourceAuth[] }>('/auth/plugin/datasource/list'),
+    retry: 0,
+  })
+}
+
+export const useInvalidDataSourceListAuth = (
+) => {
+  return useInvalid([NAME_SPACE, 'list'])
+}
+
+// !This hook is used for fetching the default data source list, which will be legacy and deprecated in the near future.
+export const useGetDefaultDataSourceListAuth = () => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'default-list'],
+    queryFn: () => get<{ result: DataSourceAuth[] }>('/auth/plugin/datasource/default-list'),
+    retry: 0,
+  })
+}
+
+export const useInvalidDefaultDataSourceListAuth = (
+) => {
+  return useInvalid([NAME_SPACE, 'default-list'])
+}
+
+export const useGetDataSourceOAuthUrl = (
+  provider: string,
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'oauth-url', provider],
+    mutationFn: (credentialId?: string) => {
+      return get<
+        {
+          authorization_url: string
+          state: string
+          context_id: string
+        }>(`/oauth/plugin/${provider}/datasource/get-authorization-url?credential_id=${credentialId}`)
+    },
+  })
+}
+
+export const useGetDataSourceAuth = ({
+  pluginId,
+  provider,
+}: {
+  pluginId: string
+  provider: string
+}) => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'specific-data-source', pluginId, provider],
+    queryFn: () => get<{ result: DataSourceCredential[] }>(`/auth/plugin/datasource/${pluginId}/${provider}`),
+    retry: 0,
+  })
+}
+
+export const useInvalidDataSourceAuth = ({
+  pluginId,
+  provider,
+}: {
+  pluginId: string
+  provider: string
+}) => {
+  return useInvalid([NAME_SPACE, 'specific-data-source', pluginId, provider])
+}
--- a/web/service/use-flow.ts
+++ b/web/service/use-flow.ts
@@ -0,0 +1,33 @@
+import type { FlowType } from '@/types/common'
+import {
+  useDeleteAllInspectorVars as useDeleteAllInspectorVarsInner,
+  useDeleteInspectVar as useDeleteInspectVarInner,
+  useDeleteNodeInspectorVars as useDeleteNodeInspectorVarsInner,
+  useEditInspectorVar as useEditInspectorVarInner,
+  useInvalidateConversationVarValues as useInvalidateConversationVarValuesInner,
+  useInvalidateSysVarValues as useInvalidateSysVarValuesInner,
+  useResetConversationVar as useResetConversationVarInner,
+  useResetToLastRunValue as useResetToLastRunValueInner,
+} from './use-workflow'
+import { curry } from 'lodash-es'
+
+type Params = {
+  flowType: FlowType
+}
+
+const useFLow = ({
+  flowType,
+}: Params) => {
+  return {
+    useInvalidateConversationVarValues: curry(useInvalidateConversationVarValuesInner)(flowType),
+    useInvalidateSysVarValues: curry(useInvalidateSysVarValuesInner)(flowType),
+    useResetConversationVar: curry(useResetConversationVarInner)(flowType),
+    useResetToLastRunValue: curry(useResetToLastRunValueInner)(flowType),
+    useDeleteAllInspectorVars: curry(useDeleteAllInspectorVarsInner)(flowType),
+    useDeleteNodeInspectorVars: curry(useDeleteNodeInspectorVarsInner)(flowType),
+    useDeleteInspectVar: curry(useDeleteInspectVarInner)(flowType),
+    useEditInspectorVar: curry(useEditInspectorVarInner)(flowType),
+  }
+}
+
+export default useFLow
--- a/web/service/use-pipeline.ts
+++ b/web/service/use-pipeline.ts
@@ -0,0 +1,386 @@
+import type { MutationOptions } from '@tanstack/react-query'
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
+import { del, get, patch, post } from './base'
+import { DatasourceType } from '@/models/pipeline'
+import type {
+  ConversionResponse,
+  DatasourceNodeSingleRunRequest,
+  DatasourceNodeSingleRunResponse,
+  DeleteTemplateResponse,
+  ExportTemplateDSLResponse,
+  ImportPipelineDSLConfirmResponse,
+  ImportPipelineDSLRequest,
+  ImportPipelineDSLResponse,
+  OnlineDocumentPreviewRequest,
+  OnlineDocumentPreviewResponse,
+  PipelineCheckDependenciesResponse,
+  PipelineExecutionLogRequest,
+  PipelineExecutionLogResponse,
+  PipelinePreProcessingParamsRequest,
+  PipelinePreProcessingParamsResponse,
+  PipelineProcessingParamsRequest,
+  PipelineProcessingParamsResponse,
+  PipelineTemplateByIdRequest,
+  PipelineTemplateByIdResponse,
+  PipelineTemplateListParams,
+  PipelineTemplateListResponse,
+  PublishedPipelineInfoResponse,
+  PublishedPipelineRunPreviewResponse,
+  PublishedPipelineRunRequest,
+  PublishedPipelineRunResponse,
+  UpdateTemplateInfoRequest,
+  UpdateTemplateInfoResponse,
+} from '@/models/pipeline'
+import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
+import type { ToolCredential } from '@/app/components/tools/types'
+import type { IconInfo } from '@/models/datasets'
+import { useInvalid } from './use-base'
+
+const NAME_SPACE = 'pipeline'
+
+export const PipelineTemplateListQueryKeyPrefix = [NAME_SPACE, 'template-list']
+export const usePipelineTemplateList = (params: PipelineTemplateListParams) => {
+  return useQuery<PipelineTemplateListResponse>({
+    queryKey: [...PipelineTemplateListQueryKeyPrefix, params.type],
+    queryFn: () => {
+      return get<PipelineTemplateListResponse>('/rag/pipeline/templates', { params })
+    },
+  })
+}
+
+export const usePipelineTemplateById = (params: PipelineTemplateByIdRequest, enabled: boolean) => {
+  const { template_id, type } = params
+  return useQuery<PipelineTemplateByIdResponse>({
+    queryKey: [NAME_SPACE, 'template', template_id],
+    queryFn: () => {
+      return get<PipelineTemplateByIdResponse>(`/rag/pipeline/templates/${template_id}`, {
+        params: {
+          type,
+        },
+      })
+    },
+    enabled,
+  })
+}
+
+export const useUpdateTemplateInfo = (
+  mutationOptions: MutationOptions<UpdateTemplateInfoResponse, Error, UpdateTemplateInfoRequest> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'template-update'],
+    mutationFn: (request: UpdateTemplateInfoRequest) => {
+      const { template_id, ...rest } = request
+      return patch<UpdateTemplateInfoResponse>(`/rag/pipeline/customized/templates/${template_id}`, {
+        body: rest,
+      })
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useDeleteTemplate = (
+  mutationOptions: MutationOptions<DeleteTemplateResponse, Error, string> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'template-delete'],
+    mutationFn: (templateId: string) => {
+      return del<DeleteTemplateResponse>(`/rag/pipeline/customized/templates/${templateId}`)
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useExportTemplateDSL = (
+  mutationOptions: MutationOptions<ExportTemplateDSLResponse, Error, string> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'template-dsl-export'],
+    mutationFn: (templateId: string) => {
+      return post<ExportTemplateDSLResponse>(`/rag/pipeline/customized/templates/${templateId}`)
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useImportPipelineDSL = (
+  mutationOptions: MutationOptions<ImportPipelineDSLResponse, Error, ImportPipelineDSLRequest> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'dsl-import'],
+    mutationFn: (request: ImportPipelineDSLRequest) => {
+      return post<ImportPipelineDSLResponse>('/rag/pipelines/imports', { body: request })
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useImportPipelineDSLConfirm = (
+  mutationOptions: MutationOptions<ImportPipelineDSLConfirmResponse, Error, string> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'dsl-import-confirm'],
+    mutationFn: (importId: string) => {
+      return post<ImportPipelineDSLConfirmResponse>(`/rag/pipelines/imports/${importId}/confirm`)
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useCheckPipelineDependencies = (
+  mutationOptions: MutationOptions<PipelineCheckDependenciesResponse, Error, string> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'check-dependencies'],
+    mutationFn: (pipelineId: string) => {
+      return get<PipelineCheckDependenciesResponse>(`/rag/pipelines/imports/${pipelineId}/check-dependencies`)
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useDraftPipelineProcessingParams = (params: PipelineProcessingParamsRequest, enabled = true) => {
+  const { pipeline_id, node_id } = params
+  return useQuery<PipelineProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'draft-pipeline-processing-params', pipeline_id, node_id],
+    queryFn: () => {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, {
+        params: {
+          node_id,
+        },
+      })
+    },
+    staleTime: 0,
+    enabled,
+  })
+}
+
+export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
+  const { pipeline_id, node_id } = params
+  return useQuery<PipelineProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'published-pipeline-processing-params', pipeline_id, node_id],
+    queryFn: () => {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
+        params: {
+          node_id,
+        },
+      })
+    },
+    staleTime: 0,
+  })
+}
+
+export const useDataSourceList = (enabled: boolean, onSuccess?: (v: DataSourceItem[]) => void) => {
+  return useQuery<DataSourceItem[]>({
+    enabled,
+    queryKey: [NAME_SPACE, 'datasource'],
+    staleTime: 0,
+    queryFn: async () => {
+      const data = await get<DataSourceItem[]>('/rag/pipelines/datasource-plugins')
+      onSuccess?.(data)
+      return data
+    },
+    retry: false,
+  })
+}
+
+export const useInvalidDataSourceList = () => {
+  return useInvalid([NAME_SPACE, 'datasource'])
+}
+
+export const publishedPipelineInfoQueryKeyPrefix = [NAME_SPACE, 'published-pipeline']
+
+export const usePublishedPipelineInfo = (pipelineId: string) => {
+  return useQuery<PublishedPipelineInfoResponse>({
+    queryKey: [...publishedPipelineInfoQueryKeyPrefix, pipelineId],
+    queryFn: () => {
+      return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`)
+    },
+    enabled: !!pipelineId,
+  })
+}
+
+export const useRunPublishedPipeline = (
+  mutationOptions: MutationOptions<PublishedPipelineRunPreviewResponse | PublishedPipelineRunResponse, Error, PublishedPipelineRunRequest> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'run-published-pipeline'],
+    mutationFn: (request: PublishedPipelineRunRequest) => {
+      const { pipeline_id: pipelineId, is_preview, ...rest } = request
+      return post<PublishedPipelineRunPreviewResponse | PublishedPipelineRunResponse>(`/rag/pipelines/${pipelineId}/workflows/published/run`, {
+        body: {
+          ...rest,
+          is_preview,
+          response_mode: 'blocking',
+        },
+      })
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useDataSourceCredentials = (provider: string, pluginId: string, onSuccess: (value: ToolCredential[]) => void) => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'datasource-credentials', provider, pluginId],
+    queryFn: async () => {
+      const result = await get<{ result: ToolCredential[] }>(`/auth/plugin/datasource?provider=${provider}&plugin_id=${pluginId}`)
+      onSuccess(result.result)
+      return result.result
+    },
+    enabled: !!provider && !!pluginId,
+    retry: 2,
+  })
+}
+
+export const useUpdateDataSourceCredentials = (
+) => {
+  const queryClient = useQueryClient()
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'update-datasource-credentials'],
+    mutationFn: ({
+      provider,
+      pluginId,
+      credentials,
+      name,
+    }: { provider: string; pluginId: string; credentials: Record<string, any>; name: string; }) => {
+      return post('/auth/plugin/datasource', {
+        body: {
+          provider,
+          plugin_id: pluginId,
+          credentials,
+          name,
+        },
+      }).then(() => {
+        queryClient.invalidateQueries({
+          queryKey: [NAME_SPACE, 'datasource'],
+        })
+      })
+    },
+  })
+}
+
+export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
+  const { pipeline_id, node_id } = params
+  return useQuery<PipelinePreProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'draft-pipeline-pre-processing-params', pipeline_id, node_id],
+    queryFn: () => {
+      return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/pre-processing/parameters`, {
+        params: {
+          node_id,
+        },
+      })
+    },
+    staleTime: 0,
+    enabled,
+  })
+}
+
+export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
+  const { pipeline_id, node_id } = params
+  return useQuery<PipelinePreProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'published-pipeline-pre-processing-params', pipeline_id, node_id],
+    queryFn: () => {
+      return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/pre-processing/parameters`, {
+        params: {
+          node_id,
+        },
+      })
+    },
+    staleTime: 0,
+    enabled,
+  })
+}
+
+export const useExportPipelineDSL = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'export-pipeline-dsl'],
+    mutationFn: ({
+      pipelineId,
+      include = false,
+    }: { pipelineId: string; include?: boolean }) => {
+      return get<ExportTemplateDSLResponse>(`/rag/pipelines/${pipelineId}/exports?include_secret=${include}`)
+    },
+  })
+}
+
+export const usePublishAsCustomizedPipeline = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'publish-as-customized-pipeline'],
+    mutationFn: ({
+      pipelineId,
+      name,
+      icon_info,
+      description,
+    }: {
+      pipelineId: string,
+      name: string,
+      icon_info: IconInfo,
+      description?: string,
+    }) => {
+      return post(`/rag/pipelines/${pipelineId}/customized/publish`, {
+        body: {
+          name,
+          icon_info,
+          description,
+        },
+      })
+    },
+  })
+}
+
+export const usePipelineExecutionLog = (params: PipelineExecutionLogRequest) => {
+  const { dataset_id, document_id } = params
+  return useQuery<PipelineExecutionLogResponse>({
+    queryKey: [NAME_SPACE, 'pipeline-execution-log', dataset_id, document_id],
+    queryFn: () => {
+      return get<PipelineExecutionLogResponse>(`/datasets/${dataset_id}/documents/${document_id}/pipeline-execution-log`)
+    },
+    staleTime: 0,
+  })
+}
+
+export const usePreviewOnlineDocument = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'preview-online-document'],
+    mutationFn: (params: OnlineDocumentPreviewRequest) => {
+      const { pipelineId, datasourceNodeId, workspaceID, pageID, pageType, credentialId } = params
+      return post<OnlineDocumentPreviewResponse>(
+        `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${datasourceNodeId}/preview`,
+        {
+          body: {
+            datasource_type: DatasourceType.onlineDocument,
+            credential_id: credentialId,
+            inputs: {
+              workspace_id: workspaceID,
+              page_id: pageID,
+              type: pageType,
+            },
+          },
+        },
+      )
+    },
+  })
+}
+
+export const useConvertDatasetToPipeline = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'convert-dataset-to-pipeline'],
+    mutationFn: (datasetId: string) => {
+      return post<ConversionResponse>(`/rag/pipelines/transform/datasets/${datasetId}`)
+    },
+  })
+}
+
+export const useDatasourceSingleRun = (
+  mutationOptions: MutationOptions<DatasourceNodeSingleRunResponse, Error, DatasourceNodeSingleRunRequest> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'datasource-node-single-run'],
+    mutationFn: (params: DatasourceNodeSingleRunRequest) => {
+      const { pipeline_id: pipelineId, ...rest } = params
+      return post<DatasourceNodeSingleRunResponse>(`/rag/pipelines/${pipelineId}/workflows/draft/datasource/variables-inspect`, {
+        body: rest,
+      })
+    },
+    ...mutationOptions,
+  })
+}
--- a/web/service/use-plugins-auth.ts
+++ b/web/service/use-plugins-auth.ts
@@ -95,6 +95,7 @@ export const useGetPluginCredentialSchema = (
  url: string,
 ) => {
  return useQuery({
+    enabled: !!url,
    queryKey: [NAME_SPACE, 'credential-schema', url],
    queryFn: () => get<FormSchema[]>(url),
  })
@@ -120,6 +121,7 @@ export const useGetPluginOAuthClientSchema = (
  url: string,
 ) => {
  return useQuery({
+    enabled: !!url,
    queryKey: [NAME_SPACE, 'oauth-client-schema', url],
    queryFn: () => get<{
      schema: FormSchema[]
--- a/web/service/use-plugins.ts
+++ b/web/service/use-plugins.ts
@@ -39,7 +39,7 @@ import {
  useQuery,
  useQueryClient,
 } from '@tanstack/react-query'
-import { useInvalidateAllBuiltInTools } from './use-tools'
+import { useInvalidateAllBuiltInTools, useInvalidateRAGRecommendedPlugins } from './use-tools'
 import useReferenceSetting from '@/app/components/plugins/plugin-page/use-reference-setting'
 import { uninstallPlugin } from '@/service/plugins'
 import useRefreshPluginList from '@/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list'
@@ -135,12 +135,14 @@ export const useInstalledLatestVersion = (pluginIds: string[]) => {
 export const useInvalidateInstalledPluginList = () => {
  const queryClient = useQueryClient()
  const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools()
+  const invalidateRAGRecommendedPlugins = useInvalidateRAGRecommendedPlugins()
  return () => {
    queryClient.invalidateQueries(
      {
        queryKey: useInstalledPluginListKey,
      })
    invalidateAllBuiltInTools()
+    invalidateRAGRecommendedPlugins()
  }
 }

--- a/web/service/use-tools.ts
+++ b/web/service/use-tools.ts
@@ -4,7 +4,7 @@ import type {
  MCPServerDetail,
  Tool,
 } from '@/app/components/tools/types'
-import type { ToolWithProvider } from '@/app/components/workflow/types'
+import type { RAGRecommendedPlugins, ToolWithProvider } from '@/app/components/workflow/types'
 import type { AppIconType } from '@/types/app'
 import { useInvalid } from './use-base'
 import {
@@ -313,3 +313,16 @@ export const useRemoveProviderCredentials = ({
    onSuccess,
  })
 }
+
+const useRAGRecommendedPluginListKey = [NAME_SPACE, 'rag-recommended-plugins']
+
+export const useRAGRecommendedPlugins = () => {
+  return useQuery<RAGRecommendedPlugins>({
+    queryKey: useRAGRecommendedPluginListKey,
+    queryFn: () => get<RAGRecommendedPlugins>('/rag/pipelines/recommended-plugins'),
+  })
+}
+
+export const useInvalidateRAGRecommendedPlugins = () => {
+  return useInvalid(useRAGRecommendedPluginListKey)
+}
--- a/web/service/use-workflow.ts
+++ b/web/service/use-workflow.ts
@@ -12,6 +12,8 @@ import type {
 } from '@/types/workflow'
 import type { CommonResponse } from '@/models/common'
 import { useInvalid, useReset } from './use-base'
+import type { FlowType } from '@/types/common'
+import { getFlowPrefix } from './utils'

 const NAME_SPACE = 'workflow'

@@ -33,11 +35,13 @@ export const useInvalidateAppWorkflow = () => {
  }
 }

-export const useWorkflowConfig = (appId: string, onSuccess: (v: WorkflowConfigResponse) => void) => {
+export const useWorkflowConfig = <T = WorkflowConfigResponse>(url: string, onSuccess: (v: T) => void) => {
  return useQuery({
-    queryKey: [NAME_SPACE, 'config', appId],
+    enabled: !!url,
+    queryKey: [NAME_SPACE, 'config', url],
+    staleTime: 0,
    queryFn: async () => {
-      const data = await get<WorkflowConfigResponse>(`/apps/${appId}/workflows/draft/config`)
+      const data = await get<T>(url)
      onSuccess(data)
      return data
    },
@@ -47,10 +51,11 @@ export const useWorkflowConfig = (appId: string, onSuccess: (v: WorkflowConfigRe
 const WorkflowVersionHistoryKey = [NAME_SPACE, 'versionHistory']

 export const useWorkflowVersionHistory = (params: FetchWorkflowDraftPageParams) => {
-  const { appId, initialPage, limit, userId, namedOnly } = params
+  const { url, initialPage, limit, userId, namedOnly } = params
  return useInfiniteQuery({
-    queryKey: [...WorkflowVersionHistoryKey, appId, initialPage, limit, userId, namedOnly],
-    queryFn: ({ pageParam = 1 }) => get<FetchWorkflowDraftPageResponse>(`/apps/${appId}/workflows`, {
+    enabled: !!url,
+    queryKey: [...WorkflowVersionHistoryKey, url, initialPage, limit, userId, namedOnly],
+    queryFn: ({ pageParam = 1 }) => get<FetchWorkflowDraftPageResponse>(url, {
      params: {
        page: pageParam,
        limit,
@@ -63,14 +68,14 @@ export const useWorkflowVersionHistory = (params: FetchWorkflowDraftPageParams)
  })
 }

-export const useResetWorkflowVersionHistory = (appId: string) => {
-  return useReset([...WorkflowVersionHistoryKey, appId])
+export const useResetWorkflowVersionHistory = () => {
+  return useReset([...WorkflowVersionHistoryKey])
 }

-export const useUpdateWorkflow = (appId: string) => {
+export const useUpdateWorkflow = () => {
  return useMutation({
    mutationKey: [NAME_SPACE, 'update'],
-    mutationFn: (params: UpdateWorkflowParams) => patch(`/apps/${appId}/workflows/${params.workflowId}`, {
+    mutationFn: (params: UpdateWorkflowParams) => patch(params.url, {
      body: {
        marked_name: params.title,
        marked_comment: params.releaseNotes,
@@ -79,17 +84,17 @@ export const useUpdateWorkflow = (appId: string) => {
  })
 }

-export const useDeleteWorkflow = (appId: string) => {
+export const useDeleteWorkflow = () => {
  return useMutation({
    mutationKey: [NAME_SPACE, 'delete'],
-    mutationFn: (workflowId: string) => del(`/apps/${appId}/workflows/${workflowId}`),
+    mutationFn: (url: string) => del(url),
  })
 }

-export const usePublishWorkflow = (appId: string) => {
+export const usePublishWorkflow = () => {
  return useMutation({
    mutationKey: [NAME_SPACE, 'publish'],
-    mutationFn: (params: PublishWorkflowParams) => post<CommonResponse & { created_at: number }>(`/apps/${appId}/workflows/publish`, {
+    mutationFn: (params: PublishWorkflowParams) => post<CommonResponse & { created_at: number }>(params.url, {
      body: {
        marked_name: params.title,
        marked_comment: params.releaseNotes,
@@ -99,12 +104,12 @@ export const usePublishWorkflow = (appId: string) => {
 }

 const useLastRunKey = [NAME_SPACE, 'last-run']
-export const useLastRun = (appID: string, nodeId: string, enabled: boolean) => {
+export const useLastRun = (flowType: FlowType, flowId: string, nodeId: string, enabled: boolean) => {
  return useQuery<NodeTracing>({
    enabled,
-    queryKey: [...useLastRunKey, appID, nodeId],
+    queryKey: [...useLastRunKey, flowType, flowId, nodeId],
    queryFn: async () => {
-      return get(`apps/${appID}/workflows/draft/nodes/${nodeId}/last-run`, {}, {
+      return get(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/nodes/${nodeId}/last-run`, {}, {
        silent: true,
      })
    },
@@ -112,103 +117,101 @@ export const useLastRun = (appID: string, nodeId: string, enabled: boolean) => {
  })
 }

-export const useInvalidLastRun = (appId: string, nodeId: string) => {
-  return useInvalid([NAME_SPACE, 'last-run', appId, nodeId])
+export const useInvalidLastRun = (flowType: FlowType, flowId: string, nodeId: string) => {
+  return useInvalid([NAME_SPACE, flowType, 'last-run', flowId, nodeId])
 }

 // Rerun workflow or change the version of workflow
-export const useInvalidAllLastRun = (appId: string) => {
-  return useInvalid([NAME_SPACE, 'last-run', appId])
+export const useInvalidAllLastRun = (flowType?: FlowType, flowId?: string) => {
+  return useInvalid([NAME_SPACE, flowType, 'last-run', flowId])
 }

-const useConversationVarValuesKey = [NAME_SPACE, 'conversation-variable']
-
-export const useConversationVarValues = (url?: string) => {
+export const useConversationVarValues = (flowType?: FlowType, flowId?: string) => {
  return useQuery({
-    enabled: !!url,
-    queryKey: [...useConversationVarValuesKey, url],
+    enabled: !!flowId,
+    queryKey: [NAME_SPACE, flowType, 'conversation var values', flowId],
    queryFn: async () => {
-      const { items } = (await get(url || '')) as { items: VarInInspect[] }
+      const { items } = (await get(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/conversation-variables`)) as { items: VarInInspect[] }
      return items
    },
  })
 }

-export const useInvalidateConversationVarValues = (url: string) => {
-  return useInvalid([...useConversationVarValuesKey, url])
+export const useInvalidateConversationVarValues = (flowType: FlowType, flowId: string) => {
+  return useInvalid([NAME_SPACE, flowType, 'conversation var values', flowId])
 }

-export const useResetConversationVar = (appId: string) => {
+export const useResetConversationVar = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'reset conversation var', appId],
+    mutationKey: [NAME_SPACE, flowType, 'reset conversation var', flowId],
    mutationFn: async (varId: string) => {
-      return put(`apps/${appId}/workflows/draft/variables/${varId}/reset`)
+      return put(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables/${varId}/reset`)
    },
  })
 }

-export const useResetToLastRunValue = (appId: string) => {
+export const useResetToLastRunValue = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'reset to last run value', appId],
+    mutationKey: [NAME_SPACE, flowType, 'reset to last run value', flowId],
    mutationFn: async (varId: string): Promise<{ value: any }> => {
-      return put(`apps/${appId}/workflows/draft/variables/${varId}/reset`)
+      return put(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables/${varId}/reset`)
    },
  })
 }

 export const useSysVarValuesKey = [NAME_SPACE, 'sys-variable']
-export const useSysVarValues = (url?: string) => {
+export const useSysVarValues = (flowType?: FlowType, flowId?: string) => {
  return useQuery({
-    enabled: !!url,
-    queryKey: [...useSysVarValuesKey, url],
+    enabled: !!flowId,
+    queryKey: [NAME_SPACE, flowType, 'sys var values', flowId],
    queryFn: async () => {
-      const { items } = (await get(url || '')) as { items: VarInInspect[] }
+      const { items } = (await get(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/system-variables`)) as { items: VarInInspect[] }
      return items
    },
  })
 }

-export const useInvalidateSysVarValues = (url: string) => {
-  return useInvalid([...useSysVarValuesKey, url])
+export const useInvalidateSysVarValues = (flowType: FlowType, flowId: string) => {
+  return useInvalid([NAME_SPACE, flowType, 'sys var values', flowId])
 }

-export const useDeleteAllInspectorVars = (appId: string) => {
+export const useDeleteAllInspectorVars = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'delete all inspector vars', appId],
+    mutationKey: [NAME_SPACE, flowType, 'delete all inspector vars', flowId],
    mutationFn: async () => {
-      return del(`apps/${appId}/workflows/draft/variables`)
+      return del(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables`)
    },
  })
 }

-export const useDeleteNodeInspectorVars = (appId: string) => {
+export const useDeleteNodeInspectorVars = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'delete node inspector vars', appId],
+    mutationKey: [NAME_SPACE, flowType, 'delete node inspector vars', flowId],
    mutationFn: async (nodeId: string) => {
-      return del(`apps/${appId}/workflows/draft/nodes/${nodeId}/variables`)
+      return del(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/nodes/${nodeId}/variables`)
    },
  })
 }

-export const useDeleteInspectVar = (appId: string) => {
+export const useDeleteInspectVar = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'delete inspector var', appId],
+    mutationKey: [NAME_SPACE, flowType, 'delete inspector var', flowId],
    mutationFn: async (varId: string) => {
-      return del(`apps/${appId}/workflows/draft/variables/${varId}`)
+      return del(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables/${varId}`)
    },
  })
 }

 // edit the name or value of the inspector var
-export const useEditInspectorVar = (appId: string) => {
+export const useEditInspectorVar = (flowType: FlowType, flowId: string) => {
  return useMutation({
-    mutationKey: [NAME_SPACE, 'edit inspector var', appId],
+    mutationKey: [NAME_SPACE, flowType, 'edit inspector var', flowId],
    mutationFn: async ({ varId, ...rest }: {
      varId: string
      name?: string
      value?: any
    }) => {
-      return patch(`apps/${appId}/workflows/draft/variables/${varId}`, {
+      return patch(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables/${varId}`, {
        body: rest,
      })
    },
--- a/web/service/utils.ts
+++ b/web/service/utils.ts
@@ -0,0 +1,10 @@
+import { FlowType } from '@/types/common'
+
+export const flowPrefixMap = {
+  [FlowType.appFlow]: 'apps',
+  [FlowType.ragPipeline]: 'rag/pipelines',
+}
+
+export const getFlowPrefix = (type?: FlowType) => {
+  return flowPrefixMap[type!] || flowPrefixMap[FlowType.appFlow]
+}
--- a/web/service/workflow.ts
+++ b/web/service/workflow.ts
@@ -10,6 +10,8 @@ import type {
 } from '@/types/workflow'
 import type { BlockEnum } from '@/app/components/workflow/types'
 import type { VarInInspect } from '@/types/workflow'
+import type { FlowType } from '@/types/common'
+import { getFlowPrefix } from './utils'

 export const fetchWorkflowDraft = (url: string) => {
  return get(url, {}, { silent: true }) as Promise<FetchWorkflowDraftResponse>
@@ -34,16 +36,16 @@ export const fetchChatRunHistory: Fetcher<ChatRunHistoryResponse, string> = (url
  return get<ChatRunHistoryResponse>(url)
 }

-export const singleNodeRun = (appId: string, nodeId: string, params: object) => {
-  return post(`apps/${appId}/workflows/draft/nodes/${nodeId}/run`, { body: params })
+export const singleNodeRun = (flowType: FlowType, flowId: string, nodeId: string, params: object) => {
+  return post(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/nodes/${nodeId}/run`, { body: params })
 }

-export const getIterationSingleNodeRunUrl = (isChatFlow: boolean, appId: string, nodeId: string) => {
-  return `apps/${appId}/${isChatFlow ? 'advanced-chat/' : ''}workflows/draft/iteration/nodes/${nodeId}/run`
+export const getIterationSingleNodeRunUrl = (flowType: FlowType, isChatFlow: boolean, flowId: string, nodeId: string) => {
+  return `${getFlowPrefix(flowType)}/${flowId}/${isChatFlow ? 'advanced-chat/' : ''}workflows/draft/iteration/nodes/${nodeId}/run`
 }

-export const getLoopSingleNodeRunUrl = (isChatFlow: boolean, appId: string, nodeId: string) => {
-  return `apps/${appId}/${isChatFlow ? 'advanced-chat/' : ''}workflows/draft/loop/nodes/${nodeId}/run`
+export const getLoopSingleNodeRunUrl = (flowType: FlowType, isChatFlow: boolean, flowId: string, nodeId: string) => {
+  return `${getFlowPrefix(flowType)}/${flowId}/${isChatFlow ? 'advanced-chat/' : ''}workflows/draft/loop/nodes/${nodeId}/run`
 }

 export const fetchPublishedWorkflow: Fetcher<FetchWorkflowDraftResponse, string> = (url) => {
@@ -60,6 +62,12 @@ export const fetchNodeDefault = (appId: string, blockType: BlockEnum, query = {}
  })
 }

+export const fetchPipelineNodeDefault = (pipelineId: string, blockType: BlockEnum, query = {}) => {
+  return get(`rag/pipelines/${pipelineId}/workflows/default-workflow-block-configs/${blockType}`, {
+    params: { q: JSON.stringify(query) },
+  })
+}
+
 // TODO: archived
 export const updateWorkflowDraftFromDSL = (appId: string, data: string) => {
  return post<FetchWorkflowDraftResponse>(`apps/${appId}/workflows/draft/import`, { body: { data } })
@@ -72,13 +80,13 @@ export const fetchCurrentValueOfConversationVariable: Fetcher<ConversationVariab
  return get<ConversationVariableResponse>(url, { params })
 }

-const fetchAllInspectVarsOnePage = async (appId: string, page: number): Promise<{ total: number, items: VarInInspect[] }> => {
-  return get(`apps/${appId}/workflows/draft/variables`, {
+const fetchAllInspectVarsOnePage = async (flowType: FlowType, flowId: string, page: number): Promise<{ total: number, items: VarInInspect[] }> => {
+  return get(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/variables`, {
    params: { page, limit: 100 },
  })
 }
-export const fetchAllInspectVars = async (appId: string): Promise<VarInInspect[]> => {
-  const res = await fetchAllInspectVarsOnePage(appId, 1)
+export const fetchAllInspectVars = async (flowType: FlowType, flowId: string): Promise<VarInInspect[]> => {
+  const res = await fetchAllInspectVarsOnePage(flowType, flowId, 1)
  const { items, total } = res
  if (total <= 100)
    return items
@@ -86,7 +94,7 @@ export const fetchAllInspectVars = async (appId: string): Promise<VarInInspect[]
  const pageCount = Math.ceil(total / 100)
  const promises = []
  for (let i = 2; i <= pageCount; i++)
-    promises.push(fetchAllInspectVarsOnePage(appId, i))
+    promises.push(fetchAllInspectVarsOnePage(flowType, flowId, i))

  const restData = await Promise.all(promises)
  restData.forEach(({ items: item }) => {
@@ -95,7 +103,7 @@ export const fetchAllInspectVars = async (appId: string): Promise<VarInInspect[]
  return items
 }

-export const fetchNodeInspectVars = async (appId: string, nodeId: string): Promise<VarInInspect[]> => {
-  const { items } = (await get(`apps/${appId}/workflows/draft/nodes/${nodeId}/variables`)) as { items: VarInInspect[] }
+export const fetchNodeInspectVars = async (flowType: FlowType, flowId: string, nodeId: string): Promise<VarInInspect[]> => {
+  const { items } = (await get(`${getFlowPrefix(flowType)}/${flowId}/workflows/draft/nodes/${nodeId}/variables`)) as { items: VarInInspect[] }
  return items
 }