feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/service/knowledge/use-create-dataset.ts

@@ -3,11 +3,30 @@ import type { MutationOptions } from '@tanstack/react-query'
 import { useMutation } from '@tanstack/react-query'
 import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from '../datasets'
 import type { IndexingType } from '@/app/components/datasets/create/step-two'
-import type { ChunkingMode, CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets'
+import type {
+  ChunkingMode,
+  CrawlOptions,
+  CrawlResultItem,
+  CreateDatasetReq,
+  CreateDatasetResponse,
+  CreateDocumentReq,
+  CustomFile,
+  DataSourceType,
+  FileIndexingEstimateResponse,
+  IndexingEstimateParams,
+  NotionInfo,
+  ProcessRule,
+  ProcessRuleResponse,
+  createDocumentResponse,
+} from '@/models/datasets'
 import type { DataSourceProvider, NotionPage } from '@/models/common'
+import { post } from '../base'
+
+const NAME_SPACE = 'knowledge/create-dataset'
 
 export const getNotionInfo = (
   notionPages: NotionPage[],
+  credentialId: string,
 ) => {
   const workspacesMap = groupBy(notionPages, 'workspace_id')
   const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
@@ -18,6 +37,7 @@ export const getNotionInfo = (
   })
   return workspaces.map((workspace) => {
     return {
+      credential_id: credentialId,
       workspace_id: workspace.workspaceId,
       pages: workspace.pages.map((page) => {
         const { page_id, page_name, page_icon, type } = page
@@ -101,6 +121,7 @@ export const useFetchFileIndexingEstimateForFile = (
 type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & {
   dataSourceType: DataSourceType.NOTION
   notionPages: NotionPage[]
+  credential_id: string
 }
 
 const getFileIndexingEstimateParamsForNotion = ({
@@ -111,11 +132,12 @@ const getFileIndexingEstimateParamsForNotion = ({
   indexingTechnique,
   processRule,
   dataset_id,
+  credential_id,
 }: GetFileIndexingEstimateParamsOptionNotion): IndexingEstimateParams => {
   return {
     info_list: {
       data_source_type: dataSourceType,
-      notion_info_list: getNotionInfo(notionPages),
+      notion_info_list: getNotionInfo(notionPages, credential_id),
     },
     indexing_technique: indexingTechnique,
     process_rule: processRule,
@@ -221,3 +243,27 @@ export const useFetchDefaultProcessRule = (
     ...mutationOptions,
   })
 }
+
+export const useCreatePipelineDataset = (
+  mutationOptions: MutationOptions<CreateDatasetResponse, Error> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'create-pipeline-empty-dataset'],
+    mutationFn: () => {
+      return post<CreateDatasetResponse>('/rag/pipeline/empty-dataset')
+    },
+    ...mutationOptions,
+  })
+}
+
+export const useCreatePipelineDatasetFromCustomized = (
+  mutationOptions: MutationOptions<CreateDatasetResponse, Error, CreateDatasetReq> = {},
+) => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'create-pipeline-dataset'],
+    mutationFn: (req: CreateDatasetReq) => {
+      return post<CreateDatasetResponse>('/rag/pipeline/dataset', { body: req })
+    },
+    ...mutationOptions,
+  })
+}

web/service/knowledge/use-dataset.ts

@@ -0,0 +1,99 @@
+import type { MutationOptions } from '@tanstack/react-query'
+import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query'
+import type {
+  DataSet,
+  DataSetListResponse,
+  DatasetListRequest,
+  IndexingStatusBatchRequest,
+  IndexingStatusBatchResponse,
+  ProcessRuleResponse,
+  RelatedAppResponse,
+} from '@/models/datasets'
+import { get, post } from '../base'
+import { useInvalid } from '../use-base'
+import qs from 'qs'
+import type { CommonResponse } from '@/models/common'
+
+const NAME_SPACE = 'dataset'
+
+const DatasetListKey = [NAME_SPACE, 'list']
+
+export const useDatasetList = (params: DatasetListRequest) => {
+  const { initialPage, tag_ids, limit, include_all, keyword } = params
+  return useInfiniteQuery({
+    queryKey: [...DatasetListKey, initialPage, tag_ids, limit, include_all, keyword],
+    queryFn: ({ pageParam = 1 }) => {
+      const urlParams = qs.stringify({
+        tag_ids,
+        limit,
+        include_all,
+        keyword,
+        page: pageParam,
+      }, { indices: false })
+      return get<DataSetListResponse>(`/datasets?${urlParams}`)
+    },
+    getNextPageParam: lastPage => lastPage.has_more ? lastPage.page + 1 : null,
+    initialPageParam: initialPage,
+  })
+}
+
+export const useInvalidDatasetList = () => {
+  return useInvalid([...DatasetListKey])
+}
+
+export const datasetDetailQueryKeyPrefix = [NAME_SPACE, 'detail']
+
+export const useDatasetDetail = (datasetId: string) => {
+  return useQuery({
+    queryKey: [...datasetDetailQueryKeyPrefix, datasetId],
+    queryFn: () => get<DataSet>(`/datasets/${datasetId}`),
+    enabled: !!datasetId,
+  })
+}
+
+export const useDatasetRelatedApps = (datasetId: string) => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'related-apps', datasetId],
+    queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`),
+  })
+}
+
+export const useIndexingStatusBatch = (
+  params: IndexingStatusBatchRequest,
+  mutationOptions: MutationOptions<IndexingStatusBatchResponse, Error> = {},
+) => {
+  const { datasetId, batchId } = params
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'indexing-status-batch', datasetId, batchId],
+    mutationFn: () => get<IndexingStatusBatchResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-status`),
+    ...mutationOptions,
+  })
+}
+
+export const useProcessRule = (documentId: string) => {
+  return useQuery<ProcessRuleResponse>({
+    queryKey: [NAME_SPACE, 'process-rule', documentId],
+    queryFn: () => get<ProcessRuleResponse>('/datasets/process-rule', { params: { document_id: documentId } }),
+  })
+}
+
+export const useDatasetApiBaseUrl = () => {
+  return useQuery<{ api_base_url: string }>({
+    queryKey: [NAME_SPACE, 'api-base-info'],
+    queryFn: () => get<{ api_base_url: string }>('/datasets/api-base-info'),
+  })
+}
+
+export const useEnableDatasetServiceApi = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'enable-api'],
+    mutationFn: (datasetId: string) => post<CommonResponse>(`/datasets/${datasetId}/api-keys/enable`),
+  })
+}
+
+export const useDisableDatasetServiceApi = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'disable-api'],
+    mutationFn: (datasetId: string) => post<CommonResponse>(`/datasets/${datasetId}/api-keys/disable`),
+  })
+}

web/service/knowledge/use-document.ts

@@ -128,7 +128,7 @@ export const useDocumentMetadata = (payload: {
   })
 }
 
-export const useInvalidDocumentDetailKey = () => {
+export const useInvalidDocumentDetail = () => {
   return useInvalid(useDocumentDetailKey)
 }
 

web/service/knowledge/use-import.ts

@@ -0,0 +1,42 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query'
+import { get } from '../base'
+import type { DataSourceNotionWorkspace } from '@/models/common'
+
+type PreImportNotionPagesParams = {
+  datasetId: string
+  credentialId: string
+}
+
+const PRE_IMPORT_NOTION_PAGES_QUERY_KEY = 'notion-pre-import-pages'
+
+export const usePreImportNotionPages = ({
+  datasetId,
+  credentialId,
+}: PreImportNotionPagesParams) => {
+  return useQuery({
+    queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
+    queryFn: async () => {
+      return get<{ notion_info: DataSourceNotionWorkspace[] }>('/notion/pre-import/pages', {
+        params: {
+          dataset_id: datasetId,
+          credential_id: credentialId,
+        },
+      })
+    },
+    retry: 0,
+  })
+}
+
+export const useInvalidPreImportNotionPages = () => {
+  const queryClient = useQueryClient()
+  return ({
+    datasetId,
+    credentialId,
+  }: PreImportNotionPagesParams) => {
+    queryClient.invalidateQueries(
+      {
+        queryKey: [PRE_IMPORT_NOTION_PAGES_QUERY_KEY, datasetId, credentialId],
+      },
+    )
+  }
+}