feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/datasets/documents/create-from-pipeline/data-source/store/index.ts

@@ -0,0 +1,45 @@
+import { useContext } from 'react'
+import { createStore, useStore } from 'zustand'
+import { DataSourceContext } from './provider'
+import type { CommonShape } from './slices/common'
+import { createCommonSlice } from './slices/common'
+import type { LocalFileSliceShape } from './slices/local-file'
+import { createLocalFileSlice } from './slices/local-file'
+import type { OnlineDocumentSliceShape } from './slices/online-document'
+import { createOnlineDocumentSlice } from './slices/online-document'
+import type { WebsiteCrawlSliceShape } from './slices/website-crawl'
+import { createWebsiteCrawlSlice } from './slices/website-crawl'
+import type { OnlineDriveSliceShape } from './slices/online-drive'
+import { createOnlineDriveSlice } from './slices/online-drive'
+
+export type DataSourceShape = CommonShape
+  & LocalFileSliceShape
+  & OnlineDocumentSliceShape
+  & WebsiteCrawlSliceShape
+  & OnlineDriveSliceShape
+
+export const createDataSourceStore = () => {
+  return createStore<DataSourceShape>((...args) => ({
+    ...createCommonSlice(...args),
+    ...createLocalFileSlice(...args),
+    ...createOnlineDocumentSlice(...args),
+    ...createWebsiteCrawlSlice(...args),
+    ...createOnlineDriveSlice(...args),
+  }))
+}
+
+export const useDataSourceStoreWithSelector = <T>(selector: (state: DataSourceShape) => T): T => {
+  const store = useContext(DataSourceContext)
+  if (!store)
+    throw new Error('Missing DataSourceContext.Provider in the tree')
+
+  return useStore(store, selector)
+}
+
+export const useDataSourceStore = () => {
+  const store = useContext(DataSourceContext)
+  if (!store)
+    throw new Error('Missing DataSourceContext.Provider in the tree')
+
+  return store
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/provider.tsx

@@ -0,0 +1,29 @@
+import { createContext, useRef } from 'react'
+import { createDataSourceStore } from './'
+
+type DataSourceStoreApi = ReturnType<typeof createDataSourceStore>
+
+type DataSourceContextType = DataSourceStoreApi | null
+
+export const DataSourceContext = createContext<DataSourceContextType>(null)
+
+type DataSourceProviderProps = {
+  children: React.ReactNode
+}
+
+const DataSourceProvider = ({
+  children,
+}: DataSourceProviderProps) => {
+  const storeRef = useRef<DataSourceStoreApi>(null)
+
+  if (!storeRef.current)
+    storeRef.current = createDataSourceStore()
+
+  return (
+    <DataSourceContext.Provider value={storeRef.current!}>
+      {children}
+    </DataSourceContext.Provider>
+  )
+}
+
+export default DataSourceProvider

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/common.ts

@@ -0,0 +1,19 @@
+import type { StateCreator } from 'zustand'
+
+export type CommonShape = {
+  currentNodeIdRef: React.RefObject<string>
+  currentCredentialId: string
+  setCurrentCredentialId: (credentialId: string) => void
+  currentCredentialIdRef: React.RefObject<string>
+}
+
+export const createCommonSlice: StateCreator<CommonShape> = (set) => {
+  return ({
+    currentNodeIdRef: { current: '' },
+    currentCredentialId: '',
+    setCurrentCredentialId: (credentialId: string) => {
+      set({ currentCredentialId: credentialId })
+    },
+    currentCredentialIdRef: { current: '' },
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts

@@ -0,0 +1,28 @@
+import type { StateCreator } from 'zustand'
+import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
+
+export type LocalFileSliceShape = {
+  localFileList: FileItem[]
+  setLocalFileList: (fileList: FileItem[]) => void
+  currentLocalFile: File | undefined
+  setCurrentLocalFile: (file: File | undefined) => void
+  previewLocalFileRef: React.RefObject<DocumentItem | undefined>
+}
+
+export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set, get) => {
+  return ({
+    localFileList: [],
+    setLocalFileList: (fileList: FileItem[]) => {
+      set(() => ({
+        localFileList: fileList,
+      }))
+      const { previewLocalFileRef } = get()
+      previewLocalFileRef.current = fileList[0]?.file as DocumentItem
+    },
+    currentLocalFile: undefined,
+    setCurrentLocalFile: (file: File | undefined) => set(() => ({
+      currentLocalFile: file,
+    })),
+    previewLocalFileRef: { current: undefined },
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts

@@ -0,0 +1,46 @@
+import type { StateCreator } from 'zustand'
+import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
+
+export type OnlineDocumentSliceShape = {
+  documentsData: DataSourceNotionWorkspace[]
+  setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
+  searchValue: string
+  setSearchValue: (searchValue: string) => void
+  onlineDocuments: NotionPage[]
+  setOnlineDocuments: (documents: NotionPage[]) => void
+  currentDocument: NotionPage | undefined
+  setCurrentDocument: (document: NotionPage | undefined) => void
+  selectedPagesId: Set<string>
+  setSelectedPagesId: (selectedPagesId: Set<string>) => void
+  previewOnlineDocumentRef: React.RefObject<NotionPage | undefined>
+}
+
+export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
+  return ({
+    documentsData: [],
+    setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({
+      documentsData,
+    })),
+    searchValue: '',
+    setSearchValue: (searchValue: string) => set(() => ({
+      searchValue,
+    })),
+    onlineDocuments: [],
+    setOnlineDocuments: (documents: NotionPage[]) => {
+      set(() => ({
+        onlineDocuments: documents,
+      }))
+      const { previewOnlineDocumentRef } = get()
+      previewOnlineDocumentRef.current = documents[0]
+    },
+    currentDocument: undefined,
+    setCurrentDocument: (document: NotionPage | undefined) => set(() => ({
+      currentDocument: document,
+    })),
+    selectedPagesId: new Set(),
+    setSelectedPagesId: (selectedPagesId: Set<string>) => set(() => ({
+      selectedPagesId,
+    })),
+    previewOnlineDocumentRef: { current: undefined },
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -0,0 +1,69 @@
+import type { StateCreator } from 'zustand'
+import type { OnlineDriveFile } from '@/models/pipeline'
+
+export type OnlineDriveSliceShape = {
+  breadcrumbs: string[]
+  setBreadcrumbs: (breadcrumbs: string[]) => void
+  prefix: string[]
+  setPrefix: (prefix: string[]) => void
+  keywords: string
+  setKeywords: (keywords: string) => void
+  selectedFileIds: string[]
+  setSelectedFileIds: (selectedFileIds: string[]) => void
+  onlineDriveFileList: OnlineDriveFile[]
+  setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void
+  bucket: string
+  setBucket: (bucket: string) => void
+  nextPageParameters: Record<string, any>
+  currentNextPageParametersRef: React.RefObject<Record<string, any>>
+  setNextPageParameters: (nextPageParameters: Record<string, any>) => void
+  isTruncated: React.RefObject<boolean>
+  previewOnlineDriveFileRef: React.RefObject<OnlineDriveFile | undefined>
+  hasBucket: boolean
+  setHasBucket: (hasBucket: boolean) => void
+}
+
+export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set, get) => {
+  return ({
+    breadcrumbs: [],
+    setBreadcrumbs: (breadcrumbs: string[]) => set(() => ({
+      breadcrumbs,
+    })),
+    prefix: [],
+    setPrefix: (prefix: string[]) => set(() => ({
+      prefix,
+    })),
+    keywords: '',
+    setKeywords: (keywords: string) => set(() => ({
+      keywords,
+    })),
+    selectedFileIds: [],
+    setSelectedFileIds: (selectedFileIds: string[]) => {
+      set(() => ({
+        selectedFileIds,
+      }))
+      const id = selectedFileIds[0]
+      const { onlineDriveFileList, previewOnlineDriveFileRef } = get()
+      previewOnlineDriveFileRef.current = onlineDriveFileList.find(file => file.id === id)
+    },
+    onlineDriveFileList: [],
+    setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => set(() => ({
+      onlineDriveFileList,
+    })),
+    bucket: '',
+    setBucket: (bucket: string) => set(() => ({
+      bucket,
+    })),
+    nextPageParameters: {},
+    currentNextPageParametersRef: { current: {} },
+    setNextPageParameters: (nextPageParameters: Record<string, any>) => set(() => ({
+      nextPageParameters,
+    })),
+    isTruncated: { current: false },
+    previewOnlineDriveFileRef: { current: undefined },
+    hasBucket: false,
+    setHasBucket: (hasBucket: boolean) => set(() => ({
+      hasBucket,
+    })),
+  })
+}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts

@@ -0,0 +1,47 @@
+import type { StateCreator } from 'zustand'
+import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
+import { CrawlStep } from '@/models/datasets'
+
+export type WebsiteCrawlSliceShape = {
+  websitePages: CrawlResultItem[]
+  setWebsitePages: (pages: CrawlResultItem[]) => void
+  currentWebsite: CrawlResultItem | undefined
+  setCurrentWebsite: (website: CrawlResultItem | undefined) => void
+  crawlResult: CrawlResult | undefined
+  setCrawlResult: (result: CrawlResult | undefined) => void
+  step: CrawlStep
+  setStep: (step: CrawlStep) => void
+  previewIndex: number
+  setPreviewIndex: (index: number) => void
+  previewWebsitePageRef: React.RefObject<CrawlResultItem | undefined>
+}
+
+export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set, get) => {
+  return ({
+    websitePages: [],
+    setWebsitePages: (pages: CrawlResultItem[]) => {
+      set(() => ({
+        websitePages: pages,
+      }))
+      const { previewWebsitePageRef } = get()
+      previewWebsitePageRef.current = pages[0]
+    },
+    currentWebsite: undefined,
+    setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({
+      currentWebsite: website,
+    })),
+    crawlResult: undefined,
+    setCrawlResult: (result: CrawlResult | undefined) => set(() => ({
+      crawlResult: result,
+    })),
+    step: CrawlStep.init,
+    setStep: (step: CrawlStep) => set(() => ({
+      step,
+    })),
+    previewIndex: -1,
+    setPreviewIndex: (index: number) => set(() => ({
+      previewIndex: index,
+    })),
+    previewWebsitePageRef: { current: undefined },
+  })
+}