feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/header/dataset-nav/index.tsx

@@ -1,64 +1,97 @@
 'use client'
 
-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useParams, useRouter } from 'next/navigation'
 import {
   RiBook2Fill,
   RiBook2Line,
 } from '@remixicon/react'
-import useSWR from 'swr'
-import useSWRInfinite from 'swr/infinite'
 import { flatten } from 'lodash-es'
 import Nav from '../nav'
 import type { NavItem } from '../nav/nav-selector'
-import { fetchDatasetDetail, fetchDatasets } from '@/service/datasets'
-import type { DataSetListResponse } from '@/models/datasets'
-
-const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
-  if (!pageIndex || previousPageData.has_more)
-    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
-  return null
-}
+import { basePath } from '@/utils/var'
+import { useDatasetDetail, useDatasetList } from '@/service/knowledge/use-dataset'
+import type { DataSet } from '@/models/datasets'
 
 const DatasetNav = () => {
   const { t } = useTranslation()
   const router = useRouter()
   const { datasetId } = useParams()
-  const { data: currentDataset } = useSWR(
-    datasetId
-      ? {
-        url: 'fetchDatasetDetail',
-        datasetId,
-      }
-      : null,
-    apiParams => fetchDatasetDetail(apiParams.datasetId as string))
-  const { data: datasetsData, setSize } = useSWRInfinite(datasetId ? getKey : () => null, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true })
-  const datasetItems = flatten(datasetsData?.map(datasetData => datasetData.data))
+  const { data: currentDataset } = useDatasetDetail(datasetId as string)
+  const {
+    data: datasetList,
+    fetchNextPage,
+    hasNextPage,
+  } = useDatasetList({
+    initialPage: 1,
+    limit: 30,
+  })
+  const datasetItems = flatten(datasetList?.pages.map(datasetData => datasetData.data))
 
-  const handleLoadmore = useCallback(() => {
-    setSize(size => size + 1)
-  }, [setSize])
+  const curNav = useMemo(() => {
+    if (!currentDataset) return
+    return {
+      id: currentDataset.id,
+      name: currentDataset.name,
+      icon: currentDataset.icon_info.icon,
+      icon_type: currentDataset.icon_info.icon_type,
+      icon_background: currentDataset.icon_info.icon_background,
+      icon_url: currentDataset.icon_info.icon_url,
+    } as Omit<NavItem, 'link'>
+  }, [currentDataset?.id, currentDataset?.name, currentDataset?.icon_info])
+
+  const getDatasetLink = useCallback((dataset: DataSet) => {
+    const isPipelineUnpublished = dataset.runtime_mode === 'rag_pipeline' && !dataset.is_published
+    const link = isPipelineUnpublished
+      ? `/datasets/${dataset.id}/pipeline`
+      : `/datasets/${dataset.id}/documents`
+    return dataset.provider === 'external'
+      ? `/datasets/${dataset.id}/hitTesting`
+      : link
+  }, [])
+
+  const navigationItems = useMemo(() => {
+    return datasetItems.map((dataset) => {
+      const link = getDatasetLink(dataset)
+      return {
+        id: dataset.id,
+        name: dataset.name,
+        link,
+        icon: dataset.icon_info.icon,
+        icon_type: dataset.icon_info.icon_type,
+        icon_background: dataset.icon_info.icon_background,
+        icon_url: dataset.icon_info.icon_url,
+      }
+    }) as NavItem[]
+  }, [datasetItems, getDatasetLink])
+
+  const createRoute = useMemo(() => {
+    const runtimeMode = currentDataset?.runtime_mode
+    if (runtimeMode === 'rag_pipeline')
+      return `${basePath}/datasets/create-from-pipeline`
+    else
+      return `${basePath}/datasets/create`
+  }, [currentDataset?.runtime_mode])
+
+  const handleLoadMore = useCallback(() => {
+    if (hasNextPage)
+      fetchNextPage()
+  }, [hasNextPage, fetchNextPage])
 
   return (
     <Nav
+      isApp={false}
       icon={<RiBook2Line className='h-4 w-4' />}
       activeIcon={<RiBook2Fill className='h-4 w-4' />}
       text={t('common.menus.datasets')}
       activeSegment='datasets'
       link='/datasets'
-      curNav={currentDataset as any}
-      navs={datasetItems.map(dataset => ({
-        id: dataset.id,
-        name: dataset.name,
-        link: dataset.provider === 'external' ? `/datasets/${dataset.id}/hitTesting` : `/datasets/${dataset.id}/documents`,
-        icon: dataset.icon,
-        icon_background: dataset.icon_background,
-      })) as NavItem[]}
+      curNav={curNav}
+      navigationItems={navigationItems}
       createText={t('common.menus.newDataset')}
-      onCreate={() => router.push('/datasets/create')}
-      onLoadmore={handleLoadmore}
-      isApp={false}
+      onCreate={() => router.push(createRoute)}
+      onLoadMore={handleLoadMore}
     />
   )
 }