feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/datasets/documents/detail/settings/index.tsx

@@ -1,96 +1,36 @@
 'use client'
-import React, { useMemo } from 'react'
-import { useTranslation } from 'react-i18next'
-import { useBoolean } from 'ahooks'
-import { useContext } from 'use-context-selector'
-import { useRouter } from 'next/navigation'
-import DatasetDetailContext from '@/context/dataset-detail'
-import type { CrawlOptions, CustomFile } from '@/models/datasets'
+import React from 'react'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import DocumentSettings from './document-settings'
+import PipelineSettings from './pipeline-settings'
 
-import Loading from '@/app/components/base/loading'
-import StepTwo from '@/app/components/datasets/create/step-two'
-import AccountSetting from '@/app/components/header/account-setting'
-import AppUnavailable from '@/app/components/base/app-unavailable'
-import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
-import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
-import type { NotionPage } from '@/models/common'
-import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
-
-type DocumentSettingsProps = {
+type SettingsProps = {
   datasetId: string
   documentId: string
 }
 
-const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
-  const { t } = useTranslation()
-  const router = useRouter()
-  const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
-  const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
-  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
+const Settings = ({
+  datasetId,
+  documentId,
+}: SettingsProps) => {
+  const runtimeMode = useDatasetDetailContextWithSelector(s => s.dataset?.runtime_mode)
+  const isGeneralDataset = runtimeMode === 'general'
 
-  const invalidDocumentDetail = useInvalidDocumentDetailKey()
-  const saveHandler = () => {
-    invalidDocumentDetail()
-    router.push(`/datasets/${datasetId}/documents/${documentId}`)
+  if (isGeneralDataset) {
+    return (
+      <DocumentSettings
+        datasetId={datasetId}
+        documentId={documentId}
+      />
+    )
   }
 
-  const cancelHandler = () => router.back()
-
-  const { data: documentDetail, error } = useDocumentDetail({
-    datasetId,
-    documentId,
-    params: { metadata: 'without' },
-  })
-
-  const currentPage = useMemo(() => {
-    return {
-      workspace_id: documentDetail?.data_source_info.notion_workspace_id,
-      page_id: documentDetail?.data_source_info.notion_page_id,
-      page_name: documentDetail?.name,
-      page_icon: documentDetail?.data_source_info.notion_page_icon,
-      type: documentDetail?.data_source_type,
-    }
-  }, [documentDetail])
-
-  if (error)
-    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
-
   return (
-    <div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
-      <div className="grow">
-        {!documentDetail && <Loading type='app' />}
-        {dataset && documentDetail && (
-          <StepTwo
-            isAPIKeySet={!!embeddingsDefaultModel}
-            onSetting={showSetAPIKey}
-            datasetId={datasetId}
-            dataSourceType={documentDetail.data_source_type}
-            notionPages={[currentPage as unknown as NotionPage]}
-            websitePages={[
-              {
-                title: documentDetail.name,
-                source_url: documentDetail.data_source_info?.url,
-                markdown: '',
-                description: '',
-              },
-            ]}
-            websiteCrawlProvider={documentDetail.data_source_info?.provider}
-            websiteCrawlJobId={documentDetail.data_source_info?.job_id}
-            crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
-            indexingType={indexingTechnique}
-            isSetting
-            documentDetail={documentDetail}
-            files={[documentDetail.data_source_info.upload_file as CustomFile]}
-            onSave={saveHandler}
-            onCancel={cancelHandler}
-          />
-        )}
-      </div>
-      {isShowSetAPIKey && <AccountSetting activeTab="provider" onCancel={async () => {
-        hideSetAPIkey()
-      }} />}
-    </div>
+    <PipelineSettings
+      datasetId={datasetId}
+      documentId={documentId}
+    />
   )
 }
 
-export default DocumentSettings
+export default Settings