feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/datasets/create/embedding-process/index.tsx

@@ -4,8 +4,8 @@ import useSWR from 'swr'
 import { useRouter } from 'next/navigation'
 import { useTranslation } from 'react-i18next'
 import { omit } from 'lodash-es'
-import { ArrowRightIcon } from '@heroicons/react/24/solid'
 import {
+  RiArrowRightLine,
   RiCheckboxCircleFill,
   RiErrorWarningFill,
   RiLoader2Fill,
@@ -31,6 +31,9 @@ import { sleep } from '@/utils'
 import { RETRIEVE_METHOD } from '@/types/app'
 import Tooltip from '@/app/components/base/tooltip'
 import { useInvalidDocumentList } from '@/service/knowledge/use-document'
+import Divider from '@/app/components/base/divider'
+import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
+import Link from 'next/link'
 
 type Props = {
   datasetId: string
@@ -129,7 +132,7 @@ const RuleDetail: FC<{
     <FieldInfo
       label={t('datasetSettings.form.retrievalSetting.title')}
       // displayedValue={t(`datasetSettings.form.retrievalSetting.${retrievalMethod}`) as string}
-      displayedValue={t(`dataset.retrieval.${indexingType === IndexingType.ECONOMICAL ? 'invertedIndex' : retrievalMethod}.title`) as string}
+      displayedValue={t(`dataset.retrieval.${indexingType === IndexingType.ECONOMICAL ? 'keyword_search' : retrievalMethod}.title`) as string}
       valueIcon={
         <Image
           className='size-4'
@@ -211,9 +214,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
     invalidDocumentList()
     router.push(`/datasets/${datasetId}/documents`)
   }
-  const navToApiDocs = () => {
-    router.push('/datasets?category=api')
-  }
+  const apiReferenceUrl = useDatasetApiAccessUrl()
 
   const isEmbedding = useMemo(() => {
     return indexingStatusBatchDetail.some(indexingStatusDetail => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''))
@@ -250,102 +251,118 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
 
   return (
     <>
-      <div className="mb-3 flex h-5 items-center">
-        <div className="mr-2 flex items-center justify-between text-sm font-medium text-text-secondary">
-          {isEmbedding && <div className='flex items-center'>
-            <RiLoader2Fill className='mr-1 size-4 animate-spin text-text-secondary' />
-            {t('datasetDocuments.embedding.processing')}
-          </div>}
+      <div className='flex flex-col gap-y-3'>
+        <div className='system-md-semibold-uppercase flex items-center gap-x-1 text-text-secondary'>
+          {isEmbedding && (
+            <>
+              <RiLoader2Fill className='size-4 animate-spin' />
+              <span>{t('datasetDocuments.embedding.processing')}</span>
+            </>
+          )}
           {isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
         </div>
-      </div>
-      {
-        enableBilling && plan.type !== Plan.team && (
-          <div className='mb-3 flex h-14 items-center rounded-xl border-[0.5px] border-black/5 bg-white p-3 shadow-md'>
-            <div className='flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-[#FFF6ED]'>
-              <ZapFast className='h-4 w-4 text-[#FB6514]' />
-            </div>
-            <div className='mx-3 grow text-[13px] font-medium text-gray-700'>
-              {t('billing.plansCommon.documentProcessingPriorityUpgrade')}
-            </div>
-            <UpgradeBtn loc='knowledge-speed-up' />
-          </div>
-        )
-      }
-      <div className="flex flex-col gap-0.5 pb-2">
-        {indexingStatusBatchDetail.map(indexingStatusDetail => (
-          <div key={indexingStatusDetail.id} className={cn(
-            'relative h-[26px] overflow-hidden rounded-md bg-components-progress-bar-bg',
-            indexingStatusDetail.indexing_status === 'error' && 'bg-state-destructive-hover-alt',
-            // indexingStatusDetail.indexing_status === 'completed' && 's.success',
-          )}>
-            {isSourceEmbedding(indexingStatusDetail) && (
-              <div className="absolute left-0 top-0 h-full min-w-0.5 border-r-[2px] border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress" style={{ width: `${getSourcePercent(indexingStatusDetail)}%` }} />
-            )}
-            <div className="z-[1] flex h-full items-center gap-1 pl-[6px] pr-2">
-              {getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && (
-                // <div className={cn(
-                //   'shrink-0 marker:size-4 bg-center bg-no-repeat bg-contain',
-                //   s[getFileType(getSourceName(indexingStatusDetail.id))] || s.unknownFileIcon,
-                // )} />
-                <DocumentFileIcon
-                  className="size-4 shrink-0"
-                  name={getSourceName(indexingStatusDetail.id)}
-                  extension={getFileType(getSourceName(indexingStatusDetail.id))}
-                />
-              )}
-              {getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && (
-                <NotionIcon
-                  className='shrink-0'
-                  type='page'
-                  src={getIcon(indexingStatusDetail.id)}
-                />
-              )}
-              <div className="flex w-0 grow items-center gap-1" title={getSourceName(indexingStatusDetail.id)}>
-                <div className="system-xs-medium truncate text-text-secondary">
-                  {getSourceName(indexingStatusDetail.id)}
-                </div>
-                {
-                  enableBilling && (
-                    <PriorityLabel className='ml-0' />
-                  )
-                }
+        {
+          enableBilling && plan.type !== Plan.team && (
+            <div className='flex h-14 items-center rounded-xl border-[0.5px] border-black/5 bg-white p-3 shadow-md'>
+              <div className='flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-[#FFF6ED]'>
+                <ZapFast className='h-4 w-4 text-[#FB6514]' />
               </div>
-              {isSourceEmbedding(indexingStatusDetail) && (
-                <div className="shrink-0 text-xs text-text-secondary">{`${getSourcePercent(indexingStatusDetail)}%`}</div>
-              )}
-              {indexingStatusDetail.indexing_status === 'error' && (
-                <Tooltip
-                  popupClassName='px-4 py-[14px] max-w-60 text-sm leading-4 text-text-secondary border-[0.5px] border-components-panel-border rounded-xl'
-                  offset={4}
-                  popupContent={indexingStatusDetail.error}
-                >
-                  <span>
-                    <RiErrorWarningFill className='size-4 shrink-0 text-text-destructive' />
-                  </span>
-                </Tooltip>
-              )}
-              {indexingStatusDetail.indexing_status === 'completed' && (
-                <RiCheckboxCircleFill className='size-4 shrink-0 text-text-success' />
-              )}
+              <div className='mx-3 grow text-[13px] font-medium text-gray-700'>
+                {t('billing.plansCommon.documentProcessingPriorityUpgrade')}
+              </div>
+              <UpgradeBtn loc='knowledge-speed-up' />
             </div>
-          </div>
-        ))}
+          )
+        }
+        <div className='flex flex-col gap-0.5 pb-2'>
+          {indexingStatusBatchDetail.map(indexingStatusDetail => (
+            <div
+              key={indexingStatusDetail.id}
+              className={cn(
+                'relative h-[26px] overflow-hidden rounded-md bg-components-progress-bar-bg',
+                indexingStatusDetail.indexing_status === 'error' && 'bg-state-destructive-hover-alt',
+              )}
+            >
+              {isSourceEmbedding(indexingStatusDetail) && (
+                <div
+                  className='absolute left-0 top-0 h-full min-w-0.5 border-r-[2px] border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress'
+                  style={{ width: `${getSourcePercent(indexingStatusDetail)}%` }}
+                />
+              )}
+              <div className='z-[1] flex h-full items-center gap-1 pl-[6px] pr-2'>
+                {getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && (
+                  <DocumentFileIcon
+                    size='sm'
+                    className='shrink-0'
+                    name={getSourceName(indexingStatusDetail.id)}
+                    extension={getFileType(getSourceName(indexingStatusDetail.id))}
+                  />
+                )}
+                {getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && (
+                  <NotionIcon
+                    className='shrink-0'
+                    type='page'
+                    src={getIcon(indexingStatusDetail.id)}
+                  />
+                )}
+                <div className='flex w-0 grow items-center gap-1' title={getSourceName(indexingStatusDetail.id)}>
+                  <div className='system-xs-medium truncate text-text-secondary'>
+                    {getSourceName(indexingStatusDetail.id)}
+                  </div>
+                  {
+                    enableBilling && (
+                      <PriorityLabel className='ml-0' />
+                    )
+                  }
+                </div>
+                {isSourceEmbedding(indexingStatusDetail) && (
+                  <div className='shrink-0 text-xs text-text-secondary'>{`${getSourcePercent(indexingStatusDetail)}%`}</div>
+                )}
+                {indexingStatusDetail.indexing_status === 'error' && (
+                  <Tooltip
+                    popupClassName='px-4 py-[14px] max-w-60 body-xs-regular text-text-secondary border-[0.5px] border-components-panel-border rounded-xl'
+                    offset={4}
+                    popupContent={indexingStatusDetail.error}
+                  >
+                    <span>
+                      <RiErrorWarningFill className='size-4 shrink-0 text-text-destructive' />
+                    </span>
+                  </Tooltip>
+                )}
+                {indexingStatusDetail.indexing_status === 'completed' && (
+                  <RiCheckboxCircleFill className='size-4 shrink-0 text-text-success' />
+                )}
+              </div>
+            </div>
+          ))}
+        </div>
+        <Divider type='horizontal' className='my-0 bg-divider-subtle' />
+        <RuleDetail
+          sourceData={ruleDetail}
+          indexingType={indexingType}
+          retrievalMethod={retrievalMethod}
+        />
       </div>
-      <hr className="my-3 h-px border-0 bg-divider-subtle" />
-      <RuleDetail
-        sourceData={ruleDetail}
-        indexingType={indexingType}
-        retrievalMethod={retrievalMethod}
-      />
-      <div className='my-10 flex items-center gap-2'>
-        <Button className='w-fit' onClick={navToApiDocs}>
-          <RiTerminalBoxLine className='mr-2 size-4' />
-          <span>Access the API</span>
-        </Button>
-        <Button className='w-fit' variant='primary' onClick={navToDocumentList}>
-          <span>{t('datasetCreation.stepThree.navTo')}</span>
-          <ArrowRightIcon className='ml-2 size-4 stroke-current stroke-1' />
+      <div className='mt-6 flex items-center gap-x-2 py-2'>
+        <Link
+          href={apiReferenceUrl}
+          target='_blank'
+          rel='noopener noreferrer'
+        >
+          <Button
+            className='w-fit gap-x-0.5 px-3'
+          >
+            <RiTerminalBoxLine className='size-4' />
+            <span className='px-0.5'>Access the API</span>
+          </Button>
+        </Link>
+        <Button
+          className='w-fit gap-x-0.5 px-3'
+          variant='primary'
+          onClick={navToDocumentList}
+        >
+          <span className='px-0.5'>{t('datasetCreation.stepThree.navTo')}</span>
+          <RiArrowRightLine className='size-4 stroke-current stroke-1' />
         </Button>
       </div>
     </>

web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx

@@ -11,6 +11,7 @@ import Button from '@/app/components/base/button'
 
 import { ToastContext } from '@/app/components/base/toast'
 import { createEmptyDataset } from '@/service/datasets'
+import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
 
 type IProps = {
   show: boolean
@@ -25,6 +26,7 @@ const EmptyDatasetCreationModal = ({
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const router = useRouter()
+  const invalidDatasetList = useInvalidDatasetList()
 
   const submit = async () => {
     if (!inputValue) {
@@ -37,6 +39,7 @@ const EmptyDatasetCreationModal = ({
     }
     try {
       const dataset = await createEmptyDataset({ name: inputValue })
+      invalidDatasetList()
       onHide()
       router.push(`/datasets/${dataset.id}/documents`)
     }

web/app/components/datasets/create/file-uploader/index.tsx

@@ -322,6 +322,7 @@ const FileUploader = ({
           <div>{t('datasetCreation.stepOne.uploader.tip', {
             size: fileUploadConfig.file_size_limit,
             supportTypes: supportTypesShowNames,
+            batchCount: fileUploadConfig.batch_count_limit,
           })}</div>
           {dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
         </div>
@@ -339,7 +340,8 @@ const FileUploader = ({
           >
             <div className="flex w-12 shrink-0 items-center justify-center">
               <DocumentFileIcon
-                className="size-6 shrink-0"
+                size='xl'
+                className="shrink-0"
                 name={fileItem.file.name}
                 extension={getFileType(fileItem.file)}
               />

web/app/components/datasets/create/index.tsx

@@ -1,5 +1,5 @@
 'use client'
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import AppUnavailable from '../../base/app-unavailable'
 import { ModelTypeEnum } from '../../header/account-setting/model-provider-page/declarations'
@@ -8,12 +8,14 @@ import StepTwo from './step-two'
 import StepThree from './step-three'
 import { TopBar } from './top-bar'
 import { DataSourceType } from '@/models/datasets'
-import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
-import { fetchDataSource } from '@/service/common'
-import { fetchDatasetDetail } from '@/service/datasets'
+import type { CrawlOptions, CrawlResultItem, FileItem, createDocumentResponse } from '@/models/datasets'
 import { DataSourceProvider, type NotionPage } from '@/models/common'
-import { useModalContext } from '@/context/modal-context'
+import { useModalContextSelector } from '@/context/modal-context'
 import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useGetDefaultDataSourceListAuth } from '@/service/use-datasource'
+import produce from 'immer'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import Loading from '@/app/components/base/loading'
 
 type DatasetUpdateFormProps = {
   datasetId?: string
@@ -31,59 +33,63 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
 
 const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const { t } = useTranslation()
-  const { setShowAccountSettingModal } = useModalContext()
-  const [hasConnection, setHasConnection] = useState(true)
+  const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
+  const datasetDetail = useDatasetDetailContextWithSelector(state => state.dataset)
+  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
+
   const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.FILE)
   const [step, setStep] = useState(1)
   const [indexingTypeCache, setIndexTypeCache] = useState('')
   const [retrievalMethodCache, setRetrievalMethodCache] = useState('')
   const [fileList, setFiles] = useState<FileItem[]>([])
   const [result, setResult] = useState<createDocumentResponse | undefined>()
-  const [hasError, setHasError] = useState(false)
-  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
-
   const [notionPages, setNotionPages] = useState<NotionPage[]>([])
-  const updateNotionPages = (value: NotionPage[]) => {
-    setNotionPages(value)
-  }
-
+  const [notionCredentialId, setNotionCredentialId] = useState<string>('')
   const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
   const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
-
-  const updateFileList = (preparedFiles: FileItem[]) => {
-    setFiles(preparedFiles)
-  }
-  const [websiteCrawlProvider, setWebsiteCrawlProvider] = useState<DataSourceProvider>(DataSourceProvider.fireCrawl)
+  const [websiteCrawlProvider, setWebsiteCrawlProvider] = useState<DataSourceProvider>(DataSourceProvider.jinaReader)
   const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
 
-  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
+  const {
+    data: dataSourceList,
+    isLoading: isLoadingAuthedDataSourceList,
+    isError: fetchingAuthedDataSourceListError,
+  } = useGetDefaultDataSourceListAuth()
+
+  const updateNotionPages = useCallback((value: NotionPage[]) => {
+    setNotionPages(value)
+  }, [])
+
+  const updateNotionCredentialId = useCallback((credentialId: string) => {
+    setNotionCredentialId(credentialId)
+  }, [])
+
+  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
+    setFiles(preparedFiles)
+  }, [])
+
+  const updateFile = useCallback((fileItem: FileItem, progress: number, list: FileItem[]) => {
     const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID)
-    list[targetIndex] = {
-      ...list[targetIndex],
-      progress,
-    }
-    setFiles([...list])
-    // use follow code would cause dirty list update problem
-    // const newList = list.map((file) => {
-    //   if (file.fileID === fileItem.fileID) {
-    //     return {
-    //       ...fileItem,
-    //       progress,
-    //     }
-    //   }
-    //   return file
-    // })
-    // setFiles(newList)
-  }
-  const updateIndexingTypeCache = (type: string) => {
+    const newList = produce(list, (draft) => {
+      draft[targetIndex] = {
+        ...draft[targetIndex],
+        progress,
+      }
+    })
+    setFiles(newList)
+  }, [])
+
+  const updateIndexingTypeCache = useCallback((type: string) => {
     setIndexTypeCache(type)
-  }
-  const updateResultCache = (res?: createDocumentResponse) => {
+  }, [])
+
+  const updateResultCache = useCallback((res?: createDocumentResponse) => {
     setResult(res)
-  }
-  const updateRetrievalMethodCache = (method: string) => {
+  }, [])
+
+  const updateRetrievalMethodCache = useCallback((method: string) => {
     setRetrievalMethodCache(method)
-  }
+  }, [])
 
   const nextStep = useCallback(() => {
     setStep(step + 1)
@@ -93,82 +99,77 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
     setStep(step + delta)
   }, [step, setStep])
 
-  const checkNotionConnection = async () => {
-    const { data } = await fetchDataSource({ url: '/data-source/integrates' })
-    const hasConnection = data.filter(item => item.provider === 'notion') || []
-    setHasConnection(hasConnection.length > 0)
-  }
-
-  useEffect(() => {
-    checkNotionConnection()
-  }, [])
-
-  const [detail, setDetail] = useState<DataSet | null>(null)
-  useEffect(() => {
-    (async () => {
-      if (datasetId) {
-        try {
-          const detail = await fetchDatasetDetail(datasetId)
-          setDetail(detail)
-        }
-        catch {
-          setHasError(true)
-        }
-      }
-    })()
-  }, [datasetId])
-
-  if (hasError)
+  if (fetchingAuthedDataSourceListError)
     return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
 
   return (
     <div className='flex flex-col overflow-hidden bg-components-panel-bg' style={{ height: 'calc(100vh - 56px)' }}>
       <TopBar activeIndex={step - 1} datasetId={datasetId} />
       <div style={{ height: 'calc(100% - 52px)' }}>
-        {step === 1 && <StepOne
-          hasConnection={hasConnection}
-          onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
-          datasetId={datasetId}
-          dataSourceType={dataSourceType}
-          dataSourceTypeDisable={!!detail?.data_source_type}
-          changeType={setDataSourceType}
-          files={fileList}
-          updateFile={updateFile}
-          updateFileList={updateFileList}
-          notionPages={notionPages}
-          updateNotionPages={updateNotionPages}
-          onStepChange={nextStep}
-          websitePages={websitePages}
-          updateWebsitePages={setWebsitePages}
-          onWebsiteCrawlProviderChange={setWebsiteCrawlProvider}
-          onWebsiteCrawlJobIdChange={setWebsiteCrawlJobId}
-          crawlOptions={crawlOptions}
-          onCrawlOptionsChange={setCrawlOptions}
-        />}
-        {(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
-          isAPIKeySet={!!embeddingsDefaultModel}
-          onSetting={() => setShowAccountSettingModal({ payload: 'provider' })}
-          indexingType={detail?.indexing_technique}
-          datasetId={datasetId}
-          dataSourceType={dataSourceType}
-          files={fileList.map(file => file.file)}
-          notionPages={notionPages}
-          websitePages={websitePages}
-          websiteCrawlProvider={websiteCrawlProvider}
-          websiteCrawlJobId={websiteCrawlJobId}
-          onStepChange={changeStep}
-          updateIndexingTypeCache={updateIndexingTypeCache}
-          updateRetrievalMethodCache={updateRetrievalMethodCache}
-          updateResultCache={updateResultCache}
-          crawlOptions={crawlOptions}
-        />}
-        {step === 3 && <StepThree
-          datasetId={datasetId}
-          datasetName={detail?.name}
-          indexingType={detail?.indexing_technique || indexingTypeCache}
-          retrievalMethod={detail?.retrieval_model_dict?.search_method || retrievalMethodCache}
-          creationCache={result}
-        />}
+        {
+          isLoadingAuthedDataSourceList && (
+            <Loading type='app' />
+          )
+        }
+        {
+          !isLoadingAuthedDataSourceList && (
+            <>
+              {step === 1 && (
+                <StepOne
+                  authedDataSourceList={dataSourceList?.result || []}
+                  onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
+                  datasetId={datasetId}
+                  dataSourceType={dataSourceType}
+                  dataSourceTypeDisable={!!datasetDetail?.data_source_type}
+                  changeType={setDataSourceType}
+                  files={fileList}
+                  updateFile={updateFile}
+                  updateFileList={updateFileList}
+                  notionPages={notionPages}
+                  notionCredentialId={notionCredentialId}
+                  updateNotionPages={updateNotionPages}
+                  updateNotionCredentialId={updateNotionCredentialId}
+                  onStepChange={nextStep}
+                  websitePages={websitePages}
+                  updateWebsitePages={setWebsitePages}
+                  onWebsiteCrawlProviderChange={setWebsiteCrawlProvider}
+                  onWebsiteCrawlJobIdChange={setWebsiteCrawlJobId}
+                  crawlOptions={crawlOptions}
+                  onCrawlOptionsChange={setCrawlOptions}
+                />
+              )}
+              {(step === 2 && (!datasetId || (datasetId && !!datasetDetail))) && (
+                <StepTwo
+                  isAPIKeySet={!!embeddingsDefaultModel}
+                  onSetting={() => setShowAccountSettingModal({ payload: 'provider' })}
+                  indexingType={datasetDetail?.indexing_technique}
+                  datasetId={datasetId}
+                  dataSourceType={dataSourceType}
+                  files={fileList.map(file => file.file)}
+                  notionPages={notionPages}
+                  notionCredentialId={notionCredentialId}
+                  websitePages={websitePages}
+                  websiteCrawlProvider={websiteCrawlProvider}
+                  websiteCrawlJobId={websiteCrawlJobId}
+                  onStepChange={changeStep}
+                  updateIndexingTypeCache={updateIndexingTypeCache}
+                  updateRetrievalMethodCache={updateRetrievalMethodCache}
+                  updateResultCache={updateResultCache}
+                  crawlOptions={crawlOptions}
+                />
+              )}
+              {step === 3 && (
+                <StepThree
+                  datasetId={datasetId}
+                  datasetName={datasetDetail?.name}
+                  indexingType={datasetDetail?.indexing_technique || indexingTypeCache}
+                  retrievalMethod={datasetDetail?.retrieval_model_dict?.search_method || retrievalMethodCache}
+                  creationCache={result}
+                />
+              )}
+            </>
+          )
+        }
       </div>
     </div>
   )

web/app/components/datasets/create/notion-page-preview/index.tsx

@@ -11,11 +11,13 @@ import { fetchNotionPagePreview } from '@/service/datasets'
 
 type IProps = {
   currentPage?: NotionPage
+  notionCredentialId: string
   hidePreview: () => void
 }
 
 const NotionPagePreview = ({
   currentPage,
+  notionCredentialId,
   hidePreview,
 }: IProps) => {
   const { t } = useTranslation()
@@ -30,6 +32,7 @@ const NotionPagePreview = ({
         workspaceID: currentPage.workspace_id,
         pageID: currentPage.page_id,
         pageType: currentPage.type,
+        credentialID: notionCredentialId,
       })
       setPreviewContent(res.content)
       setLoading(false)

web/app/components/datasets/create/step-one/index.tsx

@@ -15,24 +15,26 @@ import type { DataSourceProvider, NotionPage } from '@/models/common'
 import { DataSourceType } from '@/models/datasets'
 import Button from '@/app/components/base/button'
 import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
-import { useDatasetDetailContext } from '@/context/dataset-detail'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { useProviderContext } from '@/context/provider-context'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import classNames from '@/utils/classnames'
-import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
 import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import NotionConnector from '@/app/components/base/notion-connector'
+import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'
 
 type IStepOneProps = {
   datasetId?: string
   dataSourceType?: DataSourceType
   dataSourceTypeDisable: boolean
-  hasConnection: boolean
   onSetting: () => void
   files: FileItem[]
   updateFileList: (files: FileItem[]) => void
   updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
   notionPages?: NotionPage[]
+  notionCredentialId: string
   updateNotionPages: (value: NotionPage[]) => void
+  updateNotionCredentialId: (credentialId: string) => void
   onStepChange: () => void
   changeType: (type: DataSourceType) => void
   websitePages?: CrawlResultItem[]
@@ -41,28 +43,7 @@ type IStepOneProps = {
   onWebsiteCrawlJobIdChange: (jobId: string) => void
   crawlOptions: CrawlOptions
   onCrawlOptionsChange: (payload: CrawlOptions) => void
-}
-
-type NotionConnectorProps = {
-  onSetting: () => void
-}
-export const NotionConnector = (props: NotionConnectorProps) => {
-  const { onSetting } = props
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex w-[640px] flex-col items-start rounded-2xl bg-workflow-process-bg p-6'>
-      <span className={cn(s.notionIcon, 'mb-2 h-12 w-12 rounded-[10px] border-[0.5px] border-components-card-border p-3 shadow-lg shadow-shadow-shadow-5')} />
-      <div className='mb-1 flex flex-col gap-y-1 pb-3 pt-1'>
-        <span className='system-md-semibold text-text-secondary'>
-          {t('datasetCreation.stepOne.notionSyncTitle')}
-          <Icon3Dots className='relative -left-1.5 -top-2.5 inline h-4 w-4 text-text-secondary' />
-        </span>
-        <div className='system-sm-regular text-text-tertiary'>{t('datasetCreation.stepOne.notionSyncTip')}</div>
-      </div>
-      <Button className='h-8' variant='primary' onClick={onSetting}>{t('datasetCreation.stepOne.connect')}</Button>
-    </div>
-  )
+  authedDataSourceList: DataSourceAuth[]
 }
 
 const StepOne = ({
@@ -70,22 +51,24 @@ const StepOne = ({
   dataSourceType: inCreatePageDataSourceType,
   dataSourceTypeDisable,
   changeType,
-  hasConnection,
   onSetting,
   onStepChange,
   files,
   updateFileList,
   updateFile,
   notionPages = [],
+  notionCredentialId,
   updateNotionPages,
+  updateNotionCredentialId,
   websitePages = [],
   updateWebsitePages,
   onWebsiteCrawlProviderChange,
   onWebsiteCrawlJobIdChange,
   crawlOptions,
   onCrawlOptionsChange,
+  authedDataSourceList,
 }: IStepOneProps) => {
-  const { dataset } = useDatasetDetailContext()
+  const dataset = useDatasetDetailContextWithSelector(state => state.dataset)
   const [showModal, setShowModal] = useState(false)
   const [currentFile, setCurrentFile] = useState<File | undefined>()
   const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
@@ -136,6 +119,17 @@ const StepOne = ({
     return isShowVectorSpaceFull
   }, [files, isShowVectorSpaceFull])
 
+  const isNotionAuthed = useMemo(() => {
+    if (!authedDataSourceList) return false
+    const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource')
+    if (!notionSource) return false
+    return notionSource.credentials_list.length > 0
+  }, [authedDataSourceList])
+
+  const notionCredentialList = useMemo(() => {
+    return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || []
+  }, [authedDataSourceList])
+
   return (
     <div className='h-full w-full overflow-x-auto'>
       <div className='flex h-full w-full min-w-[1440px]'>
@@ -243,7 +237,6 @@ const StepOne = ({
                     </div>
                   )}
                   <div className="flex max-w-[640px] justify-end gap-2">
-                    {/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
                     <Button disabled={nextDisabled} variant='primary' onClick={onStepChange}>
                       <span className="flex gap-0.5 px-[10px]">
                         <span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
@@ -255,14 +248,17 @@ const StepOne = ({
               )}
               {dataSourceType === DataSourceType.NOTION && (
                 <>
-                  {!hasConnection && <NotionConnector onSetting={onSetting} />}
-                  {hasConnection && (
+                  {!isNotionAuthed && <NotionConnector onSetting={onSetting} />}
+                  {isNotionAuthed && (
                     <>
                       <div className='mb-8 w-[640px]'>
                         <NotionPageSelector
                           value={notionPages.map(page => page.page_id)}
                           onSelect={updateNotionPages}
                           onPreview={updateCurrentPage}
+                          credentialList={notionCredentialList}
+                          onSelectCredential={updateNotionCredentialId}
+                          datasetId={datasetId}
                         />
                       </div>
                       {isShowVectorSpaceFull && (
@@ -271,7 +267,6 @@ const StepOne = ({
                         </div>
                       )}
                       <div className="flex max-w-[640px] justify-end gap-2">
-                        {/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
                         <Button disabled={isShowVectorSpaceFull || !notionPages.length} variant='primary' onClick={onStepChange}>
                           <span className="flex gap-0.5 px-[10px]">
                             <span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
@@ -294,6 +289,7 @@ const StepOne = ({
                       onJobIdChange={onWebsiteCrawlJobIdChange}
                       crawlOptions={crawlOptions}
                       onCrawlOptionsChange={onCrawlOptionsChange}
+                      authedDataSourceList={authedDataSourceList}
                     />
                   </div>
                   {isShowVectorSpaceFull && (
@@ -302,7 +298,6 @@ const StepOne = ({
                     </div>
                   )}
                   <div className="flex max-w-[640px] justify-end gap-2">
-                    {/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
                     <Button disabled={isShowVectorSpaceFull || !websitePages.length} variant='primary' onClick={onStepChange}>
                       <span className="flex gap-0.5 px-[10px]">
                         <span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
@@ -327,7 +322,13 @@ const StepOne = ({
         </div>
         <div className='h-full w-1/2 overflow-y-auto'>
           {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
-          {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
+          {currentNotionPage && (
+            <NotionPagePreview
+              currentPage={currentNotionPage}
+              hidePreview={hideNotionPagePreview}
+              notionCredentialId={notionCredentialId}
+            />
+          )}
           {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
         </div>
       </div>

web/app/components/datasets/create/step-three/index.tsx

@@ -7,6 +7,8 @@ import EmbeddingProcess from '../embedding-process'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
 import type { FullDocumentDetail, createDocumentResponse } from '@/models/datasets'
 import AppIcon from '@/app/components/base/app-icon'
+import Divider from '@/app/components/base/divider'
+import { useDocLink } from '@/context/i18n'
 
 type StepThreeProps = {
   datasetId?: string
@@ -18,34 +20,52 @@ type StepThreeProps = {
 
 const StepThree = ({ datasetId, datasetName, indexingType, creationCache, retrievalMethod }: StepThreeProps) => {
   const { t } = useTranslation()
+  const docLink = useDocLink()
 
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
+  const iconInfo = creationCache?.dataset?.icon_info || {
+    icon: '📙',
+    icon_type: 'emoji',
+    icon_background: '#FFF4ED',
+    icon_url: '',
+  }
 
   return (
-    <div className="flex h-full max-h-full w-full justify-center overflow-y-auto">
-      <div className="h-full max-w-[960px] shrink-0 grow overflow-y-auto px-14 sm:px-16">
-        <div className="mx-auto max-w-[640px]">
+    <div className='flex h-full max-h-full w-full justify-center overflow-y-auto'>
+      <div className='h-full max-w-[960px] shrink-0 grow overflow-y-auto px-14 sm:px-16'>
+        <div className='mx-auto max-w-[640px] pb-8 pt-10'>
           {!datasetId && (
             <>
-              <div className="pt-10">
-                <div className="mb-1 text-xl font-semibold leading-[22px] text-text-primary">{t('datasetCreation.stepThree.creationTitle')}</div>
-                <div className="mb-7 text-[13px] leading-4 text-text-tertiary">{t('datasetCreation.stepThree.creationContent')}</div>
-                <div className="flex gap-4">
-                  <AppIcon {...creationCache?.dataset} className="size-14 self-center text-2xl" />
-                  <div className="flex grow flex-col gap-1">
-                    <div className="text-[13px] font-semibold leading-6 text-text-secondary">{t('datasetCreation.stepThree.label')}</div>
-                    <div className="w-full truncate rounded-lg bg-components-input-bg-normal px-3 py-2 text-[13px] leading-4 text-components-input-text-filled">{datasetName || creationCache?.dataset?.name}</div>
+              <div className='flex flex-col gap-y-1 pb-3'>
+                <div className='title-2xl-semi-bold text-text-primary'>{t('datasetCreation.stepThree.creationTitle')}</div>
+                <div className='system-sm-regular text-text-tertiary'>{t('datasetCreation.stepThree.creationContent')}</div>
+              </div>
+              <div className='flex items-center gap-x-4'>
+                <AppIcon
+                  size='xxl'
+                  iconType={iconInfo.icon_type}
+                  icon={iconInfo.icon}
+                  background={iconInfo.icon_background}
+                  imageUrl={iconInfo.icon_url}
+                  className='shrink-0'
+                />
+                <div className='flex grow flex-col gap-y-1'>
+                  <div className='system-sm-semibold flex h-6 items-center text-text-secondary'>
+                    {t('datasetCreation.stepThree.label')}
+                  </div>
+                  <div className='system-sm-regular w-full truncate rounded-lg bg-components-input-bg-normal p-2 text-components-input-text-filled'>
+                    <span className='px-1'>{datasetName || creationCache?.dataset?.name}</span>
                   </div>
                 </div>
               </div>
-              <hr className="my-6 h-px border-0 bg-divider-subtle" />
+              <Divider type='horizontal' className='my-6 bg-divider-subtle' />
             </>
           )}
           {datasetId && (
-            <div className="pt-10">
-              <div className="mb-1 text-xl font-semibold leading-[22px] text-text-primary">{t('datasetCreation.stepThree.additionTitle')}</div>
-              <div className="mb-7 text-[13px] leading-4 text-text-tertiary">{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}</div>
+            <div className='flex flex-col gap-y-1 pb-3'>
+              <div className='title-2xl-semi-bold text-text-primary'>{t('datasetCreation.stepThree.additionTitle')}</div>
+              <div className='system-sm-regular text-text-tertiary'>{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}</div>
             </div>
           )}
           <EmbeddingProcess
@@ -58,13 +78,21 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache, retrie
         </div>
       </div>
       {!isMobile && (
-        <div className="shrink-0 pr-8 pt-[88px] text-xs">
-          <div className="flex w-[328px] flex-col gap-3 rounded-xl bg-background-section p-6 text-text-tertiary">
-            <div className="flex size-10 items-center justify-center rounded-[10px] bg-components-card-bg shadow-lg">
-              <RiBookOpenLine className="size-5 text-text-accent" />
+        <div className='shrink-0 pr-8 pt-[88px] text-xs'>
+          <div className='flex w-[328px] flex-col gap-3 rounded-xl bg-background-section p-6 text-text-tertiary'>
+            <div className='flex size-10 items-center justify-center rounded-[10px] bg-components-card-bg shadow-lg'>
+              <RiBookOpenLine className='size-5 text-text-accent' />
             </div>
-            <div className="text-base font-semibold text-text-secondary">{t('datasetCreation.stepThree.sideTipTitle')}</div>
-            <div className="text-text-tertiary">{t('datasetCreation.stepThree.sideTipContent')}</div>
+            <div className='text-base font-semibold text-text-secondary'>{t('datasetCreation.stepThree.sideTipTitle')}</div>
+            <div className='text-text-tertiary'>{t('datasetCreation.stepThree.sideTipContent')}</div>
+            <a
+              href={docLink('/guides/knowledge-base/integrate-knowledge-within-application')}
+              target='_blank'
+              rel='noreferrer noopener'
+              className='system-sm-regular text-text-accent'
+            >
+              {t('datasetPipeline.addDocuments.stepThree.learnMore')}
+            </a>
           </div>
         </div>
       )}

web/app/components/datasets/create/step-two/index.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC, PropsWithChildren } from 'react'
-import React, { useCallback, useEffect, useRef, useState } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import {
@@ -10,14 +10,13 @@ import {
 } from '@remixicon/react'
 import Link from 'next/link'
 import Image from 'next/image'
-import { useHover } from 'ahooks'
 import SettingCog from '../assets/setting-gear-mod.svg'
 import OrangeEffect from '../assets/option-card-effect-orange.svg'
 import FamilyMod from '../assets/family-mod.svg'
 import Note from '../assets/note-mod.svg'
 import FileList from '../assets/file-list-3-fill.svg'
 import { indexMethodIcon } from '../icons'
-import { PreviewContainer } from '../../preview/container'
+import PreviewContainer from '../../preview/container'
 import { ChunkContainer, QAPreview } from '../../chunk'
 import { PreviewHeader } from '../../preview/header'
 import { FormattedText } from '../../formatted-text/formatted'
@@ -42,7 +41,7 @@ import { isReRankModelSelected } from '@/app/components/datasets/common/check-re
 import Toast from '@/app/components/base/toast'
 import type { NotionPage } from '@/models/common'
 import { DataSourceProvider } from '@/models/common'
-import { useDatasetDetailContext } from '@/context/dataset-detail'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import I18n from '@/context/i18n'
 import { RETRIEVE_METHOD } from '@/types/app'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
@@ -60,10 +59,10 @@ import Badge from '@/app/components/base/badge'
 import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
 import Tooltip from '@/app/components/base/tooltip'
 import CustomDialog from '@/app/components/base/dialog'
-import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
 import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
 import { noop } from 'lodash-es'
 import { useDocLink } from '@/context/i18n'
+import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
 
 const TextLabel: FC<PropsWithChildren> = (props) => {
   return <label className='system-sm-semibold text-text-secondary'>{props.children}</label>
@@ -80,6 +79,7 @@ type StepTwoProps = {
   dataSourceType: DataSourceType
   files: CustomFile[]
   notionPages?: NotionPage[]
+  notionCredentialId: string
   websitePages?: CrawlResultItem[]
   crawlOptions?: CrawlOptions
   websiteCrawlProvider?: DataSourceProvider
@@ -135,9 +135,10 @@ const StepTwo = ({
   dataSourceType: inCreatePageDataSourceType,
   files,
   notionPages = [],
+  notionCredentialId,
   websitePages = [],
   crawlOptions,
-  websiteCrawlProvider = DataSourceProvider.fireCrawl,
+  websiteCrawlProvider = DataSourceProvider.jinaReader,
   websiteCrawlJobId = '',
   onStepChange,
   updateIndexingTypeCache,
@@ -152,7 +153,8 @@ const StepTwo = ({
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
 
-  const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext()
+  const currentDataset = useDatasetDetailContextWithSelector(state => state.dataset)
+  const mutateDatasetRes = useDatasetDetailContextWithSelector(state => state.mutateDatasetRes)
 
   const isInUpload = Boolean(currentDataset)
   const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form
@@ -282,6 +284,7 @@ const StepTwo = ({
     indexingTechnique: getIndexing_technique() as any,
     processRule: getProcessRule(),
     dataset_id: datasetId || '',
+    credential_id: notionCredentialId,
   })
 
   const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({
@@ -469,7 +472,7 @@ const StepTwo = ({
         }
       }
       if (dataSourceType === DataSourceType.NOTION)
-        params.data_source.info_list.notion_info_list = getNotionInfo(notionPages)
+        params.data_source.info_list.notion_info_list = getNotionInfo(notionPages, notionCredentialId)
 
       if (dataSourceType === DataSourceType.WEB) {
         params.data_source.info_list.website_info_list = getWebsiteInfo({
@@ -507,7 +510,7 @@ const StepTwo = ({
       const max = rules.segmentation.max_tokens
       const overlap = rules.segmentation.chunk_overlap
       const isHierarchicalDocument = documentDetail.doc_form === ChunkingMode.parentChild
-                              || (rules.parent_mode && rules.subchunk_segmentation)
+        || (rules.parent_mode && rules.subchunk_segmentation)
       setSegmentIdentifier(separator)
       setMaxChunkLength(max)
       setOverlap(overlap!)
@@ -553,6 +556,7 @@ const StepTwo = ({
   })
 
   const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
+  const invalidDatasetList = useInvalidDatasetList()
 
   const createHandle = async () => {
     const params = getCreationParams()
@@ -582,6 +586,7 @@ const StepTwo = ({
     }
     if (mutateDatasetRes)
       mutateDatasetRes()
+    invalidDatasetList()
     onStepChange && onStepChange(+1)
     isSetting && onSave && onSave()
   }
@@ -605,9 +610,6 @@ const StepTwo = ({
       setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
   }, [isAPIKeySet, indexingType, datasetId])
 
-  const economyDomRef = useRef<HTMLDivElement>(null)
-  const isHoveringEconomy = useHover(economyDomRef)
-
   const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type
 
   return (
@@ -867,7 +869,8 @@ const StepTwo = ({
         <div className={'system-md-semibold mb-1 text-text-secondary'}>{t('datasetCreation.stepTwo.indexMode')}</div>
         <div className='flex items-center gap-2'>
           {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
-            <OptionCard className='flex-1 self-stretch'
+            <OptionCard
+              className='flex-1 self-stretch'
               title={<div className='flex items-center'>
                 {t('datasetCreation.stepTwo.qualified')}
                 <Badge className={cn('ml-1 h-[18px]', (!hasSetIndexType && indexType === IndexingType.QUALIFIED) ? 'border-text-accent-secondary text-text-accent-secondary' : '')} uppercase>
@@ -913,26 +916,8 @@ const StepTwo = ({
                   </Button>
                 </div>
               </CustomDialog>
-              <PortalToFollowElem
-                open={
-                  isHoveringEconomy && docForm !== ChunkingMode.text
-                }
-                placement={'top'}
-              >
-                <PortalToFollowElemTrigger asChild>
-                  <OptionCard className='flex-1 self-stretch'
-                    title={t('datasetCreation.stepTwo.economical')}
-                    description={t('datasetCreation.stepTwo.economicalTip')}
-                    icon={<Image src={indexMethodIcon.economical} alt='' />}
-                    isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
-                    disabled={hasSetIndexType || docForm !== ChunkingMode.text}
-                    ref={economyDomRef}
-                    onSwitched={() => {
-                      setIndexType(IndexingType.ECONOMICAL)
-                    }}
-                  />
-                </PortalToFollowElemTrigger>
-                <PortalToFollowElemContent>
+              <Tooltip
+                popupContent={
                   <div className='rounded-lg border-components-panel-border bg-components-tooltip-bg p-3 text-xs font-medium text-text-secondary shadow-lg'>
                     {
                       docForm === ChunkingMode.qa
@@ -940,8 +925,24 @@ const StepTwo = ({
                         : t('datasetCreation.stepTwo.notAvailableForParentChild')
                     }
                   </div>
-                </PortalToFollowElemContent>
-              </PortalToFollowElem>
+                }
+                noDecoration
+                position='top'
+                asChild={false}
+                triggerClassName='flex-1 self-stretch'
+              >
+                <OptionCard
+                  className='h-full'
+                  title={t('datasetCreation.stepTwo.economical')}
+                  description={t('datasetCreation.stepTwo.economicalTip')}
+                  icon={<Image src={indexMethodIcon.economical} alt='' />}
+                  isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
+                  disabled={hasSetIndexType || docForm !== ChunkingMode.text}
+                  onSwitched={() => {
+                    setIndexType(IndexingType.ECONOMICAL)
+                  }}
+                />
+              </Tooltip>
             </>)}
         </div>
         {!hasSetIndexType && indexType === IndexingType.QUALIFIED && (

web/app/components/datasets/create/step-two/option-card.tsx

@@ -64,43 +64,45 @@ export const OptionCard: FC<OptionCardProps> = (
   },
 ) => {
   const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSwitched, noHighlight, disabled, ...rest } = props
-  return <div
-    className={classNames(
-      'rounded-xl bg-components-option-card-option-bg shadow-xs',
-      (isActive && !noHighlight)
-        ? 'border-[1.5px] border-components-option-card-option-selected-border'
-        : 'border border-components-option-card-option-border',
-      disabled && 'pointer-events-none opacity-50',
-      className,
-    )}
-    style={{
-      ...style,
-    }}
-    onClick={() => {
-      if (!isActive && !disabled)
-        onSwitched?.()
-    }}
-    {...rest}
-    ref={ref}
-  >
-    <OptionCardHeader
-      icon={icon}
-      title={title}
-      description={description}
-      isActive={isActive && !noHighlight}
-      activeClassName={activeHeaderClassName}
-      effectImg={effectImg}
-      disabled={disabled}
-    />
-    {/** Body */}
-    {isActive && (children || actions) && <div className='rounded-b-xl bg-components-panel-bg px-4 py-3'>
-      {children}
-      {actions && <div className='mt-4 flex gap-2'>
-        {actions}
-      </div>
-      }
-    </div>}
-  </div>
+  return (
+    <div
+      className={classNames(
+        'rounded-xl bg-components-option-card-option-bg shadow-xs',
+        (isActive && !noHighlight)
+          ? 'border-[1.5px] border-components-option-card-option-selected-border'
+          : 'border border-components-option-card-option-border',
+        disabled && 'pointer-events-none opacity-50',
+        className,
+      )}
+      style={{
+        ...style,
+      }}
+      onClick={() => {
+        if (!isActive && !disabled)
+          onSwitched?.()
+      }}
+      {...rest}
+      ref={ref}
+    >
+      <OptionCardHeader
+        icon={icon}
+        title={title}
+        description={description}
+        isActive={isActive && !noHighlight}
+        activeClassName={activeHeaderClassName}
+        effectImg={effectImg}
+        disabled={disabled}
+      />
+      {/** Body */}
+      {isActive && (children || actions) && <div className='rounded-b-xl bg-components-panel-bg px-4 py-3'>
+        {children}
+        {actions && <div className='mt-4 flex gap-2'>
+          {actions}
+        </div>
+        }
+      </div>}
+    </div>
+  )
 }
 
 OptionCard.displayName = 'OptionCard'

web/app/components/datasets/create/website/base/header.tsx

@@ -0,0 +1,61 @@
+import React from 'react'
+import Divider from '@/app/components/base/divider'
+import Button from '@/app/components/base/button'
+import cn from '@/utils/classnames'
+import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
+
+type HeaderProps = {
+  isInPipeline?: boolean
+  onClickConfiguration?: () => void
+  title: string
+  buttonText?: string
+  docTitle: string
+  docLink: string
+}
+
+const Header = ({
+  isInPipeline = false,
+  onClickConfiguration,
+  title,
+  buttonText,
+  docTitle,
+  docLink,
+}: HeaderProps) => {
+  return (
+    <div className='flex items-center gap-x-2'>
+      <div className='flex shrink-0 grow items-center gap-x-1'>
+        <div className={cn(
+          'text-text-secondary',
+          isInPipeline ? 'system-sm-semibold' : 'system-md-semibold',
+        )}>
+          {title}
+        </div>
+        <Divider type='vertical' className='mx-1 h-3.5' />
+        <Button
+          variant='secondary'
+          size='small'
+          className={cn(isInPipeline ? 'size-6 px-1' : 'gap-x-0.5 px-1.5')}
+          onClick={onClickConfiguration}
+        >
+          <RiEqualizer2Line className='size-4' />
+          {!isInPipeline && (
+            <span className='system-xs-medium'>
+              {buttonText}
+            </span>
+          )}
+        </Button>
+      </div>
+      <a
+        className='system-xs-medium flex items-center gap-x-1 overflow-hidden text-text-accent'
+        href={docLink}
+        target='_blank'
+        rel='noopener noreferrer'
+      >
+        <RiBookOpenLine className='size-3.5 shrink-0' />
+        <span className='grow truncate' title={docTitle}>{docTitle}</span>
+      </a>
+    </div>
+  )
+}
+
+export default React.memo(Header)

web/app/components/datasets/create/website/base/input.tsx

@@ -21,7 +21,7 @@ const Input: FC<Props> = ({
     const value = e.target.value
     if (isNumber) {
       let numberValue = Number.parseInt(value, 10) // integer only
-      if (isNaN(numberValue)) {
+      if (Number.isNaN(numberValue)) {
         onChange('')
         return
       }

web/app/components/datasets/create/website/base/mock-crawl-result.ts

@@ -3,19 +3,19 @@ import type { CrawlResultItem } from '@/models/datasets'
 const result: CrawlResultItem[] = [
   {
     title: 'Start the frontend Docker container separately',
-    markdown: 'Markdown 1',
+    content: 'Markdown 1',
     description: 'Description 1',
     source_url: 'https://example.com/1',
   },
   {
     title: 'Advanced Tool Integration',
-    markdown: 'Markdown 2',
+    content: 'Markdown 2',
     description: 'Description 2',
     source_url: 'https://example.com/2',
   },
   {
     title: 'Local Source Code Start | English | Dify',
-    markdown: 'Markdown 3',
+    content: 'Markdown 3',
     description: 'Description 3',
     source_url: 'https://example.com/3',
   },

web/app/components/datasets/create/website/firecrawl/header.tsx

@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureFirecrawl`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://docs.firecrawl.dev/introduction'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.firecrawlDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)

web/app/components/datasets/create/website/firecrawl/index.tsx

@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
-import { useModalContext } from '@/context/modal-context'
+import { useModalContextSelector } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
+import Header from '../base/header'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -48,7 +48,7 @@ const FireCrawl: FC<Props> = ({
     if (step !== Step.init)
       setControlFoldOptions(Date.now())
   }, [step])
-  const { setShowAccountSettingModal } = useModalContext()
+  const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
   const handleSetting = useCallback(() => {
     setShowAccountSettingModal({
       payload: 'data-source',
@@ -112,6 +112,10 @@ const FireCrawl: FC<Props> = ({
           },
         }
       }
+      res.data = res.data.map((item: any) => ({
+        ...item,
+        content: item.markdown,
+      }))
       // update the progress
       setCrawlResult({
         ...res,
@@ -131,7 +135,7 @@ const FireCrawl: FC<Props> = ({
         },
       }
     }
-  }, [crawlOptions.limit])
+  }, [crawlOptions.limit, onCheckedCrawlResultChange])
 
   const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
@@ -161,6 +165,10 @@ const FireCrawl: FC<Props> = ({
         setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
       }
       else {
+        data.data = data.data.map((item: any) => ({
+          ...item,
+          content: item.markdown,
+        }))
         setCrawlResult(data)
         onCheckedCrawlResultChange(data.data || []) // default select the crawl result
         setCrawlErrorMessage('')
@@ -173,11 +181,17 @@ const FireCrawl: FC<Props> = ({
     finally {
       setStep(Step.finished)
     }
-  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
 
   return (
     <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.firecrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
+        docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
+        docLink={'https://docs.firecrawl.dev/introduction'}
+      />
       <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
         <UrlInput onRun={handleRun} isRunning={isRunning} />
         <OptionsWrap

web/app/components/datasets/create/website/index.module.css

@@ -6,7 +6,7 @@
 }
 
 .watercrawlLogo {
-  @apply w-5 h-5 bg-center bg-no-repeat inline-block;
+  @apply w-4 h-4 bg-center bg-no-repeat inline-block;
   /*background-color: #F5FAFF;*/
   background-image: url(../assets/watercrawl.svg);
   background-size: 16px;

web/app/components/datasets/create/website/index.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import s from './index.module.css'
 import NoData from './no-data'
@@ -10,9 +10,9 @@ import JinaReader from './jina-reader'
 import cn from '@/utils/classnames'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
-import { fetchDataSources } from '@/service/datasets'
-import { type DataSourceItem, DataSourceProvider } from '@/models/common'
+import { DataSourceProvider } from '@/models/common'
 import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'
 
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
@@ -22,6 +22,7 @@ type Props = {
   onJobIdChange: (jobId: string) => void
   crawlOptions: CrawlOptions
   onCrawlOptionsChange: (payload: CrawlOptions) => void
+  authedDataSourceList: DataSourceAuth[]
 }
 
 const Website: FC<Props> = ({
@@ -32,58 +33,34 @@ const Website: FC<Props> = ({
   onJobIdChange,
   crawlOptions,
   onCrawlOptionsChange,
+  authedDataSourceList,
 }) => {
   const { t } = useTranslation()
   const { setShowAccountSettingModal } = useModalContext()
-  const [isLoaded, setIsLoaded] = useState(false)
   const [selectedProvider, setSelectedProvider] = useState<DataSourceProvider>(DataSourceProvider.jinaReader)
-  const [sources, setSources] = useState<DataSourceItem[]>([])
 
-  useEffect(() => {
-    onCrawlProviderChange(selectedProvider)
-  }, [selectedProvider, onCrawlProviderChange])
+  const availableProviders = useMemo(() => authedDataSourceList.filter((item) => {
+    return [
+      DataSourceProvider.jinaReader,
+      DataSourceProvider.fireCrawl,
+      DataSourceProvider.waterCrawl].includes(item.provider as DataSourceProvider) && item.credentials_list.length > 0
+  }), [authedDataSourceList])
 
-  const checkSetApiKey = useCallback(async () => {
-    const res = await fetchDataSources() as any
-    setSources(res.sources)
-
-    // If users have configured one of the providers, select it.
-    const availableProviders = res.sources.filter((item: DataSourceItem) =>
-      [
-        DataSourceProvider.jinaReader,
-        DataSourceProvider.fireCrawl,
-        DataSourceProvider.waterCrawl,
-      ].includes(item.provider),
-    )
-
-    if (availableProviders.length > 0)
-      setSelectedProvider(availableProviders[0].provider)
-  }, [])
-
-  useEffect(() => {
-    checkSetApiKey().then(() => {
-      setIsLoaded(true)
-    })
-  }, [])
   const handleOnConfig = useCallback(() => {
     setShowAccountSettingModal({
       payload: 'data-source',
-      onCancelCallback: checkSetApiKey,
     })
-  }, [checkSetApiKey, setShowAccountSettingModal])
+  }, [setShowAccountSettingModal])
 
-  if (!isLoaded)
-    return null
-
-  const source = sources.find(source => source.provider === selectedProvider)
+  const source = availableProviders.find(source => source.provider === selectedProvider)
 
   return (
     <div>
-      <div className="mb-4">
-        <div className="system-md-medium mb-2 text-text-secondary">
+      <div className='mb-4'>
+        <div className='system-md-medium mb-2 text-text-secondary'>
           {t('datasetCreation.stepOne.website.chooseProvider')}
         </div>
-        <div className="flex space-x-2">
+        <div className='flex space-x-2'>
           {ENABLE_WEBSITE_JINAREADER && <button
             className={cn('flex items-center justify-center rounded-lg px-4 py-2',
               selectedProvider === DataSourceProvider.jinaReader
@@ -91,9 +68,12 @@ const Website: FC<Props> = ({
                 : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                 hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
             )}
-            onClick={() => setSelectedProvider(DataSourceProvider.jinaReader)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.jinaReader)
+              onCrawlProviderChange(DataSourceProvider.jinaReader)
+            }}
           >
-            <span className={cn(s.jinaLogo, 'mr-2')}/>
+            <span className={cn(s.jinaLogo, 'mr-2')} />
             <span>Jina Reader</span>
           </button>}
           {ENABLE_WEBSITE_FIRECRAWL && <button
@@ -103,7 +83,10 @@ const Website: FC<Props> = ({
                 : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                 hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
             )}
-            onClick={() => setSelectedProvider(DataSourceProvider.fireCrawl)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.fireCrawl)
+              onCrawlProviderChange(DataSourceProvider.fireCrawl)
+            }}
           >
             🔥 Firecrawl
           </button>}
@@ -114,9 +97,12 @@ const Website: FC<Props> = ({
                 : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                 hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
             )}
-            onClick={() => setSelectedProvider(DataSourceProvider.waterCrawl)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.waterCrawl)
+              onCrawlProviderChange(DataSourceProvider.waterCrawl)
+            }}
           >
-            <span className={cn(s.watercrawlLogo, 'mr-2')}/>
+            <span className={cn(s.watercrawlLogo, 'mr-2')} />
             <span>WaterCrawl</span>
           </button>}
         </div>
@@ -152,7 +138,7 @@ const Website: FC<Props> = ({
         />
       )}
       {!source && (
-        <NoData onConfig={handleOnConfig} provider={selectedProvider}/>
+        <NoData onConfig={handleOnConfig} provider={selectedProvider} />
       )}
     </div>
   )

web/app/components/datasets/create/website/jina-reader/base/checkbox-with-label.tsx

@@ -1,40 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import cn from '@/utils/classnames'
-import Checkbox from '@/app/components/base/checkbox'
-import Tooltip from '@/app/components/base/tooltip'
-
-type Props = {
-  className?: string
-  isChecked: boolean
-  onChange: (isChecked: boolean) => void
-  label: string
-  labelClassName?: string
-  tooltip?: string
-}
-
-const CheckboxWithLabel: FC<Props> = ({
-  className = '',
-  isChecked,
-  onChange,
-  label,
-  labelClassName,
-  tooltip,
-}) => {
-  return (
-    <label className={cn(className, 'flex h-7 items-center space-x-2')}>
-      <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
-      <div className={cn('text-sm font-normal text-gray-800', labelClassName)}>{label}</div>
-      {tooltip && (
-        <Tooltip
-          popupContent={
-            <div className='w-[200px]'>{tooltip}</div>
-          }
-          triggerClassName='ml-0.5 w-4 h-4'
-        />
-      )}
-    </label>
-  )
-}
-export default React.memo(CheckboxWithLabel)

web/app/components/datasets/create/website/jina-reader/base/error-message.tsx

@@ -1,30 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import cn from '@/utils/classnames'
-import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
-
-type Props = {
-  className?: string
-  title: string
-  errorMsg?: string
-}
-
-const ErrorMessage: FC<Props> = ({
-  className,
-  title,
-  errorMsg,
-}) => {
-  return (
-    <div className={cn(className, 'border-t border-gray-200 bg-[#FFFAEB] px-4 py-2')}>
-      <div className='flex h-5 items-center'>
-        <AlertTriangle className='mr-2 h-4 w-4 text-[#F79009]' />
-        <div className='text-sm font-medium text-[#DC6803]'>{title}</div>
-      </div>
-      {errorMsg && (
-        <div className='mt-1 pl-6 text-xs font-normal leading-[18px] text-gray-700'>{errorMsg}</div>
-      )}
-    </div>
-  )
-}
-export default React.memo(ErrorMessage)

web/app/components/datasets/create/website/jina-reader/base/field.tsx

@@ -1,54 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import Input from './input'
-import cn from '@/utils/classnames'
-import Tooltip from '@/app/components/base/tooltip'
-
-type Props = {
-  className?: string
-  label: string
-  labelClassName?: string
-  value: string | number
-  onChange: (value: string | number) => void
-  isRequired?: boolean
-  placeholder?: string
-  isNumber?: boolean
-  tooltip?: string
-}
-
-const Field: FC<Props> = ({
-  className,
-  label,
-  labelClassName,
-  value,
-  onChange,
-  isRequired = false,
-  placeholder = '',
-  isNumber = false,
-  tooltip,
-}) => {
-  return (
-    <div className={cn(className)}>
-      <div className='flex py-[7px]'>
-        <div className={cn(labelClassName, 'flex h-[18px] items-center text-[13px] font-medium text-gray-900')}>{label} </div>
-        {isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
-        {tooltip && (
-          <Tooltip
-            popupContent={
-              <div className='w-[200px]'>{tooltip}</div>
-            }
-            triggerClassName='ml-0.5 w-4 h-4'
-          />
-        )}
-      </div>
-      <Input
-        value={value}
-        onChange={onChange}
-        placeholder={placeholder}
-        isNumber={isNumber}
-      />
-    </div>
-  )
-}
-export default React.memo(Field)

web/app/components/datasets/create/website/jina-reader/base/input.tsx

@@ -1,58 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React, { useCallback } from 'react'
-
-type Props = {
-  value: string | number
-  onChange: (value: string | number) => void
-  placeholder?: string
-  isNumber?: boolean
-}
-
-const MIN_VALUE = 0
-
-const Input: FC<Props> = ({
-  value,
-  onChange,
-  placeholder = '',
-  isNumber = false,
-}) => {
-  const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
-    const value = e.target.value
-    if (isNumber) {
-      let numberValue = Number.parseInt(value, 10) // integer only
-      if (isNaN(numberValue)) {
-        onChange('')
-        return
-      }
-      if (numberValue < MIN_VALUE)
-        numberValue = MIN_VALUE
-
-      onChange(numberValue)
-      return
-    }
-    onChange(value)
-  }, [isNumber, onChange])
-
-  const otherOption = (() => {
-    if (isNumber) {
-      return {
-        min: MIN_VALUE,
-      }
-    }
-    return {
-
-    }
-  })()
-  return (
-    <input
-      type={isNumber ? 'number' : 'text'}
-      {...otherOption}
-      value={value}
-      onChange={handleChange}
-      className='flex h-9 w-full rounded-lg bg-gray-100 px-2 py-1 text-xs leading-normal caret-primary-600 placeholder:text-gray-400 hover:bg-gray-100 focus:bg-gray-50 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none'
-      placeholder={placeholder}
-    />
-  )
-}
-export default React.memo(Input)

web/app/components/datasets/create/website/jina-reader/base/options-wrap.tsx

@@ -1,54 +0,0 @@
-'use client'
-import { useBoolean } from 'ahooks'
-import type { FC } from 'react'
-import React, { useEffect } from 'react'
-import { useTranslation } from 'react-i18next'
-import cn from '@/utils/classnames'
-import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
-import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  className?: string
-  children: React.ReactNode
-  controlFoldOptions?: number
-}
-
-const OptionsWrap: FC<Props> = ({
-  className = '',
-  children,
-  controlFoldOptions,
-}) => {
-  const { t } = useTranslation()
-
-  const [fold, {
-    toggle: foldToggle,
-    setTrue: foldHide,
-  }] = useBoolean(false)
-
-  useEffect(() => {
-    if (controlFoldOptions)
-      foldHide()
-  }, [controlFoldOptions])
-  return (
-    <div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
-      <div
-        className='flex h-[26px] cursor-pointer select-none items-center justify-between py-1'
-        onClick={foldToggle}
-      >
-        <div className='flex items-center text-gray-700'>
-          <Settings04 className='mr-1 h-4 w-4' />
-          <div className='text-[13px] font-semibold uppercase text-gray-800'>{t(`${I18N_PREFIX}.options`)}</div>
-        </div>
-        <ChevronRight className={cn(!fold && 'rotate-90', 'h-4 w-4 text-gray-500')} />
-      </div>
-      {!fold && (
-        <div className='mb-4'>
-          {children}
-        </div>
-      )}
-
-    </div>
-  )
-}
-export default React.memo(OptionsWrap)

web/app/components/datasets/create/website/jina-reader/base/url-input.tsx

@@ -2,7 +2,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import Input from './input'
+import Input from '../../base/input'
 import Button from '@/app/components/base/button'
 import { useDocLink } from '@/context/i18n'
 

web/app/components/datasets/create/website/jina-reader/header.tsx

@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-sm font-semibold text-text-secondary'>{t(`${I18N_PREFIX}.jinaReaderTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureJinaReader`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://jina.ai/reader'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.jinaReaderDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)

web/app/components/datasets/create/website/jina-reader/index.tsx

@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
 import { useModalContext } from '@/context/modal-context'
 import Toast from '@/app/components/base/toast'
 import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
 import { sleep } from '@/utils'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import Header from '../base/header'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -150,14 +150,15 @@ const JinaReader: FC<Props> = ({
       }) as any
 
       if (res.data) {
+        const { title, content, description, url } = res.data
         const data = {
           current: 1,
           total: 1,
           data: [{
-            title: res.data.title,
-            markdown: res.data.content,
-            description: res.data.description,
-            source_url: res.data.url,
+            title,
+            content,
+            description,
+            source_url: url,
           }],
           time_consuming: (Date.now() - startTime) / 1000,
         }
@@ -190,7 +191,13 @@ const JinaReader: FC<Props> = ({
 
   return (
     <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.jinaReaderTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureJinaReader`)}
+        docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`)}
+        docLink={'https://jina.ai/reader'}
+      />
       <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
         <UrlInput onRun={handleRun} isRunning={isRunning} />
         <OptionsWrap

web/app/components/datasets/create/website/no-data.tsx

@@ -5,8 +5,8 @@ import { useTranslation } from 'react-i18next'
 import s from './index.module.css'
 import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
 import Button from '@/app/components/base/button'
-import { DataSourceProvider } from '@/models/common'
 import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import { DataSourceProvider } from '@/models/common'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
@@ -21,7 +21,11 @@ const NoData: FC<Props> = ({
 }) => {
   const { t } = useTranslation()
 
-  const providerConfig = {
+  const providerConfig: Record<DataSourceProvider, {
+    emoji: React.ReactNode
+    title: string
+    description: string
+  } | null> = {
     [DataSourceProvider.jinaReader]: ENABLE_WEBSITE_JINAREADER ? {
       emoji: <span className={s.jinaLogo} />,
       title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`),
@@ -39,7 +43,7 @@ const NoData: FC<Props> = ({
     } : null,
   }
 
-  const currentProvider = providerConfig[provider] || providerConfig[DataSourceProvider.jinaReader]
+  const currentProvider = providerConfig[provider] || providerConfig.jinareader
 
   if (!currentProvider) return null
 

web/app/components/datasets/create/website/preview.tsx

@@ -32,7 +32,7 @@ const WebsitePreview = ({
         <div className='system-xs-medium truncate text-text-tertiary' title={payload.source_url}>{payload.source_url}</div>
       </div>
       <div className={cn(s.previewContent, 'body-md-regular')}>
-        <div className={cn(s.fileContent)}>{payload.markdown}</div>
+        <div className={cn(s.fileContent)}>{payload.content}</div>
       </div>
     </div>
   )

web/app/components/datasets/create/website/watercrawl/header.tsx

@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.watercrawlTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureWatercrawl`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://docs.watercrawl.dev/'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.watercrawlDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)

web/app/components/datasets/create/website/watercrawl/index.tsx

@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
+import Header from '../base/header'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -177,7 +177,13 @@ const WaterCrawl: FC<Props> = ({
 
   return (
     <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.watercrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureWatercrawl`)}
+        docTitle={t(`${I18N_PREFIX}.watercrawlDoc`)}
+        docLink={'https://docs.watercrawl.dev/'}
+      />
       <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
         <UrlInput onRun={handleRun} isRunning={isRunning} />
         <OptionsWrap