feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/datasets/create/website/firecrawl/header.tsx

@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureFirecrawl`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://docs.firecrawl.dev/introduction'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.firecrawlDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)

web/app/components/datasets/create/website/firecrawl/index.tsx

@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
-import { useModalContext } from '@/context/modal-context'
+import { useModalContextSelector } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
+import Header from '../base/header'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -48,7 +48,7 @@ const FireCrawl: FC<Props> = ({
     if (step !== Step.init)
       setControlFoldOptions(Date.now())
   }, [step])
-  const { setShowAccountSettingModal } = useModalContext()
+  const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
   const handleSetting = useCallback(() => {
     setShowAccountSettingModal({
       payload: 'data-source',
@@ -112,6 +112,10 @@ const FireCrawl: FC<Props> = ({
           },
         }
       }
+      res.data = res.data.map((item: any) => ({
+        ...item,
+        content: item.markdown,
+      }))
       // update the progress
       setCrawlResult({
         ...res,
@@ -131,7 +135,7 @@ const FireCrawl: FC<Props> = ({
         },
       }
     }
-  }, [crawlOptions.limit])
+  }, [crawlOptions.limit, onCheckedCrawlResultChange])
 
   const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
@@ -161,6 +165,10 @@ const FireCrawl: FC<Props> = ({
         setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
       }
       else {
+        data.data = data.data.map((item: any) => ({
+          ...item,
+          content: item.markdown,
+        }))
         setCrawlResult(data)
         onCheckedCrawlResultChange(data.data || []) // default select the crawl result
         setCrawlErrorMessage('')
@@ -173,11 +181,17 @@ const FireCrawl: FC<Props> = ({
     finally {
       setStep(Step.finished)
     }
-  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
 
   return (
     <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.firecrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
+        docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
+        docLink={'https://docs.firecrawl.dev/introduction'}
+      />
       <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
         <UrlInput onRun={handleRun} isRunning={isRunning} />
         <OptionsWrap