feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/datasets/documents/detail/completed/common/action-buttons.tsx

@@ -1,9 +1,10 @@
 import React, { type FC, useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useKeyPress } from 'ahooks'
-import { useDocumentContext } from '../../index'
+import { useDocumentContext } from '../../context'
 import Button from '@/app/components/base/button'
 import { getKeyboardKeyCodeBySystem, getKeyboardKeyNameBySystem } from '@/app/components/workflow/utils'
+import { ChunkingMode } from '@/models/datasets'
 
 type IActionButtonsProps = {
   handleCancel: () => void
@@ -23,7 +24,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
   isChildChunk = false,
 }) => {
   const { t } = useTranslation()
-  const mode = useDocumentContext(s => s.mode)
+  const docForm = useDocumentContext(s => s.docForm)
   const parentMode = useDocumentContext(s => s.parentMode)
 
   useKeyPress(['esc'], (e) => {
@@ -40,8 +41,8 @@ const ActionButtons: FC<IActionButtonsProps> = ({
   { exactMatch: true, useCapture: true })
 
   const isParentChildParagraphMode = useMemo(() => {
-    return mode === 'hierarchical' && parentMode === 'paragraph'
-  }, [mode, parentMode])
+    return docForm === ChunkingMode.parentChild && parentMode === 'paragraph'
+  }, [docForm, parentMode])
 
   return (
     <div className='flex items-center gap-x-2'>

web/app/components/datasets/documents/detail/completed/common/batch-action.tsx

@@ -3,8 +3,9 @@ import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLin
 import { useTranslation } from 'react-i18next'
 import { useBoolean } from 'ahooks'
 import Divider from '@/app/components/base/divider'
-import classNames from '@/utils/classnames'
+import cn from '@/utils/classnames'
 import Confirm from '@/app/components/base/confirm'
+import Button from '@/app/components/base/button'
 
 const i18nPrefix = 'dataset.batchAction'
 type IBatchActionProps = {
@@ -43,55 +44,70 @@ const BatchAction: FC<IBatchActionProps> = ({
     hideDeleteConfirm()
   }
   return (
-    <div className={classNames('pointer-events-none flex w-full justify-center gap-x-2', className)}>
-      <div className='pointer-events-auto flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5 backdrop-blur-[5px]'>
+    <div className={cn('flex w-full justify-center gap-x-2', className)}>
+      <div className='flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5'>
         <div className='inline-flex items-center gap-x-2 py-1 pl-2 pr-3'>
-          <span className='flex h-5 w-5 items-center justify-center rounded-md bg-text-accent px-1 py-0.5 text-xs font-medium text-text-primary-on-surface'>
+          <span className='system-xs-medium flex h-5 w-5 items-center justify-center rounded-md bg-text-accent text-text-primary-on-surface'>
             {selectedIds.length}
           </span>
-          <span className='text-[13px] font-semibold leading-[16px] text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
+          <span className='system-sm-semibold text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
         </div>
         <Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
-        <div className='flex items-center gap-x-0.5 px-3 py-2'>
-          <RiCheckboxCircleLine className='h-4 w-4 text-components-button-ghost-text' />
-          <button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchEnable}>
-            {t(`${i18nPrefix}.enable`)}
-          </button>
-        </div>
-        <div className='flex items-center gap-x-0.5 px-3 py-2'>
-          <RiCloseCircleLine className='h-4 w-4 text-components-button-ghost-text' />
-          <button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchDisable}>
-            {t(`${i18nPrefix}.disable`)}
-          </button>
-        </div>
+        <Button
+          variant='ghost'
+          className='gap-x-0.5 px-3'
+          onClick={onBatchEnable}
+        >
+          <RiCheckboxCircleLine className='size-4' />
+          <span className='px-0.5'>{t(`${i18nPrefix}.enable`)}</span>
+        </Button>
+        <Button
+          variant='ghost'
+          className='gap-x-0.5 px-3'
+          onClick={onBatchDisable}
+        >
+          <RiCloseCircleLine className='size-4' />
+          <span className='px-0.5'>{t(`${i18nPrefix}.disable`)}</span>
+        </Button>
         {onEditMetadata && (
-          <div className='flex items-center gap-x-0.5 px-3 py-2'>
-            <RiDraftLine className='h-4 w-4 text-components-button-ghost-text' />
-            <button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onEditMetadata}>
-              {t('dataset.metadata.metadata')}
-            </button>
-          </div>
+          <Button
+            variant='ghost'
+            className='gap-x-0.5 px-3'
+            onClick={onEditMetadata}
+          >
+            <RiDraftLine className='size-4' />
+            <span className='px-0.5'>{t('dataset.metadata.metadata')}</span>
+          </Button>
         )}
 
         {onArchive && (
-          <div className='flex items-center gap-x-0.5 px-3 py-2'>
-            <RiArchive2Line className='h-4 w-4 text-components-button-ghost-text' />
-            <button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onArchive}>
-              {t(`${i18nPrefix}.archive`)}
-            </button>
-          </div>
+          <Button
+            variant='ghost'
+            className='gap-x-0.5 px-3'
+            onClick={onArchive}
+          >
+            <RiArchive2Line className='size-4' />
+            <span className='px-0.5'>{t(`${i18nPrefix}.archive`)}</span>
+          </Button>
         )}
-        <div className='flex items-center gap-x-0.5 px-3 py-2'>
-          <RiDeleteBinLine className='h-4 w-4 text-components-button-destructive-ghost-text' />
-          <button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-destructive-ghost-text' onClick={showDeleteConfirm}>
-            {t(`${i18nPrefix}.delete`)}
-          </button>
-        </div>
+        <Button
+          variant='ghost'
+          destructive
+          className='gap-x-0.5 px-3'
+          onClick={showDeleteConfirm}
+        >
+          <RiDeleteBinLine className='size-4' />
+          <span className='px-0.5'>{t(`${i18nPrefix}.delete`)}</span>
+        </Button>
 
         <Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
-        <button type='button' className='px-3.5 py-2 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onCancel}>
-          {t(`${i18nPrefix}.cancel`)}
-        </button>
+        <Button
+          variant='ghost'
+          className='px-3'
+          onClick={onCancel}
+        >
+          <span className='px-0.5'>{t(`${i18nPrefix}.cancel`)}</span>
+        </Button>
       </div>
       {
         isShowDeleteConfirm && (

web/app/components/datasets/documents/detail/completed/common/chunk-content.tsx

@@ -31,8 +31,8 @@ const Textarea: FC<IContentProps> = React.memo(({
 Textarea.displayName = 'Textarea'
 
 type IAutoResizeTextAreaProps = ComponentProps<'textarea'> & {
-  containerRef: React.RefObject<HTMLDivElement>
-  labelRef: React.RefObject<HTMLDivElement>
+  containerRef: React.RefObject<HTMLDivElement | null>
+  labelRef: React.RefObject<HTMLDivElement | null>
 }
 
 const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
@@ -45,7 +45,7 @@ const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
   ...rest
 }) => {
   const textareaRef = useRef<HTMLTextAreaElement>(null)
-  const observerRef = useRef<ResizeObserver>()
+  const observerRef = useRef<ResizeObserver>(null)
   const [maxHeight, setMaxHeight] = useState(0)
 
   useEffect(() => {

web/app/components/datasets/documents/detail/completed/common/drawer.tsx

@@ -0,0 +1,111 @@
+import React, { useCallback, useEffect, useRef } from 'react'
+import { createPortal } from 'react-dom'
+import cn from '@/utils/classnames'
+import { useKeyPress } from 'ahooks'
+import { useSegmentListContext } from '..'
+
+type DrawerProps = {
+  open: boolean
+  onClose: () => void
+  side?: 'right' | 'left' | 'bottom' | 'top'
+  showOverlay?: boolean
+  modal?: boolean // click outside event can pass through if modal is false
+  closeOnOutsideClick?: boolean
+  panelClassName?: string
+  panelContentClassName?: string
+  needCheckChunks?: boolean
+}
+
+const Drawer = ({
+  open,
+  onClose,
+  side = 'right',
+  showOverlay = true,
+  modal = false,
+  needCheckChunks = false,
+  children,
+  panelClassName,
+  panelContentClassName,
+}: React.PropsWithChildren<DrawerProps>) => {
+  const panelContentRef = useRef<HTMLDivElement>(null)
+  const currSegment = useSegmentListContext(s => s.currSegment)
+  const currChildChunk = useSegmentListContext(s => s.currChildChunk)
+
+  useKeyPress('esc', (e) => {
+    if (!open) return
+    e.preventDefault()
+    onClose()
+  }, { exactMatch: true, useCapture: true })
+
+  const shouldCloseDrawer = useCallback((target: Node | null) => {
+    const panelContent = panelContentRef.current
+    if (!panelContent) return false
+    const chunks = document.querySelectorAll('.chunk-card')
+    const childChunks = document.querySelectorAll('.child-chunk')
+    const isClickOnChunk = Array.from(chunks).some((chunk) => {
+      return chunk && chunk.contains(target)
+    })
+    const isClickOnChildChunk = Array.from(childChunks).some((chunk) => {
+      return chunk && chunk.contains(target)
+    })
+    const reopenChunkDetail = (currSegment.showModal && isClickOnChildChunk)
+      || (currChildChunk.showModal && isClickOnChunk && !isClickOnChildChunk) || (!isClickOnChunk && !isClickOnChildChunk)
+    return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail)
+  }, [currSegment, currChildChunk, needCheckChunks])
+
+  const onDownCapture = useCallback((e: PointerEvent) => {
+    if (!open || modal) return
+    const panelContent = panelContentRef.current
+    if (!panelContent) return
+    const target = e.target as Node | null
+    if (shouldCloseDrawer(target))
+      queueMicrotask(onClose)
+  }, [shouldCloseDrawer, onClose, open, modal])
+
+  useEffect(() => {
+    window.addEventListener('pointerdown', onDownCapture, { capture: true })
+    return () =>
+      window.removeEventListener('pointerdown', onDownCapture, { capture: true })
+  }, [onDownCapture])
+
+  const isHorizontal = side === 'left' || side === 'right'
+
+  const content = (
+    <div className='pointer-events-none fixed inset-0 z-[9999]'>
+      {showOverlay ? (
+        <div
+          onClick={modal ? onClose : undefined}
+          aria-hidden='true'
+          className={cn(
+            'fixed inset-0 bg-black/30 opacity-0 transition-opacity duration-200 ease-in',
+            open && 'opacity-100',
+            modal && open ? 'pointer-events-auto' : 'pointer-events-none',
+          )}
+        />
+      ) : null}
+
+      {/* Drawer panel */}
+      <div
+        role='dialog'
+        aria-modal={modal ? 'true' : 'false'}
+        className={cn(
+          'pointer-events-auto fixed flex flex-col',
+          side === 'right' && 'right-0',
+          side === 'left' && 'left-0',
+          side === 'bottom' && 'bottom-0',
+          side === 'top' && 'top-0',
+          isHorizontal ? 'h-screen' : 'w-screen',
+          panelClassName,
+        )}
+      >
+        <div ref={panelContentRef} className={cn('flex grow flex-col', panelContentClassName)}>
+          {children}
+        </div>
+      </div>
+    </div>
+  )
+
+  return open && createPortal(content, document.body)
+}
+
+export default Drawer

web/app/components/datasets/documents/detail/completed/common/empty.tsx

@@ -22,13 +22,13 @@ const Line = React.memo(({
   className,
 }: LineProps) => {
   return (
-    <svg xmlns="http://www.w3.org/2000/svg" width="2" height="241" viewBox="0 0 2 241" fill="none" className={className}>
-      <path d="M1 0.5L1 240.5" stroke="url(#paint0_linear_1989_74474)"/>
+    <svg xmlns='http://www.w3.org/2000/svg' width='2' height='241' viewBox='0 0 2 241' fill='none' className={className}>
+      <path d='M1 0.5L1 240.5' stroke='url(#paint0_linear_1989_74474)' />
       <defs>
-        <linearGradient id="paint0_linear_1989_74474" x1="-7.99584" y1="240.5" x2="-7.88094" y2="0.50004" gradientUnits="userSpaceOnUse">
-          <stop stopColor="white" stopOpacity="0.01"/>
-          <stop offset="0.503965" stopColor="#101828" stopOpacity="0.08"/>
-          <stop offset="1" stopColor="white" stopOpacity="0.01"/>
+        <linearGradient id='paint0_linear_1989_74474' x1='-7.99584' y1='240.5' x2='-7.88094' y2='0.50004' gradientUnits='userSpaceOnUse'>
+          <stop stopColor='white' stopOpacity='0.01' />
+          <stop offset='0.503965' stopColor='#101828' stopOpacity='0.08' />
+          <stop offset='1' stopColor='white' stopOpacity='0.01' />
         </linearGradient>
       </defs>
     </svg>
@@ -47,8 +47,8 @@ const Empty: FC<IEmptyProps> = ({
       <div className='flex flex-col items-center'>
         <div className='relative z-10 flex h-14 w-14 items-center justify-center rounded-xl border border-divider-subtle bg-components-card-bg shadow-lg shadow-shadow-shadow-5'>
           <RiFileList2Line className='h-6 w-6 text-text-secondary' />
-          <Line className='absolute -right-[1px] top-1/2 -translate-y-1/2' />
-          <Line className='absolute -left-[1px] top-1/2 -translate-y-1/2' />
+          <Line className='absolute -right-px top-1/2 -translate-y-1/2' />
+          <Line className='absolute -left-px top-1/2 -translate-y-1/2' />
           <Line className='absolute left-1/2 top-0 -translate-x-1/2 -translate-y-1/2 rotate-90' />
           <Line className='absolute left-1/2 top-full -translate-x-1/2 -translate-y-1/2 rotate-90' />
         </div>

web/app/components/datasets/documents/detail/completed/common/full-screen-drawer.tsx

@@ -1,33 +1,42 @@
-import React, { type FC } from 'react'
-import Drawer from '@/app/components/base/drawer'
-import classNames from '@/utils/classnames'
+import React from 'react'
+import Drawer from './drawer'
+import cn from '@/utils/classnames'
 import { noop } from 'lodash-es'
 
 type IFullScreenDrawerProps = {
   isOpen: boolean
   onClose?: () => void
   fullScreen: boolean
-  children: React.ReactNode
+  showOverlay?: boolean
+  needCheckChunks?: boolean
+  modal?: boolean
 }
 
-const FullScreenDrawer: FC<IFullScreenDrawerProps> = ({
+const FullScreenDrawer = ({
   isOpen,
   onClose = noop,
   fullScreen,
   children,
-}) => {
+  showOverlay = true,
+  needCheckChunks = false,
+  modal = false,
+}: React.PropsWithChildren<IFullScreenDrawerProps>) => {
   return (
     <Drawer
-      isOpen={isOpen}
+      open={isOpen}
       onClose={onClose}
-      panelClassName={classNames('bg-components-panel-bg !p-0',
+      panelClassName={cn(
         fullScreen
-          ? '!w-full !max-w-full'
-          : 'mb-2 mr-2 mt-16 !w-[560px] !max-w-[560px] rounded-xl border-[0.5px] border-components-panel-border',
+          ? 'w-full'
+          : 'w-[560px] pb-2 pr-2 pt-16',
       )}
-      mask={false}
-      unmount
-      footer={null}
+      panelContentClassName={cn(
+        'bg-components-panel-bg',
+        !fullScreen && 'rounded-xl border-[0.5px] border-components-panel-border',
+      )}
+      showOverlay={showOverlay}
+      needCheckChunks={needCheckChunks}
+      modal={modal}
     >
       {children}
     </Drawer>)

web/app/components/datasets/documents/detail/completed/common/segment-index-tag.tsx

@@ -1,5 +1,5 @@
 import React, { type FC, useMemo } from 'react'
-import { Chunk } from '@/app/components/base/icons/src/public/knowledge'
+import { Chunk } from '@/app/components/base/icons/src/vender/knowledge'
 import cn from '@/utils/classnames'
 
 type ISegmentIndexTagProps = {