feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/hooks.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/hooks.tsx
@@ -0,0 +1,71 @@
+import { useTranslation } from 'react-i18next'
+import {
+  GeneralChunk,
+  ParentChildChunk,
+  QuestionAndAnswer,
+} from '@/app/components/base/icons/src/vender/knowledge'
+import cn from '@/utils/classnames'
+import { ChunkStructureEnum } from '../../types'
+import type { Option } from './type'
+
+export const useChunkStructure = () => {
+  const { t } = useTranslation()
+  const GeneralOption: Option = {
+    id: ChunkStructureEnum.general,
+    icon: (isActive: boolean) => (
+      <GeneralChunk
+        className={cn(
+          'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-indigo-indigo-600',
+          isActive && 'text-util-colors-indigo-indigo-600',
+        )} />
+    ),
+    title: t('datasetCreation.stepTwo.general'),
+    description: t('datasetCreation.stepTwo.generalTip'),
+    effectColor: 'blue',
+  }
+  const ParentChildOption: Option = {
+    id: ChunkStructureEnum.parent_child,
+    icon: (isActive: boolean) => (
+      <ParentChildChunk
+        className={cn(
+          'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-blue-light-blue-light-500',
+          isActive && 'text-util-colors-blue-light-blue-light-500',
+        )}
+      />
+    ),
+    title: t('datasetCreation.stepTwo.parentChild'),
+    description: t('datasetCreation.stepTwo.parentChildTip'),
+    effectColor: 'blue-light',
+  }
+  const QuestionAnswerOption: Option = {
+    id: ChunkStructureEnum.question_answer,
+    icon: (isActive: boolean) => (
+      <QuestionAndAnswer
+        className={cn(
+          'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-teal-teal-600',
+          isActive && 'text-util-colors-teal-teal-600',
+        )}
+      />
+    ),
+    title: 'Q&A',
+    description: t('datasetCreation.stepTwo.qaTip'),
+    effectColor: 'teal',
+  }
+
+  const optionMap: Record<ChunkStructureEnum, Option> = {
+    [ChunkStructureEnum.general]: GeneralOption,
+    [ChunkStructureEnum.parent_child]: ParentChildOption,
+    [ChunkStructureEnum.question_answer]: QuestionAnswerOption,
+  }
+
+  const options = [
+    GeneralOption,
+    ParentChildOption,
+    QuestionAnswerOption,
+  ]
+
+  return {
+    options,
+    optionMap,
+  }
+}
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/index.tsx
@@ -0,0 +1,78 @@
+import { memo } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiAddLine } from '@remixicon/react'
+import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
+import type { ChunkStructureEnum } from '../../types'
+import OptionCard from '../option-card'
+import Selector from './selector'
+import { useChunkStructure } from './hooks'
+import Button from '@/app/components/base/button'
+import Instruction from './instruction'
+
+type ChunkStructureProps = {
+  chunkStructure?: ChunkStructureEnum
+  onChunkStructureChange: (value: ChunkStructureEnum) => void
+  readonly?: boolean
+}
+const ChunkStructure = ({
+  chunkStructure,
+  onChunkStructureChange,
+  readonly = false,
+}: ChunkStructureProps) => {
+  const { t } = useTranslation()
+  const {
+    options,
+    optionMap,
+  } = useChunkStructure()
+
+  return (
+    <Field
+      fieldTitleProps={{
+        title: t('workflow.nodes.knowledgeBase.chunkStructure'),
+        tooltip: t('workflow.nodes.knowledgeBase.chunkStructure'),
+        operation: chunkStructure && (
+          <Selector
+            options={options}
+            value={chunkStructure}
+            onChange={onChunkStructureChange}
+            readonly={readonly}
+          />
+        ),
+      }}
+    >
+      {
+        chunkStructure && (
+          <OptionCard
+            {...optionMap[chunkStructure]}
+            selectedId={chunkStructure}
+            enableSelect={false}
+            enableHighlightBorder={false}
+          />
+        )
+      }
+      {
+        !chunkStructure && (
+          <>
+            <Selector
+              options={options}
+              onChange={onChunkStructureChange}
+              readonly={readonly}
+              trigger={(
+                <Button
+                  className='w-full'
+                  variant='secondary-accent'
+                >
+                  <RiAddLine className='mr-1 h-4 w-4' />
+                  {t('workflow.nodes.knowledgeBase.chooseChunkStructure')}
+                </Button>
+              )}
+            />
+            <Instruction className='mt-2' />
+          </>
+        )
+      }
+    </Field>
+  )
+}
+
+export default memo(ChunkStructure)
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/instruction/index.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/instruction/index.tsx
@@ -0,0 +1,47 @@
+import React from 'react'
+import { AddChunks } from '@/app/components/base/icons/src/vender/knowledge'
+import Line from './line'
+import cn from '@/utils/classnames'
+import { useTranslation } from 'react-i18next'
+import { useDocLink } from '@/context/i18n'
+
+type InstructionProps = {
+  className?: string
+}
+
+const Instruction = ({
+  className,
+}: InstructionProps) => {
+  const { t } = useTranslation()
+  const docLink = useDocLink()
+
+  return (
+    <div className={cn('flex flex-col gap-y-2 overflow-hidden rounded-[10px] bg-workflow-process-bg p-4', className)}>
+      <div className='relative flex size-10 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg shadow-lg backdrop-blur-[5px]'>
+        <AddChunks className='size-5 text-text-accent' />
+        <Line className='absolute -left-px bottom-[-76px]' type='vertical' />
+        <Line className='absolute -right-px bottom-[-76px]' type='vertical' />
+        <Line className='absolute -top-px right-[-184px]' type='horizontal' />
+        <Line className='absolute -bottom-px right-[-184px]' type='horizontal' />
+      </div>
+      <div className='flex flex-col gap-y-1'>
+        <div className='system-sm-medium text-text-secondary'>
+          {t('workflow.nodes.knowledgeBase.chunkStructureTip.title')}
+        </div>
+        <div className='system-xs-regular'>
+          <p className='text-text-tertiary'>{t('workflow.nodes.knowledgeBase.chunkStructureTip.message')}</p>
+          <a
+            href={docLink('/guides/knowledge-base/create-knowledge-and-upload-documents/chunking-and-cleaning-text')}
+            target='_blank'
+            rel='noopener noreferrer'
+            className='text-text-accent'
+          >
+            {t('workflow.nodes.knowledgeBase.chunkStructureTip.learnMore')}
+          </a>
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default React.memo(Instruction)
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/instruction/line.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/instruction/line.tsx
@@ -0,0 +1,41 @@
+import React from 'react'
+
+type LineProps = {
+  type?: 'vertical' | 'horizontal'
+  className?: string
+}
+
+const Line = ({
+  type = 'vertical',
+  className,
+}: LineProps) => {
+  if (type === 'vertical') {
+    return (
+      <svg xmlns='http://www.w3.org/2000/svg' width='2' height='132' viewBox='0 0 2 132' fill='none' className={className}>
+        <path d='M1 0L1 132' stroke='url(#paint0_linear_10882_18766)' />
+        <defs>
+          <linearGradient id='paint0_linear_10882_18766' x1='-7.99584' y1='132' x2='-7.96108' y2='6.4974e-07' gradientUnits='userSpaceOnUse'>
+            <stop stopColor='var(--color-background-gradient-mask-transparent)' />
+            <stop offset='0.877606' stopColor='var(--color-divider-subtle)' />
+            <stop offset='1' stopColor='var(--color-background-gradient-mask-transparent)' />
+          </linearGradient>
+        </defs>
+      </svg>
+    )
+  }
+
+  return (
+    <svg xmlns='http://www.w3.org/2000/svg' width='240' height='2' viewBox='0 0 240 2' fill='none' className={className}>
+      <path d='M0 1H240' stroke='url(#paint0_linear_10882_18763)' />
+      <defs>
+        <linearGradient id='paint0_linear_10882_18763' x1='240' y1='9.99584' x2='3.95539e-05' y2='9.88094' gradientUnits='userSpaceOnUse'>
+          <stop stopColor='var(--color-background-gradient-mask-transparent)' />
+          <stop offset='0.9031' stopColor='var(--color-divider-subtle)' />
+          <stop offset='1' stopColor='var(--color-background-gradient-mask-transparent)' />
+        </linearGradient>
+      </defs>
+    </svg>
+  )
+}
+
+export default React.memo(Line)
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/selector.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/selector.tsx
@@ -0,0 +1,93 @@
+import type { ReactNode } from 'react'
+import { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+import Button from '@/app/components/base/button'
+import type { ChunkStructureEnum } from '../../types'
+import OptionCard from '../option-card'
+import type { Option } from './type'
+
+type SelectorProps = {
+  options: Option[]
+  value?: ChunkStructureEnum
+  onChange: (key: ChunkStructureEnum) => void
+  readonly?: boolean
+  trigger?: ReactNode
+}
+const Selector = ({
+  options,
+  value,
+  onChange,
+  readonly,
+  trigger,
+}: SelectorProps) => {
+  const { t } = useTranslation()
+  const [open, setOpen] = useState(false)
+
+  const handleSelect = useCallback((optionId: ChunkStructureEnum) => {
+    onChange(optionId)
+    setOpen(false)
+  }, [onChange])
+
+  return (
+    <PortalToFollowElem
+      placement='bottom-end'
+      offset={{
+        mainAxis: 0,
+        crossAxis: -8,
+      }}
+      open={open}
+      onOpenChange={setOpen}
+    >
+      <PortalToFollowElemTrigger
+        asChild
+        onClick={() => {
+          if (readonly)
+            return
+          setOpen(!open)
+        }}
+      >
+        {
+          trigger || (
+            <Button
+              size='small'
+              variant='ghost-accent'
+            >
+              {t('workflow.panel.change')}
+            </Button>
+          )
+        }
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent className='z-10'>
+        <div className='w-[404px] rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur shadow-xl backdrop-blur-[5px]'>
+          <div className='system-sm-semibold px-3 pt-3.5 text-text-primary'>
+            {t('workflow.nodes.knowledgeBase.changeChunkStructure')}
+          </div>
+          <div className='space-y-1 p-3 pt-2'>
+            {
+              options.map(option => (
+                <OptionCard
+                  key={option.id}
+                  id={option.id}
+                  selectedId={value}
+                  icon={option.icon}
+                  title={option.title}
+                  description={option.description}
+                  readonly={readonly}
+                  onClick={handleSelect}
+                  effectColor={option.effectColor}
+                ></OptionCard>
+              ))
+            }
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+
+export default Selector
--- a/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/type.ts
+++ b/web/app/components/workflow/nodes/knowledge-base/components/chunk-structure/type.ts
@@ -0,0 +1,10 @@
+import type { ReactNode } from 'react'
+import type { ChunkStructureEnum } from '../../types'
+
+export type Option = {
+  id: ChunkStructureEnum
+  icon: ReactNode | ((isActive: boolean) => ReactNode)
+  title: string
+  description: string
+  effectColor?: string
+}