feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -0,0 +1,71 @@
import { useTranslation } from 'react-i18next'
import {
GeneralChunk,
ParentChildChunk,
QuestionAndAnswer,
} from '@/app/components/base/icons/src/vender/knowledge'
import cn from '@/utils/classnames'
import { ChunkStructureEnum } from '../../types'
import type { Option } from './type'
export const useChunkStructure = () => {
const { t } = useTranslation()
const GeneralOption: Option = {
id: ChunkStructureEnum.general,
icon: (isActive: boolean) => (
<GeneralChunk
className={cn(
'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-indigo-indigo-600',
isActive && 'text-util-colors-indigo-indigo-600',
)} />
),
title: t('datasetCreation.stepTwo.general'),
description: t('datasetCreation.stepTwo.generalTip'),
effectColor: 'blue',
}
const ParentChildOption: Option = {
id: ChunkStructureEnum.parent_child,
icon: (isActive: boolean) => (
<ParentChildChunk
className={cn(
'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-blue-light-blue-light-500',
isActive && 'text-util-colors-blue-light-blue-light-500',
)}
/>
),
title: t('datasetCreation.stepTwo.parentChild'),
description: t('datasetCreation.stepTwo.parentChildTip'),
effectColor: 'blue-light',
}
const QuestionAnswerOption: Option = {
id: ChunkStructureEnum.question_answer,
icon: (isActive: boolean) => (
<QuestionAndAnswer
className={cn(
'h-[18px] w-[18px] text-text-tertiary group-hover:text-util-colors-teal-teal-600',
isActive && 'text-util-colors-teal-teal-600',
)}
/>
),
title: 'Q&A',
description: t('datasetCreation.stepTwo.qaTip'),
effectColor: 'teal',
}
const optionMap: Record<ChunkStructureEnum, Option> = {
[ChunkStructureEnum.general]: GeneralOption,
[ChunkStructureEnum.parent_child]: ParentChildOption,
[ChunkStructureEnum.question_answer]: QuestionAnswerOption,
}
const options = [
GeneralOption,
ParentChildOption,
QuestionAnswerOption,
]
return {
options,
optionMap,
}
}

View File

@@ -0,0 +1,78 @@
import { memo } from 'react'
import { useTranslation } from 'react-i18next'
import { RiAddLine } from '@remixicon/react'
import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
import type { ChunkStructureEnum } from '../../types'
import OptionCard from '../option-card'
import Selector from './selector'
import { useChunkStructure } from './hooks'
import Button from '@/app/components/base/button'
import Instruction from './instruction'
type ChunkStructureProps = {
chunkStructure?: ChunkStructureEnum
onChunkStructureChange: (value: ChunkStructureEnum) => void
readonly?: boolean
}
const ChunkStructure = ({
chunkStructure,
onChunkStructureChange,
readonly = false,
}: ChunkStructureProps) => {
const { t } = useTranslation()
const {
options,
optionMap,
} = useChunkStructure()
return (
<Field
fieldTitleProps={{
title: t('workflow.nodes.knowledgeBase.chunkStructure'),
tooltip: t('workflow.nodes.knowledgeBase.chunkStructure'),
operation: chunkStructure && (
<Selector
options={options}
value={chunkStructure}
onChange={onChunkStructureChange}
readonly={readonly}
/>
),
}}
>
{
chunkStructure && (
<OptionCard
{...optionMap[chunkStructure]}
selectedId={chunkStructure}
enableSelect={false}
enableHighlightBorder={false}
/>
)
}
{
!chunkStructure && (
<>
<Selector
options={options}
onChange={onChunkStructureChange}
readonly={readonly}
trigger={(
<Button
className='w-full'
variant='secondary-accent'
>
<RiAddLine className='mr-1 h-4 w-4' />
{t('workflow.nodes.knowledgeBase.chooseChunkStructure')}
</Button>
)}
/>
<Instruction className='mt-2' />
</>
)
}
</Field>
)
}
export default memo(ChunkStructure)

View File

@@ -0,0 +1,47 @@
import React from 'react'
import { AddChunks } from '@/app/components/base/icons/src/vender/knowledge'
import Line from './line'
import cn from '@/utils/classnames'
import { useTranslation } from 'react-i18next'
import { useDocLink } from '@/context/i18n'
type InstructionProps = {
className?: string
}
const Instruction = ({
className,
}: InstructionProps) => {
const { t } = useTranslation()
const docLink = useDocLink()
return (
<div className={cn('flex flex-col gap-y-2 overflow-hidden rounded-[10px] bg-workflow-process-bg p-4', className)}>
<div className='relative flex size-10 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg shadow-lg backdrop-blur-[5px]'>
<AddChunks className='size-5 text-text-accent' />
<Line className='absolute -left-px bottom-[-76px]' type='vertical' />
<Line className='absolute -right-px bottom-[-76px]' type='vertical' />
<Line className='absolute -top-px right-[-184px]' type='horizontal' />
<Line className='absolute -bottom-px right-[-184px]' type='horizontal' />
</div>
<div className='flex flex-col gap-y-1'>
<div className='system-sm-medium text-text-secondary'>
{t('workflow.nodes.knowledgeBase.chunkStructureTip.title')}
</div>
<div className='system-xs-regular'>
<p className='text-text-tertiary'>{t('workflow.nodes.knowledgeBase.chunkStructureTip.message')}</p>
<a
href={docLink('/guides/knowledge-base/create-knowledge-and-upload-documents/chunking-and-cleaning-text')}
target='_blank'
rel='noopener noreferrer'
className='text-text-accent'
>
{t('workflow.nodes.knowledgeBase.chunkStructureTip.learnMore')}
</a>
</div>
</div>
</div>
)
}
export default React.memo(Instruction)

View File

@@ -0,0 +1,41 @@
import React from 'react'
type LineProps = {
type?: 'vertical' | 'horizontal'
className?: string
}
const Line = ({
type = 'vertical',
className,
}: LineProps) => {
if (type === 'vertical') {
return (
<svg xmlns='http://www.w3.org/2000/svg' width='2' height='132' viewBox='0 0 2 132' fill='none' className={className}>
<path d='M1 0L1 132' stroke='url(#paint0_linear_10882_18766)' />
<defs>
<linearGradient id='paint0_linear_10882_18766' x1='-7.99584' y1='132' x2='-7.96108' y2='6.4974e-07' gradientUnits='userSpaceOnUse'>
<stop stopColor='var(--color-background-gradient-mask-transparent)' />
<stop offset='0.877606' stopColor='var(--color-divider-subtle)' />
<stop offset='1' stopColor='var(--color-background-gradient-mask-transparent)' />
</linearGradient>
</defs>
</svg>
)
}
return (
<svg xmlns='http://www.w3.org/2000/svg' width='240' height='2' viewBox='0 0 240 2' fill='none' className={className}>
<path d='M0 1H240' stroke='url(#paint0_linear_10882_18763)' />
<defs>
<linearGradient id='paint0_linear_10882_18763' x1='240' y1='9.99584' x2='3.95539e-05' y2='9.88094' gradientUnits='userSpaceOnUse'>
<stop stopColor='var(--color-background-gradient-mask-transparent)' />
<stop offset='0.9031' stopColor='var(--color-divider-subtle)' />
<stop offset='1' stopColor='var(--color-background-gradient-mask-transparent)' />
</linearGradient>
</defs>
</svg>
)
}
export default React.memo(Line)

View File

@@ -0,0 +1,93 @@
import type { ReactNode } from 'react'
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import {
PortalToFollowElem,
PortalToFollowElemContent,
PortalToFollowElemTrigger,
} from '@/app/components/base/portal-to-follow-elem'
import Button from '@/app/components/base/button'
import type { ChunkStructureEnum } from '../../types'
import OptionCard from '../option-card'
import type { Option } from './type'
type SelectorProps = {
options: Option[]
value?: ChunkStructureEnum
onChange: (key: ChunkStructureEnum) => void
readonly?: boolean
trigger?: ReactNode
}
const Selector = ({
options,
value,
onChange,
readonly,
trigger,
}: SelectorProps) => {
const { t } = useTranslation()
const [open, setOpen] = useState(false)
const handleSelect = useCallback((optionId: ChunkStructureEnum) => {
onChange(optionId)
setOpen(false)
}, [onChange])
return (
<PortalToFollowElem
placement='bottom-end'
offset={{
mainAxis: 0,
crossAxis: -8,
}}
open={open}
onOpenChange={setOpen}
>
<PortalToFollowElemTrigger
asChild
onClick={() => {
if (readonly)
return
setOpen(!open)
}}
>
{
trigger || (
<Button
size='small'
variant='ghost-accent'
>
{t('workflow.panel.change')}
</Button>
)
}
</PortalToFollowElemTrigger>
<PortalToFollowElemContent className='z-10'>
<div className='w-[404px] rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur shadow-xl backdrop-blur-[5px]'>
<div className='system-sm-semibold px-3 pt-3.5 text-text-primary'>
{t('workflow.nodes.knowledgeBase.changeChunkStructure')}
</div>
<div className='space-y-1 p-3 pt-2'>
{
options.map(option => (
<OptionCard
key={option.id}
id={option.id}
selectedId={value}
icon={option.icon}
title={option.title}
description={option.description}
readonly={readonly}
onClick={handleSelect}
effectColor={option.effectColor}
></OptionCard>
))
}
</div>
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
)
}
export default Selector

View File

@@ -0,0 +1,10 @@
import type { ReactNode } from 'react'
import type { ChunkStructureEnum } from '../../types'
export type Option = {
id: ChunkStructureEnum
icon: ReactNode | ((isActive: boolean) => ReactNode)
title: string
description: string
effectColor?: string
}

View File

@@ -0,0 +1,64 @@
import {
memo,
useCallback,
useMemo,
} from 'react'
import { useTranslation } from 'react-i18next'
import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
type EmbeddingModelProps = {
embeddingModel?: string
embeddingModelProvider?: string
onEmbeddingModelChange?: (model: {
embeddingModel: string
embeddingModelProvider: string
}) => void
readonly?: boolean
}
const EmbeddingModel = ({
embeddingModel,
embeddingModelProvider,
onEmbeddingModelChange,
readonly = false,
}: EmbeddingModelProps) => {
const { t } = useTranslation()
const {
data: embeddingModelList,
} = useModelList(ModelTypeEnum.textEmbedding)
const embeddingModelConfig = useMemo(() => {
if (!embeddingModel || !embeddingModelProvider)
return undefined
return {
providerName: embeddingModelProvider,
modelName: embeddingModel,
}
}, [embeddingModel, embeddingModelProvider])
const handleEmbeddingModelChange = useCallback((model: DefaultModel) => {
onEmbeddingModelChange?.({
embeddingModelProvider: model.provider,
embeddingModel: model.model,
})
}, [onEmbeddingModelChange])
return (
<Field
fieldTitleProps={{
title: t('datasetSettings.form.embeddingModel'),
}}
>
<ModelSelector
defaultModel={embeddingModelConfig && { provider: embeddingModelConfig.providerName, model: embeddingModelConfig.modelName }}
modelList={embeddingModelList}
onSelect={handleEmbeddingModelChange}
readonly={readonly}
/>
</Field>
)
}
export default memo(EmbeddingModel)

View File

@@ -0,0 +1,128 @@
import {
memo,
useCallback,
} from 'react'
import { useTranslation } from 'react-i18next'
import { RiQuestionLine } from '@remixicon/react'
import {
Economic,
HighQuality,
} from '@/app/components/base/icons/src/vender/knowledge'
import Tooltip from '@/app/components/base/tooltip'
import Slider from '@/app/components/base/slider'
import Input from '@/app/components/base/input'
import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
import OptionCard from './option-card'
import cn from '@/utils/classnames'
import {
ChunkStructureEnum,
IndexMethodEnum,
} from '../types'
type IndexMethodProps = {
chunkStructure: ChunkStructureEnum
indexMethod?: IndexMethodEnum
onIndexMethodChange: (value: IndexMethodEnum) => void
keywordNumber: number
onKeywordNumberChange: (value: number) => void
readonly?: boolean
}
const IndexMethod = ({
chunkStructure,
indexMethod,
onIndexMethodChange,
keywordNumber,
onKeywordNumberChange,
readonly = false,
}: IndexMethodProps) => {
const { t } = useTranslation()
const isHighQuality = indexMethod === IndexMethodEnum.QUALIFIED
const isEconomy = indexMethod === IndexMethodEnum.ECONOMICAL
const handleIndexMethodChange = useCallback((newIndexMethod: IndexMethodEnum) => {
onIndexMethodChange(newIndexMethod)
}, [onIndexMethodChange])
const handleInputChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const value = Number(e.target.value)
if (!Number.isNaN(value))
onKeywordNumberChange(value)
}, [onKeywordNumberChange])
return (
<Field
fieldTitleProps={{
title: t('datasetCreation.stepTwo.indexMode'),
}}
>
<div className='space-y-1'>
<OptionCard<IndexMethodEnum>
id={IndexMethodEnum.QUALIFIED}
selectedId={indexMethod}
icon={
<HighQuality
className={cn(
'h-[15px] w-[15px] text-text-tertiary group-hover:text-util-colors-orange-orange-500',
isHighQuality && 'text-util-colors-orange-orange-500',
)}
/>
}
title={t('datasetCreation.stepTwo.qualified')}
description={t('datasetSettings.form.indexMethodHighQualityTip')}
onClick={handleIndexMethodChange}
isRecommended
effectColor='orange'
></OptionCard>
{
chunkStructure === ChunkStructureEnum.general && (
<OptionCard
id={IndexMethodEnum.ECONOMICAL}
selectedId={indexMethod}
icon={
<Economic
className={cn(
'h-[15px] w-[15px] text-text-tertiary group-hover:text-util-colors-indigo-indigo-500',
isEconomy && 'text-util-colors-indigo-indigo-500',
)}
/>
}
title={t('datasetSettings.form.indexMethodEconomy')}
description={t('datasetSettings.form.indexMethodEconomyTip', { count: keywordNumber })}
onClick={handleIndexMethodChange}
effectColor='blue'
>
<div className='flex items-center'>
<div className='flex grow items-center'>
<div className='system-xs-medium truncate text-text-secondary'>
{t('datasetSettings.form.numberOfKeywords')}
</div>
<Tooltip
popupContent='number of keywords'
>
<RiQuestionLine className='ml-0.5 h-3.5 w-3.5 text-text-quaternary' />
</Tooltip>
</div>
<Slider
disabled={readonly}
className='mr-3 w-24 shrink-0'
value={keywordNumber}
onChange={onKeywordNumberChange}
/>
<Input
disabled={readonly}
className='shrink-0'
wrapperClassName='shrink-0 w-[72px]'
type='number'
value={keywordNumber}
onChange={handleInputChange}
/>
</div>
</OptionCard>
)
}
</div>
</Field>
)
}
export default memo(IndexMethod)

View File

@@ -0,0 +1,146 @@
import type { ReactNode } from 'react'
import {
memo,
useMemo,
} from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import Badge from '@/app/components/base/badge'
import {
OptionCardEffectBlue,
OptionCardEffectBlueLight,
OptionCardEffectOrange,
OptionCardEffectPurple,
OptionCardEffectTeal,
} from '@/app/components/base/icons/src/public/knowledge'
import { ArrowShape } from '@/app/components/base/icons/src/vender/knowledge'
const HEADER_EFFECT_MAP: Record<string, ReactNode> = {
'blue': <OptionCardEffectBlue />,
'blue-light': <OptionCardEffectBlueLight />,
'orange': <OptionCardEffectOrange />,
'purple': <OptionCardEffectPurple />,
'teal': <OptionCardEffectTeal />,
}
type OptionCardProps<T> = {
id?: T
selectedId?: T
enableSelect?: boolean
enableHighlightBorder?: boolean
enableRadio?: boolean
wrapperClassName?: string | ((isActive: boolean) => string)
className?: string | ((isActive: boolean) => string)
icon?: ReactNode | ((isActive: boolean) => ReactNode)
title: string
description?: string
isRecommended?: boolean
children?: ReactNode
effectColor?: string
onClick?: (id: T) => void
readonly?: boolean
}
const OptionCard = memo(({
id,
selectedId,
enableSelect = true,
enableHighlightBorder = true,
enableRadio,
wrapperClassName,
className,
icon,
title,
description,
isRecommended,
children,
effectColor,
onClick,
readonly,
}) => {
const { t } = useTranslation()
const isActive = useMemo(() => {
return id === selectedId
}, [id, selectedId])
const effectElement = useMemo(() => {
if (effectColor) {
return (
<div className={cn(
'absolute left-[-2px] top-[-2px] hidden h-14 w-14 rounded-full',
'group-hover:block',
isActive && 'block',
)}>
{HEADER_EFFECT_MAP[effectColor]}
</div>
)
}
return null
}, [effectColor, isActive])
return (
<div
className={cn(
'group overflow-hidden rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg',
isActive && enableHighlightBorder && 'border-[1.5px] border-components-option-card-option-selected-border',
enableSelect && 'cursor-pointer hover:shadow-xs',
readonly && 'cursor-not-allowed',
wrapperClassName && (typeof wrapperClassName === 'function' ? wrapperClassName(isActive) : wrapperClassName),
)}
onClick={() => !readonly && enableSelect && id && onClick?.(id)}
>
<div className={cn(
'relative flex rounded-t-xl p-2',
className && (typeof className === 'function' ? className(isActive) : className),
)}>
{effectElement}
{
icon && (
<div className='mr-1 flex h-[18px] w-[18px] shrink-0 items-center justify-center'>
{typeof icon === 'function' ? icon(isActive) : icon}
</div>
)
}
<div className='grow py-1 pt-[1px]'>
<div className='flex items-center'>
<div className='system-sm-medium flex grow items-center text-text-secondary'>
{title}
{
isRecommended && (
<Badge className='ml-1 h-4 border-text-accent-secondary text-text-accent-secondary'>
{t('datasetCreation.stepTwo.recommend')}
</Badge>
)
}
</div>
{
enableRadio && (
<div className={cn(
'ml-2 h-4 w-4 shrink-0 rounded-full border border-components-radio-border bg-components-radio-bg',
isActive && 'border-[5px] border-components-radio-border-checked',
)}>
</div>
)
}
</div>
{
description && (
<div className='system-xs-regular mt-1 text-text-tertiary'>
{description}
</div>
)
}
</div>
</div>
{
children && isActive && (
<div className='relative rounded-b-xl bg-components-panel-bg p-3'>
<ArrowShape className='absolute left-[14px] top-[-11px] h-4 w-4 text-components-panel-bg' />
{children}
</div>
)
}
</div>
)
}) as <T>(props: OptionCardProps<T>) => React.ReactElement
export default OptionCard

View File

@@ -0,0 +1,93 @@
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import {
FullTextSearch,
HybridSearch,
VectorSearch,
} from '@/app/components/base/icons/src/vender/knowledge'
import {
HybridSearchModeEnum,
IndexMethodEnum,
RetrievalSearchMethodEnum,
} from '../../types'
import type {
HybridSearchModeOption,
Option,
} from './type'
export const useRetrievalSetting = (indexMethod?: IndexMethodEnum) => {
const { t } = useTranslation()
const VectorSearchOption: Option = useMemo(() => {
return {
id: RetrievalSearchMethodEnum.semantic,
icon: VectorSearch as any,
title: t('dataset.retrieval.semantic_search.title'),
description: t('dataset.retrieval.semantic_search.description'),
effectColor: 'purple',
}
}, [t])
const FullTextSearchOption: Option = useMemo(() => {
return {
id: RetrievalSearchMethodEnum.fullText,
icon: FullTextSearch as any,
title: t('dataset.retrieval.full_text_search.title'),
description: t('dataset.retrieval.full_text_search.description'),
effectColor: 'purple',
}
}, [t])
const HybridSearchOption: Option = useMemo(() => {
return {
id: RetrievalSearchMethodEnum.hybrid,
icon: HybridSearch as any,
title: t('dataset.retrieval.hybrid_search.title'),
description: t('dataset.retrieval.hybrid_search.description'),
effectColor: 'purple',
}
}, [t])
const InvertedIndexOption: Option = useMemo(() => {
return {
id: RetrievalSearchMethodEnum.keywordSearch,
icon: HybridSearch as any,
title: t('dataset.retrieval.keyword_search.title'),
description: t('dataset.retrieval.keyword_search.description'),
effectColor: 'purple',
}
}, [t])
const WeightedScoreModeOption: HybridSearchModeOption = useMemo(() => {
return {
id: HybridSearchModeEnum.WeightedScore,
title: t('dataset.weightedScore.title'),
description: t('dataset.weightedScore.description'),
}
}, [t])
const RerankModelModeOption: HybridSearchModeOption = useMemo(() => {
return {
id: HybridSearchModeEnum.RerankingModel,
title: t('common.modelProvider.rerankModel.key'),
description: t('common.modelProvider.rerankModel.tip'),
}
}, [t])
return useMemo(() => ({
options: indexMethod === IndexMethodEnum.ECONOMICAL ? [
InvertedIndexOption,
] : [
VectorSearchOption,
FullTextSearchOption,
HybridSearchOption,
],
hybridSearchModeOptions: [
WeightedScoreModeOption,
RerankModelModeOption,
],
}), [
VectorSearchOption,
FullTextSearchOption,
HybridSearchOption,
InvertedIndexOption,
indexMethod,
WeightedScoreModeOption,
RerankModelModeOption,
])
}

View File

@@ -0,0 +1,102 @@
import {
memo,
} from 'react'
import { useTranslation } from 'react-i18next'
import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
import type {
HybridSearchModeEnum,
RetrievalSearchMethodEnum,
} from '../../types'
import type {
IndexMethodEnum,
WeightedScore,
} from '../../types'
import { useRetrievalSetting } from './hooks'
import type { TopKAndScoreThresholdProps } from './top-k-and-score-threshold'
import type { RerankingModelSelectorProps } from './reranking-model-selector'
import SearchMethodOption from './search-method-option'
type RetrievalSettingProps = {
indexMethod?: IndexMethodEnum
readonly?: boolean
searchMethod?: RetrievalSearchMethodEnum
onRetrievalSearchMethodChange: (value: RetrievalSearchMethodEnum) => void
hybridSearchMode?: HybridSearchModeEnum
onHybridSearchModeChange: (value: HybridSearchModeEnum) => void
rerankingModelEnabled?: boolean
onRerankingModelEnabledChange?: (value: boolean) => void
weightedScore?: WeightedScore
onWeightedScoreChange: (value: { value: number[] }) => void
} & RerankingModelSelectorProps & TopKAndScoreThresholdProps
const RetrievalSetting = ({
indexMethod,
readonly,
searchMethod,
onRetrievalSearchMethodChange,
hybridSearchMode,
onHybridSearchModeChange,
weightedScore,
onWeightedScoreChange,
rerankingModelEnabled,
onRerankingModelEnabledChange,
rerankingModel,
onRerankingModelChange,
topK,
onTopKChange,
scoreThreshold,
onScoreThresholdChange,
isScoreThresholdEnabled,
onScoreThresholdEnabledChange,
}: RetrievalSettingProps) => {
const { t } = useTranslation()
const {
options,
hybridSearchModeOptions,
} = useRetrievalSetting(indexMethod)
return (
<Field
fieldTitleProps={{
title: t('datasetSettings.form.retrievalSetting.title'),
subTitle: (
<div className='body-xs-regular flex items-center text-text-tertiary'>
<a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-text-accent'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
&nbsp;{t('workflow.nodes.knowledgeBase.aboutRetrieval')}
</div>
),
}}
>
<div className='space-y-1'>
{
options.map(option => (
<SearchMethodOption
key={option.id}
option={option}
hybridSearchModeOptions={hybridSearchModeOptions}
searchMethod={searchMethod}
onRetrievalSearchMethodChange={onRetrievalSearchMethodChange}
hybridSearchMode={hybridSearchMode}
onHybridSearchModeChange={onHybridSearchModeChange}
weightedScore={weightedScore}
onWeightedScoreChange={onWeightedScoreChange}
topK={topK}
onTopKChange={onTopKChange}
scoreThreshold={scoreThreshold}
onScoreThresholdChange={onScoreThresholdChange}
isScoreThresholdEnabled={isScoreThresholdEnabled}
onScoreThresholdEnabledChange={onScoreThresholdEnabledChange}
rerankingModelEnabled={rerankingModelEnabled}
onRerankingModelEnabledChange={onRerankingModelEnabledChange}
rerankingModel={rerankingModel}
onRerankingModelChange={onRerankingModelChange}
readonly={readonly}
/>
))
}
</div>
</Field>
)
}
export default memo(RetrievalSetting)

View File

@@ -0,0 +1,51 @@
import {
memo,
useMemo,
} from 'react'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import { useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { RerankingModel } from '../../types'
export type RerankingModelSelectorProps = {
rerankingModel?: RerankingModel
onRerankingModelChange?: (model: RerankingModel) => void
readonly?: boolean
}
const RerankingModelSelector = ({
rerankingModel,
onRerankingModelChange,
readonly = false,
}: RerankingModelSelectorProps) => {
const {
modelList: rerankModelList,
} = useModelListAndDefaultModel(ModelTypeEnum.rerank)
const rerankModel = useMemo(() => {
if (!rerankingModel)
return undefined
return {
providerName: rerankingModel.reranking_provider_name,
modelName: rerankingModel.reranking_model_name,
}
}, [rerankingModel])
const handleRerankingModelChange = (model: DefaultModel) => {
onRerankingModelChange?.({
reranking_provider_name: model.provider,
reranking_model_name: model.model,
})
}
return (
<ModelSelector
defaultModel={rerankModel && { provider: rerankModel.providerName, model: rerankModel.modelName }}
modelList={rerankModelList}
onSelect={handleRerankingModelChange}
readonly={readonly}
/>
)
}
export default memo(RerankingModelSelector)

View File

@@ -0,0 +1,204 @@
import {
memo,
useCallback,
useMemo,
} from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import WeightedScoreComponent from '@/app/components/app/configuration/dataset-config/params-config/weighted-score'
import { DEFAULT_WEIGHTED_SCORE } from '@/models/datasets'
import Switch from '@/app/components/base/switch'
import Tooltip from '@/app/components/base/tooltip'
import {
HybridSearchModeEnum,
RetrievalSearchMethodEnum,
} from '../../types'
import type {
WeightedScore,
} from '../../types'
import OptionCard from '../option-card'
import type {
HybridSearchModeOption,
Option,
} from './type'
import type { TopKAndScoreThresholdProps } from './top-k-and-score-threshold'
import TopKAndScoreThreshold from './top-k-and-score-threshold'
import type { RerankingModelSelectorProps } from './reranking-model-selector'
import RerankingModelSelector from './reranking-model-selector'
type SearchMethodOptionProps = {
readonly?: boolean
option: Option
hybridSearchModeOptions: HybridSearchModeOption[]
searchMethod?: RetrievalSearchMethodEnum
onRetrievalSearchMethodChange: (value: RetrievalSearchMethodEnum) => void
hybridSearchMode?: HybridSearchModeEnum
onHybridSearchModeChange: (value: HybridSearchModeEnum) => void
weightedScore?: WeightedScore
onWeightedScoreChange: (value: { value: number[] }) => void
rerankingModelEnabled?: boolean
onRerankingModelEnabledChange?: (value: boolean) => void
} & RerankingModelSelectorProps & TopKAndScoreThresholdProps
const SearchMethodOption = ({
readonly,
option,
hybridSearchModeOptions,
searchMethod,
onRetrievalSearchMethodChange,
hybridSearchMode,
onHybridSearchModeChange,
weightedScore,
onWeightedScoreChange,
rerankingModelEnabled,
onRerankingModelEnabledChange,
rerankingModel,
onRerankingModelChange,
topK,
onTopKChange,
scoreThreshold,
onScoreThresholdChange,
isScoreThresholdEnabled,
onScoreThresholdEnabledChange,
}: SearchMethodOptionProps) => {
const { t } = useTranslation()
const Icon = option.icon
const isHybridSearch = option.id === RetrievalSearchMethodEnum.hybrid
const isHybridSearchWeightedScoreMode = hybridSearchMode === HybridSearchModeEnum.WeightedScore
const weightedScoreValue = useMemo(() => {
const sematicWeightedScore = weightedScore?.vector_setting.vector_weight ?? DEFAULT_WEIGHTED_SCORE.other.semantic
const keywordWeightedScore = weightedScore?.keyword_setting.keyword_weight ?? DEFAULT_WEIGHTED_SCORE.other.keyword
const mergedValue = [sematicWeightedScore, keywordWeightedScore]
return {
value: mergedValue,
}
}, [weightedScore])
const icon = useCallback((isActive: boolean) => {
return (
<Icon
className={cn(
'h-[15px] w-[15px] text-text-tertiary group-hover:text-util-colors-purple-purple-600',
isActive && 'text-util-colors-purple-purple-600',
)}
/>
)
}, [Icon])
const hybridSearchModeWrapperClassName = useCallback((isActive: boolean) => {
return isActive ? 'border-[1.5px] bg-components-option-card-option-selected-bg' : ''
}, [])
const showRerankModelSelectorSwitch = useMemo(() => {
if (searchMethod === RetrievalSearchMethodEnum.semantic)
return true
if (searchMethod === RetrievalSearchMethodEnum.fullText)
return true
return false
}, [searchMethod])
const showRerankModelSelector = useMemo(() => {
if (searchMethod === RetrievalSearchMethodEnum.semantic)
return true
if (searchMethod === RetrievalSearchMethodEnum.fullText)
return true
if (searchMethod === RetrievalSearchMethodEnum.hybrid && hybridSearchMode !== HybridSearchModeEnum.WeightedScore)
return true
return false
}, [hybridSearchMode, searchMethod])
return (
<OptionCard
key={option.id}
id={option.id}
selectedId={searchMethod}
icon={icon}
title={option.title}
description={option.description}
effectColor={option.effectColor}
isRecommended={option.id === RetrievalSearchMethodEnum.hybrid}
onClick={onRetrievalSearchMethodChange}
readonly={readonly}
>
<div className='space-y-3'>
{
isHybridSearch && (
<div className='space-y-1'>
{
hybridSearchModeOptions.map(hybridOption => (
<OptionCard
key={hybridOption.id}
id={hybridOption.id}
selectedId={hybridSearchMode}
enableHighlightBorder={false}
enableRadio
wrapperClassName={hybridSearchModeWrapperClassName}
className='p-3'
title={hybridOption.title}
description={hybridOption.description}
onClick={onHybridSearchModeChange}
readonly={readonly}
/>
))
}
</div>
)
}
{
isHybridSearch && isHybridSearchWeightedScoreMode && (
<WeightedScoreComponent
value={weightedScoreValue}
onChange={onWeightedScoreChange}
readonly={readonly}
/>
)
}
{
showRerankModelSelector && (
<div>
{
showRerankModelSelectorSwitch && (
<div className='system-sm-semibold mb-1 flex items-center text-text-secondary'>
<Switch
className='mr-1'
defaultValue={rerankingModelEnabled}
onChange={onRerankingModelEnabledChange}
disabled={readonly}
/>
{t('common.modelProvider.rerankModel.key')}
<Tooltip
triggerClassName='ml-0.5 shrink-0 w-3.5 h-3.5'
popupContent={t('common.modelProvider.rerankModel.tip')}
/>
</div>
)
}
<RerankingModelSelector
rerankingModel={rerankingModel}
onRerankingModelChange={onRerankingModelChange}
readonly={readonly}
/>
</div>
)
}
<TopKAndScoreThreshold
topK={topK}
onTopKChange={onTopKChange}
scoreThreshold={scoreThreshold}
onScoreThresholdChange={onScoreThresholdChange}
isScoreThresholdEnabled={isScoreThresholdEnabled}
onScoreThresholdEnabledChange={onScoreThresholdEnabledChange}
readonly={readonly}
hiddenScoreThreshold={searchMethod === RetrievalSearchMethodEnum.keywordSearch}
/>
</div>
</OptionCard>
)
}
export default memo(SearchMethodOption)

View File

@@ -0,0 +1,90 @@
import { memo } from 'react'
import { useTranslation } from 'react-i18next'
import Tooltip from '@/app/components/base/tooltip'
import Input from '@/app/components/base/input'
import Switch from '@/app/components/base/switch'
export type TopKAndScoreThresholdProps = {
topK: number
onTopKChange: (value: number) => void
scoreThreshold?: number
onScoreThresholdChange?: (value: number) => void
isScoreThresholdEnabled?: boolean
onScoreThresholdEnabledChange?: (value: boolean) => void
readonly?: boolean
hiddenScoreThreshold?: boolean
}
const TopKAndScoreThreshold = ({
topK,
onTopKChange,
scoreThreshold,
onScoreThresholdChange,
isScoreThresholdEnabled,
onScoreThresholdEnabledChange,
readonly,
hiddenScoreThreshold,
}: TopKAndScoreThresholdProps) => {
const { t } = useTranslation()
const handleTopKChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const value = Number(e.target.value)
if (Number.isNaN(value))
return
onTopKChange?.(value)
}
const handleScoreThresholdChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const value = Number(e.target.value)
if (Number.isNaN(value))
return
onScoreThresholdChange?.(value)
}
return (
<div className='grid grid-cols-2 gap-4'>
<div>
<div className='system-xs-medium mb-0.5 flex h-6 items-center text-text-secondary'>
{t('appDebug.datasetConfig.top_k')}
<Tooltip
triggerClassName='ml-0.5 shrink-0 w-3.5 h-3.5'
popupContent={t('appDebug.datasetConfig.top_kTip')}
/>
</div>
<Input
type='number'
value={topK}
onChange={handleTopKChange}
disabled={readonly}
/>
</div>
{
!hiddenScoreThreshold && (
<div>
<div className='mb-0.5 flex h-6 items-center'>
<Switch
className='mr-2'
defaultValue={isScoreThresholdEnabled}
onChange={onScoreThresholdEnabledChange}
disabled={readonly}
/>
<div className='system-sm-medium grow truncate text-text-secondary'>
{t('appDebug.datasetConfig.score_threshold')}
</div>
<Tooltip
triggerClassName='shrink-0 ml-0.5 w-3.5 h-3.5'
popupContent={t('appDebug.datasetConfig.score_thresholdTip')}
/>
</div>
<Input
type='number'
value={scoreThreshold}
onChange={handleScoreThresholdChange}
disabled={readonly || !isScoreThresholdEnabled}
/>
</div>
)
}
</div>
)
}
export default memo(TopKAndScoreThreshold)

View File

@@ -0,0 +1,20 @@
import type { ComponentType } from 'react'
import type {
HybridSearchModeEnum,
RetrievalSearchMethodEnum,
} from '../../types'
export type Option = {
id: RetrievalSearchMethodEnum
icon: ComponentType<any>
title: any
description: string
effectColor?: string
showEffectColor?: boolean,
}
export type HybridSearchModeOption = {
id: HybridSearchModeEnum
title: string
description: string
}