admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: multimodal support (image) (#27793)

Browse Source 
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wu Tianwei
2025-12-09 11:44:50 +08:00
committed by GitHub
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx
View File

@@ -15,6 +15,8 @@ import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import Badge from '@/app/components/base/badge'
import { useKnowledge } from '@/hooks/use-knowledge'
import AppIcon from '@/app/components/base/app-icon'
import FeatureIcon from '@/app/components/header/account-setting/model-provider-page/model-selector/feature-icon'
import { ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
type Props = {
payload: DataSet
@@ -98,6 +100,11 @@ const DatasetItem: FC<Props> = ({
</ActionButton>
</div>
)}
{payload.is_multimodal && (
<div className='mr-1 shrink-0 group-hover/dataset-item:hidden'>
<FeatureIcon feature={ModelFeatureEnum.vision} />
</div>
)}
{
payload.indexing_technique && <Badge
className='shrink-0 group-hover/dataset-item:hidden'

3
web/app/components/workflow/nodes/knowledge-retrieval/default.ts
View File

@@ -15,6 +15,7 @@ const nodeDefault: NodeDefault<KnowledgeRetrievalNodeType> = {
metaData,
defaultValue: {
query_variable_selector: [],
query_attachment_selector: [],
dataset_ids: [],
retrieval_mode: RETRIEVE_TYPE.multiWay,
multiple_retrieval_config: {
@@ -25,8 +26,6 @@ const nodeDefault: NodeDefault<KnowledgeRetrievalNodeType> = {
},
checkValid(payload: KnowledgeRetrievalNodeType, t: any) {
let errorMessages = ''
if (!errorMessages && (!payload.query_variable_selector || payload.query_variable_selector.length === 0))
errorMessages = t(`${i18nPrefix}.errorMsg.fieldRequired`, { field: t(`${i18nPrefix}.nodes.knowledgeRetrieval.queryVariable`) })
if (!errorMessages && (!payload.dataset_ids || payload.dataset_ids.length === 0))
errorMessages = t(`${i18nPrefix}.errorMsg.fieldRequired`, { field: t(`${i18nPrefix}.nodes.knowledgeRetrieval.knowledge`) })

30
web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx
View File

@@ -29,7 +29,9 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
readOnly,
inputs,
handleQueryVarChange,
filterVar,
handleQueryAttachmentChange,
filterStringVar,
filterFileVar,
handleModelChanged,
handleCompletionParamsChange,
handleRetrievalModeChange,
@@ -50,6 +52,7 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
availableStringNodesWithParent,
availableNumberVars,
availableNumberNodesWithParent,
showImageQueryVarSelector,
} = useConfig(id, data)
const metadataList = useMemo(() => {
@@ -63,20 +66,30 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
return (
<div className='pt-2'>
<div className='space-y-4 px-4 pb-2'>
<Field
title={t(`${i18nPrefix}.queryVariable`)}
required
>
<Field title={t(`${i18nPrefix}.queryText`)}>
<VarReferencePicker
nodeId={id}
readonly={readOnly}
isShowNodeName
value={inputs.query_variable_selector}
onChange={handleQueryVarChange}
filterVar={filterVar}
filterVar={filterStringVar}
/>
</Field>
{showImageQueryVarSelector && (
<Field title={t(`${i18nPrefix}.queryAttachment`)}>
<VarReferencePicker
nodeId={id}
readonly={readOnly}
isShowNodeName
value={inputs.query_attachment_selector}
onChange={handleQueryAttachmentChange}
filterVar={filterFileVar}
/>
</Field>
)}
<Field
title={t(`${i18nPrefix}.knowledge`)}
required
@@ -170,6 +183,11 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
type: 'object',
description: t(`${i18nPrefix}.outputVars.metadata`),
},
{
name: 'files',
type: 'Array[File]',
description: t(`${i18nPrefix}.outputVars.files`),
},
]}
/>

1
web/app/components/workflow/nodes/knowledge-retrieval/types.ts
View File

@@ -97,6 +97,7 @@ export type MetadataFilteringConditions = {
export type KnowledgeRetrievalNodeType = CommonNodeType & {
query_variable_selector: ValueSelector
query_attachment_selector: ValueSelector
dataset_ids: string[]
retrieval_mode: RETRIEVE_TYPE
multiple_retrieval_config?: MultipleRetrievalConfig

39
web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts
View File

@@ -1,6 +1,7 @@
import {
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react'
@@ -72,6 +73,13 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
setInputs(newInputs)
}, [inputs, setInputs])
const handleQueryAttachmentChange = useCallback((newVar: ValueSelector | string) => {
const newInputs = produce(inputs, (draft) => {
draft.query_attachment_selector = newVar as ValueSelector
})
setInputs(newInputs)
}, [inputs, setInputs])
const {
currentProvider,
currentModel,
@@ -250,6 +258,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
allInternal,
allExternal,
} = getSelectedDatasetsMode(newDatasets)
const noMultiModalDatasets = newDatasets.every(d => !d.is_multimodal)
const newInputs = produce(inputs, (draft) => {
draft.dataset_ids = newDatasets.map(d => d.id)
@@ -261,6 +270,9 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
})
draft.multiple_retrieval_config = newMultipleRetrievalConfig
}
if (noMultiModalDatasets)
draft.query_attachment_selector = []
})
updateDatasetsDetail(newDatasets)
setInputs(newInputs)
@@ -274,10 +286,18 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
setRerankModelOpen(true)
}, [inputs, setInputs, payload.retrieval_mode, selectedDatasets, currentRerankModel, currentRerankProvider, updateDatasetsDetail])
const filterVar = useCallback((varPayload: Var) => {
const filterStringVar = useCallback((varPayload: Var) => {
return varPayload.type === VarType.string
}, [])
const filterNumberVar = useCallback((varPayload: Var) => {
return varPayload.type === VarType.number
}, [])
const filterFileVar = useCallback((varPayload: Var) => {
return varPayload.type === VarType.file || varPayload.type === VarType.arrayFile
}, [])
const handleMetadataFilterModeChange = useCallback((newMode: MetadataFilteringModeEnum) => {
setInputs(produce(inputRef.current, (draft) => {
draft.metadata_filtering_mode = newMode
@@ -361,10 +381,6 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
setInputs(newInputs)
}, [setInputs])
const filterStringVar = useCallback((varPayload: Var) => {
return [VarType.string].includes(varPayload.type)
}, [])
const {
availableVars: availableStringVars,
availableNodesWithParent: availableStringNodesWithParent,
@@ -373,10 +389,6 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
filterVar: filterStringVar,
})
const filterNumberVar = useCallback((varPayload: Var) => {
return [VarType.number].includes(varPayload.type)
}, [])
const {
availableVars: availableNumberVars,
availableNodesWithParent: availableNumberNodesWithParent,
@@ -385,11 +397,17 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
filterVar: filterNumberVar,
})
const showImageQueryVarSelector = useMemo(() => {
return selectedDatasets.some(d => d.is_multimodal)
}, [selectedDatasets])
return {
readOnly,
inputs,
handleQueryVarChange,
filterVar,
handleQueryAttachmentChange,
filterStringVar,
filterFileVar,
handleRetrievalModeChange,
handleMultipleRetrievalConfigChange,
handleModelChanged,
@@ -410,6 +428,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
availableStringNodesWithParent,
availableNumberVars,
availableNumberNodesWithParent,
showImageQueryVarSelector,
}
}

74
web/app/components/workflow/nodes/knowledge-retrieval/use-single-run-form-params.ts
View File

@@ -1,9 +1,14 @@
import type { RefObject } from 'react'
import { useTranslation } from 'react-i18next'
import type { InputVar, Variable } from '@/app/components/workflow/types'
import { InputVarType } from '@/app/components/workflow/types'
import type { InputVar, Var, Variable } from '@/app/components/workflow/types'
import { InputVarType, VarType } from '@/app/components/workflow/types'
import { useCallback, useMemo } from 'react'
import type { KnowledgeRetrievalNodeType } from './types'
import type { Props as FormProps } from '@/app/components/workflow/nodes/_base/components/before-run-form/form'
import { useDatasetsDetailStore } from '../../datasets-detail-store/store'
import type { DataSet } from '@/models/datasets'
import useAvailableVarList from '../_base/hooks/use-available-var-list'
import { findVariableWhenOnLLMVision } from '../utils'
const i18nPrefix = 'workflow.nodes.knowledgeRetrieval'
@@ -17,40 +22,89 @@ type Params = {
toVarInputs: (variables: Variable[]) => InputVar[]
}
const useSingleRunFormParams = ({
id,
payload,
runInputData,
runInputDataRef,
setRunInputData,
}: Params) => {
const { t } = useTranslation()
const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail)
const query = runInputData.query
const queryAttachment = runInputData.queryAttachment
const setQuery = useCallback((newQuery: string) => {
setRunInputData({
...runInputData,
...runInputDataRef.current,
query: newQuery,
})
}, [runInputData, setRunInputData])
}, [runInputDataRef, setRunInputData])
const setQueryAttachment = useCallback((newQueryAttachment: string) => {
setRunInputData({
...runInputDataRef.current,
queryAttachment: newQueryAttachment,
})
}, [runInputDataRef, setRunInputData])
const filterFileVar = useCallback((varPayload: Var) => {
return [VarType.file, VarType.arrayFile].includes(varPayload.type)
}, [])
// Get all variables from previous nodes that are file or array of file
const {
availableVars: availableFileVars,
} = useAvailableVarList(id, {
onlyLeafNodeVar: false,
filterVar: filterFileVar,
})
const forms = useMemo(() => {
return [
const datasetIds = payload.dataset_ids
const datasets = datasetIds.reduce<DataSet[]>((acc, id) => {
if (datasetsDetail[id])
acc.push(datasetsDetail[id])
return acc
}, [])
const hasMultiModalDatasets = datasets.some(d => d.is_multimodal)
const inputFields: FormProps[] = [
{
inputs: [{
label: t(`${i18nPrefix}.queryVariable`)!,
label: t(`${i18nPrefix}.queryText`)!,
variable: 'query',
type: InputVarType.paragraph,
required: true,
required: false,
}],
values: { query },
onChange: (keyValue: Record<string, any>) => setQuery(keyValue.query),
},
]
}, [query, setQuery, t])
if (hasMultiModalDatasets) {
const currentVariable = findVariableWhenOnLLMVision(payload.query_attachment_selector, availableFileVars)
inputFields.push(
{
inputs: [{
label: t(`${i18nPrefix}.queryAttachment`)!,
variable: 'queryAttachment',
type: currentVariable?.formType as InputVarType,
required: false,
}],
values: { queryAttachment },
onChange: (keyValue: Record<string, any>) => setQueryAttachment(keyValue.queryAttachment),
},
)
}
return inputFields
}, [query, setQuery, t, datasetsDetail, payload.dataset_ids, payload.query_attachment_selector, availableFileVars, queryAttachment, setQueryAttachment])
const getDependentVars = () => {
return [payload.query_variable_selector]
return [payload.query_variable_selector, payload.query_attachment_selector]
}
const getDependentVar = (variable: string) => {
if(variable === 'query')
if (variable === 'query')
return payload.query_variable_selector
if (variable === 'queryAttachment')
return payload.query_attachment_selector
}
return {