feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-09 11:44:50 +08:00
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions
--- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts
+++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts
@@ -591,16 +591,8 @@ const formatItem = (
            variable: outputKey,
            type:
              output.type === 'array'
-                ? (`Array[${output.items?.type
-                  ? output.items.type.slice(0, 1).toLocaleUpperCase()
-                  + output.items.type.slice(1)
-                  : 'Unknown'
-                }]` as VarType)
-                : (`${output.type
-                  ? output.type.slice(0, 1).toLocaleUpperCase()
-                  + output.type.slice(1)
-                  : 'Unknown'
-                }` as VarType),
+                ? (`Array[${output.items?.type ? output.items.type.slice(0, 1).toLocaleUpperCase() + output.items.type.slice(1) : 'Unknown'}]` as VarType)
+                : (`${output.type ? output.type.slice(0, 1).toLocaleUpperCase() + output.type.slice(1) : 'Unknown'}` as VarType),
          })
        },
      )
@@ -858,13 +850,14 @@ export const toNodeOutputVars = (
          filterVar,
          allPluginInfoList,
          ragVariablesInDataSource.map(
-            (ragVariable: RAGPipelineVariable) =>
-              ({
+            (ragVariable: RAGPipelineVariable) => {
+              return {
                variable: `rag.${node.id}.${ragVariable.variable}`,
                type: inputVarTypeToVarType(ragVariable.type as any),
                description: ragVariable.label,
                isRagVariable: true,
-              } as Var),
+              } as Var
+            },
          ),
          schemaTypeDefinitions,
        ),
@@ -1301,7 +1294,11 @@ export const getNodeUsedVars = (node: Node): ValueSelector[] => {
      break
    }
    case BlockEnum.KnowledgeRetrieval: {
-      res = [(data as KnowledgeRetrievalNodeType).query_variable_selector]
+      const {
+        query_variable_selector,
+        query_attachment_selector,
+      } = data as KnowledgeRetrievalNodeType
+      res = [query_variable_selector, query_attachment_selector]
      break
    }
    case BlockEnum.IfElse: {
@@ -1640,6 +1637,10 @@ export const updateNodeVars = (
          payload.query_variable_selector.join('.') === oldVarSelector.join('.')
        )
          payload.query_variable_selector = newVarSelector
+        if (
+          payload.query_attachment_selector.join('.') === oldVarSelector.join('.')
+        )
+          payload.query_attachment_selector = newVarSelector
        break
      }
      case BlockEnum.IfElse: {
--- a/web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/index.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/index.tsx
@@ -27,6 +27,7 @@ type RetrievalSettingProps = {
  onRerankingModelEnabledChange?: (value: boolean) => void
  weightedScore?: WeightedScore
  onWeightedScoreChange: (value: { value: number[] }) => void
+  showMultiModalTip?: boolean
 } & RerankingModelSelectorProps & TopKAndScoreThresholdProps

 const RetrievalSetting = ({
@@ -48,6 +49,7 @@ const RetrievalSetting = ({
  onScoreThresholdChange,
  isScoreThresholdEnabled,
  onScoreThresholdEnabledChange,
+  showMultiModalTip,
 }: RetrievalSettingProps) => {
  const { t } = useTranslation()
  const {
@@ -91,6 +93,7 @@ const RetrievalSetting = ({
              rerankingModel={rerankingModel}
              onRerankingModelChange={onRerankingModelChange}
              readonly={readonly}
+              showMultiModalTip={showMultiModalTip}
            />
          ))
        }
--- a/web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/search-method-option.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/search-method-option.tsx
@@ -25,6 +25,7 @@ import type { TopKAndScoreThresholdProps } from './top-k-and-score-threshold'
 import TopKAndScoreThreshold from './top-k-and-score-threshold'
 import type { RerankingModelSelectorProps } from './reranking-model-selector'
 import RerankingModelSelector from './reranking-model-selector'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'

 type SearchMethodOptionProps = {
  readonly?: boolean
@@ -38,6 +39,7 @@ type SearchMethodOptionProps = {
  onWeightedScoreChange: (value: { value: number[] }) => void
  rerankingModelEnabled?: boolean
  onRerankingModelEnabledChange?: (value: boolean) => void
+  showMultiModalTip?: boolean
 } & RerankingModelSelectorProps & TopKAndScoreThresholdProps
 const SearchMethodOption = ({
  readonly,
@@ -59,6 +61,7 @@ const SearchMethodOption = ({
  onScoreThresholdChange,
  isScoreThresholdEnabled,
  onScoreThresholdEnabledChange,
+  showMultiModalTip = false,
 }: SearchMethodOptionProps) => {
  const { t } = useTranslation()
  const Icon = option.icon
@@ -183,6 +186,17 @@ const SearchMethodOption = ({
                onRerankingModelChange={onRerankingModelChange}
                readonly={readonly}
              />
+              {showMultiModalTip && (
+                <div className='mt-2 flex h-10 items-center gap-x-0.5 overflow-hidden rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-2 shadow-xs backdrop-blur-[5px]'>
+                  <div className='absolute bottom-0 left-0 right-0 top-0 bg-dataset-warning-message-bg opacity-40' />
+                  <div className='p-1'>
+                    <AlertTriangle className='size-4 text-text-warning-secondary' />
+                  </div>
+                  <span className='system-xs-medium text-text-primary'>
+                    {t('datasetSettings.form.retrievalSetting.multiModalTip')}
+                  </span>
+                </div>
+              )}
            </div>
          )
        }
--- a/web/app/components/workflow/nodes/knowledge-base/panel.tsx
+++ b/web/app/components/workflow/nodes/knowledge-base/panel.tsx
@@ -25,6 +25,9 @@ import Split from '../_base/components/split'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker'
 import type { Var } from '@/app/components/workflow/types'
+import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'

 const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
  id,
@@ -32,6 +35,9 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
 }) => {
  const { t } = useTranslation()
  const { nodesReadOnly } = useNodesReadOnly()
+  const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
+  const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
+
  const {
    handleChunkStructureChange,
    handleIndexMethodChange,
@@ -52,9 +58,9 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
    if (!data.chunk_structure) return false
    switch (data.chunk_structure) {
      case ChunkStructureEnum.general:
-        return variable.schemaType === 'general_structure'
+        return variable.schemaType === 'general_structure' || variable.schemaType === 'multimodal_general_structure'
      case ChunkStructureEnum.parent_child:
-        return variable.schemaType === 'parent_child_structure'
+        return variable.schemaType === 'parent_child_structure' || variable.schemaType === 'multimodal_parent_child_structure'
      case ChunkStructureEnum.question_answer:
        return variable.schemaType === 'qa_structure'
      default:
@@ -67,10 +73,10 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
    let placeholder = ''
    switch (data.chunk_structure) {
      case ChunkStructureEnum.general:
-        placeholder = 'general_structure'
+        placeholder = '(multimodal_)general_structure'
        break
      case ChunkStructureEnum.parent_child:
-        placeholder = 'parent_child_structure'
+        placeholder = '(multimodal_)parent_child_structure'
        break
      case ChunkStructureEnum.question_answer:
        placeholder = 'qa_structure'
@@ -81,6 +87,23 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
    return placeholder.charAt(0).toUpperCase() + placeholder.slice(1)
  }, [data.chunk_structure])

+  const showMultiModalTip = useMemo(() => {
+    return checkShowMultiModalTip({
+      embeddingModel: {
+        provider: data.embedding_model_provider ?? '',
+        model: data.embedding_model ?? '',
+      },
+      rerankingEnable: !!data.retrieval_model?.reranking_enable,
+      rerankModel: {
+        rerankingProviderName: data.retrieval_model?.reranking_model?.reranking_provider_name ?? '',
+        rerankingModelName: data.retrieval_model?.reranking_model?.reranking_model_name ?? '',
+      },
+      indexMethod: data.indexing_technique,
+      embeddingModelList,
+      rerankModelList,
+    })
+  }, [data.embedding_model_provider, data.embedding_model, data.retrieval_model?.reranking_enable, data.retrieval_model?.reranking_model, data.indexing_technique, embeddingModelList, rerankModelList])
+
  return (
    <div>
      <Group
@@ -161,6 +184,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
                  onScoreThresholdChange={handleScoreThresholdChange}
                  isScoreThresholdEnabled={data.retrieval_model.score_threshold_enabled}
                  onScoreThresholdEnabledChange={handleScoreThresholdEnabledChange}
+                  showMultiModalTip={showMultiModalTip}
                  readonly={nodesReadOnly}
                />
              </div>
--- a/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx
@@ -15,6 +15,8 @@ import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
 import Badge from '@/app/components/base/badge'
 import { useKnowledge } from '@/hooks/use-knowledge'
 import AppIcon from '@/app/components/base/app-icon'
+import FeatureIcon from '@/app/components/header/account-setting/model-provider-page/model-selector/feature-icon'
+import { ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'

 type Props = {
  payload: DataSet
@@ -98,6 +100,11 @@ const DatasetItem: FC<Props> = ({
          </ActionButton>
        </div>
      )}
+      {payload.is_multimodal && (
+        <div className='mr-1 shrink-0 group-hover/dataset-item:hidden'>
+          <FeatureIcon feature={ModelFeatureEnum.vision} />
+        </div>
+      )}
      {
        payload.indexing_technique && <Badge
          className='shrink-0 group-hover/dataset-item:hidden'
--- a/web/app/components/workflow/nodes/knowledge-retrieval/default.ts
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/default.ts
@@ -15,6 +15,7 @@ const nodeDefault: NodeDefault<KnowledgeRetrievalNodeType> = {
  metaData,
  defaultValue: {
    query_variable_selector: [],
+    query_attachment_selector: [],
    dataset_ids: [],
    retrieval_mode: RETRIEVE_TYPE.multiWay,
    multiple_retrieval_config: {
@@ -25,8 +26,6 @@ const nodeDefault: NodeDefault<KnowledgeRetrievalNodeType> = {
  },
  checkValid(payload: KnowledgeRetrievalNodeType, t: any) {
    let errorMessages = ''
-    if (!errorMessages && (!payload.query_variable_selector || payload.query_variable_selector.length === 0))
-      errorMessages = t(`${i18nPrefix}.errorMsg.fieldRequired`, { field: t(`${i18nPrefix}.nodes.knowledgeRetrieval.queryVariable`) })

    if (!errorMessages && (!payload.dataset_ids || payload.dataset_ids.length === 0))
      errorMessages = t(`${i18nPrefix}.errorMsg.fieldRequired`, { field: t(`${i18nPrefix}.nodes.knowledgeRetrieval.knowledge`) })
--- a/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx
@@ -29,7 +29,9 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
    readOnly,
    inputs,
    handleQueryVarChange,
-    filterVar,
+    handleQueryAttachmentChange,
+    filterStringVar,
+    filterFileVar,
    handleModelChanged,
    handleCompletionParamsChange,
    handleRetrievalModeChange,
@@ -50,6 +52,7 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
    availableStringNodesWithParent,
    availableNumberVars,
    availableNumberNodesWithParent,
+    showImageQueryVarSelector,
  } = useConfig(id, data)

  const metadataList = useMemo(() => {
@@ -63,20 +66,30 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
  return (
    <div className='pt-2'>
      <div className='space-y-4 px-4 pb-2'>
-        <Field
-          title={t(`${i18nPrefix}.queryVariable`)}
-          required
-        >
+        <Field title={t(`${i18nPrefix}.queryText`)}>
          <VarReferencePicker
            nodeId={id}
            readonly={readOnly}
            isShowNodeName
            value={inputs.query_variable_selector}
            onChange={handleQueryVarChange}
-            filterVar={filterVar}
+            filterVar={filterStringVar}
          />
        </Field>

+        {showImageQueryVarSelector && (
+          <Field title={t(`${i18nPrefix}.queryAttachment`)}>
+            <VarReferencePicker
+              nodeId={id}
+              readonly={readOnly}
+              isShowNodeName
+              value={inputs.query_attachment_selector}
+              onChange={handleQueryAttachmentChange}
+              filterVar={filterFileVar}
+            />
+          </Field>
+        )}
+
        <Field
          title={t(`${i18nPrefix}.knowledge`)}
          required
@@ -170,6 +183,11 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
                  type: 'object',
                  description: t(`${i18nPrefix}.outputVars.metadata`),
                },
+                {
+                  name: 'files',
+                  type: 'Array[File]',
+                  description: t(`${i18nPrefix}.outputVars.files`),
+                },
              ]}
            />

--- a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts
@@ -97,6 +97,7 @@ export type MetadataFilteringConditions = {

 export type KnowledgeRetrievalNodeType = CommonNodeType & {
  query_variable_selector: ValueSelector
+  query_attachment_selector: ValueSelector
  dataset_ids: string[]
  retrieval_mode: RETRIEVE_TYPE
  multiple_retrieval_config?: MultipleRetrievalConfig
--- a/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts
@@ -1,6 +1,7 @@
 import {
  useCallback,
  useEffect,
+  useMemo,
  useRef,
  useState,
 } from 'react'
@@ -72,6 +73,13 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
    setInputs(newInputs)
  }, [inputs, setInputs])

+  const handleQueryAttachmentChange = useCallback((newVar: ValueSelector | string) => {
+    const newInputs = produce(inputs, (draft) => {
+      draft.query_attachment_selector = newVar as ValueSelector
+    })
+    setInputs(newInputs)
+  }, [inputs, setInputs])
+
  const {
    currentProvider,
    currentModel,
@@ -250,6 +258,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
      allInternal,
      allExternal,
    } = getSelectedDatasetsMode(newDatasets)
+    const noMultiModalDatasets = newDatasets.every(d => !d.is_multimodal)
    const newInputs = produce(inputs, (draft) => {
      draft.dataset_ids = newDatasets.map(d => d.id)

@@ -261,6 +270,9 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
        })
        draft.multiple_retrieval_config = newMultipleRetrievalConfig
      }
+
+      if (noMultiModalDatasets)
+        draft.query_attachment_selector = []
    })
    updateDatasetsDetail(newDatasets)
    setInputs(newInputs)
@@ -274,10 +286,18 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
      setRerankModelOpen(true)
  }, [inputs, setInputs, payload.retrieval_mode, selectedDatasets, currentRerankModel, currentRerankProvider, updateDatasetsDetail])

-  const filterVar = useCallback((varPayload: Var) => {
+  const filterStringVar = useCallback((varPayload: Var) => {
    return varPayload.type === VarType.string
  }, [])

+  const filterNumberVar = useCallback((varPayload: Var) => {
+    return varPayload.type === VarType.number
+  }, [])
+
+  const filterFileVar = useCallback((varPayload: Var) => {
+    return varPayload.type === VarType.file || varPayload.type === VarType.arrayFile
+  }, [])
+
  const handleMetadataFilterModeChange = useCallback((newMode: MetadataFilteringModeEnum) => {
    setInputs(produce(inputRef.current, (draft) => {
      draft.metadata_filtering_mode = newMode
@@ -361,10 +381,6 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
    setInputs(newInputs)
  }, [setInputs])

-  const filterStringVar = useCallback((varPayload: Var) => {
-    return [VarType.string].includes(varPayload.type)
-  }, [])
-
  const {
    availableVars: availableStringVars,
    availableNodesWithParent: availableStringNodesWithParent,
@@ -373,10 +389,6 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
    filterVar: filterStringVar,
  })

-  const filterNumberVar = useCallback((varPayload: Var) => {
-    return [VarType.number].includes(varPayload.type)
-  }, [])
-
  const {
    availableVars: availableNumberVars,
    availableNodesWithParent: availableNumberNodesWithParent,
@@ -385,11 +397,17 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
    filterVar: filterNumberVar,
  })

+  const showImageQueryVarSelector = useMemo(() => {
+    return selectedDatasets.some(d => d.is_multimodal)
+  }, [selectedDatasets])
+
  return {
    readOnly,
    inputs,
    handleQueryVarChange,
-    filterVar,
+    handleQueryAttachmentChange,
+    filterStringVar,
+    filterFileVar,
    handleRetrievalModeChange,
    handleMultipleRetrievalConfigChange,
    handleModelChanged,
@@ -410,6 +428,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
    availableStringNodesWithParent,
    availableNumberVars,
    availableNumberNodesWithParent,
+    showImageQueryVarSelector,
  }
 }

--- a/web/app/components/workflow/nodes/knowledge-retrieval/use-single-run-form-params.ts
+++ b/web/app/components/workflow/nodes/knowledge-retrieval/use-single-run-form-params.ts
@@ -1,9 +1,14 @@
 import type { RefObject } from 'react'
 import { useTranslation } from 'react-i18next'
-import type { InputVar, Variable } from '@/app/components/workflow/types'
-import { InputVarType } from '@/app/components/workflow/types'
+import type { InputVar, Var, Variable } from '@/app/components/workflow/types'
+import { InputVarType, VarType } from '@/app/components/workflow/types'
 import { useCallback, useMemo } from 'react'
 import type { KnowledgeRetrievalNodeType } from './types'
+import type { Props as FormProps } from '@/app/components/workflow/nodes/_base/components/before-run-form/form'
+import { useDatasetsDetailStore } from '../../datasets-detail-store/store'
+import type { DataSet } from '@/models/datasets'
+import useAvailableVarList from '../_base/hooks/use-available-var-list'
+import { findVariableWhenOnLLMVision } from '../utils'

 const i18nPrefix = 'workflow.nodes.knowledgeRetrieval'

@@ -17,40 +22,89 @@ type Params = {
  toVarInputs: (variables: Variable[]) => InputVar[]
 }
 const useSingleRunFormParams = ({
+  id,
  payload,
  runInputData,
+  runInputDataRef,
  setRunInputData,
 }: Params) => {
  const { t } = useTranslation()
+  const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail)
  const query = runInputData.query
+  const queryAttachment = runInputData.queryAttachment
+
  const setQuery = useCallback((newQuery: string) => {
    setRunInputData({
-      ...runInputData,
+      ...runInputDataRef.current,
      query: newQuery,
    })
-  }, [runInputData, setRunInputData])
+  }, [runInputDataRef, setRunInputData])
+
+  const setQueryAttachment = useCallback((newQueryAttachment: string) => {
+    setRunInputData({
+      ...runInputDataRef.current,
+      queryAttachment: newQueryAttachment,
+    })
+  }, [runInputDataRef, setRunInputData])
+
+  const filterFileVar = useCallback((varPayload: Var) => {
+    return [VarType.file, VarType.arrayFile].includes(varPayload.type)
+  }, [])
+
+  // Get all variables from previous nodes that are file or array of file
+  const {
+    availableVars: availableFileVars,
+  } = useAvailableVarList(id, {
+    onlyLeafNodeVar: false,
+    filterVar: filterFileVar,
+  })

  const forms = useMemo(() => {
-    return [
+    const datasetIds = payload.dataset_ids
+    const datasets = datasetIds.reduce<DataSet[]>((acc, id) => {
+      if (datasetsDetail[id])
+        acc.push(datasetsDetail[id])
+      return acc
+    }, [])
+    const hasMultiModalDatasets = datasets.some(d => d.is_multimodal)
+    const inputFields: FormProps[] = [
      {
        inputs: [{
-          label: t(`${i18nPrefix}.queryVariable`)!,
+          label: t(`${i18nPrefix}.queryText`)!,
          variable: 'query',
          type: InputVarType.paragraph,
-          required: true,
+          required: false,
        }],
        values: { query },
        onChange: (keyValue: Record<string, any>) => setQuery(keyValue.query),
      },
    ]
-  }, [query, setQuery, t])
+    if (hasMultiModalDatasets) {
+      const currentVariable = findVariableWhenOnLLMVision(payload.query_attachment_selector, availableFileVars)
+      inputFields.push(
+        {
+          inputs: [{
+            label: t(`${i18nPrefix}.queryAttachment`)!,
+            variable: 'queryAttachment',
+            type: currentVariable?.formType as InputVarType,
+            required: false,
+          }],
+          values: { queryAttachment },
+          onChange: (keyValue: Record<string, any>) => setQueryAttachment(keyValue.queryAttachment),
+        },
+      )
+    }
+    return inputFields
+  }, [query, setQuery, t, datasetsDetail, payload.dataset_ids, payload.query_attachment_selector, availableFileVars, queryAttachment, setQueryAttachment])

  const getDependentVars = () => {
-    return [payload.query_variable_selector]
+    return [payload.query_variable_selector, payload.query_attachment_selector]
  }
  const getDependentVar = (variable: string) => {
-    if(variable === 'query')
+    if (variable === 'query')
      return payload.query_variable_selector
+    if (variable === 'queryAttachment')
+      return payload.query_attachment_selector
  }

  return {