feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/index.tsx

@@ -27,6 +27,7 @@ type RetrievalSettingProps = {
   onRerankingModelEnabledChange?: (value: boolean) => void
   weightedScore?: WeightedScore
   onWeightedScoreChange: (value: { value: number[] }) => void
+  showMultiModalTip?: boolean
 } & RerankingModelSelectorProps & TopKAndScoreThresholdProps
 
 const RetrievalSetting = ({
@@ -48,6 +49,7 @@ const RetrievalSetting = ({
   onScoreThresholdChange,
   isScoreThresholdEnabled,
   onScoreThresholdEnabledChange,
+  showMultiModalTip,
 }: RetrievalSettingProps) => {
   const { t } = useTranslation()
   const {
@@ -91,6 +93,7 @@ const RetrievalSetting = ({
               rerankingModel={rerankingModel}
               onRerankingModelChange={onRerankingModelChange}
               readonly={readonly}
+              showMultiModalTip={showMultiModalTip}
             />
           ))
         }

web/app/components/workflow/nodes/knowledge-base/components/retrieval-setting/search-method-option.tsx

@@ -25,6 +25,7 @@ import type { TopKAndScoreThresholdProps } from './top-k-and-score-threshold'
 import TopKAndScoreThreshold from './top-k-and-score-threshold'
 import type { RerankingModelSelectorProps } from './reranking-model-selector'
 import RerankingModelSelector from './reranking-model-selector'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
 
 type SearchMethodOptionProps = {
   readonly?: boolean
@@ -38,6 +39,7 @@ type SearchMethodOptionProps = {
   onWeightedScoreChange: (value: { value: number[] }) => void
   rerankingModelEnabled?: boolean
   onRerankingModelEnabledChange?: (value: boolean) => void
+  showMultiModalTip?: boolean
 } & RerankingModelSelectorProps & TopKAndScoreThresholdProps
 const SearchMethodOption = ({
   readonly,
@@ -59,6 +61,7 @@ const SearchMethodOption = ({
   onScoreThresholdChange,
   isScoreThresholdEnabled,
   onScoreThresholdEnabledChange,
+  showMultiModalTip = false,
 }: SearchMethodOptionProps) => {
   const { t } = useTranslation()
   const Icon = option.icon
@@ -183,6 +186,17 @@ const SearchMethodOption = ({
                 onRerankingModelChange={onRerankingModelChange}
                 readonly={readonly}
               />
+              {showMultiModalTip && (
+                <div className='mt-2 flex h-10 items-center gap-x-0.5 overflow-hidden rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-2 shadow-xs backdrop-blur-[5px]'>
+                  <div className='absolute bottom-0 left-0 right-0 top-0 bg-dataset-warning-message-bg opacity-40' />
+                  <div className='p-1'>
+                    <AlertTriangle className='size-4 text-text-warning-secondary' />
+                  </div>
+                  <span className='system-xs-medium text-text-primary'>
+                    {t('datasetSettings.form.retrievalSetting.multiModalTip')}
+                  </span>
+                </div>
+              )}
             </div>
           )
         }

web/app/components/workflow/nodes/knowledge-base/panel.tsx

@@ -25,6 +25,9 @@ import Split from '../_base/components/split'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker'
 import type { Var } from '@/app/components/workflow/types'
+import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 
 const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
   id,
@@ -32,6 +35,9 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
 }) => {
   const { t } = useTranslation()
   const { nodesReadOnly } = useNodesReadOnly()
+  const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
+  const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
+
   const {
     handleChunkStructureChange,
     handleIndexMethodChange,
@@ -52,9 +58,9 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
     if (!data.chunk_structure) return false
     switch (data.chunk_structure) {
       case ChunkStructureEnum.general:
-        return variable.schemaType === 'general_structure'
+        return variable.schemaType === 'general_structure' || variable.schemaType === 'multimodal_general_structure'
       case ChunkStructureEnum.parent_child:
-        return variable.schemaType === 'parent_child_structure'
+        return variable.schemaType === 'parent_child_structure' || variable.schemaType === 'multimodal_parent_child_structure'
       case ChunkStructureEnum.question_answer:
         return variable.schemaType === 'qa_structure'
       default:
@@ -67,10 +73,10 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
     let placeholder = ''
     switch (data.chunk_structure) {
       case ChunkStructureEnum.general:
-        placeholder = 'general_structure'
+        placeholder = '(multimodal_)general_structure'
         break
       case ChunkStructureEnum.parent_child:
-        placeholder = 'parent_child_structure'
+        placeholder = '(multimodal_)parent_child_structure'
         break
       case ChunkStructureEnum.question_answer:
         placeholder = 'qa_structure'
@@ -81,6 +87,23 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
     return placeholder.charAt(0).toUpperCase() + placeholder.slice(1)
   }, [data.chunk_structure])
 
+  const showMultiModalTip = useMemo(() => {
+    return checkShowMultiModalTip({
+      embeddingModel: {
+        provider: data.embedding_model_provider ?? '',
+        model: data.embedding_model ?? '',
+      },
+      rerankingEnable: !!data.retrieval_model?.reranking_enable,
+      rerankModel: {
+        rerankingProviderName: data.retrieval_model?.reranking_model?.reranking_provider_name ?? '',
+        rerankingModelName: data.retrieval_model?.reranking_model?.reranking_model_name ?? '',
+      },
+      indexMethod: data.indexing_technique,
+      embeddingModelList,
+      rerankModelList,
+    })
+  }, [data.embedding_model_provider, data.embedding_model, data.retrieval_model?.reranking_enable, data.retrieval_model?.reranking_model, data.indexing_technique, embeddingModelList, rerankModelList])
+
   return (
     <div>
       <Group
@@ -161,6 +184,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
                   onScoreThresholdChange={handleScoreThresholdChange}
                   isScoreThresholdEnabled={data.retrieval_model.score_threshold_enabled}
                   onScoreThresholdEnabledChange={handleScoreThresholdEnabledChange}
+                  showMultiModalTip={showMultiModalTip}
                   readonly={nodesReadOnly}
                 />
               </div>