feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/datasets/settings/form/index.tsx

@@ -1,5 +1,5 @@
 'use client'
-import { useCallback, useRef, useState } from 'react'
+import { useCallback, useMemo, useRef, useState } from 'react'
 import { useMount } from 'ahooks'
 import { useTranslation } from 'react-i18next'
 import PermissionSelector from '../permission-selector'
@@ -20,10 +20,7 @@ import type { AppIconType, RetrievalConfig } from '@/types/app'
 import { useSelector as useAppContextWithSelector } from '@/context/app-context'
 import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
-import {
-  useModelList,
-  useModelListAndDefaultModelAndCurrentProviderAndModel,
-} from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { fetchMembers } from '@/service/common'
@@ -37,6 +34,7 @@ import Toast from '@/app/components/base/toast'
 import { RiAlertFill } from '@remixicon/react'
 import { useDocLink } from '@/context/i18n'
 import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
+import { checkShowMultiModalTip } from '../utils'
 
 const rowClass = 'flex gap-x-1'
 const labelClass = 'flex items-center shrink-0 w-[180px] h-7 pt-1'
@@ -79,9 +77,7 @@ const Form = () => {
         model: '',
       },
   )
-  const {
-    modelList: rerankModelList,
-  } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
+  const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
   const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
   const previousAppIcon = useRef(DEFAULT_APP_ICON)
 
@@ -203,6 +199,20 @@ const Form = () => {
 
   const isShowIndexMethod = currentDataset && currentDataset.doc_form !== ChunkingMode.parentChild && currentDataset.indexing_technique && indexMethod
 
+  const showMultiModalTip = useMemo(() => {
+    return checkShowMultiModalTip({
+      embeddingModel,
+      rerankingEnable: retrievalConfig.reranking_enable,
+      rerankModel: {
+        rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name,
+        rerankingModelName: retrievalConfig.reranking_model.reranking_model_name,
+      },
+      indexMethod,
+      embeddingModelList,
+      rerankModelList,
+    })
+  }, [embeddingModel, rerankModelList, retrievalConfig.reranking_enable, retrievalConfig.reranking_model, embeddingModelList, indexMethod])
+
   return (
     <div className='flex w-full flex-col gap-y-4 px-20 py-8 sm:w-[960px]'>
       {/* Dataset name and icon */}
@@ -434,6 +444,7 @@ const Form = () => {
                       <RetrievalMethodConfig
                         value={retrievalConfig}
                         onChange={setRetrievalConfig}
+                        showMultiModalTip={showMultiModalTip}
                       />
                     )
                     : (

web/app/components/datasets/settings/utils/index.tsx

@@ -0,0 +1,46 @@
+import { type DefaultModel, type Model, ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import { IndexingType } from '../../create/step-two'
+
+type ShowMultiModalTipProps = {
+  embeddingModel: DefaultModel
+  rerankingEnable: boolean
+  rerankModel: {
+    rerankingProviderName: string
+    rerankingModelName: string
+  }
+  indexMethod: IndexingType | undefined
+  embeddingModelList: Model[]
+  rerankModelList: Model[]
+}
+
+export const checkShowMultiModalTip = ({
+  embeddingModel,
+  rerankingEnable,
+  rerankModel,
+  indexMethod,
+  embeddingModelList,
+  rerankModelList,
+}: ShowMultiModalTipProps) => {
+  if (indexMethod !== IndexingType.QUALIFIED || !embeddingModel.provider || !embeddingModel.model)
+    return false
+  const currentEmbeddingModelProvider = embeddingModelList.find(model => model.provider === embeddingModel.provider)
+  if (!currentEmbeddingModelProvider)
+    return false
+  const currentEmbeddingModel = currentEmbeddingModelProvider.models.find(model => model.model === embeddingModel.model)
+  if (!currentEmbeddingModel)
+    return false
+  const isCurrentEmbeddingModelSupportMultiModal = !!currentEmbeddingModel.features?.includes(ModelFeatureEnum.vision)
+  if (!isCurrentEmbeddingModelSupportMultiModal)
+    return false
+  const { rerankingModelName, rerankingProviderName } = rerankModel
+  if (!rerankingEnable || !rerankingModelName || !rerankingProviderName)
+    return false
+  const currentRerankingModelProvider = rerankModelList.find(model => model.provider === rerankingProviderName)
+  if (!currentRerankingModelProvider)
+    return false
+  const currentRerankingModel = currentRerankingModelProvider.models.find(model => model.model === rerankingModelName)
+  if (!currentRerankingModel)
+    return false
+  const isRerankingModelSupportMultiModal = !!currentRerankingModel.features?.includes(ModelFeatureEnum.vision)
+  return !isRerankingModelSupportMultiModal
+}