feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/app/configuration/dataset-config/index.tsx

@@ -77,7 +77,7 @@ const DatasetConfig: FC = () => {
     const oldRetrievalConfig = {
       top_k,
       score_threshold,
-      reranking_model: (reranking_model.reranking_provider_name && reranking_model.reranking_model_name) ? {
+      reranking_model: (reranking_model && reranking_model.reranking_provider_name && reranking_model.reranking_model_name) ? {
         provider: reranking_model.reranking_provider_name,
         model: reranking_model.reranking_model_name,
       } : undefined,

web/app/components/app/configuration/dataset-config/select-dataset/index.tsx

@@ -13,6 +13,8 @@ import Badge from '@/app/components/base/badge'
 import { useKnowledge } from '@/hooks/use-knowledge'
 import cn from '@/utils/classnames'
 import AppIcon from '@/app/components/base/app-icon'
+import { ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import FeatureIcon from '@/app/components/header/account-setting/model-provider-page/model-selector/feature-icon'
 
 export type ISelectDataSetProps = {
   isShow: boolean
@@ -121,7 +123,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
               <div
                 key={item.id}
                 className={cn(
-                  'flex h-10 cursor-pointer items-center justify-between rounded-lg border-[0.5px] border-components-panel-border-subtle bg-components-panel-on-panel-item-bg px-2 shadow-xs hover:border-components-panel-border hover:bg-components-panel-on-panel-item-bg-hover hover:shadow-sm',
+                  'flex h-10 cursor-pointer items-center rounded-lg border-[0.5px] border-components-panel-border-subtle bg-components-panel-on-panel-item-bg px-2 shadow-xs hover:border-components-panel-border hover:bg-components-panel-on-panel-item-bg-hover hover:shadow-sm',
                   selected.some(i => i.id === item.id) && 'border-[1.5px] border-components-option-card-option-selected-border bg-state-accent-hover shadow-xs hover:border-components-option-card-option-selected-border hover:bg-state-accent-hover hover:shadow-xs',
                   !item.embedding_available && 'hover:border-components-panel-border-subtle hover:bg-components-panel-on-panel-item-bg hover:shadow-xs',
                 )}
@@ -131,7 +133,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                   toggleSelect(item)
                 }}
               >
-                <div className='mr-1 flex items-center overflow-hidden'>
+                <div className='mr-1 flex grow items-center overflow-hidden'>
                   <div className={cn('mr-2', !item.embedding_available && 'opacity-30')}>
                     <AppIcon
                       size='tiny'
@@ -146,6 +148,11 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                     <span className='ml-1 shrink-0 rounded-md border border-divider-deep px-1 text-xs font-normal leading-[18px] text-text-tertiary'>{t('dataset.unavailable')}</span>
                   )}
                 </div>
+                {item.is_multimodal && (
+                  <div className='mr-1 shrink-0'>
+                    <FeatureIcon feature={ModelFeatureEnum.vision} />
+                  </div>
+                )}
                 {
                   item.indexing_technique && (
                     <Badge

web/app/components/app/configuration/dataset-config/settings-modal/index.tsx

@@ -1,5 +1,5 @@
 import type { FC } from 'react'
-import { useRef, useState } from 'react'
+import { useMemo, useRef, useState } from 'react'
 import { useMount } from 'ahooks'
 import { useTranslation } from 'react-i18next'
 import { isEqual } from 'lodash-es'
@@ -25,15 +25,13 @@ import { isReRankModelSelected } from '@/app/components/datasets/common/check-re
 import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
 import PermissionSelector from '@/app/components/datasets/settings/permission-selector'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
-import {
-  useModelList,
-  useModelListAndDefaultModelAndCurrentProviderAndModel,
-} from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { fetchMembers } from '@/service/common'
 import type { Member } from '@/models/common'
 import { IndexingType } from '@/app/components/datasets/create/step-two'
 import { useDocLink } from '@/context/i18n'
+import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils'
 
 type SettingsModalProps = {
   currentDataset: DataSet
@@ -54,10 +52,8 @@ const SettingsModal: FC<SettingsModalProps> = ({
   onCancel,
   onSave,
 }) => {
-  const { data: embeddingsModelList } = useModelList(ModelTypeEnum.textEmbedding)
-  const {
-    modelList: rerankModelList,
-  } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
+  const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
+  const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
   const { t } = useTranslation()
   const docLink = useDocLink()
   const { notify } = useToastContext()
@@ -181,6 +177,23 @@ const SettingsModal: FC<SettingsModalProps> = ({
     getMembers()
   })
 
+  const showMultiModalTip = useMemo(() => {
+    return checkShowMultiModalTip({
+      embeddingModel: {
+        provider: localeCurrentDataset.embedding_model_provider,
+        model: localeCurrentDataset.embedding_model,
+      },
+      rerankingEnable: retrievalConfig.reranking_enable,
+      rerankModel: {
+        rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name,
+        rerankingModelName: retrievalConfig.reranking_model.reranking_model_name,
+      },
+      indexMethod,
+      embeddingModelList,
+      rerankModelList,
+    })
+  }, [localeCurrentDataset.embedding_model, localeCurrentDataset.embedding_model_provider, retrievalConfig.reranking_enable, retrievalConfig.reranking_model, indexMethod, embeddingModelList, rerankModelList])
+
   return (
     <div
       className='flex w-full flex-col overflow-hidden rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl'
@@ -273,7 +286,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
                     provider: localeCurrentDataset.embedding_model_provider,
                     model: localeCurrentDataset.embedding_model,
                   }}
-                  modelList={embeddingsModelList}
+                  modelList={embeddingModelList}
                 />
               </div>
               <div className='mt-2 w-full text-xs leading-6 text-text-tertiary'>
@@ -344,6 +357,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
                   <RetrievalMethodConfig
                     value={retrievalConfig}
                     onChange={setRetrievalConfig}
+                    showMultiModalTip={showMultiModalTip}
                   />
                 )
                 : (

web/app/components/app/configuration/index.tsx

@@ -307,7 +307,7 @@ const Configuration: FC = () => {
     const oldRetrievalConfig = {
       top_k,
       score_threshold,
-      reranking_model: (reranking_model.reranking_provider_name && reranking_model.reranking_model_name) ? {
+      reranking_model: (reranking_model?.reranking_provider_name && reranking_model?.reranking_model_name) ? {
         provider: reranking_model.reranking_provider_name,
         model: reranking_model.reranking_model_name,
       } : undefined,