feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wu Tianwei
2025-12-09 11:44:50 +08:00
committed by GitHub
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions

View File

@@ -1,5 +1,5 @@
'use client'
import { useCallback, useRef, useState } from 'react'
import { useCallback, useMemo, useRef, useState } from 'react'
import { useMount } from 'ahooks'
import { useTranslation } from 'react-i18next'
import PermissionSelector from '../permission-selector'
@@ -20,10 +20,7 @@ import type { AppIconType, RetrievalConfig } from '@/types/app'
import { useSelector as useAppContextWithSelector } from '@/context/app-context'
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import {
useModelList,
useModelListAndDefaultModelAndCurrentProviderAndModel,
} from '@/app/components/header/account-setting/model-provider-page/hooks'
import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { fetchMembers } from '@/service/common'
@@ -37,6 +34,7 @@ import Toast from '@/app/components/base/toast'
import { RiAlertFill } from '@remixicon/react'
import { useDocLink } from '@/context/i18n'
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
import { checkShowMultiModalTip } from '../utils'
const rowClass = 'flex gap-x-1'
const labelClass = 'flex items-center shrink-0 w-[180px] h-7 pt-1'
@@ -79,9 +77,7 @@ const Form = () => {
model: '',
},
)
const {
modelList: rerankModelList,
} = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
const previousAppIcon = useRef(DEFAULT_APP_ICON)
@@ -203,6 +199,20 @@ const Form = () => {
const isShowIndexMethod = currentDataset && currentDataset.doc_form !== ChunkingMode.parentChild && currentDataset.indexing_technique && indexMethod
const showMultiModalTip = useMemo(() => {
return checkShowMultiModalTip({
embeddingModel,
rerankingEnable: retrievalConfig.reranking_enable,
rerankModel: {
rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name,
rerankingModelName: retrievalConfig.reranking_model.reranking_model_name,
},
indexMethod,
embeddingModelList,
rerankModelList,
})
}, [embeddingModel, rerankModelList, retrievalConfig.reranking_enable, retrievalConfig.reranking_model, embeddingModelList, indexMethod])
return (
<div className='flex w-full flex-col gap-y-4 px-20 py-8 sm:w-[960px]'>
{/* Dataset name and icon */}
@@ -434,6 +444,7 @@ const Form = () => {
<RetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
showMultiModalTip={showMultiModalTip}
/>
)
: (