feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wu Tianwei
2025-12-09 11:44:50 +08:00
committed by GitHub
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions

View File

@@ -1,6 +1,5 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import React, { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { SegmentIndexTag } from '../../documents/detail/completed/common/segment-index-tag'
import Dot from '../../documents/detail/completed/common/dot'
@@ -13,25 +12,42 @@ import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader
import cn from '@/utils/classnames'
import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
import { Markdown } from '@/app/components/base/markdown'
import ImageList from '../../common/image-list'
import Mask from './mask'
const i18nPrefix = 'datasetHitTesting'
type Props = {
type ChunkDetailModalProps = {
payload: HitTesting
onHide: () => void
}
const ChunkDetailModal: FC<Props> = ({
const ChunkDetailModal = ({
payload,
onHide,
}) => {
}: ChunkDetailModalProps) => {
const { t } = useTranslation()
const { segment, score, child_chunks } = payload
const { segment, score, child_chunks, files } = payload
const { position, content, sign_content, keywords, document, answer } = segment
const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
const extension = document.name.split('.').slice(-1)[0] as FileAppearanceTypeEnum
const heighClassName = isParentChildRetrieval ? 'h-[min(627px,_80vh)] overflow-y-auto' : 'h-[min(539px,_80vh)] overflow-y-auto'
const labelPrefix = isParentChildRetrieval ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
const images = useMemo(() => {
if (!files) return []
return files.map(file => ({
name: file.name,
mimeType: file.mime_type,
sourceUrl: file.source_url,
size: file.size,
extension: file.extension,
}))
}, [files])
const showImages = images.length > 0
const showKeywords = !isParentChildRetrieval && keywords && keywords.length > 0
return (
<Modal
title={t(`${i18nPrefix}.chunkDetail`)}
@@ -58,37 +74,49 @@ const ChunkDetailModal: FC<Props> = ({
</div>
<Score value={score} />
</div>
{!answer && (
<Markdown
className={cn('!mt-2 !text-text-secondary', heighClassName)}
content={sign_content || content}
customDisallowedElements={['input']}
/>
)}
{answer && (
<div className='break-all'>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
<div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
{content}
{/* Content */}
<div className='relative'>
{!answer && (
<Markdown
className={cn('!mt-2 !text-text-secondary', heighClassName)}
content={sign_content || content}
customDisallowedElements={['input']}
/>
)}
{answer && (
<div className='break-all'>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
<div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
{content}
</div>
</div>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
<div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
{answer}
</div>
</div>
</div>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
<div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
{answer}
)}
{/* Mask */}
<Mask className='absolute inset-x-0 bottom-0' />
</div>
{(showImages || showKeywords) && (
<div className='flex flex-col gap-y-3 pt-3'>
{showImages && (
<ImageList images={images} size='md' className='py-1' />
)}
{showKeywords && (
<div className='flex flex-col gap-y-1'>
<div className='text-xs font-medium uppercase text-text-tertiary'>{t(`${i18nPrefix}.keyword`)}</div>
<div className='flex flex-wrap gap-x-2'>
{keywords.map(keyword => (
<Tag key={keyword} text={keyword} />
))}
</div>
</div>
</div>
</div>
)}
{!isParentChildRetrieval && keywords && keywords.length > 0 && (
<div className='mt-6'>
<div className='text-xs font-medium uppercase text-text-tertiary'>{t(`${i18nPrefix}.keyword`)}</div>
<div className='mt-1 flex flex-wrap'>
{keywords.map(keyword => (
<Tag key={keyword} text={keyword} className='mr-2' />
))}
</div>
)}
</div>
)}
</div>

View File

@@ -0,0 +1,15 @@
import { RiHistoryLine } from '@remixicon/react'
import React from 'react'
import { useTranslation } from 'react-i18next'
const EmptyRecords = () => {
const { t } = useTranslation()
return <div className='rounded-2xl bg-workflow-process-bg p-5'>
<div className='flex h-10 w-10 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'>
<RiHistoryLine className='h-5 w-5 text-text-tertiary' />
</div>
<div className='my-2 text-[13px] font-medium leading-4 text-text-tertiary'>{t('datasetHitTesting.noRecentTip')}</div>
</div>
}
export default React.memo(EmptyRecords)

View File

@@ -0,0 +1,19 @@
import React from 'react'
import cn from '@/utils/classnames'
type MaskProps = {
className?: string
}
export const Mask = ({
className,
}: MaskProps) => {
return (
<div className={cn(
'h-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg',
className,
)} />
)
}
export default React.memo(Mask)

View File

@@ -0,0 +1,257 @@
import type { ChangeEvent } from 'react'
import React, { useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import {
RiEqualizer2Line,
RiPlayCircleLine,
} from '@remixicon/react'
import Image from 'next/image'
import Button from '@/app/components/base/button'
import { getIcon } from '@/app/components/datasets/common/retrieval-method-info'
import ModifyExternalRetrievalModal from '@/app/components/datasets/hit-testing/modify-external-retrieval-modal'
import cn from '@/utils/classnames'
import type {
Attachment,
ExternalKnowledgeBaseHitTestingRequest,
ExternalKnowledgeBaseHitTestingResponse,
HitTestingRequest,
HitTestingResponse,
Query,
} from '@/models/datasets'
import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'
import type { UseMutateAsyncFunction } from '@tanstack/react-query'
import ImageUploaderInRetrievalTesting from '@/app/components/datasets/common/image-uploader/image-uploader-in-retrieval-testing'
import Textarea from './textarea'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
import { v4 as uuid4 } from 'uuid'
type QueryInputProps = {
onUpdateList: () => void
setHitResult: (res: HitTestingResponse) => void
setExternalHitResult: (res: ExternalKnowledgeBaseHitTestingResponse) => void
loading: boolean
queries: Query[]
setQueries: (v: Query[]) => void
isExternal?: boolean
onClickRetrievalMethod: () => void
retrievalConfig: RetrievalConfig
isEconomy: boolean
onSubmit?: () => void
hitTestingMutation: UseMutateAsyncFunction<HitTestingResponse, Error, HitTestingRequest, unknown>
externalKnowledgeBaseHitTestingMutation: UseMutateAsyncFunction<
ExternalKnowledgeBaseHitTestingResponse,
Error,
ExternalKnowledgeBaseHitTestingRequest,
unknown
>
}
const QueryInput = ({
onUpdateList,
setHitResult,
setExternalHitResult,
loading,
queries,
setQueries,
isExternal = false,
onClickRetrievalMethod,
retrievalConfig,
isEconomy,
onSubmit: _onSubmit,
hitTestingMutation,
externalKnowledgeBaseHitTestingMutation,
}: QueryInputProps) => {
const { t } = useTranslation()
const isMultimodal = useDatasetDetailContextWithSelector(s => !!s.dataset?.is_multimodal)
const [isSettingsOpen, setIsSettingsOpen] = useState(false)
const [externalRetrievalSettings, setExternalRetrievalSettings] = useState({
top_k: 4,
score_threshold: 0.5,
score_threshold_enabled: false,
})
const text = useMemo(() => {
return queries.find(query => query.content_type === 'text_query')?.content ?? ''
}, [queries])
const images = useMemo(() => {
const imageQueries = queries
.filter(query => query.content_type === 'image_query')
.map(query => query.file_info)
.filter(Boolean) as Attachment[]
return imageQueries.map(item => ({
id: uuid4(),
name: item.name,
size: item.size,
mimeType: item.mime_type,
extension: item.extension,
sourceUrl: item.source_url,
uploadedId: item.id,
progress: 100,
})) || []
}, [queries])
const isAllUploaded = useMemo(() => {
return images.every(image => !!image.uploadedId)
}, [images])
const handleSaveExternalRetrievalSettings = useCallback((data: {
top_k: number
score_threshold: number
score_threshold_enabled: boolean
}) => {
setExternalRetrievalSettings(data)
setIsSettingsOpen(false)
}, [])
const handleTextChange = useCallback((event: ChangeEvent<HTMLTextAreaElement>) => {
const newQueries = [...queries]
const textQuery = newQueries.find(query => query.content_type === 'text_query')
if (!textQuery) {
newQueries.push({
content: event.target.value,
content_type: 'text_query',
file_info: null,
})
}
else {
textQuery.content = event.target.value
}
setQueries(newQueries)
}, [queries, setQueries])
const handleImageChange = useCallback((files: FileEntity[]) => {
let newQueries = [...queries]
newQueries = newQueries.filter(query => query.content_type !== 'image_query')
files.forEach((file) => {
newQueries.push({
content: file.sourceUrl || '',
content_type: 'image_query',
file_info: {
id: file.uploadedId || '',
mime_type: file.mimeType,
source_url: file.sourceUrl || '',
name: file.name,
size: file.size,
extension: file.extension,
},
})
})
setQueries(newQueries)
}, [queries, setQueries])
const onSubmit = useCallback(async () => {
await hitTestingMutation({
query: text,
attachment_ids: images.map(image => image.uploadedId),
retrieval_model: {
...retrievalConfig,
search_method: isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method,
},
}, {
onSuccess: (data) => {
setHitResult(data)
onUpdateList?.()
if (_onSubmit)
_onSubmit()
},
})
}, [text, retrievalConfig, isEconomy, hitTestingMutation, onUpdateList, _onSubmit, images, setHitResult])
const externalRetrievalTestingOnSubmit = useCallback(async () => {
await externalKnowledgeBaseHitTestingMutation({
query: text,
external_retrieval_model: {
top_k: externalRetrievalSettings.top_k,
score_threshold: externalRetrievalSettings.score_threshold,
score_threshold_enabled: externalRetrievalSettings.score_threshold_enabled,
},
}, {
onSuccess: (data) => {
setExternalHitResult(data)
onUpdateList?.()
},
})
}, [text, externalRetrievalSettings, externalKnowledgeBaseHitTestingMutation, onUpdateList, setExternalHitResult])
const retrievalMethod = isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method
const icon = <Image className='size-3.5 text-util-colors-purple-purple-600' src={getIcon(retrievalMethod)} alt='' />
const TextAreaComp = useMemo(() => {
return (
<Textarea
text={text}
handleTextChange={handleTextChange}
/>
)
}, [text, handleTextChange])
const ActionButtonComp = useMemo(() => {
return (
<Button
onClick={isExternal ? externalRetrievalTestingOnSubmit : onSubmit}
variant='primary'
loading={loading}
disabled={(text.length === 0 && images.length === 0) || text.length > 200 || (images.length > 0 && !isAllUploaded)}
className='w-[88px]'
>
<RiPlayCircleLine className='mr-1 size-4' />
{t('datasetHitTesting.input.testing')}
</Button>
)
}, [isExternal, externalRetrievalTestingOnSubmit, onSubmit, text, loading, t, images, isAllUploaded])
return (
<div className={cn('relative flex h-80 shrink-0 flex-col overflow-hidden rounded-xl bg-gradient-to-r from-components-input-border-active-prompt-1 to-components-input-border-active-prompt-2 p-0.5 shadow-xs')}>
<div className='flex h-full flex-col overflow-hidden rounded-[10px] bg-background-section-burn'>
<div className='relative flex shrink-0 items-center justify-between p-1.5 pb-1 pl-3'>
<span className='system-sm-semibold-uppercase text-text-secondary'>
{t('datasetHitTesting.input.title')}
</span>
{isExternal ? (
<Button
variant='secondary'
size='small'
onClick={() => setIsSettingsOpen(!isSettingsOpen)}
>
<RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
<div className='flex items-center justify-center gap-1 px-[3px]'>
<span className='system-xs-medium text-components-button-secondary-text'>{t('datasetHitTesting.settingTitle')}</span>
</div>
</Button>
) : (
<div
onClick={onClickRetrievalMethod}
className='flex h-7 cursor-pointer items-center space-x-0.5 rounded-lg border-[0.5px] border-components-button-secondary-bg bg-components-button-secondary-bg px-1.5 shadow-xs backdrop-blur-[5px] hover:bg-components-button-secondary-bg-hover'
>
{icon}
<div className='text-xs font-medium uppercase text-text-secondary'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
<RiEqualizer2Line className='size-4 text-components-menu-item-text'></RiEqualizer2Line>
</div>
)}
{
isSettingsOpen && (
<ModifyExternalRetrievalModal
onClose={() => setIsSettingsOpen(false)}
onSave={handleSaveExternalRetrievalSettings}
initialTopK={externalRetrievalSettings.top_k}
initialScoreThreshold={externalRetrievalSettings.score_threshold}
initialScoreThresholdEnabled={externalRetrievalSettings.score_threshold_enabled}
/>
)
}
</div>
<ImageUploaderInRetrievalTesting
textArea={TextAreaComp}
actionButton={ActionButtonComp}
onChange={handleImageChange}
value={images}
showUploader={isMultimodal}
className='grow'
actionAreaClassName='px-4 py-2 shrink-0 bg-background-default'
/>
</div>
</div>
)
}
export default QueryInput

View File

@@ -0,0 +1,61 @@
import type { ChangeEvent } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import { Corner } from '@/app/components/base/icons/src/vender/solid/shapes'
import Tooltip from '@/app/components/base/tooltip'
type TextareaProps = {
text: string
handleTextChange: (e: ChangeEvent<HTMLTextAreaElement>) => void
}
const Textarea = ({
text,
handleTextChange,
}: TextareaProps) => {
const { t } = useTranslation()
return (
<div className={cn(
'relative flex-1 overflow-hidden rounded-t-[10px] border-t-[0.5px] border-components-panel-border-subtle bg-background-default px-4 pb-0 pt-3',
text.length > 200 && 'border-state-destructive-active',
)}>
<textarea
className='system-md-regular h-full w-full resize-none border-none bg-transparent text-text-secondary caret-[#295EFF] placeholder:text-components-input-text-placeholder focus-visible:outline-none'
value={text}
onChange={handleTextChange}
placeholder={t('datasetHitTesting.input.placeholder') as string}
/>
<div className='absolute right-0 top-0 flex items-center'>
<Corner className={cn(
'text-background-section-burn',
text.length > 200 && 'text-util-colors-red-red-100',
)} />
{text.length > 200
? (
<Tooltip
popupContent={t('datasetHitTesting.input.countWarning')}
>
<div
className={cn('system-2xs-medium-uppercase bg-util-colors-red-red-100 py-1 pr-2 text-util-colors-red-red-600')}
>
{`${text.length}/200`}
</div>
</Tooltip>
)
: (
<div
className={cn(
'system-2xs-medium-uppercase bg-background-section-burn py-1 pr-2 text-text-tertiary',
)}
>
{`${text.length}/200`}
</div>
)}
</div>
</div>
)
}
export default React.memo(Textarea)

View File

@@ -0,0 +1,117 @@
import React, { useCallback, useMemo, useState } from 'react'
import useTimestamp from '@/hooks/use-timestamp'
import type { Attachment, HitTestingRecord, Query } from '@/models/datasets'
import { RiApps2Line, RiArrowDownLine, RiFocus2Line } from '@remixicon/react'
import { useTranslation } from 'react-i18next'
import ImageList from '../../common/image-list'
import cn from '@/utils/classnames'
type RecordsProps = {
records: HitTestingRecord[]
onClickRecord: (record: HitTestingRecord) => void
}
const Records = ({
records,
onClickRecord,
}: RecordsProps) => {
const { t } = useTranslation()
const { formatTime } = useTimestamp()
const [sortTimeOrder, setTimeOrder] = useState<'asc' | 'desc'>('desc')
const handleSortTime = useCallback(() => {
setTimeOrder(prev => prev === 'asc' ? 'desc' : 'asc')
}, [])
const sortedRecords = useMemo(() => {
return [...records].sort((a, b) => {
return sortTimeOrder === 'asc' ? a.created_at - b.created_at : b.created_at - a.created_at
})
}, [records, sortTimeOrder])
const getImageList = (queries: Query[]) => {
const imageQueries = queries
.filter(query => query.content_type === 'image_query')
.map(query => query.file_info)
.filter(Boolean) as Attachment[]
return imageQueries.map(image => ({
name: image.name,
mimeType: image.mime_type,
sourceUrl: image.source_url,
size: image.size,
extension: image.extension,
}))
}
return (
<div className='grow overflow-y-auto'>
<table className={'w-full border-collapse border-0 text-[13px] leading-4 text-text-secondary '}>
<thead className='sticky top-0 h-7 text-xs font-medium uppercase leading-7 text-text-tertiary backdrop-blur-[5px]'>
<tr>
<td className='rounded-l-lg bg-background-section-burn pl-3'>{t('datasetHitTesting.table.header.queryContent')}</td>
<td className='w-[128px] bg-background-section-burn pl-3'>{t('datasetHitTesting.table.header.source')}</td>
<td className='w-48 rounded-r-lg bg-background-section-burn pl-3'>
<div
className='flex cursor-pointer items-center'
onClick={handleSortTime}
>
{t('datasetHitTesting.table.header.time')}
<RiArrowDownLine
className={cn(
'ml-0.5 size-3.5',
sortTimeOrder === 'asc' ? 'rotate-180' : '',
)}
/>
</div>
</td>
</tr>
</thead>
<tbody>
{sortedRecords.map((record) => {
const { id, source, created_at, queries } = record
const SourceIcon = record.source === 'app' ? RiApps2Line : RiFocus2Line
const content = queries.find(query => query.content_type === 'text_query')?.content || ''
const images = getImageList(queries)
return (
<tr
key={id}
className='group cursor-pointer border-b border-divider-subtle hover:bg-background-default-hover'
onClick={() => onClickRecord(record)}
>
<td className='max-w-xs p-3 pr-2'>
<div className='flex flex-col gap-y-1'>
{content && (
<div className='line-clamp-2'>
{content}
</div>
)}
{images.length > 0 && (
<ImageList
images={images}
size='md'
className='py-1'
limit={5}
/>
)}
</div>
</td>
<td className='w-[128px] p-3 pr-2'>
<div className='flex items-center'>
<SourceIcon className='mr-1 size-4 text-text-tertiary' />
<span className='capitalize'>{source.replace('_', ' ').replace('hit testing', 'retrieval test')}</span>
</div>
</td>
<td className='w-48 p-3 pr-2'>
{formatTime(created_at, t('datasetHitTesting.dateTimeFormat') as string)}
</td>
</tr>
)
})}
</tbody>
</table>
</div>
)
}
export default React.memo(Records)

View File

@@ -1,6 +1,5 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import React, { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
import { useBoolean } from 'ahooks'
@@ -14,17 +13,18 @@ import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader
import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
import { extensionToFileType } from '@/app/components/datasets/hit-testing/utils/extension-to-file-type'
import { Markdown } from '@/app/components/base/markdown'
import ImageList from '../../common/image-list'
const i18nPrefix = 'datasetHitTesting'
type Props = {
type ResultItemProps = {
payload: HitTesting
}
const ResultItem: FC<Props> = ({
const ResultItem = ({
payload,
}) => {
}: ResultItemProps) => {
const { t } = useTranslation()
const { segment, score, child_chunks } = payload
const { segment, score, child_chunks, files } = payload
const data = segment
const { position, word_count, content, sign_content, keywords, document } = data
const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
@@ -40,6 +40,17 @@ const ResultItem: FC<Props> = ({
setFalse: hideDetailModal,
}] = useBoolean(false)
const images = useMemo(() => {
if (!files) return []
return files.map(file => ({
name: file.name,
mimeType: file.mime_type,
sourceUrl: file.source_url,
size: file.size,
extension: file.extension,
}))
}, [files])
return (
<div className={cn('cursor-pointer rounded-xl bg-chat-bubble-bg pt-3 hover:shadow-lg')} onClick={showDetailModal}>
{/* Meta info */}
@@ -47,11 +58,14 @@ const ResultItem: FC<Props> = ({
{/* Main */}
<div className='mt-1 px-3'>
<Markdown
{<Markdown
className='line-clamp-2'
content={sign_content || content}
customDisallowedElements={['input']}
/>
/>}
{images.length > 0 && (
<ImageList images={images} size='md' className='py-1' />
)}
{isParentChildRetrieval && (
<div className='mt-1'>
<div