feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wu Tianwei
2025-12-09 11:44:50 +08:00
committed by GitHub
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions

View File

@@ -21,8 +21,6 @@ import dynamic from 'next/dynamic'
const SimplePieChart = dynamic(() => import('@/app/components/base/simple-pie-chart'), { ssr: false })
const FILES_NUMBER_LIMIT = 20
export type LocalFileProps = {
allowedExtensions: string[]
notSupportBatchUpload?: boolean
@@ -64,10 +62,11 @@ const LocalFile = ({
.join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
}, [locale, allowedExtensions])
const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`)
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
file_size_limit: 15,
batch_count_limit: 5,
}, [fileUploadConfigResponse])
const fileUploadConfig = useMemo(() => ({
file_size_limit: fileUploadConfigResponse?.file_size_limit ?? 15,
batch_count_limit: fileUploadConfigResponse?.batch_count_limit ?? 5,
file_upload_limit: fileUploadConfigResponse?.file_upload_limit ?? 5,
}), [fileUploadConfigResponse])
const updateFile = useCallback((fileItem: FileItem, progress: number, list: FileItem[]) => {
const { setLocalFileList } = dataSourceStore.getState()
@@ -186,11 +185,12 @@ const LocalFile = ({
}, [fileUploadConfig, uploadBatchFiles])
const initialUpload = useCallback((files: File[]) => {
const filesCountLimit = fileUploadConfig.file_upload_limit
if (!files.length)
return false
if (files.length + localFileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
if (files.length + localFileList.length > filesCountLimit && !IS_CE_EDITION) {
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: filesCountLimit }) })
return false
}
@@ -203,7 +203,7 @@ const LocalFile = ({
updateFileList(newFiles)
fileListRef.current = newFiles
uploadMultipleFiles(preparedFiles)
}, [updateFileList, uploadMultipleFiles, notify, t, localFileList])
}, [fileUploadConfig.file_upload_limit, localFileList.length, updateFileList, uploadMultipleFiles, notify, t])
const handleDragEnter = (e: DragEvent) => {
e.preventDefault()
@@ -250,9 +250,10 @@ const LocalFile = ({
updateFileList([...fileListRef.current])
}
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const files = [...(e.target.files ?? [])] as File[]
let files = [...(e.target.files ?? [])] as File[]
files = files.slice(0, fileUploadConfig.batch_count_limit)
initialUpload(files.filter(isValid))
}, [isValid, initialUpload])
}, [isValid, initialUpload, fileUploadConfig.batch_count_limit])
const { theme } = useTheme()
const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme])
@@ -305,6 +306,7 @@ const LocalFile = ({
size: fileUploadConfig.file_size_limit,
supportTypes: supportTypesShowNames,
batchCount: notSupportBatchUpload ? 1 : fileUploadConfig.batch_count_limit,
totalCount: fileUploadConfig.file_upload_limit,
})}</div>
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
</div>

View File

@@ -13,6 +13,7 @@ type IActionButtonsProps = {
actionType?: 'edit' | 'add'
handleRegeneration?: () => void
isChildChunk?: boolean
showRegenerationButton?: boolean
}
const ActionButtons: FC<IActionButtonsProps> = ({
@@ -22,6 +23,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
actionType = 'edit',
handleRegeneration,
isChildChunk = false,
showRegenerationButton = true,
}) => {
const { t } = useTranslation()
const docForm = useDocumentContext(s => s.docForm)
@@ -54,7 +56,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
<span className='system-kbd rounded-[4px] bg-components-kbd-bg-gray px-[1px] text-text-tertiary'>ESC</span>
</div>
</Button>
{(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk)
{(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk && showRegenerationButton)
? <Button
onClick={handleRegeneration}
disabled={loading}

View File

@@ -42,6 +42,7 @@ const Drawer = ({
if (!panelContent) return false
const chunks = document.querySelectorAll('.chunk-card')
const childChunks = document.querySelectorAll('.child-chunk')
const imagePreviewer = document.querySelector('.image-previewer')
const isClickOnChunk = Array.from(chunks).some((chunk) => {
return chunk && chunk.contains(target)
})
@@ -50,7 +51,8 @@ const Drawer = ({
})
const reopenChunkDetail = (currSegment.showModal && isClickOnChildChunk)
|| (currChildChunk.showModal && isClickOnChunk && !isClickOnChildChunk) || (!isClickOnChunk && !isClickOnChildChunk)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail)
const isClickOnImagePreviewer = imagePreviewer && imagePreviewer.contains(target)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail) && !isClickOnImagePreviewer
}, [currSegment, currChildChunk, needCheckChunks])
const onDownCapture = useCallback((e: PointerEvent) => {

View File

@@ -28,7 +28,7 @@ const FullScreenDrawer = ({
panelClassName={cn(
fullScreen
? 'w-full'
: 'w-[560px] pb-2 pr-2 pt-16',
: 'w-[568px] pb-2 pr-2 pt-16',
)}
panelContentClassName={cn(
'bg-components-panel-bg',

View File

@@ -47,6 +47,7 @@ import {
} from '@/service/knowledge/use-segment'
import { useInvalid } from '@/service/use-base'
import { noop } from 'lodash-es'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
const DEFAULT_LIMIT = 10
@@ -318,9 +319,10 @@ const Completed: FC<ICompletedProps> = ({
question: string,
answer: string,
keywords: string[],
attachments: FileEntity[],
needRegenerate = false,
) => {
const params: SegmentUpdater = { content: '' }
const params: SegmentUpdater = { content: '', attachment_ids: [] }
if (docForm === ChunkingMode.qa) {
if (!question.trim())
return notify({ type: 'error', message: t('datasetDocuments.segment.questionEmpty') })
@@ -340,6 +342,13 @@ const Completed: FC<ICompletedProps> = ({
if (keywords.length)
params.keywords = keywords
if (attachments.length) {
const notAllUploaded = attachments.some(item => !item.uploadedId)
if (notAllUploaded)
return notify({ type: 'error', message: t('datasetDocuments.segment.allFilesUploaded') })
params.attachment_ids = attachments.map(item => item.uploadedId!)
}
if (needRegenerate)
params.regenerate_child_chunks = needRegenerate
@@ -355,6 +364,7 @@ const Completed: FC<ICompletedProps> = ({
seg.content = res.data.content
seg.sign_content = res.data.sign_content
seg.keywords = res.data.keywords
seg.attachments = res.data.attachments
seg.word_count = res.data.word_count
seg.hit_count = res.data.hit_count
seg.enabled = res.data.enabled

View File

@@ -18,6 +18,7 @@ import Badge from '@/app/components/base/badge'
import { isAfter } from '@/utils/time'
import Tooltip from '@/app/components/base/tooltip'
import ChunkContent from './chunk-content'
import ImageList from '@/app/components/datasets/common/image-list'
type ISegmentCardProps = {
loading: boolean
@@ -67,6 +68,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
child_chunks = [],
created_at,
updated_at,
attachments = [],
} = detail as Required<ISegmentCardProps>['detail']
const [showModal, setShowModal] = useState(false)
const docForm = useDocumentContext(s => s.docForm)
@@ -112,6 +114,16 @@ const SegmentCard: FC<ISegmentCardProps> = ({
return isParentChildMode ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
}, [isParentChildMode, t])
const images = useMemo(() => {
return attachments.map(attachment => ({
name: attachment.name,
mimeType: attachment.mime_type,
sourceUrl: attachment.source_url,
size: attachment.size,
extension: attachment.extension,
}))
}, [attachments])
if (loading)
return <ParentChunkCardSkeleton />
@@ -214,6 +226,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
isFullDocMode={isFullDocMode}
className={contentOpacity}
/>
{images.length > 0 && <ImageList images={images} size='md' className='py-1' />}
{isGeneralMode && <div className={cn('flex flex-wrap items-center gap-2 py-1.5', contentOpacity)}>
{keywords?.map(keyword => <Tag key={keyword} text={keyword} />)}
</div>}

View File

@@ -19,11 +19,21 @@ import { formatNumber } from '@/utils/format'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { IndexingType } from '../../../create/step-two'
import { IndexingType } from '@/app/components/datasets/create/step-two'
import ImageUploaderInChunk from '@/app/components/datasets/common/image-uploader/image-uploader-in-chunk'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
import { v4 as uuid4 } from 'uuid'
type ISegmentDetailProps = {
segInfo?: Partial<SegmentDetailModel> & { id: string }
onUpdate: (segmentId: string, q: string, a: string, k: string[], needRegenerate?: boolean) => void
onUpdate: (
segmentId: string,
q: string,
a: string,
k: string[],
attachments: FileEntity[],
needRegenerate?: boolean,
) => void
onCancel: () => void
isEditMode?: boolean
docForm: ChunkingMode
@@ -44,6 +54,18 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const { t } = useTranslation()
const [question, setQuestion] = useState(isEditMode ? segInfo?.content || '' : segInfo?.sign_content || '')
const [answer, setAnswer] = useState(segInfo?.answer || '')
const [attachments, setAttachments] = useState<FileEntity[]>(() => {
return segInfo?.attachments?.map(item => ({
id: uuid4(),
name: item.name,
size: item.size,
mimeType: item.mime_type,
extension: item.extension,
sourceUrl: item.source_url,
uploadedId: item.id,
progress: 100,
})) || []
})
const [keywords, setKeywords] = useState<string[]>(segInfo?.keywords || [])
const { eventEmitter } = useEventEmitterContextContext()
const [loading, setLoading] = useState(false)
@@ -52,6 +74,7 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen)
const parentMode = useDocumentContext(s => s.parentMode)
const indexingTechnique = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
const runtimeMode = useDatasetDetailContextWithSelector(s => s.dataset?.runtime_mode)
eventEmitter?.useSubscription((v) => {
if (v === 'update-segment')
@@ -65,8 +88,8 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
}, [onCancel])
const handleSave = useCallback(() => {
onUpdate(segInfo?.id || '', question, answer, keywords)
}, [onUpdate, segInfo?.id, question, answer, keywords])
onUpdate(segInfo?.id || '', question, answer, keywords, attachments)
}, [onUpdate, segInfo?.id, question, answer, keywords, attachments])
const handleRegeneration = useCallback(() => {
setShowRegenerationModal(true)
@@ -85,8 +108,12 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
}, [onCancel, onModalStateChange])
const onConfirmRegeneration = useCallback(() => {
onUpdate(segInfo?.id || '', question, answer, keywords, true)
}, [onUpdate, segInfo?.id, question, answer, keywords])
onUpdate(segInfo?.id || '', question, answer, keywords, attachments, true)
}, [onUpdate, segInfo?.id, question, answer, keywords, attachments])
const onAttachmentsChange = useCallback((attachments: FileEntity[]) => {
setAttachments(attachments)
}, [])
const wordCountText = useMemo(() => {
const contentLength = docForm === ChunkingMode.qa ? (question.length + answer.length) : question.length
@@ -102,7 +129,10 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
return (
<div className={'flex h-full flex-col'}>
<div className={cn('flex items-center justify-between', fullScreen ? 'border border-divider-subtle py-3 pl-6 pr-4' : 'pl-4 pr-3 pt-3')}>
<div className={cn(
'flex shrink-0 items-center justify-between',
fullScreen ? 'border border-divider-subtle py-3 pl-6 pr-4' : 'pl-4 pr-3 pt-3',
)}>
<div className='flex flex-col'>
<div className='system-xl-semibold text-text-primary'>{titleText}</div>
<div className='flex items-center gap-x-2'>
@@ -119,12 +149,17 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
handleRegeneration={handleRegeneration}
handleSave={handleSave}
loading={loading}
showRegenerationButton={runtimeMode === 'general'}
/>
<Divider type='vertical' className='ml-4 mr-2 h-3.5 bg-divider-regular' />
</>
)}
<div className='mr-1 flex h-8 w-8 cursor-pointer items-center justify-center p-1.5' onClick={toggleFullScreen}>
{fullScreen ? <RiCollapseDiagonalLine className='h-4 w-4 text-text-tertiary' /> : <RiExpandDiagonalLine className='h-4 w-4 text-text-tertiary' />}
{
fullScreen
? <RiCollapseDiagonalLine className='h-4 w-4 text-text-tertiary' />
: <RiExpandDiagonalLine className='h-4 w-4 text-text-tertiary' />
}
</div>
<div className='flex h-8 w-8 cursor-pointer items-center justify-center p-1.5' onClick={onCancel}>
<RiCloseLine className='h-4 w-4 text-text-tertiary' />
@@ -132,11 +167,14 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
</div>
</div>
<div className={cn(
'flex grow',
'flex h-0 grow',
fullScreen ? 'w-full flex-row justify-center gap-x-8 px-6 pt-6' : 'flex-col gap-y-1 px-4 py-3',
!isEditMode && 'overflow-hidden pb-0',
!isEditMode && 'pb-0',
)}>
<div className={cn(isEditMode ? 'overflow-hidden whitespace-pre-line break-all' : 'overflow-y-auto', fullScreen ? 'w-1/2' : 'grow')}>
<div className={cn(
isEditMode ? 'overflow-hidden whitespace-pre-line break-all' : 'overflow-y-auto',
fullScreen ? 'w-1/2' : 'h-0 grow',
)}>
<ChunkContent
docForm={docForm}
question={question}
@@ -146,14 +184,24 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
isEditMode={isEditMode}
/>
</div>
{isECOIndexing && <Keywords
className={fullScreen ? 'w-1/5' : ''}
actionType={isEditMode ? 'edit' : 'view'}
segInfo={segInfo}
keywords={keywords}
isEditMode={isEditMode}
onKeywordsChange={keywords => setKeywords(keywords)}
/>}
<div className={cn('flex shrink-0 flex-col', fullScreen ? 'w-[320px] gap-y-2' : 'w-full gap-y-1')}>
<ImageUploaderInChunk
disabled={!isEditMode}
value={attachments}
onChange={onAttachmentsChange}
/>
{isECOIndexing && (
<Keywords
className='w-full'
actionType={isEditMode ? 'edit' : 'view'}
segInfo={segInfo}
keywords={keywords}
isEditMode={isEditMode}
onKeywordsChange={keywords => setKeywords(keywords)}
/>
)}
</div>
</div>
{isEditMode && !fullScreen && (
<div className='flex items-center justify-end border-t-[1px] border-t-divider-subtle p-4 pt-3'>
@@ -162,6 +210,7 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
handleRegeneration={handleRegeneration}
handleSave={handleSave}
loading={loading}
showRegenerationButton={runtimeMode === 'general'}
/>
</div>
)}

View File

@@ -21,6 +21,8 @@ import Divider from '@/app/components/base/divider'
import { useAddSegment } from '@/service/knowledge/use-segment'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { IndexingType } from '../../create/step-two'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
import ImageUploaderInChunk from '@/app/components/datasets/common/image-uploader/image-uploader-in-chunk'
type NewSegmentModalProps = {
onCancel: () => void
@@ -39,6 +41,7 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
const { notify } = useContext(ToastContext)
const [question, setQuestion] = useState('')
const [answer, setAnswer] = useState('')
const [attachments, setAttachments] = useState<FileEntity[]>([])
const { datasetId, documentId } = useParams<{ datasetId: string; documentId: string }>()
const [keywords, setKeywords] = useState<string[]>([])
const [loading, setLoading] = useState(false)
@@ -49,6 +52,7 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
const { appSidebarExpand } = useAppStore(useShallow(state => ({
appSidebarExpand: state.appSidebarExpand,
})))
const [imageUploaderKey, setImageUploaderKey] = useState(Date.now())
const refreshTimer = useRef<any>(null)
const CustomButton = useMemo(() => (
@@ -71,10 +75,14 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
onCancel()
}, [onCancel, addAnother])
const onAttachmentsChange = useCallback((attachments: FileEntity[]) => {
setAttachments(attachments)
}, [])
const { mutateAsync: addSegment } = useAddSegment()
const handleSave = useCallback(async () => {
const params: SegmentUpdater = { content: '' }
const params: SegmentUpdater = { content: '', attachment_ids: [] }
if (docForm === ChunkingMode.qa) {
if (!question.trim()) {
return notify({
@@ -106,6 +114,9 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
if (keywords?.length)
params.keywords = keywords
if (attachments.length)
params.attachment_ids = attachments.filter(item => Boolean(item.uploadedId)).map(item => item.uploadedId!)
setLoading(true)
await addSegment({ datasetId, documentId, body: params }, {
onSuccess() {
@@ -119,6 +130,8 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
handleCancel('add')
setQuestion('')
setAnswer('')
setAttachments([])
setImageUploaderKey(Date.now())
setKeywords([])
refreshTimer.current = setTimeout(() => {
onSave()
@@ -128,7 +141,7 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
setLoading(false)
},
})
}, [docForm, keywords, addSegment, datasetId, documentId, question, answer, notify, t, appSidebarExpand, CustomButton, handleCancel, onSave])
}, [docForm, keywords, addSegment, datasetId, documentId, question, answer, attachments, notify, t, appSidebarExpand, CustomButton, handleCancel, onSave])
const wordCountText = useMemo(() => {
const count = docForm === ChunkingMode.qa ? (question.length + answer.length) : question.length
@@ -187,13 +200,22 @@ const NewSegmentModal: FC<NewSegmentModalProps> = ({
isEditMode={true}
/>
</div>
{isECOIndexing && <Keywords
className={fullScreen ? 'w-1/5' : ''}
actionType='add'
keywords={keywords}
isEditMode={true}
onKeywordsChange={keywords => setKeywords(keywords)}
/>}
<div className={classNames('flex flex-col', fullScreen ? 'w-[320px] gap-y-2' : 'w-full gap-y-1')}>
<ImageUploaderInChunk
key={imageUploaderKey}
value={attachments}
onChange={onAttachmentsChange}
/>
{isECOIndexing && (
<Keywords
className={fullScreen ? 'w-1/5' : ''}
actionType='add'
keywords={keywords}
isEditMode={true}
onKeywordsChange={keywords => setKeywords(keywords)}
/>
)}
</div>
</div>
{!fullScreen && (
<div className='flex items-center justify-between border-t-[1px] border-t-divider-subtle p-4 pt-3'>

View File

@@ -2,9 +2,9 @@
import type { FC } from 'react'
import React, { useCallback, useEffect, useMemo, useState } from 'react'
import { useBoolean } from 'ahooks'
import { ArrowDownIcon } from '@heroicons/react/24/outline'
import { pick, uniq } from 'lodash-es'
import {
RiArrowDownLine,
RiEditLine,
RiGlobalLine,
} from '@remixicon/react'
@@ -181,8 +181,8 @@ const DocumentList: FC<IDocumentListProps> = ({
return (
<div className='flex cursor-pointer items-center hover:text-text-secondary' onClick={() => handleSort(field)}>
{label}
<ArrowDownIcon
className={cn('ml-0.5 h-3 w-3 stroke-current stroke-2 transition-all',
<RiArrowDownLine
className={cn('ml-0.5 h-3 w-3 transition-all',
isActive ? 'text-text-tertiary' : 'text-text-disabled',
isActive && !isDesc ? 'rotate-180' : '',
)}