admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: multimodal support (image) (#27793)

Browse Source 
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Wu Tianwei
2025-12-09 11:44:50 +08:00
committed by GitHub
parent a44b800c85
commit 14d1b3f9b3
77 changed files with 2932 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
web/app/components/datasets/documents/detail/completed/common/action-buttons.tsx
View File

@@ -13,6 +13,7 @@ type IActionButtonsProps = {
actionType?: 'edit' | 'add'
handleRegeneration?: () => void
isChildChunk?: boolean
showRegenerationButton?: boolean
}
const ActionButtons: FC<IActionButtonsProps> = ({
@@ -22,6 +23,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
actionType = 'edit',
handleRegeneration,
isChildChunk = false,
showRegenerationButton = true,
}) => {
const { t } = useTranslation()
const docForm = useDocumentContext(s => s.docForm)
@@ -54,7 +56,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
<span className='system-kbd rounded-[4px] bg-components-kbd-bg-gray px-[1px] text-text-tertiary'>ESC</span>
</div>
</Button>
{(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk)
{(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk && showRegenerationButton)
? <Button
onClick={handleRegeneration}
disabled={loading}

4
web/app/components/datasets/documents/detail/completed/common/drawer.tsx
View File

@@ -42,6 +42,7 @@ const Drawer = ({
if (!panelContent) return false
const chunks = document.querySelectorAll('.chunk-card')
const childChunks = document.querySelectorAll('.child-chunk')
const imagePreviewer = document.querySelector('.image-previewer')
const isClickOnChunk = Array.from(chunks).some((chunk) => {
return chunk && chunk.contains(target)
})
@@ -50,7 +51,8 @@ const Drawer = ({
})
const reopenChunkDetail = (currSegment.showModal && isClickOnChildChunk)
|| (currChildChunk.showModal && isClickOnChunk && !isClickOnChildChunk) || (!isClickOnChunk && !isClickOnChildChunk)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail)
const isClickOnImagePreviewer = imagePreviewer && imagePreviewer.contains(target)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail) && !isClickOnImagePreviewer
}, [currSegment, currChildChunk, needCheckChunks])
const onDownCapture = useCallback((e: PointerEvent) => {

2
web/app/components/datasets/documents/detail/completed/common/full-screen-drawer.tsx
View File

@@ -28,7 +28,7 @@ const FullScreenDrawer = ({
panelClassName={cn(
fullScreen
? 'w-full'
: 'w-[560px] pb-2 pr-2 pt-16',
: 'w-[568px] pb-2 pr-2 pt-16',
)}
panelContentClassName={cn(
'bg-components-panel-bg',

12
web/app/components/datasets/documents/detail/completed/index.tsx
View File

@@ -47,6 +47,7 @@ import {
} from '@/service/knowledge/use-segment'
import { useInvalid } from '@/service/use-base'
import { noop } from 'lodash-es'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
const DEFAULT_LIMIT = 10
@@ -318,9 +319,10 @@ const Completed: FC<ICompletedProps> = ({
question: string,
answer: string,
keywords: string[],
attachments: FileEntity[],
needRegenerate = false,
) => {
const params: SegmentUpdater = { content: '' }
const params: SegmentUpdater = { content: '', attachment_ids: [] }
if (docForm === ChunkingMode.qa) {
if (!question.trim())
return notify({ type: 'error', message: t('datasetDocuments.segment.questionEmpty') })
@@ -340,6 +342,13 @@ const Completed: FC<ICompletedProps> = ({
if (keywords.length)
params.keywords = keywords
if (attachments.length) {
const notAllUploaded = attachments.some(item => !item.uploadedId)
if (notAllUploaded)
return notify({ type: 'error', message: t('datasetDocuments.segment.allFilesUploaded') })
params.attachment_ids = attachments.map(item => item.uploadedId!)
}
if (needRegenerate)
params.regenerate_child_chunks = needRegenerate
@@ -355,6 +364,7 @@ const Completed: FC<ICompletedProps> = ({
seg.content = res.data.content
seg.sign_content = res.data.sign_content
seg.keywords = res.data.keywords
seg.attachments = res.data.attachments
seg.word_count = res.data.word_count
seg.hit_count = res.data.hit_count
seg.enabled = res.data.enabled

13
web/app/components/datasets/documents/detail/completed/segment-card/index.tsx
View File

@@ -18,6 +18,7 @@ import Badge from '@/app/components/base/badge'
import { isAfter } from '@/utils/time'
import Tooltip from '@/app/components/base/tooltip'
import ChunkContent from './chunk-content'
import ImageList from '@/app/components/datasets/common/image-list'
type ISegmentCardProps = {
loading: boolean
@@ -67,6 +68,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
child_chunks = [],
created_at,
updated_at,
attachments = [],
} = detail as Required<ISegmentCardProps>['detail']
const [showModal, setShowModal] = useState(false)
const docForm = useDocumentContext(s => s.docForm)
@@ -112,6 +114,16 @@ const SegmentCard: FC<ISegmentCardProps> = ({
return isParentChildMode ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
}, [isParentChildMode, t])
const images = useMemo(() => {
return attachments.map(attachment => ({
name: attachment.name,
mimeType: attachment.mime_type,
sourceUrl: attachment.source_url,
size: attachment.size,
extension: attachment.extension,
}))
}, [attachments])
if (loading)
return <ParentChunkCardSkeleton />
@@ -214,6 +226,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
isFullDocMode={isFullDocMode}
className={contentOpacity}
/>
{images.length > 0 && <ImageList images={images} size='md' className='py-1' />}
{isGeneralMode && <div className={cn('flex flex-wrap items-center gap-2 py-1.5', contentOpacity)}>
{keywords?.map(keyword => <Tag key={keyword} text={keyword} />)}
</div>}

87
web/app/components/datasets/documents/detail/completed/segment-detail.tsx
View File

@@ -19,11 +19,21 @@ import { formatNumber } from '@/utils/format'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { IndexingType } from '../../../create/step-two'
import { IndexingType } from '@/app/components/datasets/create/step-two'
import ImageUploaderInChunk from '@/app/components/datasets/common/image-uploader/image-uploader-in-chunk'
import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
import { v4 as uuid4 } from 'uuid'
type ISegmentDetailProps = {
segInfo?: Partial<SegmentDetailModel> & { id: string }
onUpdate: (segmentId: string, q: string, a: string, k: string[], needRegenerate?: boolean) => void
onUpdate: (
segmentId: string,
q: string,
a: string,
k: string[],
attachments: FileEntity[],
needRegenerate?: boolean,
) => void
onCancel: () => void
isEditMode?: boolean
docForm: ChunkingMode
@@ -44,6 +54,18 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const { t } = useTranslation()
const [question, setQuestion] = useState(isEditMode ? segInfo?.content || '' : segInfo?.sign_content || '')
const [answer, setAnswer] = useState(segInfo?.answer || '')
const [attachments, setAttachments] = useState<FileEntity[]>(() => {
return segInfo?.attachments?.map(item => ({
id: uuid4(),
name: item.name,
size: item.size,
mimeType: item.mime_type,
extension: item.extension,
sourceUrl: item.source_url,
uploadedId: item.id,
progress: 100,
})) || []
})
const [keywords, setKeywords] = useState<string[]>(segInfo?.keywords || [])
const { eventEmitter } = useEventEmitterContextContext()
const [loading, setLoading] = useState(false)
@@ -52,6 +74,7 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen)
const parentMode = useDocumentContext(s => s.parentMode)
const indexingTechnique = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
const runtimeMode = useDatasetDetailContextWithSelector(s => s.dataset?.runtime_mode)
eventEmitter?.useSubscription((v) => {
if (v === 'update-segment')
@@ -65,8 +88,8 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
}, [onCancel])
const handleSave = useCallback(() => {
onUpdate(segInfo?.id || '', question, answer, keywords)
}, [onUpdate, segInfo?.id, question, answer, keywords])
onUpdate(segInfo?.id || '', question, answer, keywords, attachments)
}, [onUpdate, segInfo?.id, question, answer, keywords, attachments])
const handleRegeneration = useCallback(() => {
setShowRegenerationModal(true)
@@ -85,8 +108,12 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
}, [onCancel, onModalStateChange])
const onConfirmRegeneration = useCallback(() => {
onUpdate(segInfo?.id || '', question, answer, keywords, true)
}, [onUpdate, segInfo?.id, question, answer, keywords])
onUpdate(segInfo?.id || '', question, answer, keywords, attachments, true)
}, [onUpdate, segInfo?.id, question, answer, keywords, attachments])
const onAttachmentsChange = useCallback((attachments: FileEntity[]) => {
setAttachments(attachments)
}, [])
const wordCountText = useMemo(() => {
const contentLength = docForm === ChunkingMode.qa ? (question.length + answer.length) : question.length
@@ -102,7 +129,10 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
return (
<div className={'flex h-full flex-col'}>
<div className={cn('flex items-center justify-between', fullScreen ? 'border border-divider-subtle py-3 pl-6 pr-4' : 'pl-4 pr-3 pt-3')}>
<div className={cn(
'flex shrink-0 items-center justify-between',
fullScreen ? 'border border-divider-subtle py-3 pl-6 pr-4' : 'pl-4 pr-3 pt-3',
)}>
<div className='flex flex-col'>
<div className='system-xl-semibold text-text-primary'>{titleText}</div>
<div className='flex items-center gap-x-2'>
@@ -119,12 +149,17 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
handleRegeneration={handleRegeneration}
handleSave={handleSave}
loading={loading}
showRegenerationButton={runtimeMode === 'general'}
/>
<Divider type='vertical' className='ml-4 mr-2 h-3.5 bg-divider-regular' />
</>
)}
<div className='mr-1 flex h-8 w-8 cursor-pointer items-center justify-center p-1.5' onClick={toggleFullScreen}>
{fullScreen ? <RiCollapseDiagonalLine className='h-4 w-4 text-text-tertiary' /> : <RiExpandDiagonalLine className='h-4 w-4 text-text-tertiary' />}
{
fullScreen
? <RiCollapseDiagonalLine className='h-4 w-4 text-text-tertiary' />
: <RiExpandDiagonalLine className='h-4 w-4 text-text-tertiary' />
}
</div>
<div className='flex h-8 w-8 cursor-pointer items-center justify-center p-1.5' onClick={onCancel}>
<RiCloseLine className='h-4 w-4 text-text-tertiary' />
@@ -132,11 +167,14 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
</div>
</div>
<div className={cn(
'flex grow',
'flex h-0 grow',
fullScreen ? 'w-full flex-row justify-center gap-x-8 px-6 pt-6' : 'flex-col gap-y-1 px-4 py-3',
!isEditMode && 'overflow-hidden pb-0',
!isEditMode && 'pb-0',
)}>
<div className={cn(isEditMode ? 'overflow-hidden whitespace-pre-line break-all' : 'overflow-y-auto', fullScreen ? 'w-1/2' : 'grow')}>
<div className={cn(
isEditMode ? 'overflow-hidden whitespace-pre-line break-all' : 'overflow-y-auto',
fullScreen ? 'w-1/2' : 'h-0 grow',
)}>
<ChunkContent
docForm={docForm}
question={question}
@@ -146,14 +184,24 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
isEditMode={isEditMode}
/>
</div>
{isECOIndexing && <Keywords
className={fullScreen ? 'w-1/5' : ''}
actionType={isEditMode ? 'edit' : 'view'}
segInfo={segInfo}
keywords={keywords}
isEditMode={isEditMode}
onKeywordsChange={keywords => setKeywords(keywords)}
/>}
<div className={cn('flex shrink-0 flex-col', fullScreen ? 'w-[320px] gap-y-2' : 'w-full gap-y-1')}>
<ImageUploaderInChunk
disabled={!isEditMode}
value={attachments}
onChange={onAttachmentsChange}
/>
{isECOIndexing && (
<Keywords
className='w-full'
actionType={isEditMode ? 'edit' : 'view'}
segInfo={segInfo}
keywords={keywords}
isEditMode={isEditMode}
onKeywordsChange={keywords => setKeywords(keywords)}
/>
)}
</div>
</div>
{isEditMode && !fullScreen && (
<div className='flex items-center justify-end border-t-[1px] border-t-divider-subtle p-4 pt-3'>
@@ -162,6 +210,7 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
handleRegeneration={handleRegeneration}
handleSave={handleSave}
loading={loading}
showRegenerationButton={runtimeMode === 'general'}
/>
</div>
)}