feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/datasets/hit-testing/components/chunk-detail-modal.tsx

@@ -1,6 +1,5 @@
 'use client'
-import type { FC } from 'react'
-import React from 'react'
+import React, { useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { SegmentIndexTag } from '../../documents/detail/completed/common/segment-index-tag'
 import Dot from '../../documents/detail/completed/common/dot'
@@ -13,25 +12,42 @@ import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader
 import cn from '@/utils/classnames'
 import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
 import { Markdown } from '@/app/components/base/markdown'
+import ImageList from '../../common/image-list'
+import Mask from './mask'
 
 const i18nPrefix = 'datasetHitTesting'
 
-type Props = {
+type ChunkDetailModalProps = {
   payload: HitTesting
   onHide: () => void
 }
 
-const ChunkDetailModal: FC<Props> = ({
+const ChunkDetailModal = ({
   payload,
   onHide,
-}) => {
+}: ChunkDetailModalProps) => {
   const { t } = useTranslation()
-  const { segment, score, child_chunks } = payload
+  const { segment, score, child_chunks, files } = payload
   const { position, content, sign_content, keywords, document, answer } = segment
   const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
   const extension = document.name.split('.').slice(-1)[0] as FileAppearanceTypeEnum
   const heighClassName = isParentChildRetrieval ? 'h-[min(627px,_80vh)] overflow-y-auto' : 'h-[min(539px,_80vh)] overflow-y-auto'
   const labelPrefix = isParentChildRetrieval ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
+
+  const images = useMemo(() => {
+    if (!files) return []
+    return files.map(file => ({
+      name: file.name,
+      mimeType: file.mime_type,
+      sourceUrl: file.source_url,
+      size: file.size,
+      extension: file.extension,
+    }))
+  }, [files])
+
+  const showImages = images.length > 0
+  const showKeywords = !isParentChildRetrieval && keywords && keywords.length > 0
+
   return (
     <Modal
       title={t(`${i18nPrefix}.chunkDetail`)}
@@ -58,37 +74,49 @@ const ChunkDetailModal: FC<Props> = ({
             </div>
             <Score value={score} />
           </div>
-          {!answer && (
-            <Markdown
-              className={cn('!mt-2 !text-text-secondary', heighClassName)}
-              content={sign_content || content}
-              customDisallowedElements={['input']}
-            />
-          )}
-          {answer && (
-            <div className='break-all'>
-              <div className='flex gap-x-1'>
-                <div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
-                <div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
-                  {content}
+          {/* Content */}
+          <div className='relative'>
+            {!answer && (
+              <Markdown
+                className={cn('!mt-2 !text-text-secondary', heighClassName)}
+                content={sign_content || content}
+                customDisallowedElements={['input']}
+              />
+            )}
+            {answer && (
+              <div className='break-all'>
+                <div className='flex gap-x-1'>
+                  <div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
+                  <div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
+                    {content}
+                  </div>
+                </div>
+                <div className='flex gap-x-1'>
+                  <div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
+                  <div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
+                    {answer}
+                  </div>
                 </div>
               </div>
-              <div className='flex gap-x-1'>
-                <div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
-                <div className={cn('body-md-regular line-clamp-20 text-text-secondary')}>
-                  {answer}
+            )}
+            {/* Mask */}
+            <Mask className='absolute inset-x-0 bottom-0' />
+          </div>
+          {(showImages || showKeywords) && (
+            <div className='flex flex-col gap-y-3 pt-3'>
+              {showImages && (
+                <ImageList images={images} size='md' className='py-1' />
+              )}
+              {showKeywords && (
+                <div className='flex flex-col gap-y-1'>
+                  <div className='text-xs font-medium uppercase text-text-tertiary'>{t(`${i18nPrefix}.keyword`)}</div>
+                  <div className='flex flex-wrap gap-x-2'>
+                    {keywords.map(keyword => (
+                      <Tag key={keyword} text={keyword} />
+                    ))}
+                  </div>
                 </div>
-              </div>
-            </div>
-          )}
-          {!isParentChildRetrieval && keywords && keywords.length > 0 && (
-            <div className='mt-6'>
-              <div className='text-xs font-medium uppercase text-text-tertiary'>{t(`${i18nPrefix}.keyword`)}</div>
-              <div className='mt-1 flex flex-wrap'>
-                {keywords.map(keyword => (
-                  <Tag key={keyword} text={keyword} className='mr-2' />
-                ))}
-              </div>
+              )}
             </div>
           )}
         </div>

web/app/components/datasets/hit-testing/components/empty-records.tsx

@@ -0,0 +1,15 @@
+import { RiHistoryLine } from '@remixicon/react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+
+const EmptyRecords = () => {
+  const { t } = useTranslation()
+  return <div className='rounded-2xl bg-workflow-process-bg p-5'>
+    <div className='flex h-10 w-10 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'>
+      <RiHistoryLine className='h-5 w-5 text-text-tertiary' />
+    </div>
+    <div className='my-2 text-[13px] font-medium leading-4 text-text-tertiary'>{t('datasetHitTesting.noRecentTip')}</div>
+  </div>
+}
+
+export default React.memo(EmptyRecords)

web/app/components/datasets/hit-testing/components/mask.tsx

@@ -0,0 +1,19 @@
+import React from 'react'
+import cn from '@/utils/classnames'
+
+type MaskProps = {
+  className?: string
+}
+
+export const Mask = ({
+  className,
+}: MaskProps) => {
+  return (
+    <div className={cn(
+      'h-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg',
+      className,
+    )} />
+  )
+}
+
+export default React.memo(Mask)

web/app/components/datasets/hit-testing/components/query-input/index.tsx

@@ -0,0 +1,257 @@
+import type { ChangeEvent } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  RiEqualizer2Line,
+  RiPlayCircleLine,
+} from '@remixicon/react'
+import Image from 'next/image'
+import Button from '@/app/components/base/button'
+import { getIcon } from '@/app/components/datasets/common/retrieval-method-info'
+import ModifyExternalRetrievalModal from '@/app/components/datasets/hit-testing/modify-external-retrieval-modal'
+import cn from '@/utils/classnames'
+import type {
+  Attachment,
+  ExternalKnowledgeBaseHitTestingRequest,
+  ExternalKnowledgeBaseHitTestingResponse,
+  HitTestingRequest,
+  HitTestingResponse,
+  Query,
+} from '@/models/datasets'
+import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'
+import type { UseMutateAsyncFunction } from '@tanstack/react-query'
+import ImageUploaderInRetrievalTesting from '@/app/components/datasets/common/image-uploader/image-uploader-in-retrieval-testing'
+import Textarea from './textarea'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import type { FileEntity } from '@/app/components/datasets/common/image-uploader/types'
+import { v4 as uuid4 } from 'uuid'
+
+type QueryInputProps = {
+  onUpdateList: () => void
+  setHitResult: (res: HitTestingResponse) => void
+  setExternalHitResult: (res: ExternalKnowledgeBaseHitTestingResponse) => void
+  loading: boolean
+  queries: Query[]
+  setQueries: (v: Query[]) => void
+  isExternal?: boolean
+  onClickRetrievalMethod: () => void
+  retrievalConfig: RetrievalConfig
+  isEconomy: boolean
+  onSubmit?: () => void
+  hitTestingMutation: UseMutateAsyncFunction<HitTestingResponse, Error, HitTestingRequest, unknown>
+  externalKnowledgeBaseHitTestingMutation: UseMutateAsyncFunction<
+    ExternalKnowledgeBaseHitTestingResponse,
+    Error,
+    ExternalKnowledgeBaseHitTestingRequest,
+    unknown
+  >
+}
+
+const QueryInput = ({
+  onUpdateList,
+  setHitResult,
+  setExternalHitResult,
+  loading,
+  queries,
+  setQueries,
+  isExternal = false,
+  onClickRetrievalMethod,
+  retrievalConfig,
+  isEconomy,
+  onSubmit: _onSubmit,
+  hitTestingMutation,
+  externalKnowledgeBaseHitTestingMutation,
+}: QueryInputProps) => {
+  const { t } = useTranslation()
+  const isMultimodal = useDatasetDetailContextWithSelector(s => !!s.dataset?.is_multimodal)
+  const [isSettingsOpen, setIsSettingsOpen] = useState(false)
+  const [externalRetrievalSettings, setExternalRetrievalSettings] = useState({
+    top_k: 4,
+    score_threshold: 0.5,
+    score_threshold_enabled: false,
+  })
+
+  const text = useMemo(() => {
+    return queries.find(query => query.content_type === 'text_query')?.content ?? ''
+  }, [queries])
+
+  const images = useMemo(() => {
+    const imageQueries = queries
+      .filter(query => query.content_type === 'image_query')
+      .map(query => query.file_info)
+      .filter(Boolean) as Attachment[]
+    return imageQueries.map(item => ({
+      id: uuid4(),
+      name: item.name,
+      size: item.size,
+      mimeType: item.mime_type,
+      extension: item.extension,
+      sourceUrl: item.source_url,
+      uploadedId: item.id,
+      progress: 100,
+    })) || []
+  }, [queries])
+
+  const isAllUploaded = useMemo(() => {
+    return images.every(image => !!image.uploadedId)
+  }, [images])
+
+  const handleSaveExternalRetrievalSettings = useCallback((data: {
+    top_k: number
+    score_threshold: number
+    score_threshold_enabled: boolean
+  }) => {
+    setExternalRetrievalSettings(data)
+    setIsSettingsOpen(false)
+  }, [])
+
+  const handleTextChange = useCallback((event: ChangeEvent<HTMLTextAreaElement>) => {
+    const newQueries = [...queries]
+    const textQuery = newQueries.find(query => query.content_type === 'text_query')
+    if (!textQuery) {
+      newQueries.push({
+        content: event.target.value,
+        content_type: 'text_query',
+        file_info: null,
+      })
+    }
+    else {
+      textQuery.content = event.target.value
+    }
+    setQueries(newQueries)
+  }, [queries, setQueries])
+
+  const handleImageChange = useCallback((files: FileEntity[]) => {
+    let newQueries = [...queries]
+    newQueries = newQueries.filter(query => query.content_type !== 'image_query')
+    files.forEach((file) => {
+      newQueries.push({
+        content: file.sourceUrl || '',
+        content_type: 'image_query',
+        file_info: {
+          id: file.uploadedId || '',
+          mime_type: file.mimeType,
+          source_url: file.sourceUrl || '',
+          name: file.name,
+          size: file.size,
+          extension: file.extension,
+        },
+      })
+    })
+    setQueries(newQueries)
+  }, [queries, setQueries])
+
+  const onSubmit = useCallback(async () => {
+    await hitTestingMutation({
+      query: text,
+      attachment_ids: images.map(image => image.uploadedId),
+      retrieval_model: {
+        ...retrievalConfig,
+        search_method: isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method,
+      },
+    }, {
+      onSuccess: (data) => {
+        setHitResult(data)
+        onUpdateList?.()
+        if (_onSubmit)
+          _onSubmit()
+      },
+    })
+  }, [text, retrievalConfig, isEconomy, hitTestingMutation, onUpdateList, _onSubmit, images, setHitResult])
+
+  const externalRetrievalTestingOnSubmit = useCallback(async () => {
+    await externalKnowledgeBaseHitTestingMutation({
+      query: text,
+      external_retrieval_model: {
+        top_k: externalRetrievalSettings.top_k,
+        score_threshold: externalRetrievalSettings.score_threshold,
+        score_threshold_enabled: externalRetrievalSettings.score_threshold_enabled,
+      },
+    }, {
+      onSuccess: (data) => {
+        setExternalHitResult(data)
+        onUpdateList?.()
+      },
+    })
+  }, [text, externalRetrievalSettings, externalKnowledgeBaseHitTestingMutation, onUpdateList, setExternalHitResult])
+
+  const retrievalMethod = isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method
+  const icon = <Image className='size-3.5 text-util-colors-purple-purple-600' src={getIcon(retrievalMethod)} alt='' />
+  const TextAreaComp = useMemo(() => {
+    return (
+      <Textarea
+        text={text}
+        handleTextChange={handleTextChange}
+      />
+    )
+  }, [text, handleTextChange])
+  const ActionButtonComp = useMemo(() => {
+    return (
+      <Button
+        onClick={isExternal ? externalRetrievalTestingOnSubmit : onSubmit}
+        variant='primary'
+        loading={loading}
+        disabled={(text.length === 0 && images.length === 0) || text.length > 200 || (images.length > 0 && !isAllUploaded)}
+        className='w-[88px]'
+      >
+        <RiPlayCircleLine className='mr-1 size-4' />
+        {t('datasetHitTesting.input.testing')}
+      </Button>
+    )
+  }, [isExternal, externalRetrievalTestingOnSubmit, onSubmit, text, loading, t, images, isAllUploaded])
+
+  return (
+    <div className={cn('relative flex h-80 shrink-0 flex-col overflow-hidden rounded-xl bg-gradient-to-r from-components-input-border-active-prompt-1 to-components-input-border-active-prompt-2 p-0.5 shadow-xs')}>
+      <div className='flex h-full flex-col overflow-hidden rounded-[10px] bg-background-section-burn'>
+        <div className='relative flex shrink-0 items-center justify-between p-1.5 pb-1 pl-3'>
+          <span className='system-sm-semibold-uppercase text-text-secondary'>
+            {t('datasetHitTesting.input.title')}
+          </span>
+          {isExternal ? (
+            <Button
+              variant='secondary'
+              size='small'
+              onClick={() => setIsSettingsOpen(!isSettingsOpen)}
+            >
+              <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
+              <div className='flex items-center justify-center gap-1 px-[3px]'>
+                <span className='system-xs-medium text-components-button-secondary-text'>{t('datasetHitTesting.settingTitle')}</span>
+              </div>
+            </Button>
+          ) : (
+            <div
+              onClick={onClickRetrievalMethod}
+              className='flex h-7 cursor-pointer items-center space-x-0.5 rounded-lg border-[0.5px] border-components-button-secondary-bg bg-components-button-secondary-bg px-1.5 shadow-xs backdrop-blur-[5px] hover:bg-components-button-secondary-bg-hover'
+            >
+              {icon}
+              <div className='text-xs font-medium uppercase text-text-secondary'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
+              <RiEqualizer2Line className='size-4 text-components-menu-item-text'></RiEqualizer2Line>
+            </div>
+          )}
+          {
+            isSettingsOpen && (
+              <ModifyExternalRetrievalModal
+                onClose={() => setIsSettingsOpen(false)}
+                onSave={handleSaveExternalRetrievalSettings}
+                initialTopK={externalRetrievalSettings.top_k}
+                initialScoreThreshold={externalRetrievalSettings.score_threshold}
+                initialScoreThresholdEnabled={externalRetrievalSettings.score_threshold_enabled}
+              />
+            )
+          }
+        </div>
+        <ImageUploaderInRetrievalTesting
+          textArea={TextAreaComp}
+          actionButton={ActionButtonComp}
+          onChange={handleImageChange}
+          value={images}
+          showUploader={isMultimodal}
+          className='grow'
+          actionAreaClassName='px-4 py-2 shrink-0 bg-background-default'
+        />
+      </div>
+    </div>
+  )
+}
+
+export default QueryInput

web/app/components/datasets/hit-testing/components/query-input/textarea.tsx

@@ -0,0 +1,61 @@
+import type { ChangeEvent } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from '@/utils/classnames'
+import { Corner } from '@/app/components/base/icons/src/vender/solid/shapes'
+import Tooltip from '@/app/components/base/tooltip'
+
+type TextareaProps = {
+  text: string
+  handleTextChange: (e: ChangeEvent<HTMLTextAreaElement>) => void
+}
+
+const Textarea = ({
+  text,
+  handleTextChange,
+}: TextareaProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={cn(
+      'relative flex-1 overflow-hidden rounded-t-[10px] border-t-[0.5px] border-components-panel-border-subtle bg-background-default px-4 pb-0 pt-3',
+      text.length > 200 && 'border-state-destructive-active',
+    )}>
+      <textarea
+        className='system-md-regular h-full w-full resize-none border-none bg-transparent text-text-secondary caret-[#295EFF] placeholder:text-components-input-text-placeholder focus-visible:outline-none'
+        value={text}
+        onChange={handleTextChange}
+        placeholder={t('datasetHitTesting.input.placeholder') as string}
+      />
+      <div className='absolute right-0 top-0 flex items-center'>
+        <Corner className={cn(
+          'text-background-section-burn',
+          text.length > 200 && 'text-util-colors-red-red-100',
+        )} />
+        {text.length > 200
+          ? (
+            <Tooltip
+              popupContent={t('datasetHitTesting.input.countWarning')}
+            >
+              <div
+                className={cn('system-2xs-medium-uppercase bg-util-colors-red-red-100 py-1 pr-2 text-util-colors-red-red-600')}
+              >
+                {`${text.length}/200`}
+              </div>
+            </Tooltip>
+          )
+          : (
+            <div
+              className={cn(
+                'system-2xs-medium-uppercase bg-background-section-burn py-1 pr-2 text-text-tertiary',
+              )}
+            >
+              {`${text.length}/200`}
+            </div>
+          )}
+      </div>
+    </div>
+  )
+}
+
+export default React.memo(Textarea)

web/app/components/datasets/hit-testing/components/records.tsx

@@ -0,0 +1,117 @@
+import React, { useCallback, useMemo, useState } from 'react'
+import useTimestamp from '@/hooks/use-timestamp'
+import type { Attachment, HitTestingRecord, Query } from '@/models/datasets'
+import { RiApps2Line, RiArrowDownLine, RiFocus2Line } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import ImageList from '../../common/image-list'
+import cn from '@/utils/classnames'
+
+type RecordsProps = {
+  records: HitTestingRecord[]
+  onClickRecord: (record: HitTestingRecord) => void
+}
+
+const Records = ({
+  records,
+  onClickRecord,
+}: RecordsProps) => {
+  const { t } = useTranslation()
+  const { formatTime } = useTimestamp()
+
+  const [sortTimeOrder, setTimeOrder] = useState<'asc' | 'desc'>('desc')
+
+  const handleSortTime = useCallback(() => {
+    setTimeOrder(prev => prev === 'asc' ? 'desc' : 'asc')
+  }, [])
+
+  const sortedRecords = useMemo(() => {
+    return [...records].sort((a, b) => {
+      return sortTimeOrder === 'asc' ? a.created_at - b.created_at : b.created_at - a.created_at
+    })
+  }, [records, sortTimeOrder])
+
+  const getImageList = (queries: Query[]) => {
+    const imageQueries = queries
+      .filter(query => query.content_type === 'image_query')
+      .map(query => query.file_info)
+      .filter(Boolean) as Attachment[]
+    return imageQueries.map(image => ({
+      name: image.name,
+      mimeType: image.mime_type,
+      sourceUrl: image.source_url,
+      size: image.size,
+      extension: image.extension,
+    }))
+  }
+
+  return (
+    <div className='grow overflow-y-auto'>
+      <table className={'w-full border-collapse border-0 text-[13px] leading-4 text-text-secondary '}>
+        <thead className='sticky top-0 h-7 text-xs  font-medium uppercase leading-7 text-text-tertiary backdrop-blur-[5px]'>
+          <tr>
+            <td className='rounded-l-lg bg-background-section-burn pl-3'>{t('datasetHitTesting.table.header.queryContent')}</td>
+            <td className='w-[128px]  bg-background-section-burn pl-3'>{t('datasetHitTesting.table.header.source')}</td>
+            <td className='w-48 rounded-r-lg bg-background-section-burn pl-3'>
+              <div
+                className='flex cursor-pointer items-center'
+                onClick={handleSortTime}
+              >
+                {t('datasetHitTesting.table.header.time')}
+                <RiArrowDownLine
+                  className={cn(
+                    'ml-0.5 size-3.5',
+                    sortTimeOrder === 'asc' ? 'rotate-180' : '',
+                  )}
+                />
+              </div>
+            </td>
+          </tr>
+        </thead>
+        <tbody>
+          {sortedRecords.map((record) => {
+            const { id, source, created_at, queries } = record
+            const SourceIcon = record.source === 'app' ? RiApps2Line : RiFocus2Line
+            const content = queries.find(query => query.content_type === 'text_query')?.content || ''
+            const images = getImageList(queries)
+            return (
+              <tr
+                key={id}
+                className='group cursor-pointer border-b border-divider-subtle hover:bg-background-default-hover'
+                onClick={() => onClickRecord(record)}
+              >
+                <td className='max-w-xs p-3 pr-2'>
+                  <div className='flex flex-col gap-y-1'>
+                    {content && (
+                      <div className='line-clamp-2'>
+                        {content}
+                      </div>
+                    )}
+                    {images.length > 0 && (
+                      <ImageList
+                        images={images}
+                        size='md'
+                        className='py-1'
+                        limit={5}
+                      />
+                    )}
+                  </div>
+                </td>
+                <td className='w-[128px] p-3 pr-2'>
+                  <div className='flex items-center'>
+                    <SourceIcon className='mr-1 size-4 text-text-tertiary' />
+                    <span className='capitalize'>{source.replace('_', ' ').replace('hit testing', 'retrieval test')}</span>
+                  </div>
+                </td>
+                <td className='w-48 p-3 pr-2'>
+                  {formatTime(created_at, t('datasetHitTesting.dateTimeFormat') as string)}
+                </td>
+              </tr>
+            )
+          })}
+        </tbody>
+      </table>
+    </div>
+  )
+}
+
+export default React.memo(Records)

web/app/components/datasets/hit-testing/components/result-item.tsx

@@ -1,6 +1,5 @@
 'use client'
-import type { FC } from 'react'
-import React from 'react'
+import React, { useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
 import { useBoolean } from 'ahooks'
@@ -14,17 +13,18 @@ import type { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader
 import Tag from '@/app/components/datasets/documents/detail/completed/common/tag'
 import { extensionToFileType } from '@/app/components/datasets/hit-testing/utils/extension-to-file-type'
 import { Markdown } from '@/app/components/base/markdown'
+import ImageList from '../../common/image-list'
 
 const i18nPrefix = 'datasetHitTesting'
-type Props = {
+type ResultItemProps = {
   payload: HitTesting
 }
 
-const ResultItem: FC<Props> = ({
+const ResultItem = ({
   payload,
-}) => {
+}: ResultItemProps) => {
   const { t } = useTranslation()
-  const { segment, score, child_chunks } = payload
+  const { segment, score, child_chunks, files } = payload
   const data = segment
   const { position, word_count, content, sign_content, keywords, document } = data
   const isParentChildRetrieval = !!(child_chunks && child_chunks.length > 0)
@@ -40,6 +40,17 @@ const ResultItem: FC<Props> = ({
     setFalse: hideDetailModal,
   }] = useBoolean(false)
 
+  const images = useMemo(() => {
+    if (!files) return []
+    return files.map(file => ({
+      name: file.name,
+      mimeType: file.mime_type,
+      sourceUrl: file.source_url,
+      size: file.size,
+      extension: file.extension,
+    }))
+  }, [files])
+
   return (
     <div className={cn('cursor-pointer rounded-xl bg-chat-bubble-bg pt-3 hover:shadow-lg')} onClick={showDetailModal}>
       {/* Meta info */}
@@ -47,11 +58,14 @@ const ResultItem: FC<Props> = ({
 
       {/* Main */}
       <div className='mt-1 px-3'>
-        <Markdown
+        {<Markdown
           className='line-clamp-2'
           content={sign_content || content}
           customDisallowedElements={['input']}
-        />
+        />}
+        {images.length > 0 && (
+          <ImageList images={images} size='md' className='py-1' />
+        )}
         {isParentChildRetrieval && (
           <div className='mt-1'>
             <div

web/app/components/datasets/hit-testing/index.tsx

@@ -1,30 +1,40 @@
 'use client'
 import type { FC } from 'react'
-import React, { useEffect, useState } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import useSWR from 'swr'
-import { omit } from 'lodash-es'
 import { useBoolean } from 'ahooks'
 import { useContext } from 'use-context-selector'
-import { RiApps2Line, RiFocus2Line, RiHistoryLine } from '@remixicon/react'
-import Textarea from './textarea'
+import QueryInput from './components/query-input'
 import s from './style.module.css'
 import ModifyRetrievalModal from './modify-retrieval-modal'
 import ResultItem from './components/result-item'
 import ResultItemExternal from './components/result-item-external'
 import cn from '@/utils/classnames'
-import type { ExternalKnowledgeBaseHitTesting, ExternalKnowledgeBaseHitTestingResponse, HitTesting, HitTestingResponse } from '@/models/datasets'
+import type {
+  ExternalKnowledgeBaseHitTesting,
+  ExternalKnowledgeBaseHitTestingResponse,
+  HitTesting,
+  HitTestingRecord,
+  HitTestingResponse,
+  Query,
+} from '@/models/datasets'
 import Loading from '@/app/components/base/loading'
 import Drawer from '@/app/components/base/drawer'
 import Pagination from '@/app/components/base/pagination'
 import FloatRightContainer from '@/app/components/base/float-right-container'
-import { fetchTestingRecords } from '@/service/datasets'
 import DatasetDetailContext from '@/context/dataset-detail'
 import type { RetrievalConfig } from '@/types/app'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
-import useTimestamp from '@/hooks/use-timestamp'
 import docStyle from '@/app/components/datasets/documents/detail/completed/style.module.css'
 import { CardSkelton } from '../documents/detail/completed/skeleton/general-list-skeleton'
+import EmptyRecords from './components/empty-records'
+import Records from './components/records'
+import {
+  useExternalKnowledgeBaseHitTesting,
+  useHitTesting,
+  useHitTestingRecords,
+  useInvalidateHitTestingRecords,
+} from '@/service/knowledge/use-hit-testing'
 
 const limit = 10
 
@@ -32,34 +42,20 @@ type Props = {
   datasetId: string
 }
 
-const RecordsEmpty: FC = () => {
-  const { t } = useTranslation()
-  return <div className='rounded-2xl bg-workflow-process-bg p-5'>
-    <div className='flex h-10 w-10 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'>
-      <RiHistoryLine className='h-5 w-5 text-text-tertiary' />
-    </div>
-    <div className='my-2 text-[13px] font-medium leading-4 text-text-tertiary'>{t('datasetHitTesting.noRecentTip')}</div>
-  </div>
-}
-
 const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
   const { t } = useTranslation()
-  const { formatTime } = useTimestamp()
 
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
 
   const [hitResult, setHitResult] = useState<HitTestingResponse | undefined>() // 初始化记录为空数组
   const [externalHitResult, setExternalHitResult] = useState<ExternalKnowledgeBaseHitTestingResponse | undefined>()
-  const [submitLoading, setSubmitLoading] = useState(false)
-  const [text, setText] = useState('')
+  const [queries, setQueries] = useState<Query[]>([])
+  const [queryInputKey, setQueryInputKey] = useState(Date.now())
 
-  const [currPage, setCurrPage] = React.useState<number>(0)
-  const { data: recordsRes, error, mutate: recordsMutate } = useSWR({
-    action: 'fetchTestingRecords',
-    datasetId,
-    params: { limit, page: currPage + 1 },
-  }, apiParams => fetchTestingRecords(omit(apiParams, 'action')))
+  const [currPage, setCurrPage] = useState<number>(0)
+  const { data: recordsRes, isLoading: isRecordsLoading } = useHitTestingRecords({ datasetId, page: currPage + 1, limit })
+  const invalidateHitTestingRecords = useInvalidateHitTestingRecords(datasetId)
 
   const total = recordsRes?.total || 0
 
@@ -69,6 +65,15 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
   const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict as RetrievalConfig)
   const [isShowModifyRetrievalModal, setIsShowModifyRetrievalModal] = useState(false)
   const [isShowRightPanel, { setTrue: showRightPanel, setFalse: hideRightPanel, set: setShowRightPanel }] = useBoolean(!isMobile)
+
+  const { mutateAsync: hitTestingMutation, isPending: isHitTestingPending } = useHitTesting(datasetId)
+  const {
+    mutateAsync: externalKnowledgeBaseHitTestingMutation,
+    isPending: isExternalKnowledgeBaseHitTestingPending,
+  } = useExternalKnowledgeBaseHitTesting(datasetId)
+
+  const isRetrievalLoading = isHitTestingPending || isExternalKnowledgeBaseHitTestingPending
+
   const renderHitResults = (results: HitTesting[] | ExternalKnowledgeBaseHitTesting[]) => (
     <div className='flex h-full flex-col rounded-tl-2xl bg-background-body px-4 py-3'>
       <div className='mb-2 shrink-0 pl-2 font-semibold leading-6 text-text-primary'>
@@ -101,6 +106,12 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
     </div>
   )
 
+  const handleClickRecord = useCallback((record: HitTestingRecord) => {
+    const { queries } = record
+    setQueries(queries)
+    setQueryInputKey(Date.now())
+  }, [])
+
   useEffect(() => {
     setShowRightPanel(!isMobile)
   }, [isMobile, setShowRightPanel])
@@ -112,74 +123,50 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
           <h1 className='text-base font-semibold text-text-primary'>{t('datasetHitTesting.title')}</h1>
           <p className='mt-0.5 text-[13px] font-normal leading-4 text-text-tertiary'>{t('datasetHitTesting.desc')}</p>
         </div>
-        <Textarea
-          datasetId={datasetId}
+        <QueryInput
+          key={queryInputKey}
           setHitResult={setHitResult}
           setExternalHitResult={setExternalHitResult}
           onSubmit={showRightPanel}
-          onUpdateList={recordsMutate}
-          loading={submitLoading}
-          setLoading={setSubmitLoading}
-          setText={setText}
-          text={text}
+          onUpdateList={invalidateHitTestingRecords}
+          loading={isRetrievalLoading}
+          queries={queries}
+          setQueries={setQueries}
           isExternal={isExternal}
           onClickRetrievalMethod={() => setIsShowModifyRetrievalModal(true)}
           retrievalConfig={retrievalConfig}
           isEconomy={currentDataset?.indexing_technique === 'economy'}
+          hitTestingMutation={hitTestingMutation}
+          externalKnowledgeBaseHitTestingMutation={externalKnowledgeBaseHitTestingMutation}
         />
         <div className='mb-3 mt-6 text-base font-semibold text-text-primary'>{t('datasetHitTesting.records')}</div>
-        {(!recordsRes && !error)
-          ? (
+        {isRecordsLoading
+          && (
             <div className='flex-1'><Loading type='app' /></div>
           )
-          : recordsRes?.data?.length
-            ? (
-              <>
-                <div className='grow overflow-y-auto'>
-                  <table className={'w-full border-collapse border-0 text-[13px] leading-4 text-text-secondary '}>
-                    <thead className='sticky top-0 h-7 text-xs  font-medium uppercase leading-7 text-text-tertiary backdrop-blur-[5px]'>
-                      <tr>
-                        <td className='w-[128px] rounded-l-lg bg-background-section-burn pl-3'>{t('datasetHitTesting.table.header.source')}</td>
-                        <td className='bg-background-section-burn'>{t('datasetHitTesting.table.header.text')}</td>
-                        <td className='w-48 rounded-r-lg bg-background-section-burn pl-2'>{t('datasetHitTesting.table.header.time')}</td>
-                      </tr>
-                    </thead>
-                    <tbody>
-                      {recordsRes?.data?.map((record) => {
-                        const SourceIcon = record.source === 'app' ? RiApps2Line : RiFocus2Line
-                        return <tr
-                          key={record.id}
-                          className='group h-10 cursor-pointer border-b border-divider-subtle hover:bg-background-default-hover'
-                          onClick={() => setText(record.content)}
-                        >
-                          <td className='w-[128px] pl-3'>
-                            <div className='flex items-center'>
-                              <SourceIcon className='mr-1 size-4 text-text-tertiary' />
-                              <span className='capitalize'>{record.source.replace('_', ' ').replace('hit testing', 'retrieval test')}</span>
-                            </div>
-                          </td>
-                          <td className='max-w-xs py-2'>{record.content}</td>
-                          <td className='w-36 pl-2'>
-                            {formatTime(record.created_at, t('datasetHitTesting.dateTimeFormat') as string)}
-                          </td>
-                        </tr>
-                      })}
-                    </tbody>
-                  </table>
-                </div>
-                {(total && total > limit)
-                  ? <Pagination current={currPage} onChange={setCurrPage} total={total} limit={limit} />
-                  : null}
-              </>
-            )
-            : (
-              <RecordsEmpty />
-            )}
+        }
+        {!isRecordsLoading && recordsRes?.data && recordsRes.data.length > 0 && (
+          <>
+            <Records records={recordsRes?.data} onClickRecord={handleClickRecord}/>
+            {(total && total > limit)
+              ? <Pagination current={currPage} onChange={setCurrPage} total={total} limit={limit} />
+              : null}
+          </>
+        )}
+        {!isRecordsLoading && !recordsRes?.data?.length && (
+          <EmptyRecords />
+        )}
       </div>
-      <FloatRightContainer panelClassName='!justify-start !overflow-y-auto' showClose isMobile={isMobile} isOpen={isShowRightPanel} onClose={hideRightPanel} footer={null}>
+      <FloatRightContainer
+        panelClassName='!justify-start !overflow-y-auto'
+        showClose
+        isMobile={isMobile}
+        isOpen={isShowRightPanel}
+        onClose={hideRightPanel}
+        footer={null}
+      >
         <div className='flex flex-col pt-3'>
-          {/* {renderHitResults(generalResultData)} */}
-          {submitLoading
+          {isRetrievalLoading
             ? <div className='flex h-full flex-col rounded-tl-2xl bg-background-body px-4 py-3'>
               <CardSkelton />
             </div>
@@ -197,7 +184,14 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
           }
         </div>
       </FloatRightContainer>
-      <Drawer unmount={true} isOpen={isShowModifyRetrievalModal} onClose={() => setIsShowModifyRetrievalModal(false)} footer={null} mask={isMobile} panelClassName='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>
+      <Drawer
+        unmount={true}
+        isOpen={isShowModifyRetrievalModal}
+        onClose={() => setIsShowModifyRetrievalModal(false)}
+        footer={null}
+        mask={isMobile}
+        panelClassName='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'
+      >
         <ModifyRetrievalModal
           indexMethod={currentDataset?.indexing_technique || ''}
           value={retrievalConfig}

web/app/components/datasets/hit-testing/modify-retrieval-modal.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React, { useRef, useState } from 'react'
+import React, { useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { RiCloseLine } from '@remixicon/react'
 import Toast from '../../base/toast'
@@ -10,8 +10,11 @@ import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-me
 import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
 import Button from '@/app/components/base/button'
 import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
-import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { useDocLink } from '@/context/i18n'
+import { checkShowMultiModalTip } from '../settings/utils'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import type { IndexingType } from '../create/step-two'
 
 type Props = {
   indexMethod: string
@@ -32,15 +35,16 @@ const ModifyRetrievalModal: FC<Props> = ({
   const { t } = useTranslation()
   const docLink = useDocLink()
   const [retrievalConfig, setRetrievalConfig] = useState(value)
+  const embeddingModel = useDatasetDetailContextWithSelector(state => state.dataset?.embedding_model)
+  const embeddingModelProvider = useDatasetDetailContextWithSelector(state => state.dataset?.embedding_model_provider)
 
   // useClickAway(() => {
   //   if (ref)
   //     onHide()
   // }, ref)
 
-  const {
-    modelList: rerankModelList,
-  } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
+  const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
+  const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
 
   const handleSave = () => {
     if (
@@ -56,6 +60,23 @@ const ModifyRetrievalModal: FC<Props> = ({
     onSave(retrievalConfig)
   }
 
+  const showMultiModalTip = useMemo(() => {
+    return checkShowMultiModalTip({
+      embeddingModel: {
+        provider: embeddingModelProvider ?? '',
+        model: embeddingModel ?? '',
+      },
+      rerankingEnable: retrievalConfig.reranking_enable,
+      rerankModel: {
+        rerankingProviderName: retrievalConfig.reranking_model.reranking_provider_name,
+        rerankingModelName: retrievalConfig.reranking_model.reranking_model_name,
+      },
+      indexMethod: indexMethod as IndexingType,
+      embeddingModelList,
+      rerankModelList,
+    })
+  }, [embeddingModelProvider, embeddingModel, retrievalConfig.reranking_enable, retrievalConfig.reranking_model, indexMethod, embeddingModelList, rerankModelList])
+
   if (!isShow)
     return null
 
@@ -104,6 +125,7 @@ const ModifyRetrievalModal: FC<Props> = ({
             <RetrievalMethodConfig
               value={retrievalConfig}
               onChange={setRetrievalConfig}
+              showMultiModalTip={showMultiModalTip}
             />
           )
           : (

web/app/components/datasets/hit-testing/textarea.tsx

@@ -1,201 +0,0 @@
-import type { ChangeEvent } from 'react'
-import React, { useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import {
-  RiEqualizer2Line,
-} from '@remixicon/react'
-import Image from 'next/image'
-import Button from '../../base/button'
-import { getIcon } from '../common/retrieval-method-info'
-import ModifyExternalRetrievalModal from './modify-external-retrieval-modal'
-import Tooltip from '@/app/components/base/tooltip'
-import cn from '@/utils/classnames'
-import type { ExternalKnowledgeBaseHitTestingResponse, HitTestingResponse } from '@/models/datasets'
-import { externalKnowledgeBaseHitTesting, hitTesting } from '@/service/datasets'
-import { asyncRunSafe } from '@/utils'
-import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'
-
-type TextAreaWithButtonIProps = {
-  datasetId: string
-  onUpdateList: () => void
-  setHitResult: (res: HitTestingResponse) => void
-  setExternalHitResult: (res: ExternalKnowledgeBaseHitTestingResponse) => void
-  loading: boolean
-  setLoading: (v: boolean) => void
-  text: string
-  setText: (v: string) => void
-  isExternal?: boolean
-  onClickRetrievalMethod: () => void
-  retrievalConfig: RetrievalConfig
-  isEconomy: boolean
-  onSubmit?: () => void
-}
-
-const TextAreaWithButton = ({
-  datasetId,
-  onUpdateList,
-  setHitResult,
-  setExternalHitResult,
-  setLoading,
-  loading,
-  text,
-  setText,
-  isExternal = false,
-  onClickRetrievalMethod,
-  retrievalConfig,
-  isEconomy,
-  onSubmit: _onSubmit,
-}: TextAreaWithButtonIProps) => {
-  const { t } = useTranslation()
-  const [isSettingsOpen, setIsSettingsOpen] = useState(false)
-  const [externalRetrievalSettings, setExternalRetrievalSettings] = useState({
-    top_k: 4,
-    score_threshold: 0.5,
-    score_threshold_enabled: false,
-  })
-
-  const handleSaveExternalRetrievalSettings = (data: { top_k: number; score_threshold: number; score_threshold_enabled: boolean }) => {
-    setExternalRetrievalSettings(data)
-    setIsSettingsOpen(false)
-  }
-
-  function handleTextChange(event: ChangeEvent<HTMLTextAreaElement>) {
-    setText(event.target.value)
-  }
-
-  const onSubmit = async () => {
-    setLoading(true)
-    const [e, res] = await asyncRunSafe<HitTestingResponse>(
-      hitTesting({
-        datasetId,
-        queryText: text,
-        retrieval_model: {
-          ...retrievalConfig,
-          search_method: isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method,
-        },
-      }) as Promise<HitTestingResponse>,
-    )
-    if (!e) {
-      setHitResult(res)
-      onUpdateList?.()
-    }
-    setLoading(false)
-    if (_onSubmit)
-      _onSubmit()
-  }
-
-  const externalRetrievalTestingOnSubmit = async () => {
-    setLoading(true)
-    const [e, res] = await asyncRunSafe<ExternalKnowledgeBaseHitTestingResponse>(
-      externalKnowledgeBaseHitTesting({
-        datasetId,
-        query: text,
-        external_retrieval_model: {
-          top_k: externalRetrievalSettings.top_k,
-          score_threshold: externalRetrievalSettings.score_threshold,
-          score_threshold_enabled: externalRetrievalSettings.score_threshold_enabled,
-        },
-      }) as Promise<ExternalKnowledgeBaseHitTestingResponse>,
-    )
-    if (!e) {
-      setExternalHitResult(res)
-      onUpdateList?.()
-    }
-    setLoading(false)
-  }
-
-  const retrievalMethod = isEconomy ? RETRIEVE_METHOD.keywordSearch : retrievalConfig.search_method
-  const icon = <Image className='size-3.5 text-util-colors-purple-purple-600' src={getIcon(retrievalMethod)} alt='' />
-  return (
-    <>
-      <div className={cn('relative rounded-xl bg-gradient-to-r from-components-input-border-active-prompt-1 to-components-input-border-active-prompt-2 p-0.5 shadow-xs')}>
-        <div className='relative rounded-t-xl bg-background-section-burn pt-1.5'>
-          <div className="flex h-8 items-center justify-between pb-1 pl-4 pr-1.5">
-            <span className="text-[13px] font-semibold uppercase leading-4 text-text-secondary">
-              {t('datasetHitTesting.input.title')}
-            </span>
-            {isExternal
-              ? <Button
-                variant='secondary'
-                size='small'
-                onClick={() => setIsSettingsOpen(!isSettingsOpen)}
-              >
-                <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-                <div className='flex items-center justify-center gap-1 px-[3px]'>
-                  <span className='system-xs-medium text-components-button-secondary-text'>{t('datasetHitTesting.settingTitle')}</span>
-                </div>
-              </Button>
-              : <div
-                onClick={onClickRetrievalMethod}
-                className='flex h-7 cursor-pointer items-center space-x-0.5 rounded-lg border-[0.5px] border-components-button-secondary-bg bg-components-button-secondary-bg px-1.5 shadow-xs backdrop-blur-[5px] hover:bg-components-button-secondary-bg-hover'
-              >
-                {icon}
-                <div className='text-xs font-medium uppercase text-text-secondary'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
-                <RiEqualizer2Line className='size-4 text-components-menu-item-text'></RiEqualizer2Line>
-              </div>
-            }
-          </div>
-          {
-            isSettingsOpen && (
-              <ModifyExternalRetrievalModal
-                onClose={() => setIsSettingsOpen(false)}
-                onSave={handleSaveExternalRetrievalSettings}
-                initialTopK={externalRetrievalSettings.top_k}
-                initialScoreThreshold={externalRetrievalSettings.score_threshold}
-                initialScoreThresholdEnabled={externalRetrievalSettings.score_threshold_enabled}
-              />
-            )
-          }
-          <div className='h-2 rounded-t-xl bg-background-default'></div>
-        </div>
-        <div className='rounded-b-xl bg-background-default px-4 pb-11'>
-          <textarea
-            className='h-[220px] w-full resize-none border-none bg-transparent text-sm font-normal text-text-secondary caret-[#295EFF]  placeholder:text-sm placeholder:font-normal placeholder:text-components-input-text-placeholder focus-visible:outline-none'
-            value={text}
-            onChange={handleTextChange}
-            placeholder={t('datasetHitTesting.input.placeholder') as string}
-          />
-          <div className="absolute inset-x-0 bottom-0 mx-4 mb-2 mt-2 flex items-center justify-between">
-            {text?.length > 200
-              ? (
-                <Tooltip
-                  popupContent={t('datasetHitTesting.input.countWarning')}
-                >
-                  <div
-                    className={cn('flex h-5 items-center rounded-md bg-background-section-burn px-1 text-xs font-medium text-red-600', !text?.length && 'opacity-50')}
-                  >
-                    {text?.length}
-                    <span className="mx-0.5 text-red-300">/</span>
-                    200
-                  </div>
-                </Tooltip>
-              )
-              : (
-                <div
-                  className={cn('flex h-5 items-center rounded-md bg-background-section-burn px-1 text-xs font-medium text-text-tertiary', !text?.length && 'opacity-50')}
-                >
-                  {text?.length}
-                  <span className="mx-0.5 text-divider-deep">/</span>
-                  200
-                </div>
-              )}
-
-            <div>
-              <Button
-                onClick={isExternal ? externalRetrievalTestingOnSubmit : onSubmit}
-                variant="primary"
-                loading={loading}
-                disabled={(!text?.length || text?.length > 200)}
-                className='w-[88px]'
-              >
-                {t('datasetHitTesting.input.testing')}
-              </Button>
-            </div>
-          </div>
-        </div>
-      </div>
-    </>
-  )
-}
-
-export default TextAreaWithButton