feat: [frontend] support vision (#1518)

Co-authored-by: Joel <iamjoel007@gmail.com>

web/app/components/app/chat/index.tsx

@@ -1,6 +1,7 @@
 'use client'
 import type { FC, ReactNode } from 'react'
 import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
+import Textarea from 'rc-textarea'
 import { useContext } from 'use-context-selector'
 import cn from 'classnames'
 import Recorder from 'js-audio-recorder'
@@ -10,9 +11,8 @@ import type { DisplayScene, FeedbackFunc, IChatItem, SubmitAnnotationFunc } from
 import { TryToAskIcon, stopIcon } from './icon-component'
 import Answer from './answer'
 import Question from './question'
-import Tooltip from '@/app/components/base/tooltip'
+import TooltipPlus from '@/app/components/base/tooltip-plus'
 import { ToastContext } from '@/app/components/base/toast'
-import AutoHeightTextarea from '@/app/components/base/auto-height-textarea'
 import Button from '@/app/components/base/button'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
 import VoiceInput from '@/app/components/base/voice-input'
@@ -20,6 +20,10 @@ import { Microphone01 } from '@/app/components/base/icons/src/vender/line/mediaA
 import { Microphone01 as Microphone01Solid } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
 import { XCircle } from '@/app/components/base/icons/src/vender/solid/general'
 import type { DataSet } from '@/models/datasets'
+import ChatImageUploader from '@/app/components/base/image-uploader/chat-image-uploader'
+import ImageList from '@/app/components/base/image-uploader/image-list'
+import { TransferMethod, type VisionFile, type VisionSettings } from '@/types/app'
+import { useImageFiles } from '@/app/components/base/image-uploader/hooks'
 
 export type IChatProps = {
   configElem?: React.ReactNode
@@ -37,7 +41,7 @@ export type IChatProps = {
   onFeedback?: FeedbackFunc
   onSubmitAnnotation?: SubmitAnnotationFunc
   checkCanSend?: () => boolean
-  onSend?: (message: string) => void
+  onSend?: (message: string, files: VisionFile[]) => void
   displayScene?: DisplayScene
   useCurrentUserAvatar?: boolean
   isResponsing?: boolean
@@ -54,6 +58,7 @@ export type IChatProps = {
   dataSets?: DataSet[]
   isShowCitationHitInfo?: boolean
   isShowPromptLog?: boolean
+  visionConfig?: VisionSettings
 }
 
 const Chat: FC<IChatProps> = ({
@@ -83,9 +88,19 @@ const Chat: FC<IChatProps> = ({
   dataSets,
   isShowCitationHitInfo,
   isShowPromptLog,
+  visionConfig,
 }) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
+  const {
+    files,
+    onUpload,
+    onRemove,
+    onReUpload,
+    onImageLinkLoadError,
+    onImageLinkLoadSuccess,
+    onClear,
+  } = useImageFiles()
   const isUseInputMethod = useRef(false)
 
   const [query, setQuery] = React.useState('')
@@ -114,9 +129,18 @@ const Chat: FC<IChatProps> = ({
   const handleSend = () => {
     if (!valid() || (checkCanSend && !checkCanSend()))
       return
-    onSend(query)
-    if (!isResponsing)
-      setQuery('')
+    onSend(query, files.filter(file => file.progress !== -1).map(fileItem => ({
+      type: 'image',
+      transfer_method: fileItem.type,
+      url: fileItem.url,
+      upload_file_id: fileItem.fileId,
+    })))
+    if (!files.find(item => item.type === TransferMethod.local_file && !item.fileId)) {
+      if (files.length)
+        onClear()
+      if (!isResponsing)
+        setQuery('')
+    }
   }
 
   const handleKeyUp = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
@@ -198,6 +222,8 @@ const Chat: FC<IChatProps> = ({
               item={item}
               isShowPromptLog={isShowPromptLog}
               isResponsing={isResponsing}
+              // ['https://placekitten.com/360/360', 'https://placekitten.com/360/640']
+              imgSrcs={(item.message_files && item.message_files?.length > 0) ? item.message_files.map(item => item.url) : []}
             />
           )
         })}
@@ -246,18 +272,42 @@ const Chat: FC<IChatProps> = ({
                   </div>
                 </div>)
             }
-            <div className="relative">
-              <AutoHeightTextarea
+            <div className='p-[5.5px] max-h-[150px] bg-white border-[1.5px] border-gray-200 rounded-xl overflow-y-auto'>
+              {
+                visionConfig?.enabled && (
+                  <>
+                    <div className='absolute bottom-2 left-2 flex items-center'>
+                      <ChatImageUploader
+                        settings={visionConfig}
+                        onUpload={onUpload}
+                        disabled={files.length >= visionConfig.number_limits}
+                      />
+                      <div className='mx-1 w-[1px] h-4 bg-black/5' />
+                    </div>
+                    <div className='pl-[52px]'>
+                      <ImageList
+                        list={files}
+                        onRemove={onRemove}
+                        onReUpload={onReUpload}
+                        onImageLinkLoadSuccess={onImageLinkLoadSuccess}
+                        onImageLinkLoadError={onImageLinkLoadError}
+                      />
+                    </div>
+                  </>
+                )
+              }
+              <Textarea
+                className={`
+                  block w-full px-2 pr-[118px] py-[7px] leading-5 max-h-none text-sm text-gray-700 outline-none appearance-none resize-none
+                  ${visionConfig?.enabled && 'pl-12'}
+                `}
                 value={query}
                 onChange={handleContentChange}
                 onKeyUp={handleKeyUp}
                 onKeyDown={handleKeyDown}
-                minHeight={48}
-                autoFocus
-                controlFocus={controlFocus}
-                className={`${cn(s.textArea)} resize-none block w-full pl-3 bg-gray-50 border border-gray-200 rounded-md  focus:outline-none sm:text-sm text-gray-700`}
+                autoSize
               />
-              <div className="absolute top-0 right-2 flex items-center h-[48px]">
+              <div className="absolute bottom-2 right-2 flex items-center h-8">
                 <div className={`${s.count} mr-4 h-5 leading-5 text-sm bg-gray-50 text-gray-500`}>{query.trim().length}</div>
                 {
                   query
@@ -282,9 +332,8 @@ const Chat: FC<IChatProps> = ({
                 {isMobile
                   ? sendBtn
                   : (
-                    <Tooltip
-                      selector='send-tip'
-                      htmlContent={
+                    <TooltipPlus
+                      popupContent={
                         <div>
                           <div>{t('common.operation.send')} Enter</div>
                           <div>{t('common.operation.lineBreak')} Shift Enter</div>
@@ -292,7 +341,7 @@ const Chat: FC<IChatProps> = ({
                       }
                     >
                       {sendBtn}
-                    </Tooltip>
+                    </TooltipPlus>
                   )}
               </div>
               {

web/app/components/app/chat/question/index.tsx

@@ -8,14 +8,16 @@ import Log from '../log'
 import MoreInfo from '../more-info'
 import AppContext from '@/context/app-context'
 import { Markdown } from '@/app/components/base/markdown'
+import ImageGallery from '@/app/components/base/image-gallery'
 
 type IQuestionProps = Pick<IChatItem, 'id' | 'content' | 'more' | 'useCurrentUserAvatar'> & {
   isShowPromptLog?: boolean
   item: IChatItem
+  imgSrcs?: string[]
   isResponsing?: boolean
 }
 
-const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar, isShowPromptLog, item, isResponsing }) => {
+const Question: FC<IQuestionProps> = ({ id, content, imgSrcs, more, useCurrentUserAvatar, isShowPromptLog, item, isResponsing }) => {
   const { userProfile } = useContext(AppContext)
   const userName = userProfile?.name
   const ref = useRef(null)
@@ -23,6 +25,7 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar,
   return (
     <div className={`flex items-start justify-end ${isShowPromptLog && 'first-of-type:pt-[14px]'}`} key={id} ref={ref}>
       <div className={s.questionWrapWrap}>
+
         <div className={`${s.question} group relative text-sm text-gray-900`}>
           {
             isShowPromptLog && !isResponsing && (
@@ -32,6 +35,9 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar,
           <div
             className={'mr-2 py-3 px-4 bg-blue-500 rounded-tl-2xl rounded-b-2xl'}
           >
+            {imgSrcs && imgSrcs.length > 0 && (
+              <ImageGallery srcs={imgSrcs} />
+            )}
             <Markdown content={content} />
           </div>
         </div>

web/app/components/app/chat/type.ts

@@ -1,5 +1,5 @@
 import type { Annotation, MessageRating } from '@/models/log'
-
+import type { VisionFile } from '@/types/app'
 export type MessageMore = {
   time: string
   tokens: number
@@ -67,6 +67,7 @@ export type IChatItem = {
   useCurrentUserAvatar?: boolean
   isOpeningStatement?: boolean
   log?: { role: string; text: string }[]
+  message_files?: VisionFile[]
 }
 
 export type MessageEnd = {