admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: [frontend] support vision (#1518)

Browse Source 
Co-authored-by: Joel <iamjoel007@gmail.com>
This commit is contained in:
zxhlyh
2023-11-13 22:32:39 +08:00
committed by GitHub
parent 41d0a8b295
commit 6b15827246
74 changed files with 3159 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
web/app/components/app/chat/index.tsx
View File

@@ -1,6 +1,7 @@
'use client'
import type { FC, ReactNode } from 'react'
import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
import Textarea from 'rc-textarea'
import { useContext } from 'use-context-selector'
import cn from 'classnames'
import Recorder from 'js-audio-recorder'
@@ -10,9 +11,8 @@ import type { DisplayScene, FeedbackFunc, IChatItem, SubmitAnnotationFunc } from
import { TryToAskIcon, stopIcon } from './icon-component'
import Answer from './answer'
import Question from './question'
import Tooltip from '@/app/components/base/tooltip'
import TooltipPlus from '@/app/components/base/tooltip-plus'
import { ToastContext } from '@/app/components/base/toast'
import AutoHeightTextarea from '@/app/components/base/auto-height-textarea'
import Button from '@/app/components/base/button'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import VoiceInput from '@/app/components/base/voice-input'
@@ -20,6 +20,10 @@ import { Microphone01 } from '@/app/components/base/icons/src/vender/line/mediaA
import { Microphone01 as Microphone01Solid } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
import { XCircle } from '@/app/components/base/icons/src/vender/solid/general'
import type { DataSet } from '@/models/datasets'
import ChatImageUploader from '@/app/components/base/image-uploader/chat-image-uploader'
import ImageList from '@/app/components/base/image-uploader/image-list'
import { TransferMethod, type VisionFile, type VisionSettings } from '@/types/app'
import { useImageFiles } from '@/app/components/base/image-uploader/hooks'
export type IChatProps = {
configElem?: React.ReactNode
@@ -37,7 +41,7 @@ export type IChatProps = {
onFeedback?: FeedbackFunc
onSubmitAnnotation?: SubmitAnnotationFunc
checkCanSend?: () => boolean
onSend?: (message: string) => void
onSend?: (message: string, files: VisionFile[]) => void
displayScene?: DisplayScene
useCurrentUserAvatar?: boolean
isResponsing?: boolean
@@ -54,6 +58,7 @@ export type IChatProps = {
dataSets?: DataSet[]
isShowCitationHitInfo?: boolean
isShowPromptLog?: boolean
visionConfig?: VisionSettings
}
const Chat: FC<IChatProps> = ({
@@ -83,9 +88,19 @@ const Chat: FC<IChatProps> = ({
dataSets,
isShowCitationHitInfo,
isShowPromptLog,
visionConfig,
}) => {
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const {
files,
onUpload,
onRemove,
onReUpload,
onImageLinkLoadError,
onImageLinkLoadSuccess,
onClear,
} = useImageFiles()
const isUseInputMethod = useRef(false)
const [query, setQuery] = React.useState('')
@@ -114,9 +129,18 @@ const Chat: FC<IChatProps> = ({
const handleSend = () => {
if (!valid() || (checkCanSend && !checkCanSend()))
return
onSend(query)
if (!isResponsing)
setQuery('')
onSend(query, files.filter(file => file.progress !== -1).map(fileItem => ({
type: 'image',
transfer_method: fileItem.type,
url: fileItem.url,
upload_file_id: fileItem.fileId,
})))
if (!files.find(item => item.type === TransferMethod.local_file && !item.fileId)) {
if (files.length)
onClear()
if (!isResponsing)
setQuery('')
}
}
const handleKeyUp = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
@@ -198,6 +222,8 @@ const Chat: FC<IChatProps> = ({
item={item}
isShowPromptLog={isShowPromptLog}
isResponsing={isResponsing}
// ['https://placekitten.com/360/360', 'https://placekitten.com/360/640']
imgSrcs={(item.message_files && item.message_files?.length > 0) ? item.message_files.map(item => item.url) : []}
/>
)
})}
@@ -246,18 +272,42 @@ const Chat: FC<IChatProps> = ({
</div>
</div>)
}
<div className="relative">
<AutoHeightTextarea
<div className='p-[5.5px] max-h-[150px] bg-white border-[1.5px] border-gray-200 rounded-xl overflow-y-auto'>
{
visionConfig?.enabled && (
<>
<div className='absolute bottom-2 left-2 flex items-center'>
<ChatImageUploader
settings={visionConfig}
onUpload={onUpload}
disabled={files.length >= visionConfig.number_limits}
/>
<div className='mx-1 w-[1px] h-4 bg-black/5' />
</div>
<div className='pl-[52px]'>
<ImageList
list={files}
onRemove={onRemove}
onReUpload={onReUpload}
onImageLinkLoadSuccess={onImageLinkLoadSuccess}
onImageLinkLoadError={onImageLinkLoadError}
/>
</div>
</>
)
}
<Textarea
className={`
block w-full px-2 pr-[118px] py-[7px] leading-5 max-h-none text-sm text-gray-700 outline-none appearance-none resize-none
${visionConfig?.enabled && 'pl-12'}
`}
value={query}
onChange={handleContentChange}
onKeyUp={handleKeyUp}
onKeyDown={handleKeyDown}
minHeight={48}
autoFocus
controlFocus={controlFocus}
className={`${cn(s.textArea)} resize-none block w-full pl-3 bg-gray-50 border border-gray-200 rounded-md focus:outline-none sm:text-sm text-gray-700`}
autoSize
/>
<div className="absolute top-0 right-2 flex items-center h-[48px]">
<div className="absolute bottom-2 right-2 flex items-center h-8">
<div className={`${s.count} mr-4 h-5 leading-5 text-sm bg-gray-50 text-gray-500`}>{query.trim().length}</div>
{
query
@@ -282,9 +332,8 @@ const Chat: FC<IChatProps> = ({
{isMobile
? sendBtn
: (
<Tooltip
selector='send-tip'
htmlContent={
<TooltipPlus
popupContent={
<div>
<div>{t('common.operation.send')} Enter</div>
<div>{t('common.operation.lineBreak')} Shift Enter</div>
@@ -292,7 +341,7 @@ const Chat: FC<IChatProps> = ({
}
>
{sendBtn}
</Tooltip>
</TooltipPlus>
)}
</div>
{

8
web/app/components/app/chat/question/index.tsx
View File

@@ -8,14 +8,16 @@ import Log from '../log'
import MoreInfo from '../more-info'
import AppContext from '@/context/app-context'
import { Markdown } from '@/app/components/base/markdown'
import ImageGallery from '@/app/components/base/image-gallery'
type IQuestionProps = Pick<IChatItem, 'id' | 'content' | 'more' | 'useCurrentUserAvatar'> & {
isShowPromptLog?: boolean
item: IChatItem
imgSrcs?: string[]
isResponsing?: boolean
}
const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar, isShowPromptLog, item, isResponsing }) => {
const Question: FC<IQuestionProps> = ({ id, content, imgSrcs, more, useCurrentUserAvatar, isShowPromptLog, item, isResponsing }) => {
const { userProfile } = useContext(AppContext)
const userName = userProfile?.name
const ref = useRef(null)
@@ -23,6 +25,7 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar,
return (
<div className={`flex items-start justify-end ${isShowPromptLog && 'first-of-type:pt-[14px]'}`} key={id} ref={ref}>
<div className={s.questionWrapWrap}>
<div className={`${s.question} group relative text-sm text-gray-900`}>
{
isShowPromptLog && !isResponsing && (
@@ -32,6 +35,9 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar,
<div
className={'mr-2 py-3 px-4 bg-blue-500 rounded-tl-2xl rounded-b-2xl'}
>
{imgSrcs && imgSrcs.length > 0 && (
<ImageGallery srcs={imgSrcs} />
)}
<Markdown content={content} />
</div>
</div>

3
web/app/components/app/chat/type.ts
View File

@@ -1,5 +1,5 @@
import type { Annotation, MessageRating } from '@/models/log'
import type { VisionFile } from '@/types/app'
export type MessageMore = {
time: string
tokens: number
@@ -67,6 +67,7 @@ export type IChatItem = {
useCurrentUserAvatar?: boolean
isOpeningStatement?: boolean
log?: { role: string; text: string }[]
message_files?: VisionFile[]
}
export type MessageEnd = {

6
web/app/components/app/configuration/config-model/index.tsx
View File

@@ -33,7 +33,7 @@ export type IConfigModelProps = {
mode: string
modelId: string
provider: ProviderEnum
setModel: (model: { id: string; provider: ProviderEnum; mode: ModelModeType }) => void
setModel: (model: { id: string; provider: ProviderEnum; mode: ModelModeType; features: string[] }) => void
completionParams: CompletionParams
onCompletionParamsChange: (newParams: CompletionParams) => void
disabled: boolean
@@ -121,7 +121,7 @@ const ConfigModel: FC<IConfigModelProps> = ({
return adjustedValue
}
const handleSelectModel = ({ id, provider: nextProvider, mode }: { id: string; provider: ProviderEnum; mode: ModelModeType }) => {
const handleSelectModel = ({ id, provider: nextProvider, mode, features }: { id: string; provider: ProviderEnum; mode: ModelModeType; features: string[] }) => {
return async () => {
const prevParamsRule = getAllParams()[provider]?.[modelId]
@@ -129,6 +129,7 @@ const ConfigModel: FC<IConfigModelProps> = ({
id,
provider: nextProvider || ProviderEnum.openai,
mode,
features,
})
await ensureModelParamLoaded(nextProvider, id)
@@ -320,6 +321,7 @@ const ConfigModel: FC<IConfigModelProps> = ({
id: model.model_name,
provider: model.model_provider.provider_name as ProviderEnum,
mode: model.model_mode,
features: model.features,
})()
}}
/>

2
web/app/components/app/configuration/config-var/index.tsx
View File

@@ -169,7 +169,7 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
}
title={
<div className='flex items-center'>
<div className='ml-1 mr-1'>{t('appDebug.variableTitle')}</div>
<div className='mr-1'>{t('appDebug.variableTitle')}</div>
{!readonly && (
<Tooltip htmlContent={<div className='w-[180px]'>
{t('appDebug.variableTip')}

60
web/app/components/app/configuration/config-vision/index.tsx Normal file
View File

@@ -0,0 +1,60 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import Panel from '../base/feature-panel'
import ParamConfig from './param-config'
import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
import Tooltip from '@/app/components/base/tooltip'
import Switch from '@/app/components/base/switch'
import { Eye } from '@/app/components/base/icons/src/vender/solid/general'
import ConfigContext from '@/context/debug-configuration'
const ConfigVision: FC = () => {
const { t } = useTranslation()
const {
isShowVisionConfig,
visionConfig,
setVisionConfig,
} = useContext(ConfigContext)
if (!isShowVisionConfig)
return null
return (<>
<Panel
className="mt-4"
headerIcon={
<Eye className='w-4 h-4 text-[#6938EF]'/>
}
title={
<div className='flex items-center'>
<div className='mr-1'>{t('appDebug.vision.name')}</div>
<Tooltip htmlContent={<div className='w-[180px]' >
{t('appDebug.vision.description')}
</div>} selector='config-vision-tooltip'>
<HelpCircle className='w-[14px] h-[14px] text-gray-400' />
</Tooltip>
</div>
}
headerRight={
<div className='flex items-center'>
<ParamConfig />
<div className='ml-4 mr-3 w-[1px] h-3.5 bg-gray-200'></div>
<Switch
defaultValue={visionConfig.enabled}
onChange={value => setVisionConfig({
...visionConfig,
enabled: value,
})}
size='md'
/>
</div>
}
noBodySpacing
/>
</>
)
}
export default React.memo(ConfigVision)

132
web/app/components/app/configuration/config-vision/param-config-content.tsx Normal file
View File

@@ -0,0 +1,132 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useContext } from 'use-context-selector'
import { useTranslation } from 'react-i18next'
import RadioGroup from './radio-group'
import ConfigContext from '@/context/debug-configuration'
import { Resolution, TransferMethod } from '@/types/app'
import ParamItem from '@/app/components/base/param-item'
import Tooltip from '@/app/components/base/tooltip'
import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
const MIN = 1
const MAX = 6
const ParamConfigContent: FC = () => {
const { t } = useTranslation()
const {
visionConfig,
setVisionConfig,
} = useContext(ConfigContext)
const transferMethod = (() => {
if (!visionConfig.transfer_methods || visionConfig.transfer_methods.length === 2)
return TransferMethod.all
return visionConfig.transfer_methods[0]
})()
return (
<div>
<div>
<div className='leading-6 text-base font-semibold text-gray-800'>{t('appDebug.vision.visionSettings.title')}</div>
<div className='pt-3 space-y-6'>
<div>
<div className='mb-2 flex items-center space-x-1'>
<div className='leading-[18px] text-[13px] font-semibold text-gray-800'>{t('appDebug.vision.visionSettings.resolution')}</div>
<Tooltip htmlContent={<div className='w-[180px]' >
{t('appDebug.vision.visionSettings.resolutionTooltip').split('\n').map(item => (
<div key={item}>{item}</div>
))}
</div>} selector='config-resolution-tooltip'>
<HelpCircle className='w-[14px] h-[14px] text-gray-400' />
</Tooltip>
</div>
<RadioGroup
className='space-x-3'
options={[
{
label: t('appDebug.vision.visionSettings.high'),
value: Resolution.high,
},
{
label: t('appDebug.vision.visionSettings.low'),
value: Resolution.low,
},
]}
value={visionConfig.detail}
onChange={(value: Resolution) => {
setVisionConfig({
...visionConfig,
detail: value,
})
}}
/>
</div>
<div>
<div className='mb-2 leading-[18px] text-[13px] font-semibold text-gray-800'>{t('appDebug.vision.visionSettings.uploadMethod')}</div>
<RadioGroup
className='space-x-3'
options={[
{
label: t('appDebug.vision.visionSettings.both'),
value: TransferMethod.all,
},
{
label: t('appDebug.vision.visionSettings.localUpload'),
value: TransferMethod.local_file,
},
{
label: t('appDebug.vision.visionSettings.url'),
value: TransferMethod.remote_url,
},
]}
value={transferMethod}
onChange={(value: TransferMethod) => {
if (value === TransferMethod.all) {
setVisionConfig({
...visionConfig,
transfer_methods: [TransferMethod.remote_url, TransferMethod.local_file],
})
return
}
setVisionConfig({
...visionConfig,
transfer_methods: [value],
})
}}
/>
</div>
<div>
<ParamItem
id='upload_limit'
className=''
name={t('appDebug.vision.visionSettings.uploadLimit')}
noTooltip
{...{
default: 2,
step: 1,
min: MIN,
max: MAX,
}}
value={visionConfig.number_limits}
enable={true}
onChange={(_key: string, value: number) => {
if (!value)
return
setVisionConfig({
...visionConfig,
number_limits: value,
})
}}
/>
</div>
</div>
</div>
</div>
)
}
export default React.memo(ParamConfigContent)

41
web/app/components/app/configuration/config-vision/param-config.tsx Normal file
View File

@@ -0,0 +1,41 @@
'use client'
import type { FC } from 'react'
import { memo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import cn from 'classnames'
import ParamConfigContent from './param-config-content'
import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
import {
PortalToFollowElem,
PortalToFollowElemContent,
PortalToFollowElemTrigger,
} from '@/app/components/base/portal-to-follow-elem'
const ParamsConfig: FC = () => {
const { t } = useTranslation()
const [open, setOpen] = useState(false)
return (
<PortalToFollowElem
open={open}
onOpenChange={setOpen}
placement='bottom-end'
offset={{
mainAxis: 4,
}}
>
<PortalToFollowElemTrigger onClick={() => setOpen(v => !v)}>
<div className={cn('flex items-center rounded-md h-7 px-3 space-x-1 text-gray-700 cursor-pointer hover:bg-gray-200', open && 'bg-gray-200')}>
<Settings01 className='w-3.5 h-3.5 ' />
<div className='ml-1 leading-[18px] text-xs font-medium '>{t('appDebug.vision.settings')}</div>
</div>
</PortalToFollowElemTrigger>
<PortalToFollowElemContent style={{ zIndex: 50 }}>
<div className='w-[412px] p-4 bg-white rounded-lg border-[0.5px] border-gray-200 shadow-lg space-y-3'>
<ParamConfigContent />
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
)
}
export default memo(ParamsConfig)

40
web/app/components/app/configuration/config-vision/radio-group/index.tsx Normal file
View File

@@ -0,0 +1,40 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from 'classnames'
import s from './style.module.css'
type OPTION = {
label: string
value: any
}
type Props = {
className?: string
options: OPTION[]
value: any
onChange: (value: any) => void
}
const RadioGroup: FC<Props> = ({
className = '',
options,
value,
onChange,
}) => {
return (
<div className={cn(className, 'flex')}>
{options.map(item => (
<div
key={item.value}
className={cn(s.item, item.value === value && s.checked)}
onClick={() => onChange(item.value)}
>
<div className={s.radio}></div>
<div className='text-[13px] font-medium text-gray-900'>{item.label}</div>
</div>
))}
</div>
)
}
export default React.memo(RadioGroup)

24
web/app/components/app/configuration/config-vision/radio-group/style.module.css Normal file
View File

@@ -0,0 +1,24 @@
.item {
@apply grow flex items-center h-8 px-2.5 rounded-lg bg-gray-25 border border-gray-100 cursor-pointer space-x-2;
}
.item:hover {
background-color: #ffffff;
border-color: #B2CCFF;
box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
}
.item.checked {
background-color: #ffffff;
border-color: #528BFF;
box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.06), 0px 1px 3px 0px rgba(16, 24, 40, 0.10);
}
.radio {
@apply w-4 h-4 border-[2px] border-gray-200 rounded-full;
}
.item.checked .radio {
border-width: 5px;
border-color: #155eef;
}

3
web/app/components/app/configuration/config/index.tsx
View File

@@ -10,6 +10,7 @@ import ChatGroup from '../features/chat-group'
import ExperienceEnchanceGroup from '../features/experience-enchance-group'
import Toolbox from '../toolbox'
import HistoryPanel from '../config-prompt/conversation-histroy/history-panel'
import ConfigVision from '../config-vision'
import AddFeatureBtn from './feature/add-feature-btn'
import ChooseFeature from './feature/choose-feature'
import useFeature from './feature/use-feature'
@@ -193,6 +194,8 @@ const Config: FC = () => {
<Tools />
<ConfigVision />
{/* Chat History */}
{isAdvancedMode && isChatApp && modelModeType === ModelModeType.completion && (
<HistoryPanel

2
web/app/components/app/configuration/dataset-config/index.tsx
View File

@@ -81,7 +81,7 @@ const DatasetConfig: FC = () => {
>
{hasData
? (
<div className='flex flex-wrap mt-1 px-3 justify-between'>
<div className='flex flex-wrap mt-1 px-3 pb-3 justify-between'>
{dataSet.map(item => (
<CardItem
key={item.id}

66
web/app/components/app/configuration/debug/index.tsx
View File

@@ -1,5 +1,6 @@
'use client'
import type { FC } from 'react'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import React, { useEffect, useRef, useState } from 'react'
import cn from 'classnames'
@@ -11,7 +12,7 @@ import HasNotSetAPIKEY from '../base/warning-mask/has-not-set-api'
import FormattingChanged from '../base/warning-mask/formatting-changed'
import GroupName from '../base/group-name'
import CannotQueryDataset from '../base/warning-mask/cannot-query-dataset'
import { AppType, ModelModeType } from '@/types/app'
import { AppType, ModelModeType, TransferMethod } from '@/types/app'
import PromptValuePanel, { replaceStringWithValues } from '@/app/components/app/configuration/prompt-value-panel'
import type { IChatItem } from '@/app/components/app/chat/type'
import Chat from '@/app/components/app/chat'
@@ -19,12 +20,13 @@ import ConfigContext from '@/context/debug-configuration'
import { ToastContext } from '@/app/components/base/toast'
import { fetchConvesationMessages, fetchSuggestedQuestions, sendChatMessage, sendCompletionMessage, stopChatMessageResponding } from '@/service/debug'
import Button from '@/app/components/base/button'
import type { ModelConfig as BackendModelConfig } from '@/types/app'
import type { ModelConfig as BackendModelConfig, VisionFile } from '@/types/app'
import { promptVariablesToUserInputsForm } from '@/utils/model-config'
import TextGeneration from '@/app/components/app/text-generate/item'
import { IS_CE_EDITION } from '@/config'
import { useProviderContext } from '@/context/provider-context'
import type { Inputs } from '@/models/debug'
import { fetchFileUploadConfig } from '@/service/common'
type IDebug = {
hasSetAPIKEY: boolean
@@ -64,10 +66,12 @@ const Debug: FC<IDebug> = ({
hasSetContextVar,
datasetConfigs,
externalDataToolsConfig,
visionConfig,
} = useContext(ConfigContext)
const { speech2textDefaultModel } = useProviderContext()
const [chatList, setChatList, getChatList] = useGetState<IChatItem[]>([])
const chatListDomRef = useRef<HTMLDivElement>(null)
const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
useEffect(() => {
// scroll to bottom
if (chatListDomRef.current)
@@ -161,17 +165,28 @@ const Debug: FC<IDebug> = ({
logError(t('appDebug.errorMessage.valueOfVarRequired', { key: hasEmptyInput }))
return false
}
// eslint-disable-next-line @typescript-eslint/no-use-before-define
if (completionFiles.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
return false
}
return !hasEmptyInput
}
const doShowSuggestion = isShowSuggestion && !isResponsing
const [suggestQuestions, setSuggestQuestions] = useState<string[]>([])
const onSend = async (message: string) => {
const onSend = async (message: string, files?: VisionFile[]) => {
if (isResponsing) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') })
return false
}
if (files?.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
return false
}
const postDatasets = dataSets.map(({ id }) => ({
dataset: {
enabled: true,
@@ -207,6 +222,9 @@ const Debug: FC<IDebug> = ({
completion_params: completionParams as any,
},
dataset_configs: datasetConfigs,
file_upload: {
image: visionConfig,
},
}
if (isAdvancedMode) {
@@ -214,19 +232,32 @@ const Debug: FC<IDebug> = ({
postModelConfig.completion_prompt_config = completionPromptConfig
}
const data = {
const data: Record<string, any> = {
conversation_id: conversationId,
inputs,
query: message,
model_config: postModelConfig,
}
if (visionConfig.enabled && files && files?.length > 0) {
data.files = files.map((item) => {
if (item.transfer_method === TransferMethod.local_file) {
return {
...item,
url: '',
}
}
return item
})
}
// qustion
const questionId = `question-${Date.now()}`
const questionItem = {
id: questionId,
content: message,
isAnswer: false,
message_files: files,
}
const placeholderAnswerId = `answer-placeholder-${Date.now()}`
@@ -347,6 +378,7 @@ const Debug: FC<IDebug> = ({
const [completionRes, setCompletionRes] = useState('')
const [messageId, setMessageId] = useState<string | null>(null)
const [completionFiles, setCompletionFiles] = useState<VisionFile[]>([])
const sendTextCompletion = async () => {
if (isResponsing) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') })
@@ -394,6 +426,9 @@ const Debug: FC<IDebug> = ({
completion_params: completionParams as any,
},
dataset_configs: datasetConfigs,
file_upload: {
image: visionConfig,
},
}
if (isAdvancedMode) {
@@ -401,11 +436,23 @@ const Debug: FC<IDebug> = ({
postModelConfig.completion_prompt_config = completionPromptConfig
}
const data = {
const data: Record<string, any> = {
inputs,
model_config: postModelConfig,
}
if (visionConfig.enabled && completionFiles && completionFiles?.length > 0) {
data.files = completionFiles.map((item) => {
if (item.transfer_method === TransferMethod.local_file) {
return {
...item,
url: '',
}
}
return item
})
}
setCompletionRes('')
setMessageId('')
let res: string[] = []
@@ -448,6 +495,11 @@ const Debug: FC<IDebug> = ({
appType={mode as AppType}
onSend={sendTextCompletion}
inputs={inputs}
visionConfig={{
...visionConfig,
image_file_size_limit: fileUploadConfigResponse?.image_file_size_limit,
}}
onVisionFilesChange={setCompletionFiles}
/>
</div>
<div className="flex flex-col grow">
@@ -475,6 +527,10 @@ const Debug: FC<IDebug> = ({
isShowCitation={citationConfig.enabled}
isShowCitationHitInfo
isShowPromptLog
visionConfig={{
...visionConfig,
image_file_size_limit: fileUploadConfigResponse?.image_file_size_limit,
}}
/>
</div>
</div>

43
web/app/components/app/configuration/index.tsx
View File

@@ -25,19 +25,19 @@ import type {
} from '@/models/debug'
import type { ExternalDataTool } from '@/models/common'
import type { DataSet } from '@/models/datasets'
import type { ModelConfig as BackendModelConfig } from '@/types/app'
import type { ModelConfig as BackendModelConfig, VisionSettings } from '@/types/app'
import ConfigContext from '@/context/debug-configuration'
import ConfigModel from '@/app/components/app/configuration/config-model'
import Config from '@/app/components/app/configuration/config'
import Debug from '@/app/components/app/configuration/debug'
import Confirm from '@/app/components/base/confirm'
import { ProviderEnum } from '@/app/components/header/account-setting/model-page/declarations'
import { ModelFeature, ProviderEnum } from '@/app/components/header/account-setting/model-page/declarations'
import { ToastContext } from '@/app/components/base/toast'
import { fetchAppDetail, updateAppModelConfig } from '@/service/apps'
import { promptVariablesToUserInputsForm, userInputsFormToPromptVariables } from '@/utils/model-config'
import { fetchDatasets } from '@/service/datasets'
import { useProviderContext } from '@/context/provider-context'
import { AppType, ModelModeType } from '@/types/app'
import { AppType, ModelModeType, Resolution, TransferMethod } from '@/types/app'
import { FlipBackward } from '@/app/components/base/icons/src/vender/line/arrows'
import { PromptMode } from '@/models/debug'
import { DEFAULT_CHAT_PROMPT_CONFIG, DEFAULT_COMPLETION_PROMPT_CONFIG } from '@/config'
@@ -198,6 +198,7 @@ const Configuration: FC = () => {
}
const { textGenerationModelList } = useProviderContext()
const currModel = textGenerationModelList.find(item => item.model_name === modelConfig.model_id)
const hasSetCustomAPIKEY = !!textGenerationModelList?.find(({ model_provider: provider }) => {
if (provider.provider_type === 'system' && provider.quota_type === 'paid')
return true
@@ -271,7 +272,8 @@ const Configuration: FC = () => {
id: modelId,
provider,
mode: modeMode,
}: { id: string; provider: ProviderEnum; mode: ModelModeType }) => {
features,
}: { id: string; provider: ProviderEnum; mode: ModelModeType; features: string[] }) => {
if (isAdvancedMode) {
const appMode = mode
@@ -297,10 +299,31 @@ const Configuration: FC = () => {
})
setModelConfig(newModelConfig)
const supportVision = features && features.includes(ModelFeature.vision)
// eslint-disable-next-line @typescript-eslint/no-use-before-define
setVisionConfig({
// eslint-disable-next-line @typescript-eslint/no-use-before-define
...visionConfig,
enabled: supportVision,
}, true)
}
const isShowVisionConfig = !!currModel?.features.includes(ModelFeature.vision)
const [visionConfig, doSetVisionConfig] = useState({
enabled: false,
number_limits: 2,
detail: Resolution.low,
transfer_methods: [TransferMethod.local_file],
})
const setVisionConfig = (config: VisionSettings, notNoticeFormattingChanged?: boolean) => {
doSetVisionConfig(config)
if (!notNoticeFormattingChanged)
setFormattingChanged(true)
}
useEffect(() => {
fetchAppDetail({ url: '/apps', id: appId }).then(async (res) => {
fetchAppDetail({ url: '/apps', id: appId }).then(async (res: any) => {
setMode(res.mode)
const modelConfig = res.model_config
const promptMode = modelConfig.prompt_type === PromptMode.advanced ? PromptMode.advanced : PromptMode.simple
@@ -362,6 +385,10 @@ const Configuration: FC = () => {
},
completionParams: model.completion_params,
}
if (modelConfig.file_upload)
setVisionConfig(modelConfig.file_upload.image, true)
syncToPublishedConfig(config)
setPublishedConfig(config)
setDatasetConfigs(modelConfig.dataset_configs)
@@ -459,6 +486,9 @@ const Configuration: FC = () => {
completion_params: completionParams as any,
},
dataset_configs: datasetConfigs,
file_upload: {
image: visionConfig,
},
}
if (isAdvancedMode) {
@@ -557,6 +587,9 @@ const Configuration: FC = () => {
datasetConfigs,
setDatasetConfigs,
hasSetContextVar,
isShowVisionConfig,
visionConfig,
setVisionConfig,
}}
>
<>

24
web/app/components/app/configuration/prompt-value-panel/index.tsx
View File

@@ -14,17 +14,23 @@ import { DEFAULT_VALUE_MAX_LEN } from '@/config'
import Button from '@/app/components/base/button'
import { ChevronDown, ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
import Tooltip from '@/app/components/base/tooltip-plus'
import TextGenerationImageUploader from '@/app/components/base/image-uploader/text-generation-image-uploader'
import type { VisionFile, VisionSettings } from '@/types/app'
export type IPromptValuePanelProps = {
appType: AppType
onSend?: () => void
inputs: Inputs
visionConfig: VisionSettings
onVisionFilesChange: (files: VisionFile[]) => void
}
const PromptValuePanel: FC<IPromptValuePanelProps> = ({
appType,
onSend,
inputs,
visionConfig,
onVisionFilesChange,
}) => {
const { t } = useTranslation()
const { modelModeType, modelConfig, setInputs, mode, isAdvancedMode, completionPromptConfig, chatPromptConfig } = useContext(ConfigContext)
@@ -152,6 +158,24 @@ const PromptValuePanel: FC<IPromptValuePanelProps> = ({
<div className='text-xs text-gray-500'>{t('appDebug.inputs.noVar')}</div>
)
}
{
appType === AppType.completion && visionConfig?.enabled && (
<div className="mt-3 xl:flex justify-between">
<div className="mr-1 py-2 shrink-0 w-[120px] text-sm text-gray-900">Image Upload</div>
<div className='grow'>
<TextGenerationImageUploader
settings={visionConfig}
onFilesChange={files => onVisionFilesChange(files.filter(file => file.progress !== -1).map(fileItem => ({
type: 'image',
transfer_method: fileItem.type,
url: fileItem.url,
upload_file_id: fileItem.fileId,
})))}
/>
</div>
</div>
)
}
</>
)
}

17
web/app/components/app/log/list.tsx
View File

@@ -33,7 +33,6 @@ import { TONE_LIST } from '@/config'
import ModelIcon from '@/app/components/app/configuration/config-model/model-icon'
import ModelName from '@/app/components/app/configuration/config-model/model-name'
import ModelModeTypeLabel from '@/app/components/app/configuration/config-model/model-mode-type-label'
import { ModelModeType } from '@/types/app'
type IConversationList = {
logs?: ChatConversationsResponse | CompletionConversationsResponse
@@ -81,6 +80,7 @@ const getFormattedChatList = (messages: ChatMessage[]) => {
content: item.inputs.query || item.inputs.default_input || item.query, // text generation: item.inputs.query; chat: item.query
isAnswer: false,
log: item.message as any,
message_files: item.message_files,
})
newChatList.push({
@@ -174,9 +174,12 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
const itemContent = item[Object.keys(item)[0]]
return {
label: itemContent.variable,
value: varValues[itemContent.variable],
value: varValues[itemContent.variable] || detail.message?.inputs?.[itemContent.variable],
}
})
const message_files = (!isChatMode && detail.message.message_files && detail.message.message_files.length > 0)
? detail.message.message_files.map((item: any) => item.url)
: []
const getParamValue = (param: string) => {
const value = detail?.model_config.model?.completion_params?.[param] || '-'
@@ -209,7 +212,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
<div className='text-[13px] text-gray-900 font-medium'>
<ModelName modelId={modelName} modelDisplayName={modelName} />
</div>
<ModelModeTypeLabel type={ModelModeType.chat} isHighlight />
<ModelModeTypeLabel type={detail?.model_config.model.mode as any} isHighlight />
</div>
<Popover
position='br'
@@ -239,11 +242,15 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
</div>
{/* Panel Body */}
{varList.length > 0 && (
{(varList.length > 0 || (!isChatMode && message_files.length > 0)) && (
<div className='px-6 pt-4 pb-2'>
<VarPanel varList={varList} />
<VarPanel
varList={varList}
message_files={message_files}
/>
</div>
)}
{!isChatMode
? <div className="px-2.5 py-4">
<Chat

34
web/app/components/app/log/var-panel.tsx
View File

@@ -1,19 +1,24 @@
'use client'
import { useBoolean } from 'ahooks'
import type { FC } from 'react'
import React from 'react'
import React, { useState } from 'react'
import { useTranslation } from 'react-i18next'
import { ChevronDown, ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
import ImagePreview from '@/app/components/base/image-uploader/image-preview'
type Props = {
varList: { label: string; value: string }[]
message_files: string[]
}
const VarPanel: FC<Props> = ({
varList,
message_files,
}) => {
const { t } = useTranslation()
const [isCollapse, { toggle: toggleCollapse }] = useBoolean(false)
const [imagePreviewUrl, setImagePreviewUrl] = useState('')
return (
<div className='rounded-xl border border-color-indigo-100 bg-indigo-25'>
<div
@@ -30,7 +35,7 @@ const VarPanel: FC<Props> = ({
{!isCollapse && (
<div className='px-6 pb-3'>
{varList.map(({ label, value }, index) => (
<div key={index} className='flex py-1 leading-[18px] text-[13px]'>
<div key={index} className='flex py-2 leading-[18px] text-[13px]'>
<div className='shrink-0 w-[128px] flex text-primary-600'>
<span className='shrink-0 opacity-60'>{'{{'}</span>
<span className='truncate'>{label}</span>
@@ -39,9 +44,32 @@ const VarPanel: FC<Props> = ({
<div className='pl-2.5 break-all'>{value}</div>
</div>
))}
{message_files.length > 0 && (
<div className='mt-1 flex py-2'>
<div className='shrink-0 w-[128px] leading-[18px] text-[13px] font-medium text-gray-700'>{t('appLog.detail.uploadImages')}</div>
<div className="flex space-x-2">
{message_files.map((url, index) => (
<div
key={index}
className="ml-2.5 w-16 h-16 rounded-lg bg-no-repeat bg-cover bg-center cursor-pointer"
style={{ backgroundImage: `url(${url})` }}
onClick={() => setImagePreviewUrl(url)}
/>
))}
</div>
</div>
)}
</div>
)}
{
imagePreviewUrl && (
<ImagePreview
url={imagePreviewUrl}
onCancel={() => setImagePreviewUrl('')}
/>
)
}
</div>
)
}