feat: support LLM process document file (#10966)

Co-authored-by: -LAN- <laipz8200@outlook.com>
2024-11-22 19:32:44 +08:00
parent 556de444e8
commit 08ac36812b
37 changed files with 233 additions and 88 deletions
--- a/web/app/components/app/configuration/config-vision/index.tsx
+++ b/web/app/components/app/configuration/config-vision/index.tsx
@@ -12,34 +12,46 @@ import ConfigContext from '@/context/debug-configuration'
 // import { Resolution } from '@/types/app'
 import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks'
 import Switch from '@/app/components/base/switch'
-import type { FileUpload } from '@/app/components/base/features/types'
+import { SupportUploadFileTypes } from '@/app/components/workflow/types'

 const ConfigVision: FC = () => {
  const { t } = useTranslation()
-  const { isShowVisionConfig } = useContext(ConfigContext)
+  const { isShowVisionConfig, isAllowVideoUpload } = useContext(ConfigContext)
  const file = useFeatures(s => s.features.file)
  const featuresStore = useFeaturesStore()

-  const handleChange = useCallback((data: FileUpload) => {
+  const isImageEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.image) ?? false
+
+  const handleChange = useCallback((value: boolean) => {
    const {
      features,
      setFeatures,
    } = featuresStore!.getState()

    const newFeatures = produce(features, (draft) => {
-      draft.file = {
-        ...draft.file,
-        enabled: data.enabled,
-        image: {
-          enabled: data.enabled,
-          detail: data.image?.detail,
-          transfer_methods: data.image?.transfer_methods,
-          number_limits: data.image?.number_limits,
-        },
+      if (value) {
+        draft.file!.allowed_file_types = Array.from(new Set([
+          ...(draft.file?.allowed_file_types || []),
+          SupportUploadFileTypes.image,
+          ...(isAllowVideoUpload ? [SupportUploadFileTypes.video] : []),
+        ]))
+      }
+      else {
+        draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter(
+          type => type !== SupportUploadFileTypes.image && (isAllowVideoUpload ? type !== SupportUploadFileTypes.video : true),
+        )
+      }
+
+      if (draft.file) {
+        draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0
+        draft.file.image = {
+          ...(draft.file.image || {}),
+          enabled: value,
+        }
      }
    })
    setFeatures(newFeatures)
-  }, [featuresStore])
+  }, [featuresStore, isAllowVideoUpload])

  if (!isShowVisionConfig)
    return null
@@ -89,11 +101,8 @@ const ConfigVision: FC = () => {
        <ParamConfig />
        <div className='ml-1 mr-3 w-[1px] h-3.5 bg-divider-subtle'></div>
        <Switch
-          defaultValue={file?.enabled}
-          onChange={value => handleChange({
-            ...(file || {}),
-            enabled: value,
-          })}
+          defaultValue={isImageEnabled}
+          onChange={handleChange}
          size='md'
        />
      </div>
--- a/web/app/components/app/configuration/config/config-document.tsx
+++ b/web/app/components/app/configuration/config/config-document.tsx
@@ -0,0 +1,78 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import produce from 'immer'
+import { useContext } from 'use-context-selector'
+
+import { Document } from '@/app/components/base/icons/src/vender/features'
+import Tooltip from '@/app/components/base/tooltip'
+import ConfigContext from '@/context/debug-configuration'
+import { SupportUploadFileTypes } from '@/app/components/workflow/types'
+import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks'
+import Switch from '@/app/components/base/switch'
+
+const ConfigDocument: FC = () => {
+  const { t } = useTranslation()
+  const file = useFeatures(s => s.features.file)
+  const featuresStore = useFeaturesStore()
+  const { isShowDocumentConfig } = useContext(ConfigContext)
+
+  const isDocumentEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.document) ?? false
+
+  const handleChange = useCallback((value: boolean) => {
+    const {
+      features,
+      setFeatures,
+    } = featuresStore!.getState()
+
+    const newFeatures = produce(features, (draft) => {
+      if (value) {
+        draft.file!.allowed_file_types = Array.from(new Set([
+          ...(draft.file?.allowed_file_types || []),
+          SupportUploadFileTypes.document,
+        ]))
+      }
+      else {
+        draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter(
+          type => type !== SupportUploadFileTypes.document,
+        )
+      }
+      if (draft.file)
+        draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0
+    })
+    setFeatures(newFeatures)
+  }, [featuresStore])
+
+  if (!isShowDocumentConfig)
+    return null
+
+  return (
+    <div className='mt-2 flex items-center gap-2 p-2 rounded-xl border-t-[0.5px] border-l-[0.5px] bg-background-section-burn'>
+      <div className='shrink-0 p-1'>
+        <div className='p-1 rounded-lg border-[0.5px] border-divider-subtle shadow-xs bg-util-colors-indigo-indigo-600'>
+          <Document className='w-4 h-4 text-text-primary-on-surface' />
+        </div>
+      </div>
+      <div className='grow flex items-center'>
+        <div className='mr-1 text-text-secondary system-sm-semibold'>{t('appDebug.feature.documentUpload.title')}</div>
+        <Tooltip
+          popupContent={
+            <div className='w-[180px]' >
+              {t('appDebug.feature.documentUpload.description')}
+            </div>
+          }
+        />
+      </div>
+      <div className='shrink-0 flex items-center'>
+        <div className='ml-1 mr-3 w-[1px] h-3.5 bg-divider-subtle'></div>
+        <Switch
+          defaultValue={isDocumentEnabled}
+          onChange={handleChange}
+          size='md'
+        />
+      </div>
+    </div>
+  )
+}
+export default React.memo(ConfigDocument)
--- a/web/app/components/app/configuration/config/index.tsx
+++ b/web/app/components/app/configuration/config/index.tsx
@@ -7,6 +7,7 @@ import { useFormattingChangedDispatcher } from '../debug/hooks'
 import DatasetConfig from '../dataset-config'
 import HistoryPanel from '../config-prompt/conversation-history/history-panel'
 import ConfigVision from '../config-vision'
+import ConfigDocument from './config-document'
 import AgentTools from './agent/agent-tools'
 import ConfigContext from '@/context/debug-configuration'
 import ConfigPrompt from '@/app/components/app/configuration/config-prompt'
@@ -82,6 +83,8 @@ const Config: FC = () => {

        <ConfigVision />

+        <ConfigDocument />
+
        {/* Chat History */}
        {isAdvancedMode && isChatApp && modelModeType === ModelModeType.completion && (
          <HistoryPanel
--- a/web/app/components/app/configuration/index.tsx
+++ b/web/app/components/app/configuration/index.tsx
@@ -451,7 +451,8 @@ const Configuration: FC = () => {
  }

  const isShowVisionConfig = !!currModel?.features?.includes(ModelFeatureEnum.vision)
-
+  const isShowDocumentConfig = !!currModel?.features?.includes(ModelFeatureEnum.document)
+  const isAllowVideoUpload = !!currModel?.features?.includes(ModelFeatureEnum.video)
  // *** web app features ***
  const featuresData: FeaturesData = useMemo(() => {
    return {
@@ -472,7 +473,7 @@ const Configuration: FC = () => {
          transfer_methods: modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'],
        },
        enabled: !!(modelConfig.file_upload?.enabled || modelConfig.file_upload?.image?.enabled),
-        allowed_file_types: modelConfig.file_upload?.allowed_file_types || [SupportUploadFileTypes.image, SupportUploadFileTypes.video],
+        allowed_file_types: modelConfig.file_upload?.allowed_file_types || [],
        allowed_file_extensions: modelConfig.file_upload?.allowed_file_extensions || [...FILE_EXTS[SupportUploadFileTypes.image], ...FILE_EXTS[SupportUploadFileTypes.video]].map(ext => `.${ext}`),
        allowed_file_upload_methods: modelConfig.file_upload?.allowed_file_upload_methods || modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'],
        number_limits: modelConfig.file_upload?.number_limits || modelConfig.file_upload?.image?.number_limits || 3,
@@ -861,6 +862,8 @@ const Configuration: FC = () => {
      isShowVisionConfig,
      visionConfig,
      setVisionConfig: handleSetVisionConfig,
+      isAllowVideoUpload,
+      isShowDocumentConfig,
      rerankSettingModalOpen,
      setRerankSettingModalOpen,
    }}