Feature/mutil embedding model (#908)

Co-authored-by: JzoNg <jzongcode@gmail.com> Co-authored-by: jyong <jyong@dify.ai> Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
2023-08-18 17:37:31 +08:00
parent 4420281d96
commit db7156dafd
54 changed files with 1704 additions and 278 deletions
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
@@ -0,0 +1,108 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import {
+  useCSVDownloader,
+} from 'react-papaparse'
+import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
+import { Download02 as DownloadIcon } from '@/app/components/base/icons/src/vender/solid/general'
+import { DocForm } from '@/models/datasets'
+import I18n from '@/context/i18n'
+
+const CSV_TEMPLATE_QA_EN = [
+  ['question', 'answer'],
+  ['question1', 'answer1'],
+  ['question2', 'answer2'],
+]
+const CSV_TEMPLATE_QA_CN = [
+  ['问题', '答案'],
+  ['问题 1', '答案 1'],
+  ['问题 2', '答案 2'],
+]
+const CSV_TEMPLATE_EN = [
+  ['segment content'],
+  ['content1'],
+  ['content2'],
+]
+const CSV_TEMPLATE_CN = [
+  ['分段内容'],
+  ['内容 1'],
+  ['内容 2'],
+]
+
+const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
+  const { t } = useTranslation()
+  const { locale } = useContext(I18n)
+  const { CSVDownloader, Type } = useCSVDownloader()
+
+  const getTemplate = () => {
+    if (locale === 'en') {
+      if (docForm === DocForm.QA)
+        return CSV_TEMPLATE_QA_EN
+      return CSV_TEMPLATE_EN
+    }
+    if (docForm === DocForm.QA)
+      return CSV_TEMPLATE_QA_CN
+    return CSV_TEMPLATE_CN
+  }
+
+  return (
+    <div className='mt-6'>
+      <div className='text-sm text-gray-900 font-medium'>{t('share.generation.csvStructureTitle')}</div>
+      <div className='mt-2 max-h-[500px] overflow-auto'>
+        {docForm === DocForm.QA && (
+          <table className='table-fixed w-full border-separate border-spacing-0 border border-gray-200 rounded-lg text-xs'>
+            <thead className='text-gray-500'>
+              <tr>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.question')}</td>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.answer')}</td>
+              </tr>
+            </thead>
+            <tbody className='text-gray-700'>
+              <tr>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 1</td>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 1</td>
+              </tr>
+              <tr>
+                <td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 2</td>
+                <td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 2</td>
+              </tr>
+            </tbody>
+          </table>
+        )}
+        {docForm === DocForm.TEXT && (
+          <table className='table-fixed w-full border-separate border-spacing-0 border border-gray-200 rounded-lg text-xs'>
+            <thead className='text-gray-500'>
+              <tr>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.contentTitle')}</td>
+              </tr>
+            </thead>
+            <tbody className='text-gray-700'>
+              <tr>
+                <td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.content')} 1</td>
+              </tr>
+              <tr>
+                <td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.content')} 2</td>
+              </tr>
+            </tbody>
+          </table>
+        )}
+      </div>
+      <CSVDownloader
+        className="block mt-2 cursor-pointer"
+        type={Type.Link}
+        filename={'template'}
+        bom={true}
+        data={getTemplate()}
+      >
+        <div className='flex items-center h-[18px] space-x-1 text-[#155EEF] text-xs font-medium'>
+          <DownloadIcon className='w-3 h-3 mr-1' />
+          {t('datasetDocuments.list.batchModal.template')}
+        </div>
+      </CSVDownloader>
+    </div>
+
+  )
+}
+export default React.memo(CSVDownload)
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
@@ -0,0 +1,126 @@
+'use client'
+import type { FC } from 'react'
+import React, { useEffect, useRef, useState } from 'react'
+import cn from 'classnames'
+import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
+import { Csv as CSVIcon } from '@/app/components/base/icons/src/public/files'
+import { ToastContext } from '@/app/components/base/toast'
+import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
+import Button from '@/app/components/base/button'
+
+export type Props = {
+  file: File | undefined
+  updateFile: (file?: File) => void
+}
+
+const CSVUploader: FC<Props> = ({
+  file,
+  updateFile,
+}) => {
+  const { t } = useTranslation()
+  const { notify } = useContext(ToastContext)
+  const [dragging, setDragging] = useState(false)
+  const dropRef = useRef<HTMLDivElement>(null)
+  const dragRef = useRef<HTMLDivElement>(null)
+  const fileUploader = useRef<HTMLInputElement>(null)
+
+  const handleDragEnter = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    e.target !== dragRef.current && setDragging(true)
+  }
+  const handleDragOver = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+  }
+  const handleDragLeave = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    e.target === dragRef.current && setDragging(false)
+  }
+  const handleDrop = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    setDragging(false)
+    if (!e.dataTransfer)
+      return
+    const files = [...e.dataTransfer.files]
+    if (files.length > 1) {
+      notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.count') })
+      return
+    }
+    updateFile(files[0])
+  }
+  const selectHandle = () => {
+    if (fileUploader.current)
+      fileUploader.current.click()
+  }
+  const removeFile = () => {
+    if (fileUploader.current)
+      fileUploader.current.value = ''
+    updateFile()
+  }
+  const fileChangeHandle = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const currentFile = e.target.files?.[0]
+    updateFile(currentFile)
+  }
+
+  useEffect(() => {
+    dropRef.current?.addEventListener('dragenter', handleDragEnter)
+    dropRef.current?.addEventListener('dragover', handleDragOver)
+    dropRef.current?.addEventListener('dragleave', handleDragLeave)
+    dropRef.current?.addEventListener('drop', handleDrop)
+    return () => {
+      dropRef.current?.removeEventListener('dragenter', handleDragEnter)
+      dropRef.current?.removeEventListener('dragover', handleDragOver)
+      dropRef.current?.removeEventListener('dragleave', handleDragLeave)
+      dropRef.current?.removeEventListener('drop', handleDrop)
+    }
+  }, [])
+
+  return (
+    <div className='mt-6'>
+      <input
+        ref={fileUploader}
+        style={{ display: 'none' }}
+        type="file"
+        id="fileUploader"
+        accept='.csv'
+        onChange={fileChangeHandle}
+      />
+      <div ref={dropRef}>
+        {!file && (
+          <div className={cn('flex items-center h-20 rounded-xl bg-gray-50 border border-dashed border-gray-200 text-sm font-normal', dragging && 'bg-[#F5F8FF] border border-[#B2CCFF]')}>
+            <div className='w-full flex items-center justify-center space-x-2'>
+              <CSVIcon className="shrink-0" />
+              <div className='text-gray-500'>
+                {t('datasetDocuments.list.batchModal.csvUploadTitle')}
+                <span className='text-primary-400 cursor-pointer' onClick={selectHandle}>{t('datasetDocuments.list.batchModal.browse')}</span>
+              </div>
+            </div>
+            {dragging && <div ref={dragRef} className='absolute w-full h-full top-0 left-0'/>}
+          </div>
+        )}
+        {file && (
+          <div className={cn('flex items-center h-20 px-6 rounded-xl bg-gray-50 border border-gray-200 text-sm font-normal group', 'hover:bg-[#F5F8FF] hover:border-[#B2CCFF]')}>
+            <CSVIcon className="shrink-0" />
+            <div className='flex ml-2 w-0 grow'>
+              <span className='max-w-[calc(100%_-_30px)] text-ellipsis whitespace-nowrap overflow-hidden text-gray-800'>{file.name.replace(/.csv$/, '')}</span>
+              <span className='shrink-0 text-gray-500'>.csv</span>
+            </div>
+            <div className='hidden group-hover:flex items-center'>
+              <Button className='!h-8 !px-3 !py-[6px] bg-white !text-[13px] !leading-[18px] text-gray-700' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.change')}</Button>
+              <div className='mx-2 w-px h-4 bg-gray-200' />
+              <div className='p-2 cursor-pointer' onClick={removeFile}>
+                <Trash03 className='w-4 h-4 text-gray-500' />
+              </div>
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default React.memo(CSVUploader)
--- a/web/app/components/datasets/documents/detail/batch-modal/index.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/index.tsx
@@ -0,0 +1,65 @@
+'use client'
+import type { FC } from 'react'
+import React, { useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import CSVUploader from './csv-uploader'
+import CSVDownloader from './csv-downloader'
+import Button from '@/app/components/base/button'
+import Modal from '@/app/components/base/modal'
+import { XClose } from '@/app/components/base/icons/src/vender/line/general'
+import type { DocForm } from '@/models/datasets'
+
+export type IBatchModalProps = {
+  isShow: boolean
+  docForm: DocForm
+  onCancel: () => void
+  onConfirm: (file: File) => void
+}
+
+const BatchModal: FC<IBatchModalProps> = ({
+  isShow,
+  docForm,
+  onCancel,
+  onConfirm,
+}) => {
+  const { t } = useTranslation()
+  const [currentCSV, setCurrentCSV] = useState<File>()
+  const handleFile = (file?: File) => setCurrentCSV(file)
+
+  const handleSend = () => {
+    if (!currentCSV)
+      return
+    onCancel()
+    onConfirm(currentCSV)
+  }
+
+  useEffect(() => {
+    if (!isShow)
+      setCurrentCSV(undefined)
+  }, [isShow])
+
+  return (
+    <Modal isShow={isShow} onClose={() => {}} className='px-8 py-6 !max-w-[520px] !rounded-xl'>
+      <div className='relative pb-1 text-xl font-medium leading-[30px] text-gray-900'>{t('datasetDocuments.list.batchModal.title')}</div>
+      <div className='absolute right-4 top-4 p-2 cursor-pointer' onClick={onCancel}>
+        <XClose className='w-4 h-4 text-gray-500' />
+      </div>
+      <CSVUploader
+        file={currentCSV}
+        updateFile={handleFile}
+      />
+      <CSVDownloader
+        docForm={docForm}
+      />
+      <div className='mt-[28px] pt-6 flex justify-end'>
+        <Button className='mr-2 text-gray-700 text-sm font-medium' onClick={onCancel}>
+          {t('datasetDocuments.list.batchModal.cancel')}
+        </Button>
+        <Button className='text-sm font-medium' type="primary" onClick={handleSend} disabled={!currentCSV}>
+          {t('datasetDocuments.list.batchModal.run')}
+        </Button>
+      </div>
+    </Modal>
+  )
+}
+export default React.memo(BatchModal)