feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -2,14 +2,14 @@ import { type FC, useMemo, useState } from 'react'
import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
import { useTranslation } from 'react-i18next'
import { EditSlice } from '../../../formatted-text/flavours/edit-slice'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { FormattedText } from '../../../formatted-text/formatted'
import Empty from './common/empty'
import FullDocListSkeleton from './skeleton/full-doc-list-skeleton'
import { useSegmentListContext } from './index'
import type { ChildChunkDetail } from '@/models/datasets'
import Input from '@/app/components/base/input'
import classNames from '@/utils/classnames'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import { formatNumber } from '@/utils/format'
@@ -87,24 +87,25 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
}, [isFullDocMode, total, childChunks.length, inputValue])
return (
<div className={classNames(
<div className={cn(
'flex flex-col',
contentOpacity,
isParagraphMode ? 'pb-2 pt-1' : 'grow px-3',
(isFullDocMode && isLoading) && 'overflow-y-hidden',
)}>
{isFullDocMode ? <Divider type='horizontal' className='my-1 h-px bg-divider-subtle' /> : null}
<div className={classNames('flex items-center justify-between', isFullDocMode ? 'sticky -top-2 left-0 bg-background-default pb-3 pt-2' : '')}>
<div className={classNames(
'flex h-7 items-center rounded-lg pl-1 pr-3',
isParagraphMode && 'cursor-pointer',
(isParagraphMode && collapsed) && 'bg-dataset-child-chunk-expand-btn-bg',
isFullDocMode && 'pl-0',
)}
onClick={(event) => {
event.stopPropagation()
toggleCollapse()
}}
<div className={cn('flex items-center justify-between', isFullDocMode ? 'sticky -top-2 left-0 bg-background-default pb-3 pt-2' : '')}>
<div
className={cn(
'flex h-7 items-center rounded-lg pl-1 pr-3',
isParagraphMode && 'cursor-pointer',
(isParagraphMode && collapsed) && 'bg-dataset-child-chunk-expand-btn-bg',
isFullDocMode && 'pl-0',
)}
onClick={(event) => {
event.stopPropagation()
toggleCollapse()
}}
>
{
isParagraphMode
@@ -116,10 +117,10 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
: null
}
<span className='system-sm-semibold-uppercase text-text-secondary'>{totalText}</span>
<span className={classNames('pl-1.5 text-xs font-medium text-text-quaternary', isParagraphMode ? 'hidden group-hover/card:inline-block' : '')}>·</span>
<span className={cn('pl-1.5 text-xs font-medium text-text-quaternary', isParagraphMode ? 'hidden group-hover/card:inline-block' : '')}>·</span>
<button
type='button'
className={classNames(
className={cn(
'system-xs-semibold-uppercase px-1.5 py-1 text-components-button-secondary-accent-text',
isParagraphMode ? 'hidden group-hover/card:inline-block' : '',
(isFullDocMode && isLoading) ? 'text-components-button-secondary-accent-text-disabled' : '',
@@ -146,14 +147,14 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
</div>
{isLoading ? <FullDocListSkeleton /> : null}
{((isFullDocMode && !isLoading) || !collapsed)
? <div className={classNames('flex gap-x-0.5', isFullDocMode ? 'mb-6 grow' : 'items-center')}>
? <div className={cn('flex gap-x-0.5', isFullDocMode ? 'mb-6 grow' : 'items-center')}>
{isParagraphMode && (
<div className='self-stretch'>
<Divider type='vertical' className='mx-[7px] w-[2px] bg-text-accent-secondary' />
</div>
)}
{childChunks.length > 0
? <FormattedText className={classNames('flex w-full flex-col !leading-6', isParagraphMode ? 'gap-y-2' : 'gap-y-3')}>
? <FormattedText className={cn('flex w-full flex-col !leading-6', isParagraphMode ? 'gap-y-2' : 'gap-y-3')}>
{childChunks.map((childChunk) => {
const edited = childChunk.updated_at !== childChunk.created_at
const focused = currChildChunk?.childChunkInfo?.id === childChunk.id
@@ -162,9 +163,10 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
label={`C-${childChunk.position}${edited ? ` · ${t('datasetDocuments.segment.edited')}` : ''}`}
text={childChunk.content}
onDelete={() => onDelete?.(childChunk.segment_id, childChunk.id)}
className='child-chunk'
labelClassName={focused ? 'bg-state-accent-solid text-text-primary-on-surface' : ''}
labelInnerClassName={'text-[10px] font-semibold align-bottom leading-6'}
contentClassName={classNames('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : 'text-text-secondary')}
contentClassName={cn('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : 'text-text-secondary')}
showDivider={false}
onClick={(e) => {
e.stopPropagation()

View File

@@ -1,9 +1,10 @@
import React, { type FC, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useKeyPress } from 'ahooks'
import { useDocumentContext } from '../../index'
import { useDocumentContext } from '../../context'
import Button from '@/app/components/base/button'
import { getKeyboardKeyCodeBySystem, getKeyboardKeyNameBySystem } from '@/app/components/workflow/utils'
import { ChunkingMode } from '@/models/datasets'
type IActionButtonsProps = {
handleCancel: () => void
@@ -23,7 +24,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
isChildChunk = false,
}) => {
const { t } = useTranslation()
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
useKeyPress(['esc'], (e) => {
@@ -40,8 +41,8 @@ const ActionButtons: FC<IActionButtonsProps> = ({
{ exactMatch: true, useCapture: true })
const isParentChildParagraphMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'paragraph'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'paragraph'
}, [docForm, parentMode])
return (
<div className='flex items-center gap-x-2'>

View File

@@ -3,8 +3,9 @@ import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLin
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import Divider from '@/app/components/base/divider'
import classNames from '@/utils/classnames'
import cn from '@/utils/classnames'
import Confirm from '@/app/components/base/confirm'
import Button from '@/app/components/base/button'
const i18nPrefix = 'dataset.batchAction'
type IBatchActionProps = {
@@ -43,55 +44,70 @@ const BatchAction: FC<IBatchActionProps> = ({
hideDeleteConfirm()
}
return (
<div className={classNames('pointer-events-none flex w-full justify-center gap-x-2', className)}>
<div className='pointer-events-auto flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5 backdrop-blur-[5px]'>
<div className={cn('flex w-full justify-center gap-x-2', className)}>
<div className='flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5'>
<div className='inline-flex items-center gap-x-2 py-1 pl-2 pr-3'>
<span className='flex h-5 w-5 items-center justify-center rounded-md bg-text-accent px-1 py-0.5 text-xs font-medium text-text-primary-on-surface'>
<span className='system-xs-medium flex h-5 w-5 items-center justify-center rounded-md bg-text-accent text-text-primary-on-surface'>
{selectedIds.length}
</span>
<span className='text-[13px] font-semibold leading-[16px] text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
<span className='system-sm-semibold text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
</div>
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiCheckboxCircleLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchEnable}>
{t(`${i18nPrefix}.enable`)}
</button>
</div>
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiCloseCircleLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchDisable}>
{t(`${i18nPrefix}.disable`)}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onBatchEnable}
>
<RiCheckboxCircleLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.enable`)}</span>
</Button>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onBatchDisable}
>
<RiCloseCircleLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.disable`)}</span>
</Button>
{onEditMetadata && (
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiDraftLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onEditMetadata}>
{t('dataset.metadata.metadata')}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onEditMetadata}
>
<RiDraftLine className='size-4' />
<span className='px-0.5'>{t('dataset.metadata.metadata')}</span>
</Button>
)}
{onArchive && (
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiArchive2Line className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onArchive}>
{t(`${i18nPrefix}.archive`)}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onArchive}
>
<RiArchive2Line className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.archive`)}</span>
</Button>
)}
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiDeleteBinLine className='h-4 w-4 text-components-button-destructive-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-destructive-ghost-text' onClick={showDeleteConfirm}>
{t(`${i18nPrefix}.delete`)}
</button>
</div>
<Button
variant='ghost'
destructive
className='gap-x-0.5 px-3'
onClick={showDeleteConfirm}
>
<RiDeleteBinLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.delete`)}</span>
</Button>
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
<button type='button' className='px-3.5 py-2 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onCancel}>
{t(`${i18nPrefix}.cancel`)}
</button>
<Button
variant='ghost'
className='px-3'
onClick={onCancel}
>
<span className='px-0.5'>{t(`${i18nPrefix}.cancel`)}</span>
</Button>
</div>
{
isShowDeleteConfirm && (

View File

@@ -31,8 +31,8 @@ const Textarea: FC<IContentProps> = React.memo(({
Textarea.displayName = 'Textarea'
type IAutoResizeTextAreaProps = ComponentProps<'textarea'> & {
containerRef: React.RefObject<HTMLDivElement>
labelRef: React.RefObject<HTMLDivElement>
containerRef: React.RefObject<HTMLDivElement | null>
labelRef: React.RefObject<HTMLDivElement | null>
}
const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
@@ -45,7 +45,7 @@ const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
...rest
}) => {
const textareaRef = useRef<HTMLTextAreaElement>(null)
const observerRef = useRef<ResizeObserver>()
const observerRef = useRef<ResizeObserver>(null)
const [maxHeight, setMaxHeight] = useState(0)
useEffect(() => {

View File

@@ -0,0 +1,111 @@
import React, { useCallback, useEffect, useRef } from 'react'
import { createPortal } from 'react-dom'
import cn from '@/utils/classnames'
import { useKeyPress } from 'ahooks'
import { useSegmentListContext } from '..'
type DrawerProps = {
open: boolean
onClose: () => void
side?: 'right' | 'left' | 'bottom' | 'top'
showOverlay?: boolean
modal?: boolean // click outside event can pass through if modal is false
closeOnOutsideClick?: boolean
panelClassName?: string
panelContentClassName?: string
needCheckChunks?: boolean
}
const Drawer = ({
open,
onClose,
side = 'right',
showOverlay = true,
modal = false,
needCheckChunks = false,
children,
panelClassName,
panelContentClassName,
}: React.PropsWithChildren<DrawerProps>) => {
const panelContentRef = useRef<HTMLDivElement>(null)
const currSegment = useSegmentListContext(s => s.currSegment)
const currChildChunk = useSegmentListContext(s => s.currChildChunk)
useKeyPress('esc', (e) => {
if (!open) return
e.preventDefault()
onClose()
}, { exactMatch: true, useCapture: true })
const shouldCloseDrawer = useCallback((target: Node | null) => {
const panelContent = panelContentRef.current
if (!panelContent) return false
const chunks = document.querySelectorAll('.chunk-card')
const childChunks = document.querySelectorAll('.child-chunk')
const isClickOnChunk = Array.from(chunks).some((chunk) => {
return chunk && chunk.contains(target)
})
const isClickOnChildChunk = Array.from(childChunks).some((chunk) => {
return chunk && chunk.contains(target)
})
const reopenChunkDetail = (currSegment.showModal && isClickOnChildChunk)
|| (currChildChunk.showModal && isClickOnChunk && !isClickOnChildChunk) || (!isClickOnChunk && !isClickOnChildChunk)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail)
}, [currSegment, currChildChunk, needCheckChunks])
const onDownCapture = useCallback((e: PointerEvent) => {
if (!open || modal) return
const panelContent = panelContentRef.current
if (!panelContent) return
const target = e.target as Node | null
if (shouldCloseDrawer(target))
queueMicrotask(onClose)
}, [shouldCloseDrawer, onClose, open, modal])
useEffect(() => {
window.addEventListener('pointerdown', onDownCapture, { capture: true })
return () =>
window.removeEventListener('pointerdown', onDownCapture, { capture: true })
}, [onDownCapture])
const isHorizontal = side === 'left' || side === 'right'
const content = (
<div className='pointer-events-none fixed inset-0 z-[9999]'>
{showOverlay ? (
<div
onClick={modal ? onClose : undefined}
aria-hidden='true'
className={cn(
'fixed inset-0 bg-black/30 opacity-0 transition-opacity duration-200 ease-in',
open && 'opacity-100',
modal && open ? 'pointer-events-auto' : 'pointer-events-none',
)}
/>
) : null}
{/* Drawer panel */}
<div
role='dialog'
aria-modal={modal ? 'true' : 'false'}
className={cn(
'pointer-events-auto fixed flex flex-col',
side === 'right' && 'right-0',
side === 'left' && 'left-0',
side === 'bottom' && 'bottom-0',
side === 'top' && 'top-0',
isHorizontal ? 'h-screen' : 'w-screen',
panelClassName,
)}
>
<div ref={panelContentRef} className={cn('flex grow flex-col', panelContentClassName)}>
{children}
</div>
</div>
</div>
)
return open && createPortal(content, document.body)
}
export default Drawer

View File

@@ -22,13 +22,13 @@ const Line = React.memo(({
className,
}: LineProps) => {
return (
<svg xmlns="http://www.w3.org/2000/svg" width="2" height="241" viewBox="0 0 2 241" fill="none" className={className}>
<path d="M1 0.5L1 240.5" stroke="url(#paint0_linear_1989_74474)"/>
<svg xmlns='http://www.w3.org/2000/svg' width='2' height='241' viewBox='0 0 2 241' fill='none' className={className}>
<path d='M1 0.5L1 240.5' stroke='url(#paint0_linear_1989_74474)' />
<defs>
<linearGradient id="paint0_linear_1989_74474" x1="-7.99584" y1="240.5" x2="-7.88094" y2="0.50004" gradientUnits="userSpaceOnUse">
<stop stopColor="white" stopOpacity="0.01"/>
<stop offset="0.503965" stopColor="#101828" stopOpacity="0.08"/>
<stop offset="1" stopColor="white" stopOpacity="0.01"/>
<linearGradient id='paint0_linear_1989_74474' x1='-7.99584' y1='240.5' x2='-7.88094' y2='0.50004' gradientUnits='userSpaceOnUse'>
<stop stopColor='white' stopOpacity='0.01' />
<stop offset='0.503965' stopColor='#101828' stopOpacity='0.08' />
<stop offset='1' stopColor='white' stopOpacity='0.01' />
</linearGradient>
</defs>
</svg>
@@ -47,8 +47,8 @@ const Empty: FC<IEmptyProps> = ({
<div className='flex flex-col items-center'>
<div className='relative z-10 flex h-14 w-14 items-center justify-center rounded-xl border border-divider-subtle bg-components-card-bg shadow-lg shadow-shadow-shadow-5'>
<RiFileList2Line className='h-6 w-6 text-text-secondary' />
<Line className='absolute -right-[1px] top-1/2 -translate-y-1/2' />
<Line className='absolute -left-[1px] top-1/2 -translate-y-1/2' />
<Line className='absolute -right-px top-1/2 -translate-y-1/2' />
<Line className='absolute -left-px top-1/2 -translate-y-1/2' />
<Line className='absolute left-1/2 top-0 -translate-x-1/2 -translate-y-1/2 rotate-90' />
<Line className='absolute left-1/2 top-full -translate-x-1/2 -translate-y-1/2 rotate-90' />
</div>

View File

@@ -1,33 +1,42 @@
import React, { type FC } from 'react'
import Drawer from '@/app/components/base/drawer'
import classNames from '@/utils/classnames'
import React from 'react'
import Drawer from './drawer'
import cn from '@/utils/classnames'
import { noop } from 'lodash-es'
type IFullScreenDrawerProps = {
isOpen: boolean
onClose?: () => void
fullScreen: boolean
children: React.ReactNode
showOverlay?: boolean
needCheckChunks?: boolean
modal?: boolean
}
const FullScreenDrawer: FC<IFullScreenDrawerProps> = ({
const FullScreenDrawer = ({
isOpen,
onClose = noop,
fullScreen,
children,
}) => {
showOverlay = true,
needCheckChunks = false,
modal = false,
}: React.PropsWithChildren<IFullScreenDrawerProps>) => {
return (
<Drawer
isOpen={isOpen}
open={isOpen}
onClose={onClose}
panelClassName={classNames('bg-components-panel-bg !p-0',
panelClassName={cn(
fullScreen
? '!w-full !max-w-full'
: 'mb-2 mr-2 mt-16 !w-[560px] !max-w-[560px] rounded-xl border-[0.5px] border-components-panel-border',
? 'w-full'
: 'w-[560px] pb-2 pr-2 pt-16',
)}
mask={false}
unmount
footer={null}
panelContentClassName={cn(
'bg-components-panel-bg',
!fullScreen && 'rounded-xl border-[0.5px] border-components-panel-border',
)}
showOverlay={showOverlay}
needCheckChunks={needCheckChunks}
modal={modal}
>
{children}
</Drawer>)

View File

@@ -1,5 +1,5 @@
import React, { type FC, useMemo } from 'react'
import { Chunk } from '@/app/components/base/icons/src/public/knowledge'
import { Chunk } from '@/app/components/base/icons/src/vender/knowledge'
import cn from '@/utils/classnames'
type ISegmentIndexTagProps = {

View File

@@ -2,7 +2,7 @@ import React, { type FC } from 'react'
import { useTranslation } from 'react-i18next'
import { RiLineHeight } from '@remixicon/react'
import Tooltip from '@/app/components/base/tooltip'
import { Collapse } from '@/app/components/base/icons/src/vender/line/editor'
import { Collapse } from '@/app/components/base/icons/src/vender/knowledge'
type DisplayToggleProps = {
isCollapsed: boolean

View File

@@ -5,7 +5,7 @@ import { useDebounceFn } from 'ahooks'
import { useTranslation } from 'react-i18next'
import { createContext, useContext, useContextSelector } from 'use-context-selector'
import { usePathname } from 'next/navigation'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { ProcessStatus } from '../segment-add'
import s from './style.module.css'
import SegmentList from './segment-list'
@@ -105,7 +105,6 @@ const Completed: FC<ICompletedProps> = ({
const datasetId = useDocumentContext(s => s.datasetId) || ''
const documentId = useDocumentContext(s => s.documentId) || ''
const docForm = useDocumentContext(s => s.docForm)
const mode = useDocumentContext(s => s.mode)
const parentMode = useDocumentContext(s => s.parentMode)
// the current segment id and whether to show the modal
const [currSegment, setCurrSegment] = useState<CurrSegmentType>({ showModal: false })
@@ -151,10 +150,10 @@ const Completed: FC<ICompletedProps> = ({
}
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
}, [docForm, parentMode])
const { isFetching: isLoadingSegmentList, data: segmentListData } = useSegmentList(
const { isLoading: isLoadingSegmentList, data: segmentListData } = useSegmentList(
{
datasetId,
documentId,
@@ -184,7 +183,7 @@ const Completed: FC<ICompletedProps> = ({
}
}, [segments])
const { isFetching: isLoadingChildSegmentList, data: childChunkListData } = useChildSegmentList(
const { isLoading: isLoadingChildSegmentList, data: childChunkListData } = useChildSegmentList(
{
datasetId,
documentId,
@@ -413,7 +412,7 @@ const Completed: FC<ICompletedProps> = ({
if (!isSearch) {
const total = segmentListData?.total ? formatNumber(segmentListData.total) : '--'
const count = total === '--' ? 0 : segmentListData!.total
const translationKey = (mode === 'hierarchical' && parentMode === 'paragraph')
const translationKey = (docForm === ChunkingMode.parentChild && parentMode === 'paragraph')
? 'datasetDocuments.segment.parentChunks'
: 'datasetDocuments.segment.chunks'
return `${total} ${t(translationKey, { count })}`
@@ -423,7 +422,7 @@ const Completed: FC<ICompletedProps> = ({
const count = segmentListData?.total || 0
return `${total} ${t('datasetDocuments.segment.searchResults', { count })}`
}
}, [segmentListData, mode, parentMode, searchValue, selectedStatus, t])
}, [segmentListData, docForm, parentMode, searchValue, selectedStatus, t])
const toggleFullScreen = useCallback(() => {
setFullScreen(!fullScreen)
@@ -665,8 +664,11 @@ const Completed: FC<ICompletedProps> = ({
isOpen={currSegment.showModal}
fullScreen={fullScreen}
onClose={onCloseSegmentDetail}
showOverlay={false}
needCheckChunks
>
<SegmentDetail
key={currSegment.segInfo?.id}
segInfo={currSegment.segInfo ?? { id: '' }}
docForm={docForm}
isEditMode={currSegment.isEditMode}
@@ -679,6 +681,7 @@ const Completed: FC<ICompletedProps> = ({
isOpen={showNewSegmentModal}
fullScreen={fullScreen}
onClose={onCloseNewSegmentModal}
modal
>
<NewSegment
docForm={docForm}
@@ -692,8 +695,11 @@ const Completed: FC<ICompletedProps> = ({
isOpen={currChildChunk.showModal}
fullScreen={fullScreen}
onClose={onCloseChildSegmentDetail}
showOverlay={false}
needCheckChunks
>
<ChildSegmentDetail
key={currChildChunk.childChunkInfo?.id}
chunkId={currChunkId}
childChunkInfo={currChildChunk.childChunkInfo ?? { id: '' }}
docForm={docForm}
@@ -706,6 +712,7 @@ const Completed: FC<ICompletedProps> = ({
isOpen={showNewChildSegmentModal}
fullScreen={fullScreen}
onClose={onCloseNewChildChunkModal}
modal
>
<NewChildSegment
chunkId={currChunkId}
@@ -715,15 +722,16 @@ const Completed: FC<ICompletedProps> = ({
/>
</FullScreenDrawer>
{/* Batch Action Buttons */}
{selectedSegmentIds.length > 0
&& <BatchAction
{selectedSegmentIds.length > 0 && (
<BatchAction
className='absolute bottom-16 left-0 z-20'
selectedIds={selectedSegmentIds}
onBatchEnable={onChangeSwitch.bind(null, true, '')}
onBatchDisable={onChangeSwitch.bind(null, false, '')}
onBatchDelete={onDelete.bind(null, '')}
onCancel={onCancelBatchOperation}
/>}
/>
)}
</SegmentListContext.Provider>
)
}

View File

@@ -5,7 +5,7 @@ import { useContext } from 'use-context-selector'
import { useParams } from 'next/navigation'
import { RiCloseLine, RiExpandDiagonalLine } from '@remixicon/react'
import { useShallow } from 'zustand/react/shallow'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { SegmentIndexTag } from './common/segment-index-tag'
import ActionButtons from './common/action-buttons'
import ChunkContent from './common/chunk-content'

View File

@@ -26,32 +26,37 @@ const ChunkContent: FC<ChunkContentProps> = ({
<div className={className}>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
<div
<Markdown
className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}>
{content}
</div>
)}
content={content}
customDisallowedElements={['input']}
/>
</div>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
<div className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}>
{answer}
</div>
<Markdown
className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}
content={answer}
customDisallowedElements={['input']}
/>
</div>
</div>
)
}
return <Markdown
className={cn('!mt-0.5 !text-text-secondary',
isFullDocMode ? 'line-clamp-3' : isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
className,
)}
content={sign_content || content || ''}
customDisallowedElements={['input']}
/>
return (
<Markdown
className={cn('!mt-0.5 !text-text-secondary',
isFullDocMode ? 'line-clamp-3' : isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
className,
)}
content={sign_content || content || ''}
customDisallowedElements={['input']}
/>
)
}
export default React.memo(ChunkContent)

View File

@@ -1,14 +1,14 @@
import React, { type FC, useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { RiDeleteBinLine, RiEditLine } from '@remixicon/react'
import { StatusItem } from '../../../list'
import { useDocumentContext } from '../../index'
import StatusItem from '../../../status-item'
import { useDocumentContext } from '../../context'
import ChildSegmentList from '../child-segment-list'
import Tag from '../common/tag'
import Dot from '../common/dot'
import { SegmentIndexTag } from '../common/segment-index-tag'
import ParentChunkCardSkeleton from '../skeleton/parent-chunk-card-skeleton'
import type { ChildChunkDetail, SegmentDetailModel } from '@/models/datasets'
import { type ChildChunkDetail, ChunkingMode, type SegmentDetailModel } from '@/models/datasets'
import Switch from '@/app/components/base/switch'
import Divider from '@/app/components/base/divider'
import { formatNumber } from '@/utils/format'
@@ -69,39 +69,39 @@ const SegmentCard: FC<ISegmentCardProps> = ({
updated_at,
} = detail as Required<ISegmentCardProps>['detail']
const [showModal, setShowModal] = useState(false)
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
const isGeneralMode = useMemo(() => {
return mode === 'custom'
}, [mode])
return docForm === ChunkingMode.text
}, [docForm])
const isParentChildMode = useMemo(() => {
return mode === 'hierarchical'
}, [mode])
return docForm === ChunkingMode.parentChild
}, [docForm])
const isParagraphMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'paragraph'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'paragraph'
}, [docForm, parentMode])
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
}, [docForm, parentMode])
const chunkEdited = useMemo(() => {
if (mode === 'hierarchical' && parentMode === 'full-doc')
if (docForm === ChunkingMode.parentChild && parentMode === 'full-doc')
return false
return isAfter(updated_at * 1000, created_at * 1000)
}, [mode, parentMode, updated_at, created_at])
}, [docForm, parentMode, updated_at, created_at])
const contentOpacity = useMemo(() => {
return (enabled || focused.segmentContent) ? '' : 'opacity-50 group-hover/card:opacity-100'
}, [enabled, focused.segmentContent])
const handleClickCard = useCallback(() => {
if (mode !== 'hierarchical' || parentMode !== 'full-doc')
if (docForm !== ChunkingMode.parentChild || parentMode !== 'full-doc')
onClick?.()
}, [mode, parentMode, onClick])
}, [docForm, parentMode, onClick])
const wordCountText = useMemo(() => {
const total = formatNumber(word_count)
@@ -118,7 +118,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
return (
<div
className={cn(
'group/card w-full rounded-xl px-3',
'chunk-card group/card w-full rounded-xl px-3',
isFullDocMode ? '' : 'pb-2 pt-2.5 hover:bg-dataset-chunk-detail-card-hover-bg',
focused.segmentContent ? 'bg-dataset-chunk-detail-card-hover-bg' : '',
className,
@@ -228,15 +228,15 @@ const SegmentCard: FC<ISegmentCardProps> = ({
}
{
isParagraphMode && child_chunks.length > 0
&& <ChildSegmentList
parentChunkId={id}
childChunks={child_chunks}
enabled={enabled}
onDelete={onDeleteChildChunk!}
handleAddNewChildChunk={handleAddNewChildChunk}
onClickSlice={onClickSlice}
focused={focused.segmentContent}
/>
&& <ChildSegmentList
parentChunkId={id}
childChunks={child_chunks}
enabled={enabled}
onDelete={onDeleteChildChunk!}
handleAddNewChildChunk={handleAddNewChildChunk}
onClickSlice={onClickSlice}
focused={focused.segmentContent}
/>
}
{showModal
&& <Confirm

View File

@@ -5,7 +5,7 @@ import {
RiCollapseDiagonalLine,
RiExpandDiagonalLine,
} from '@remixicon/react'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import ActionButtons from './common/action-buttons'
import ChunkContent from './common/chunk-content'
import Keywords from './common/keywords'
@@ -48,7 +48,6 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const [showRegenerationModal, setShowRegenerationModal] = useState(false)
const fullScreen = useSegmentListContext(s => s.fullScreen)
const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen)
const mode = useDocumentContext(s => s.mode)
const parentMode = useDocumentContext(s => s.parentMode)
const indexingTechnique = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
@@ -86,9 +85,9 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
return `${total} ${t('datasetDocuments.segment.characters', { count })}`
}, [isEditMode, question.length, answer.length, docForm, segInfo, t])
const isFullDocMode = mode === 'hierarchical' && parentMode === 'full-doc'
const isFullDocMode = docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
const titleText = isEditMode ? t('datasetDocuments.segment.editChunk') : t('datasetDocuments.segment.chunkDetail')
const labelPrefix = mode === 'hierarchical' ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
const labelPrefix = docForm === ChunkingMode.parentChild ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
const isECOIndexing = indexingTechnique === IndexingType.ECONOMICAL
return (

View File

@@ -1,11 +1,11 @@
import React, { useMemo } from 'react'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import SegmentCard from './segment-card'
import Empty from './common/empty'
import GeneralListSkeleton from './skeleton/general-list-skeleton'
import ParagraphListSkeleton from './skeleton/paragraph-list-skeleton'
import { useSegmentListContext } from './index'
import type { ChildChunkDetail, SegmentDetailModel } from '@/models/datasets'
import { type ChildChunkDetail, ChunkingMode, type SegmentDetailModel } from '@/models/datasets'
import Checkbox from '@/app/components/base/checkbox'
import Divider from '@/app/components/base/divider'
@@ -45,14 +45,14 @@ const SegmentList = (
ref: React.LegacyRef<HTMLDivElement>
},
) => {
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
const currSegment = useSegmentListContext(s => s.currSegment)
const currChildChunk = useSegmentListContext(s => s.currChildChunk)
const Skeleton = useMemo(() => {
return (mode === 'hierarchical' && parentMode === 'paragraph') ? ParagraphListSkeleton : GeneralListSkeleton
}, [mode, parentMode])
return (docForm === ChunkingMode.parentChild && parentMode === 'paragraph') ? ParagraphListSkeleton : GeneralListSkeleton
}, [docForm, parentMode])
// Loading skeleton
if (isLoading)

View File

@@ -0,0 +1,15 @@
import type { ChunkingMode, ParentMode } from '@/models/datasets'
import { createContext, useContextSelector } from 'use-context-selector'
type DocumentContextValue = {
datasetId?: string
documentId?: string
docForm?: ChunkingMode
parentMode?: ParentMode
}
export const DocumentContext = createContext<DocumentContextValue>({})
export const useDocumentContext = (selector: (value: DocumentContextValue) => any) => {
return useContextSelector(DocumentContext, selector)
}

View File

@@ -0,0 +1,43 @@
import type { FC } from 'react'
import type { ChunkingMode, ParentMode } from '@/models/datasets'
import { useRouter } from 'next/navigation'
import DocumentPicker from '../../common/document-picker'
import cn from '@/utils/classnames'
type DocumentTitleProps = {
datasetId: string
extension?: string
name?: string
chunkingMode?: ChunkingMode
parent_mode?: ParentMode
iconCls?: string
textCls?: string
wrapperCls?: string
}
export const DocumentTitle: FC<DocumentTitleProps> = ({
datasetId,
extension,
name,
chunkingMode,
parent_mode,
wrapperCls,
}) => {
const router = useRouter()
return (
<div className={cn('flex flex-1 items-center justify-start', wrapperCls)}>
<DocumentPicker
datasetId={datasetId}
value={{
name,
extension,
chunkingMode,
parentMode: parent_mode || 'paragraph',
}}
onChange={(doc) => {
router.push(`/datasets/${datasetId}/documents/${doc.id}`)
}}
/>
</div>
)
}

View File

@@ -7,7 +7,7 @@ import { omit } from 'lodash-es'
import { RiLoader2Line, RiPauseCircleLine, RiPlayCircleLine } from '@remixicon/react'
import Image from 'next/image'
import { FieldInfo } from '../metadata'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { IndexingType } from '../../../create/step-two'
import { indexMethodIcon, retrievalIcon } from '../../../create/icons'
import EmbeddingSkeleton from './skeleton'
@@ -131,7 +131,7 @@ const RuleDetail: FC<IRuleDetailProps> = React.memo(({
/>
<FieldInfo
label={t('datasetSettings.form.retrievalSetting.title')}
displayedValue={t(`dataset.retrieval.${indexingType === IndexingType.ECONOMICAL ? 'invertedIndex' : retrievalMethod}.title`) as string}
displayedValue={t(`dataset.retrieval.${indexingType === IndexingType.ECONOMICAL ? 'keyword_search' : retrievalMethod}.title`) as string}
valueIcon={
<Image
className='size-4'

View File

@@ -1,12 +1,11 @@
'use client'
import type { FC } from 'react'
import React, { useMemo, useState } from 'react'
import { createContext, useContext, useContextSelector } from 'use-context-selector'
import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { RiArrowLeftLine, RiLayoutLeft2Line, RiLayoutRight2Line } from '@remixicon/react'
import { OperationAction, StatusItem } from '../list'
import DocumentPicker from '../../common/document-picker'
import Operations from '../operations'
import StatusItem from '../status-item'
import Completed from './completed'
import Embedding from './embedding'
import Metadata from '@/app/components/datasets/metadata/metadata-document'
@@ -16,74 +15,31 @@ import style from './style.module.css'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import { ToastContext } from '@/app/components/base/toast'
import type { ChunkingMode, FileItem, ParentMode, ProcessMode } from '@/models/datasets'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import Toast from '@/app/components/base/toast'
import { ChunkingMode } from '@/models/datasets'
import type { FileItem } from '@/models/datasets'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import FloatRightContainer from '@/app/components/base/float-right-container'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import { useCheckSegmentBatchImportProgress, useChildSegmentListKey, useSegmentBatchImport, useSegmentListKey } from '@/service/knowledge/use-segment'
import { useDocumentDetail, useDocumentMetadata, useInvalidDocumentList } from '@/service/knowledge/use-document'
import { useInvalid } from '@/service/use-base'
import { DocumentContext } from './context'
import { DocumentTitle } from './document-title'
type DocumentContextValue = {
datasetId?: string
documentId?: string
docForm: string
mode?: ProcessMode
parentMode?: ParentMode
}
export const DocumentContext = createContext<DocumentContextValue>({ docForm: '' })
export const useDocumentContext = (selector: (value: DocumentContextValue) => any) => {
return useContextSelector(DocumentContext, selector)
}
type DocumentTitleProps = {
datasetId: string
extension?: string
name?: string
processMode?: ProcessMode
parent_mode?: ParentMode
iconCls?: string
textCls?: string
wrapperCls?: string
}
export const DocumentTitle: FC<DocumentTitleProps> = ({ datasetId, extension, name, processMode, parent_mode, wrapperCls }) => {
const router = useRouter()
return (
<div className={cn('flex flex-1 items-center justify-start', wrapperCls)}>
<DocumentPicker
datasetId={datasetId}
value={{
name,
extension,
processMode,
parentMode: parent_mode,
}}
onChange={(doc) => {
router.push(`/datasets/${datasetId}/documents/${doc.id}`)
}}
/>
</div>
)
}
type Props = {
type DocumentDetailProps = {
datasetId: string
documentId: string
}
const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {
const router = useRouter()
const { t } = useTranslation()
const media = useBreakpoints()
const isMobile = media === MediaType.mobile
const { notify } = useContext(ToastContext)
const { dataset } = useDatasetDetailContext()
const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
const embeddingAvailable = !!dataset?.embedding_available
const [showMetadata, setShowMetadata] = useState(!isMobile)
const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false)
@@ -102,10 +58,11 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
if (res.job_status === ProcessStatus.WAITING || res.job_status === ProcessStatus.PROCESSING)
setTimeout(() => checkProcess(res.job_id), 2500)
if (res.job_status === ProcessStatus.ERROR)
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
},
onError: (e) => {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
const message = 'message' in e ? `: ${e.message}` : ''
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${message}` })
},
})
}
@@ -121,7 +78,8 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
checkProcess(res.job_id)
},
onError: (e) => {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
const message = 'message' in e ? `: ${e.message}` : ''
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${message}` })
},
})
}
@@ -172,24 +130,20 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
}
}
const mode = useMemo(() => {
return documentDetail?.document_process_rule?.mode
}, [documentDetail?.document_process_rule])
const parentMode = useMemo(() => {
return documentDetail?.document_process_rule?.rules?.parent_mode
}, [documentDetail?.document_process_rule])
return documentDetail?.document_process_rule?.rules?.parent_mode || documentDetail?.dataset_process_rule?.rules?.parent_mode || 'paragraph'
}, [documentDetail?.document_process_rule?.rules?.parent_mode, documentDetail?.dataset_process_rule?.rules?.parent_mode])
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
}, [mode, parentMode])
const chunkMode = documentDetail?.doc_form
return chunkMode === ChunkingMode.parentChild && parentMode === 'full-doc'
}, [documentDetail?.doc_form, parentMode])
return (
<DocumentContext.Provider value={{
datasetId,
documentId,
docForm: documentDetail?.doc_form || '',
mode,
docForm: documentDetail?.doc_form as ChunkingMode,
parentMode,
}}>
<div className='flex h-full flex-col bg-background-default'>
@@ -203,7 +157,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
name={documentDetail?.name}
wrapperCls='mr-2'
parent_mode={parentMode}
processMode={mode}
chunkingMode={documentDetail?.doc_form as ChunkingMode}
/>
<div className='flex flex-wrap items-center'>
{embeddingAvailable && documentDetail && !documentDetail.archived && !isFullDocMode && (
@@ -231,7 +185,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
datasetId={datasetId}
onUpdate={handleOperate}
/>
<OperationAction
<Operations
scene='detail'
embeddingAvailable={embeddingAvailable}
detail={{
@@ -262,7 +216,8 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
{isDetailLoading
? <Loading type='app' />
: <div className={cn('flex h-full min-w-0 grow flex-col',
embedding ? '' : isFullDocMode ? 'relative pl-11 pr-11 pt-4' : 'relative pl-5 pr-11 pt-3',
!embedding && isFullDocMode && 'relative pl-11 pr-11 pt-4',
!embedding && !isFullDocMode && 'relative pl-5 pr-11 pt-3',
)}>
{embedding
? <Embedding

View File

@@ -5,7 +5,7 @@ import { PencilIcon } from '@heroicons/react/24/outline'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import { get } from 'lodash-es'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import s from './style.module.css'
import cn from '@/utils/classnames'
import Input from '@/app/components/base/input'

View File

@@ -0,0 +1,96 @@
import React, { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import { useContext } from 'use-context-selector'
import { useRouter } from 'next/navigation'
import DatasetDetailContext from '@/context/dataset-detail'
import type { CrawlOptions, CustomFile, DataSourceType } from '@/models/datasets'
import Loading from '@/app/components/base/loading'
import StepTwo from '@/app/components/datasets/create/step-two'
import AccountSetting from '@/app/components/header/account-setting'
import AppUnavailable from '@/app/components/base/app-unavailable'
import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { NotionPage } from '@/models/common'
import { useDocumentDetail, useInvalidDocumentDetail, useInvalidDocumentList } from '@/service/knowledge/use-document'
type DocumentSettingsProps = {
datasetId: string
documentId: string
}
const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
const { t } = useTranslation()
const router = useRouter()
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
const invalidDocumentList = useInvalidDocumentList(datasetId)
const invalidDocumentDetail = useInvalidDocumentDetail()
const saveHandler = () => {
invalidDocumentList()
invalidDocumentDetail()
router.push(`/datasets/${datasetId}/documents/${documentId}`)
}
const cancelHandler = () => router.back()
const { data: documentDetail, error } = useDocumentDetail({
datasetId,
documentId,
params: { metadata: 'without' },
})
const currentPage = useMemo(() => {
return {
workspace_id: documentDetail?.data_source_info.notion_workspace_id,
page_id: documentDetail?.data_source_info.notion_page_id,
page_name: documentDetail?.name,
page_icon: documentDetail?.data_source_info.notion_page_icon,
type: documentDetail?.data_source_type,
}
}, [documentDetail])
if (error)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
return (
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
<div className='grow'>
{!documentDetail && <Loading type='app' />}
{dataset && documentDetail && (
<StepTwo
isAPIKeySet={!!embeddingsDefaultModel}
onSetting={showSetAPIKey}
datasetId={datasetId}
dataSourceType={documentDetail.data_source_type as DataSourceType}
notionPages={[currentPage as unknown as NotionPage]}
websitePages={[
{
title: documentDetail.name,
source_url: documentDetail.data_source_info?.url,
content: '',
description: '',
},
]}
websiteCrawlProvider={documentDetail.data_source_info?.provider}
websiteCrawlJobId={documentDetail.data_source_info?.job_id}
crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
indexingType={indexingTechnique}
isSetting
documentDetail={documentDetail}
files={[documentDetail.data_source_info.upload_file as CustomFile]}
onSave={saveHandler}
onCancel={cancelHandler}
/>
)}
</div>
{isShowSetAPIKey && <AccountSetting activeTab='provider' onCancel={async () => {
hideSetAPIkey()
}} />}
</div>
)
}
export default DocumentSettings

View File

@@ -1,96 +1,36 @@
'use client'
import React, { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import { useContext } from 'use-context-selector'
import { useRouter } from 'next/navigation'
import DatasetDetailContext from '@/context/dataset-detail'
import type { CrawlOptions, CustomFile } from '@/models/datasets'
import React from 'react'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import DocumentSettings from './document-settings'
import PipelineSettings from './pipeline-settings'
import Loading from '@/app/components/base/loading'
import StepTwo from '@/app/components/datasets/create/step-two'
import AccountSetting from '@/app/components/header/account-setting'
import AppUnavailable from '@/app/components/base/app-unavailable'
import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { NotionPage } from '@/models/common'
import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
type DocumentSettingsProps = {
type SettingsProps = {
datasetId: string
documentId: string
}
const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
const { t } = useTranslation()
const router = useRouter()
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
const Settings = ({
datasetId,
documentId,
}: SettingsProps) => {
const runtimeMode = useDatasetDetailContextWithSelector(s => s.dataset?.runtime_mode)
const isGeneralDataset = runtimeMode === 'general'
const invalidDocumentDetail = useInvalidDocumentDetailKey()
const saveHandler = () => {
invalidDocumentDetail()
router.push(`/datasets/${datasetId}/documents/${documentId}`)
if (isGeneralDataset) {
return (
<DocumentSettings
datasetId={datasetId}
documentId={documentId}
/>
)
}
const cancelHandler = () => router.back()
const { data: documentDetail, error } = useDocumentDetail({
datasetId,
documentId,
params: { metadata: 'without' },
})
const currentPage = useMemo(() => {
return {
workspace_id: documentDetail?.data_source_info.notion_workspace_id,
page_id: documentDetail?.data_source_info.notion_page_id,
page_name: documentDetail?.name,
page_icon: documentDetail?.data_source_info.notion_page_icon,
type: documentDetail?.data_source_type,
}
}, [documentDetail])
if (error)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
return (
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
<div className="grow">
{!documentDetail && <Loading type='app' />}
{dataset && documentDetail && (
<StepTwo
isAPIKeySet={!!embeddingsDefaultModel}
onSetting={showSetAPIKey}
datasetId={datasetId}
dataSourceType={documentDetail.data_source_type}
notionPages={[currentPage as unknown as NotionPage]}
websitePages={[
{
title: documentDetail.name,
source_url: documentDetail.data_source_info?.url,
markdown: '',
description: '',
},
]}
websiteCrawlProvider={documentDetail.data_source_info?.provider}
websiteCrawlJobId={documentDetail.data_source_info?.job_id}
crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
indexingType={indexingTechnique}
isSetting
documentDetail={documentDetail}
files={[documentDetail.data_source_info.upload_file as CustomFile]}
onSave={saveHandler}
onCancel={cancelHandler}
/>
)}
</div>
{isShowSetAPIKey && <AccountSetting activeTab="provider" onCancel={async () => {
hideSetAPIkey()
}} />}
</div>
<PipelineSettings
datasetId={datasetId}
documentId={documentId}
/>
)
}
export default DocumentSettings
export default Settings

View File

@@ -0,0 +1,207 @@
import { useCallback, useMemo, useRef, useState } from 'react'
import type { CrawlResultItem, CustomFile, FileIndexingEstimateResponse } from '@/models/datasets'
import type { NotionPage } from '@/models/common'
import { useTranslation } from 'react-i18next'
import AppUnavailable from '@/app/components/base/app-unavailable'
import ChunkPreview from '../../../create-from-pipeline/preview/chunk-preview'
import Loading from '@/app/components/base/loading'
import ProcessDocuments from './process-documents'
import LeftHeader from './left-header'
import { usePipelineExecutionLog, useRunPublishedPipeline } from '@/service/use-pipeline'
import type { OnlineDriveFile, PublishedPipelineRunPreviewResponse } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { noop } from 'lodash-es'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useRouter } from 'next/navigation'
import { useInvalidDocumentDetail, useInvalidDocumentList } from '@/service/knowledge/use-document'
type PipelineSettingsProps = {
datasetId: string
documentId: string
}
const PipelineSettings = ({
datasetId,
documentId,
}: PipelineSettingsProps) => {
const { t } = useTranslation()
const { push } = useRouter()
const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
const isPreview = useRef(false)
const formRef = useRef<any>(null)
const { data: lastRunData, isFetching: isFetchingLastRunData, isError } = usePipelineExecutionLog({
dataset_id: datasetId,
document_id: documentId,
})
const files = useMemo(() => {
const files: CustomFile[] = []
if (lastRunData?.datasource_type === DatasourceType.localFile) {
const { related_id, name, extension } = lastRunData.datasource_info
files.push({
id: related_id,
name,
extension,
} as CustomFile)
}
return files
}, [lastRunData])
const websitePages = useMemo(() => {
const websitePages: CrawlResultItem[] = []
if (lastRunData?.datasource_type === DatasourceType.websiteCrawl) {
const { content, description, source_url, title } = lastRunData.datasource_info
websitePages.push({
content,
description,
source_url,
title,
})
}
return websitePages
}, [lastRunData])
const onlineDocuments = useMemo(() => {
const onlineDocuments: NotionPage[] = []
if (lastRunData?.datasource_type === DatasourceType.onlineDocument) {
const { workspace_id, page } = lastRunData.datasource_info
onlineDocuments.push({
workspace_id,
...page,
})
}
return onlineDocuments
}, [lastRunData])
const onlineDriveFiles = useMemo(() => {
const onlineDriveFiles: OnlineDriveFile[] = []
if (lastRunData?.datasource_type === DatasourceType.onlineDrive) {
const { id, type, name, size } = lastRunData.datasource_info
onlineDriveFiles.push({
id,
name,
type,
size,
})
}
return onlineDriveFiles
}, [lastRunData])
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
if (!lastRunData)
return
const datasourceInfoList: Record<string, any>[] = []
const documentInfo = lastRunData.datasource_info
datasourceInfoList.push(documentInfo)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
start_node_id: lastRunData.datasource_node_id,
datasource_type: lastRunData.datasource_type,
datasource_info_list: datasourceInfoList,
is_preview: true,
}, {
onSuccess: (res) => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [lastRunData, pipelineId, runPublishedPipeline])
const invalidDocumentList = useInvalidDocumentList(datasetId)
const invalidDocumentDetail = useInvalidDocumentDetail()
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!lastRunData)
return
const datasourceInfoList: Record<string, any>[] = []
const documentInfo = lastRunData.datasource_info
datasourceInfoList.push(documentInfo)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
start_node_id: lastRunData.datasource_node_id,
datasource_type: lastRunData.datasource_type,
datasource_info_list: datasourceInfoList,
original_document_id: documentId,
is_preview: false,
}, {
onSuccess: () => {
invalidDocumentList()
invalidDocumentDetail()
push(`/datasets/${datasetId}/documents`)
},
})
}, [datasetId, invalidDocumentDetail, invalidDocumentList, lastRunData, pipelineId, push, runPublishedPipeline])
const onClickProcess = useCallback(() => {
isPreview.current = false
formRef.current?.submit()
}, [])
const onClickPreview = useCallback(() => {
isPreview.current = true
formRef.current?.submit()
}, [])
const handleSubmit = useCallback((data: Record<string, any>) => {
isPreview.current ? handlePreviewChunks(data) : handleProcess(data)
}, [handlePreviewChunks, handleProcess])
if (isFetchingLastRunData) {
return (
<Loading type='app' />
)
}
if (isError)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
return (
<div
className='relative flex h-[calc(100vh-56px)] min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
>
<div className='h-full min-w-0 flex-1'>
<div className='flex h-full flex-col px-14'>
<LeftHeader title={t('datasetPipeline.documentSettings.title')} />
<div className='grow overflow-y-auto'>
<ProcessDocuments
ref={formRef}
lastRunInputData={lastRunData!.input_data}
datasourceNodeId={lastRunData!.datasource_node_id}
onProcess={onClickProcess}
onPreview={onClickPreview}
onSubmit={handleSubmit}
isRunning={isPending}
/>
</div>
</div>
</div>
{/* Preview */}
<div className='h-full min-w-0 flex-1'>
<div className='flex h-full flex-col pl-2 pt-2'>
<ChunkPreview
dataSourceType={lastRunData!.datasource_type}
localFiles={files}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
onlineDriveFiles={onlineDriveFiles}
isIdle={isIdle}
isPending={isPending && isPreview.current}
estimateData={estimateData}
onPreview={onClickPreview}
handlePreviewFileChange={noop}
handlePreviewOnlineDocumentChange={noop}
handlePreviewWebsitePageChange={noop}
handlePreviewOnlineDriveFileChange={noop}
/>
</div>
</div>
</div>
)
}
export default PipelineSettings

View File

@@ -0,0 +1,42 @@
import React, { useCallback } from 'react'
import { RiArrowLeftLine } from '@remixicon/react'
import Button from '@/app/components/base/button'
import { useRouter } from 'next/navigation'
import Effect from '@/app/components/base/effect'
import { useTranslation } from 'react-i18next'
type LeftHeaderProps = {
title: string
}
const LeftHeader = ({
title,
}: LeftHeaderProps) => {
const { t } = useTranslation()
const { back } = useRouter()
const navigateBack = useCallback(() => {
back()
}, [back])
return (
<div className='relative flex flex-col gap-y-0.5 pb-2 pt-4'>
<div className='system-2xs-semibold-uppercase bg-pipeline-add-documents-title-bg bg-clip-text text-transparent'>
{title}
</div>
<div className='system-md-semibold text-text-primary'>
{t('datasetPipeline.addDocuments.steps.processDocuments')}
</div>
<Button
variant='secondary-accent'
className='absolute -left-11 top-3.5 size-9 rounded-full p-0'
onClick={navigateBack}
>
<RiArrowLeftLine className='size-5 ' />
</Button>
<Effect className='left-8 top-[-34px] opacity-20' />
</div>
)
}
export default React.memo(LeftHeader)

View File

@@ -0,0 +1,29 @@
import React from 'react'
import Button from '@/app/components/base/button'
import { useTranslation } from 'react-i18next'
type ActionsProps = {
runDisabled?: boolean
onProcess: () => void
}
const Actions = ({
onProcess,
runDisabled,
}: ActionsProps) => {
const { t } = useTranslation()
return (
<div className='flex items-center justify-end'>
<Button
variant='primary'
onClick={onProcess}
disabled={runDisabled}
>
{t('datasetPipeline.operations.saveAndProcess')}
</Button>
</div>
)
}
export default React.memo(Actions)

View File

@@ -0,0 +1,15 @@
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline'
export const useInputVariables = (datasourceNodeId: string) => {
const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
const { data: paramsConfig, isFetching: isFetchingParams } = usePublishedPipelineProcessingParams({
pipeline_id: pipelineId!,
node_id: datasourceNodeId,
})
return {
paramsConfig,
isFetchingParams,
}
}

View File

@@ -0,0 +1,47 @@
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
import { useInputVariables } from './hooks'
import Actions from './actions'
import Form from '../../../../create-from-pipeline/process-documents/form'
import { useConfigurations, useInitialData } from '@/app/components/rag-pipeline/hooks/use-input-fields'
type ProcessDocumentsProps = {
datasourceNodeId: string
lastRunInputData: Record<string, any>
isRunning: boolean
ref: React.RefObject<any>
onProcess: () => void
onPreview: () => void
onSubmit: (data: Record<string, any>) => void
}
const ProcessDocuments = ({
datasourceNodeId,
lastRunInputData,
isRunning,
onProcess,
onPreview,
onSubmit,
ref,
}: ProcessDocumentsProps) => {
const { isFetchingParams, paramsConfig } = useInputVariables(datasourceNodeId)
const initialData = useInitialData(paramsConfig?.variables || [], lastRunInputData)
const configurations = useConfigurations(paramsConfig?.variables || [])
const schema = generateZodSchema(configurations)
return (
<div className='flex flex-col gap-y-4 pt-4'>
<Form
ref={ref}
initialData={initialData}
configurations={configurations}
schema={schema}
onSubmit={onSubmit}
onPreview={onPreview}
isRunning={isRunning}
/>
<Actions runDisabled={isFetchingParams || isRunning} onProcess={onProcess} />
</div>
)
}
export default ProcessDocuments