feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -2,14 +2,14 @@ import { type FC, useMemo, useState } from 'react'
import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
import { useTranslation } from 'react-i18next'
import { EditSlice } from '../../../formatted-text/flavours/edit-slice'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { FormattedText } from '../../../formatted-text/formatted'
import Empty from './common/empty'
import FullDocListSkeleton from './skeleton/full-doc-list-skeleton'
import { useSegmentListContext } from './index'
import type { ChildChunkDetail } from '@/models/datasets'
import Input from '@/app/components/base/input'
import classNames from '@/utils/classnames'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import { formatNumber } from '@/utils/format'
@@ -87,24 +87,25 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
}, [isFullDocMode, total, childChunks.length, inputValue])
return (
<div className={classNames(
<div className={cn(
'flex flex-col',
contentOpacity,
isParagraphMode ? 'pb-2 pt-1' : 'grow px-3',
(isFullDocMode && isLoading) && 'overflow-y-hidden',
)}>
{isFullDocMode ? <Divider type='horizontal' className='my-1 h-px bg-divider-subtle' /> : null}
<div className={classNames('flex items-center justify-between', isFullDocMode ? 'sticky -top-2 left-0 bg-background-default pb-3 pt-2' : '')}>
<div className={classNames(
'flex h-7 items-center rounded-lg pl-1 pr-3',
isParagraphMode && 'cursor-pointer',
(isParagraphMode && collapsed) && 'bg-dataset-child-chunk-expand-btn-bg',
isFullDocMode && 'pl-0',
)}
onClick={(event) => {
event.stopPropagation()
toggleCollapse()
}}
<div className={cn('flex items-center justify-between', isFullDocMode ? 'sticky -top-2 left-0 bg-background-default pb-3 pt-2' : '')}>
<div
className={cn(
'flex h-7 items-center rounded-lg pl-1 pr-3',
isParagraphMode && 'cursor-pointer',
(isParagraphMode && collapsed) && 'bg-dataset-child-chunk-expand-btn-bg',
isFullDocMode && 'pl-0',
)}
onClick={(event) => {
event.stopPropagation()
toggleCollapse()
}}
>
{
isParagraphMode
@@ -116,10 +117,10 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
: null
}
<span className='system-sm-semibold-uppercase text-text-secondary'>{totalText}</span>
<span className={classNames('pl-1.5 text-xs font-medium text-text-quaternary', isParagraphMode ? 'hidden group-hover/card:inline-block' : '')}>·</span>
<span className={cn('pl-1.5 text-xs font-medium text-text-quaternary', isParagraphMode ? 'hidden group-hover/card:inline-block' : '')}>·</span>
<button
type='button'
className={classNames(
className={cn(
'system-xs-semibold-uppercase px-1.5 py-1 text-components-button-secondary-accent-text',
isParagraphMode ? 'hidden group-hover/card:inline-block' : '',
(isFullDocMode && isLoading) ? 'text-components-button-secondary-accent-text-disabled' : '',
@@ -146,14 +147,14 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
</div>
{isLoading ? <FullDocListSkeleton /> : null}
{((isFullDocMode && !isLoading) || !collapsed)
? <div className={classNames('flex gap-x-0.5', isFullDocMode ? 'mb-6 grow' : 'items-center')}>
? <div className={cn('flex gap-x-0.5', isFullDocMode ? 'mb-6 grow' : 'items-center')}>
{isParagraphMode && (
<div className='self-stretch'>
<Divider type='vertical' className='mx-[7px] w-[2px] bg-text-accent-secondary' />
</div>
)}
{childChunks.length > 0
? <FormattedText className={classNames('flex w-full flex-col !leading-6', isParagraphMode ? 'gap-y-2' : 'gap-y-3')}>
? <FormattedText className={cn('flex w-full flex-col !leading-6', isParagraphMode ? 'gap-y-2' : 'gap-y-3')}>
{childChunks.map((childChunk) => {
const edited = childChunk.updated_at !== childChunk.created_at
const focused = currChildChunk?.childChunkInfo?.id === childChunk.id
@@ -162,9 +163,10 @@ const ChildSegmentList: FC<IChildSegmentCardProps> = ({
label={`C-${childChunk.position}${edited ? ` · ${t('datasetDocuments.segment.edited')}` : ''}`}
text={childChunk.content}
onDelete={() => onDelete?.(childChunk.segment_id, childChunk.id)}
className='child-chunk'
labelClassName={focused ? 'bg-state-accent-solid text-text-primary-on-surface' : ''}
labelInnerClassName={'text-[10px] font-semibold align-bottom leading-6'}
contentClassName={classNames('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : 'text-text-secondary')}
contentClassName={cn('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : 'text-text-secondary')}
showDivider={false}
onClick={(e) => {
e.stopPropagation()

View File

@@ -1,9 +1,10 @@
import React, { type FC, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useKeyPress } from 'ahooks'
import { useDocumentContext } from '../../index'
import { useDocumentContext } from '../../context'
import Button from '@/app/components/base/button'
import { getKeyboardKeyCodeBySystem, getKeyboardKeyNameBySystem } from '@/app/components/workflow/utils'
import { ChunkingMode } from '@/models/datasets'
type IActionButtonsProps = {
handleCancel: () => void
@@ -23,7 +24,7 @@ const ActionButtons: FC<IActionButtonsProps> = ({
isChildChunk = false,
}) => {
const { t } = useTranslation()
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
useKeyPress(['esc'], (e) => {
@@ -40,8 +41,8 @@ const ActionButtons: FC<IActionButtonsProps> = ({
{ exactMatch: true, useCapture: true })
const isParentChildParagraphMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'paragraph'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'paragraph'
}, [docForm, parentMode])
return (
<div className='flex items-center gap-x-2'>

View File

@@ -3,8 +3,9 @@ import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLin
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import Divider from '@/app/components/base/divider'
import classNames from '@/utils/classnames'
import cn from '@/utils/classnames'
import Confirm from '@/app/components/base/confirm'
import Button from '@/app/components/base/button'
const i18nPrefix = 'dataset.batchAction'
type IBatchActionProps = {
@@ -43,55 +44,70 @@ const BatchAction: FC<IBatchActionProps> = ({
hideDeleteConfirm()
}
return (
<div className={classNames('pointer-events-none flex w-full justify-center gap-x-2', className)}>
<div className='pointer-events-auto flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5 backdrop-blur-[5px]'>
<div className={cn('flex w-full justify-center gap-x-2', className)}>
<div className='flex items-center gap-x-1 rounded-[10px] border border-components-actionbar-border-accent bg-components-actionbar-bg-accent p-1 shadow-xl shadow-shadow-shadow-5'>
<div className='inline-flex items-center gap-x-2 py-1 pl-2 pr-3'>
<span className='flex h-5 w-5 items-center justify-center rounded-md bg-text-accent px-1 py-0.5 text-xs font-medium text-text-primary-on-surface'>
<span className='system-xs-medium flex h-5 w-5 items-center justify-center rounded-md bg-text-accent text-text-primary-on-surface'>
{selectedIds.length}
</span>
<span className='text-[13px] font-semibold leading-[16px] text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
<span className='system-sm-semibold text-text-accent'>{t(`${i18nPrefix}.selected`)}</span>
</div>
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiCheckboxCircleLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchEnable}>
{t(`${i18nPrefix}.enable`)}
</button>
</div>
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiCloseCircleLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onBatchDisable}>
{t(`${i18nPrefix}.disable`)}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onBatchEnable}
>
<RiCheckboxCircleLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.enable`)}</span>
</Button>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onBatchDisable}
>
<RiCloseCircleLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.disable`)}</span>
</Button>
{onEditMetadata && (
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiDraftLine className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onEditMetadata}>
{t('dataset.metadata.metadata')}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onEditMetadata}
>
<RiDraftLine className='size-4' />
<span className='px-0.5'>{t('dataset.metadata.metadata')}</span>
</Button>
)}
{onArchive && (
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiArchive2Line className='h-4 w-4 text-components-button-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onArchive}>
{t(`${i18nPrefix}.archive`)}
</button>
</div>
<Button
variant='ghost'
className='gap-x-0.5 px-3'
onClick={onArchive}
>
<RiArchive2Line className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.archive`)}</span>
</Button>
)}
<div className='flex items-center gap-x-0.5 px-3 py-2'>
<RiDeleteBinLine className='h-4 w-4 text-components-button-destructive-ghost-text' />
<button type='button' className='px-0.5 text-[13px] font-medium leading-[16px] text-components-button-destructive-ghost-text' onClick={showDeleteConfirm}>
{t(`${i18nPrefix}.delete`)}
</button>
</div>
<Button
variant='ghost'
destructive
className='gap-x-0.5 px-3'
onClick={showDeleteConfirm}
>
<RiDeleteBinLine className='size-4' />
<span className='px-0.5'>{t(`${i18nPrefix}.delete`)}</span>
</Button>
<Divider type='vertical' className='mx-0.5 h-3.5 bg-divider-regular' />
<button type='button' className='px-3.5 py-2 text-[13px] font-medium leading-[16px] text-components-button-ghost-text' onClick={onCancel}>
{t(`${i18nPrefix}.cancel`)}
</button>
<Button
variant='ghost'
className='px-3'
onClick={onCancel}
>
<span className='px-0.5'>{t(`${i18nPrefix}.cancel`)}</span>
</Button>
</div>
{
isShowDeleteConfirm && (

View File

@@ -31,8 +31,8 @@ const Textarea: FC<IContentProps> = React.memo(({
Textarea.displayName = 'Textarea'
type IAutoResizeTextAreaProps = ComponentProps<'textarea'> & {
containerRef: React.RefObject<HTMLDivElement>
labelRef: React.RefObject<HTMLDivElement>
containerRef: React.RefObject<HTMLDivElement | null>
labelRef: React.RefObject<HTMLDivElement | null>
}
const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
@@ -45,7 +45,7 @@ const AutoResizeTextArea: FC<IAutoResizeTextAreaProps> = React.memo(({
...rest
}) => {
const textareaRef = useRef<HTMLTextAreaElement>(null)
const observerRef = useRef<ResizeObserver>()
const observerRef = useRef<ResizeObserver>(null)
const [maxHeight, setMaxHeight] = useState(0)
useEffect(() => {

View File

@@ -0,0 +1,111 @@
import React, { useCallback, useEffect, useRef } from 'react'
import { createPortal } from 'react-dom'
import cn from '@/utils/classnames'
import { useKeyPress } from 'ahooks'
import { useSegmentListContext } from '..'
type DrawerProps = {
open: boolean
onClose: () => void
side?: 'right' | 'left' | 'bottom' | 'top'
showOverlay?: boolean
modal?: boolean // click outside event can pass through if modal is false
closeOnOutsideClick?: boolean
panelClassName?: string
panelContentClassName?: string
needCheckChunks?: boolean
}
const Drawer = ({
open,
onClose,
side = 'right',
showOverlay = true,
modal = false,
needCheckChunks = false,
children,
panelClassName,
panelContentClassName,
}: React.PropsWithChildren<DrawerProps>) => {
const panelContentRef = useRef<HTMLDivElement>(null)
const currSegment = useSegmentListContext(s => s.currSegment)
const currChildChunk = useSegmentListContext(s => s.currChildChunk)
useKeyPress('esc', (e) => {
if (!open) return
e.preventDefault()
onClose()
}, { exactMatch: true, useCapture: true })
const shouldCloseDrawer = useCallback((target: Node | null) => {
const panelContent = panelContentRef.current
if (!panelContent) return false
const chunks = document.querySelectorAll('.chunk-card')
const childChunks = document.querySelectorAll('.child-chunk')
const isClickOnChunk = Array.from(chunks).some((chunk) => {
return chunk && chunk.contains(target)
})
const isClickOnChildChunk = Array.from(childChunks).some((chunk) => {
return chunk && chunk.contains(target)
})
const reopenChunkDetail = (currSegment.showModal && isClickOnChildChunk)
|| (currChildChunk.showModal && isClickOnChunk && !isClickOnChildChunk) || (!isClickOnChunk && !isClickOnChildChunk)
return target && !panelContent.contains(target) && (!needCheckChunks || reopenChunkDetail)
}, [currSegment, currChildChunk, needCheckChunks])
const onDownCapture = useCallback((e: PointerEvent) => {
if (!open || modal) return
const panelContent = panelContentRef.current
if (!panelContent) return
const target = e.target as Node | null
if (shouldCloseDrawer(target))
queueMicrotask(onClose)
}, [shouldCloseDrawer, onClose, open, modal])
useEffect(() => {
window.addEventListener('pointerdown', onDownCapture, { capture: true })
return () =>
window.removeEventListener('pointerdown', onDownCapture, { capture: true })
}, [onDownCapture])
const isHorizontal = side === 'left' || side === 'right'
const content = (
<div className='pointer-events-none fixed inset-0 z-[9999]'>
{showOverlay ? (
<div
onClick={modal ? onClose : undefined}
aria-hidden='true'
className={cn(
'fixed inset-0 bg-black/30 opacity-0 transition-opacity duration-200 ease-in',
open && 'opacity-100',
modal && open ? 'pointer-events-auto' : 'pointer-events-none',
)}
/>
) : null}
{/* Drawer panel */}
<div
role='dialog'
aria-modal={modal ? 'true' : 'false'}
className={cn(
'pointer-events-auto fixed flex flex-col',
side === 'right' && 'right-0',
side === 'left' && 'left-0',
side === 'bottom' && 'bottom-0',
side === 'top' && 'top-0',
isHorizontal ? 'h-screen' : 'w-screen',
panelClassName,
)}
>
<div ref={panelContentRef} className={cn('flex grow flex-col', panelContentClassName)}>
{children}
</div>
</div>
</div>
)
return open && createPortal(content, document.body)
}
export default Drawer

View File

@@ -22,13 +22,13 @@ const Line = React.memo(({
className,
}: LineProps) => {
return (
<svg xmlns="http://www.w3.org/2000/svg" width="2" height="241" viewBox="0 0 2 241" fill="none" className={className}>
<path d="M1 0.5L1 240.5" stroke="url(#paint0_linear_1989_74474)"/>
<svg xmlns='http://www.w3.org/2000/svg' width='2' height='241' viewBox='0 0 2 241' fill='none' className={className}>
<path d='M1 0.5L1 240.5' stroke='url(#paint0_linear_1989_74474)' />
<defs>
<linearGradient id="paint0_linear_1989_74474" x1="-7.99584" y1="240.5" x2="-7.88094" y2="0.50004" gradientUnits="userSpaceOnUse">
<stop stopColor="white" stopOpacity="0.01"/>
<stop offset="0.503965" stopColor="#101828" stopOpacity="0.08"/>
<stop offset="1" stopColor="white" stopOpacity="0.01"/>
<linearGradient id='paint0_linear_1989_74474' x1='-7.99584' y1='240.5' x2='-7.88094' y2='0.50004' gradientUnits='userSpaceOnUse'>
<stop stopColor='white' stopOpacity='0.01' />
<stop offset='0.503965' stopColor='#101828' stopOpacity='0.08' />
<stop offset='1' stopColor='white' stopOpacity='0.01' />
</linearGradient>
</defs>
</svg>
@@ -47,8 +47,8 @@ const Empty: FC<IEmptyProps> = ({
<div className='flex flex-col items-center'>
<div className='relative z-10 flex h-14 w-14 items-center justify-center rounded-xl border border-divider-subtle bg-components-card-bg shadow-lg shadow-shadow-shadow-5'>
<RiFileList2Line className='h-6 w-6 text-text-secondary' />
<Line className='absolute -right-[1px] top-1/2 -translate-y-1/2' />
<Line className='absolute -left-[1px] top-1/2 -translate-y-1/2' />
<Line className='absolute -right-px top-1/2 -translate-y-1/2' />
<Line className='absolute -left-px top-1/2 -translate-y-1/2' />
<Line className='absolute left-1/2 top-0 -translate-x-1/2 -translate-y-1/2 rotate-90' />
<Line className='absolute left-1/2 top-full -translate-x-1/2 -translate-y-1/2 rotate-90' />
</div>

View File

@@ -1,33 +1,42 @@
import React, { type FC } from 'react'
import Drawer from '@/app/components/base/drawer'
import classNames from '@/utils/classnames'
import React from 'react'
import Drawer from './drawer'
import cn from '@/utils/classnames'
import { noop } from 'lodash-es'
type IFullScreenDrawerProps = {
isOpen: boolean
onClose?: () => void
fullScreen: boolean
children: React.ReactNode
showOverlay?: boolean
needCheckChunks?: boolean
modal?: boolean
}
const FullScreenDrawer: FC<IFullScreenDrawerProps> = ({
const FullScreenDrawer = ({
isOpen,
onClose = noop,
fullScreen,
children,
}) => {
showOverlay = true,
needCheckChunks = false,
modal = false,
}: React.PropsWithChildren<IFullScreenDrawerProps>) => {
return (
<Drawer
isOpen={isOpen}
open={isOpen}
onClose={onClose}
panelClassName={classNames('bg-components-panel-bg !p-0',
panelClassName={cn(
fullScreen
? '!w-full !max-w-full'
: 'mb-2 mr-2 mt-16 !w-[560px] !max-w-[560px] rounded-xl border-[0.5px] border-components-panel-border',
? 'w-full'
: 'w-[560px] pb-2 pr-2 pt-16',
)}
mask={false}
unmount
footer={null}
panelContentClassName={cn(
'bg-components-panel-bg',
!fullScreen && 'rounded-xl border-[0.5px] border-components-panel-border',
)}
showOverlay={showOverlay}
needCheckChunks={needCheckChunks}
modal={modal}
>
{children}
</Drawer>)

View File

@@ -1,5 +1,5 @@
import React, { type FC, useMemo } from 'react'
import { Chunk } from '@/app/components/base/icons/src/public/knowledge'
import { Chunk } from '@/app/components/base/icons/src/vender/knowledge'
import cn from '@/utils/classnames'
type ISegmentIndexTagProps = {

View File

@@ -2,7 +2,7 @@ import React, { type FC } from 'react'
import { useTranslation } from 'react-i18next'
import { RiLineHeight } from '@remixicon/react'
import Tooltip from '@/app/components/base/tooltip'
import { Collapse } from '@/app/components/base/icons/src/vender/line/editor'
import { Collapse } from '@/app/components/base/icons/src/vender/knowledge'
type DisplayToggleProps = {
isCollapsed: boolean

View File

@@ -5,7 +5,7 @@ import { useDebounceFn } from 'ahooks'
import { useTranslation } from 'react-i18next'
import { createContext, useContext, useContextSelector } from 'use-context-selector'
import { usePathname } from 'next/navigation'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { ProcessStatus } from '../segment-add'
import s from './style.module.css'
import SegmentList from './segment-list'
@@ -105,7 +105,6 @@ const Completed: FC<ICompletedProps> = ({
const datasetId = useDocumentContext(s => s.datasetId) || ''
const documentId = useDocumentContext(s => s.documentId) || ''
const docForm = useDocumentContext(s => s.docForm)
const mode = useDocumentContext(s => s.mode)
const parentMode = useDocumentContext(s => s.parentMode)
// the current segment id and whether to show the modal
const [currSegment, setCurrSegment] = useState<CurrSegmentType>({ showModal: false })
@@ -151,10 +150,10 @@ const Completed: FC<ICompletedProps> = ({
}
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
}, [docForm, parentMode])
const { isFetching: isLoadingSegmentList, data: segmentListData } = useSegmentList(
const { isLoading: isLoadingSegmentList, data: segmentListData } = useSegmentList(
{
datasetId,
documentId,
@@ -184,7 +183,7 @@ const Completed: FC<ICompletedProps> = ({
}
}, [segments])
const { isFetching: isLoadingChildSegmentList, data: childChunkListData } = useChildSegmentList(
const { isLoading: isLoadingChildSegmentList, data: childChunkListData } = useChildSegmentList(
{
datasetId,
documentId,
@@ -413,7 +412,7 @@ const Completed: FC<ICompletedProps> = ({
if (!isSearch) {
const total = segmentListData?.total ? formatNumber(segmentListData.total) : '--'
const count = total === '--' ? 0 : segmentListData!.total
const translationKey = (mode === 'hierarchical' && parentMode === 'paragraph')
const translationKey = (docForm === ChunkingMode.parentChild && parentMode === 'paragraph')
? 'datasetDocuments.segment.parentChunks'
: 'datasetDocuments.segment.chunks'
return `${total} ${t(translationKey, { count })}`
@@ -423,7 +422,7 @@ const Completed: FC<ICompletedProps> = ({
const count = segmentListData?.total || 0
return `${total} ${t('datasetDocuments.segment.searchResults', { count })}`
}
}, [segmentListData, mode, parentMode, searchValue, selectedStatus, t])
}, [segmentListData, docForm, parentMode, searchValue, selectedStatus, t])
const toggleFullScreen = useCallback(() => {
setFullScreen(!fullScreen)
@@ -665,8 +664,11 @@ const Completed: FC<ICompletedProps> = ({
isOpen={currSegment.showModal}
fullScreen={fullScreen}
onClose={onCloseSegmentDetail}
showOverlay={false}
needCheckChunks
>
<SegmentDetail
key={currSegment.segInfo?.id}
segInfo={currSegment.segInfo ?? { id: '' }}
docForm={docForm}
isEditMode={currSegment.isEditMode}
@@ -679,6 +681,7 @@ const Completed: FC<ICompletedProps> = ({
isOpen={showNewSegmentModal}
fullScreen={fullScreen}
onClose={onCloseNewSegmentModal}
modal
>
<NewSegment
docForm={docForm}
@@ -692,8 +695,11 @@ const Completed: FC<ICompletedProps> = ({
isOpen={currChildChunk.showModal}
fullScreen={fullScreen}
onClose={onCloseChildSegmentDetail}
showOverlay={false}
needCheckChunks
>
<ChildSegmentDetail
key={currChildChunk.childChunkInfo?.id}
chunkId={currChunkId}
childChunkInfo={currChildChunk.childChunkInfo ?? { id: '' }}
docForm={docForm}
@@ -706,6 +712,7 @@ const Completed: FC<ICompletedProps> = ({
isOpen={showNewChildSegmentModal}
fullScreen={fullScreen}
onClose={onCloseNewChildChunkModal}
modal
>
<NewChildSegment
chunkId={currChunkId}
@@ -715,15 +722,16 @@ const Completed: FC<ICompletedProps> = ({
/>
</FullScreenDrawer>
{/* Batch Action Buttons */}
{selectedSegmentIds.length > 0
&& <BatchAction
{selectedSegmentIds.length > 0 && (
<BatchAction
className='absolute bottom-16 left-0 z-20'
selectedIds={selectedSegmentIds}
onBatchEnable={onChangeSwitch.bind(null, true, '')}
onBatchDisable={onChangeSwitch.bind(null, false, '')}
onBatchDelete={onDelete.bind(null, '')}
onCancel={onCancelBatchOperation}
/>}
/>
)}
</SegmentListContext.Provider>
)
}

View File

@@ -5,7 +5,7 @@ import { useContext } from 'use-context-selector'
import { useParams } from 'next/navigation'
import { RiCloseLine, RiExpandDiagonalLine } from '@remixicon/react'
import { useShallow } from 'zustand/react/shallow'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import { SegmentIndexTag } from './common/segment-index-tag'
import ActionButtons from './common/action-buttons'
import ChunkContent from './common/chunk-content'

View File

@@ -26,32 +26,37 @@ const ChunkContent: FC<ChunkContentProps> = ({
<div className={className}>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>Q</div>
<div
<Markdown
className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}>
{content}
</div>
)}
content={content}
customDisallowedElements={['input']}
/>
</div>
<div className='flex gap-x-1'>
<div className='w-4 shrink-0 text-[13px] font-medium leading-[20px] text-text-tertiary'>A</div>
<div className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}>
{answer}
</div>
<Markdown
className={cn('body-md-regular text-text-secondary',
isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
)}
content={answer}
customDisallowedElements={['input']}
/>
</div>
</div>
)
}
return <Markdown
className={cn('!mt-0.5 !text-text-secondary',
isFullDocMode ? 'line-clamp-3' : isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
className,
)}
content={sign_content || content || ''}
customDisallowedElements={['input']}
/>
return (
<Markdown
className={cn('!mt-0.5 !text-text-secondary',
isFullDocMode ? 'line-clamp-3' : isCollapsed ? 'line-clamp-2' : 'line-clamp-20',
className,
)}
content={sign_content || content || ''}
customDisallowedElements={['input']}
/>
)
}
export default React.memo(ChunkContent)

View File

@@ -1,14 +1,14 @@
import React, { type FC, useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { RiDeleteBinLine, RiEditLine } from '@remixicon/react'
import { StatusItem } from '../../../list'
import { useDocumentContext } from '../../index'
import StatusItem from '../../../status-item'
import { useDocumentContext } from '../../context'
import ChildSegmentList from '../child-segment-list'
import Tag from '../common/tag'
import Dot from '../common/dot'
import { SegmentIndexTag } from '../common/segment-index-tag'
import ParentChunkCardSkeleton from '../skeleton/parent-chunk-card-skeleton'
import type { ChildChunkDetail, SegmentDetailModel } from '@/models/datasets'
import { type ChildChunkDetail, ChunkingMode, type SegmentDetailModel } from '@/models/datasets'
import Switch from '@/app/components/base/switch'
import Divider from '@/app/components/base/divider'
import { formatNumber } from '@/utils/format'
@@ -69,39 +69,39 @@ const SegmentCard: FC<ISegmentCardProps> = ({
updated_at,
} = detail as Required<ISegmentCardProps>['detail']
const [showModal, setShowModal] = useState(false)
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
const isGeneralMode = useMemo(() => {
return mode === 'custom'
}, [mode])
return docForm === ChunkingMode.text
}, [docForm])
const isParentChildMode = useMemo(() => {
return mode === 'hierarchical'
}, [mode])
return docForm === ChunkingMode.parentChild
}, [docForm])
const isParagraphMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'paragraph'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'paragraph'
}, [docForm, parentMode])
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
}, [mode, parentMode])
return docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
}, [docForm, parentMode])
const chunkEdited = useMemo(() => {
if (mode === 'hierarchical' && parentMode === 'full-doc')
if (docForm === ChunkingMode.parentChild && parentMode === 'full-doc')
return false
return isAfter(updated_at * 1000, created_at * 1000)
}, [mode, parentMode, updated_at, created_at])
}, [docForm, parentMode, updated_at, created_at])
const contentOpacity = useMemo(() => {
return (enabled || focused.segmentContent) ? '' : 'opacity-50 group-hover/card:opacity-100'
}, [enabled, focused.segmentContent])
const handleClickCard = useCallback(() => {
if (mode !== 'hierarchical' || parentMode !== 'full-doc')
if (docForm !== ChunkingMode.parentChild || parentMode !== 'full-doc')
onClick?.()
}, [mode, parentMode, onClick])
}, [docForm, parentMode, onClick])
const wordCountText = useMemo(() => {
const total = formatNumber(word_count)
@@ -118,7 +118,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
return (
<div
className={cn(
'group/card w-full rounded-xl px-3',
'chunk-card group/card w-full rounded-xl px-3',
isFullDocMode ? '' : 'pb-2 pt-2.5 hover:bg-dataset-chunk-detail-card-hover-bg',
focused.segmentContent ? 'bg-dataset-chunk-detail-card-hover-bg' : '',
className,
@@ -228,15 +228,15 @@ const SegmentCard: FC<ISegmentCardProps> = ({
}
{
isParagraphMode && child_chunks.length > 0
&& <ChildSegmentList
parentChunkId={id}
childChunks={child_chunks}
enabled={enabled}
onDelete={onDeleteChildChunk!}
handleAddNewChildChunk={handleAddNewChildChunk}
onClickSlice={onClickSlice}
focused={focused.segmentContent}
/>
&& <ChildSegmentList
parentChunkId={id}
childChunks={child_chunks}
enabled={enabled}
onDelete={onDeleteChildChunk!}
handleAddNewChildChunk={handleAddNewChildChunk}
onClickSlice={onClickSlice}
focused={focused.segmentContent}
/>
}
{showModal
&& <Confirm

View File

@@ -5,7 +5,7 @@ import {
RiCollapseDiagonalLine,
RiExpandDiagonalLine,
} from '@remixicon/react'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import ActionButtons from './common/action-buttons'
import ChunkContent from './common/chunk-content'
import Keywords from './common/keywords'
@@ -48,7 +48,6 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
const [showRegenerationModal, setShowRegenerationModal] = useState(false)
const fullScreen = useSegmentListContext(s => s.fullScreen)
const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen)
const mode = useDocumentContext(s => s.mode)
const parentMode = useDocumentContext(s => s.parentMode)
const indexingTechnique = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
@@ -86,9 +85,9 @@ const SegmentDetail: FC<ISegmentDetailProps> = ({
return `${total} ${t('datasetDocuments.segment.characters', { count })}`
}, [isEditMode, question.length, answer.length, docForm, segInfo, t])
const isFullDocMode = mode === 'hierarchical' && parentMode === 'full-doc'
const isFullDocMode = docForm === ChunkingMode.parentChild && parentMode === 'full-doc'
const titleText = isEditMode ? t('datasetDocuments.segment.editChunk') : t('datasetDocuments.segment.chunkDetail')
const labelPrefix = mode === 'hierarchical' ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
const labelPrefix = docForm === ChunkingMode.parentChild ? t('datasetDocuments.segment.parentChunk') : t('datasetDocuments.segment.chunk')
const isECOIndexing = indexingTechnique === IndexingType.ECONOMICAL
return (

View File

@@ -1,11 +1,11 @@
import React, { useMemo } from 'react'
import { useDocumentContext } from '../index'
import { useDocumentContext } from '../context'
import SegmentCard from './segment-card'
import Empty from './common/empty'
import GeneralListSkeleton from './skeleton/general-list-skeleton'
import ParagraphListSkeleton from './skeleton/paragraph-list-skeleton'
import { useSegmentListContext } from './index'
import type { ChildChunkDetail, SegmentDetailModel } from '@/models/datasets'
import { type ChildChunkDetail, ChunkingMode, type SegmentDetailModel } from '@/models/datasets'
import Checkbox from '@/app/components/base/checkbox'
import Divider from '@/app/components/base/divider'
@@ -45,14 +45,14 @@ const SegmentList = (
ref: React.LegacyRef<HTMLDivElement>
},
) => {
const mode = useDocumentContext(s => s.mode)
const docForm = useDocumentContext(s => s.docForm)
const parentMode = useDocumentContext(s => s.parentMode)
const currSegment = useSegmentListContext(s => s.currSegment)
const currChildChunk = useSegmentListContext(s => s.currChildChunk)
const Skeleton = useMemo(() => {
return (mode === 'hierarchical' && parentMode === 'paragraph') ? ParagraphListSkeleton : GeneralListSkeleton
}, [mode, parentMode])
return (docForm === ChunkingMode.parentChild && parentMode === 'paragraph') ? ParagraphListSkeleton : GeneralListSkeleton
}, [docForm, parentMode])
// Loading skeleton
if (isLoading)