feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -49,10 +49,10 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
const media = useBreakpoints()
const isMobile = media === MediaType.mobile
const { isCurrentWorkspaceEditor, isLoadingCurrentWorkspace, currentWorkspace } = useAppContext()
const { appDetail, setAppDetail, setAppSiderbarExpand } = useStore(useShallow(state => ({
const { appDetail, setAppDetail, setAppSidebarExpand } = useStore(useShallow(state => ({
appDetail: state.appDetail,
setAppDetail: state.setAppDetail,
setAppSiderbarExpand: state.setAppSiderbarExpand,
setAppSidebarExpand: state.setAppSidebarExpand,
})))
const showTagManagementModal = useTagStore(s => s.showTagManagementModal)
const [isLoadingAppDetail, setIsLoadingAppDetail] = useState(false)
@@ -64,8 +64,8 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
selectedIcon: NavIcon
}>>([])
const getNavigations = useCallback((appId: string, isCurrentWorkspaceEditor: boolean, mode: string) => {
const navs = [
const getNavigationConfig = useCallback((appId: string, isCurrentWorkspaceEditor: boolean, mode: string) => {
const navConfig = [
...(isCurrentWorkspaceEditor
? [{
name: t('common.appMenus.promptEng'),
@@ -99,8 +99,8 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
selectedIcon: RiDashboard2Fill,
},
]
return navs
}, [])
return navConfig
}, [t])
useDocumentTitle(appDetail?.name || t('common.menus.appDetail'))
@@ -108,10 +108,10 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
if (appDetail) {
const localeMode = localStorage.getItem('app-detail-collapse-or-expand') || 'expand'
const mode = isMobile ? 'collapse' : 'expand'
setAppSiderbarExpand(isMobile ? mode : localeMode)
setAppSidebarExpand(isMobile ? mode : localeMode)
// TODO: consider screen size and mode
// if ((appDetail.mode === 'advanced-chat' || appDetail.mode === 'workflow') && (pathname).endsWith('workflow'))
// setAppSiderbarExpand('collapse')
// setAppSidebarExpand('collapse')
}
}, [appDetail, isMobile])
@@ -146,7 +146,7 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
}
else {
setAppDetail({ ...res, enable_sso: false })
setNavigation(getNavigations(appId, isCurrentWorkspaceEditor, res.mode))
setNavigation(getNavigationConfig(appId, isCurrentWorkspaceEditor, res.mode))
}
}, [appDetailRes, isCurrentWorkspaceEditor, isLoadingAppDetail, isLoadingCurrentWorkspace])
@@ -165,7 +165,9 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
return (
<div className={cn(s.app, 'relative flex', 'overflow-hidden')}>
{appDetail && (
<AppSideBar title={appDetail.name} icon={appDetail.icon} icon_background={appDetail.icon_background as string} desc={appDetail.mode} navigation={navigation} />
<AppSideBar
navigation={navigation}
/>
)}
<div className="grow overflow-hidden bg-components-panel-bg">
{children}

View File

@@ -0,0 +1,10 @@
import React from 'react'
import CreateFromPipeline from '@/app/components/datasets/documents/create-from-pipeline'
const CreateFromPipelinePage = async () => {
return (
<CreateFromPipeline />
)
}
export default CreateFromPipelinePage

View File

@@ -1,9 +1,9 @@
'use client'
import type { FC } from 'react'
import React, { useEffect, useMemo } from 'react'
import React, { useEffect, useMemo, useState } from 'react'
import { usePathname } from 'next/navigation'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import type { RemixiconComponentType } from '@remixicon/react'
import {
RiEqualizer2Fill,
RiEqualizer2Line,
@@ -12,188 +12,135 @@ import {
RiFocus2Fill,
RiFocus2Line,
} from '@remixicon/react'
import {
PaperClipIcon,
} from '@heroicons/react/24/outline'
import { RiApps2AddLine, RiBookOpenLine, RiInformation2Line } from '@remixicon/react'
import classNames from '@/utils/classnames'
import { fetchDatasetDetail, fetchDatasetRelatedApps } from '@/service/datasets'
import type { RelatedAppResponse } from '@/models/datasets'
import AppSideBar from '@/app/components/app-sidebar'
import Loading from '@/app/components/base/loading'
import DatasetDetailContext from '@/context/dataset-detail'
import { DataSourceType } from '@/models/datasets'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import { useStore } from '@/app/components/app/store'
import { useDocLink } from '@/context/i18n'
import { useAppContext } from '@/context/app-context'
import Tooltip from '@/app/components/base/tooltip'
import LinkedAppsPanel from '@/app/components/base/linked-apps-panel'
import { PipelineFill, PipelineLine } from '@/app/components/base/icons/src/vender/pipeline'
import { useDatasetDetail, useDatasetRelatedApps } from '@/service/knowledge/use-dataset'
import useDocumentTitle from '@/hooks/use-document-title'
import ExtraInfo from '@/app/components/datasets/extra-info'
import { useEventEmitterContextContext } from '@/context/event-emitter'
import cn from '@/utils/classnames'
export type IAppDetailLayoutProps = {
children: React.ReactNode
params: { datasetId: string }
}
type IExtraInfoProps = {
isMobile: boolean
relatedApps?: RelatedAppResponse
expand: boolean
}
const ExtraInfo = ({ isMobile, relatedApps, expand }: IExtraInfoProps) => {
const { t } = useTranslation()
const docLink = useDocLink()
const hasRelatedApps = relatedApps?.data && relatedApps?.data?.length > 0
const relatedAppsTotal = relatedApps?.data?.length || 0
return <div>
{/* Related apps for desktop */}
<div className={classNames(
'transition-all duration-200 ease-in-out',
(hasRelatedApps && !isMobile)
? 'w-auto opacity-100'
: 'pointer-events-none h-0 w-0 overflow-hidden opacity-0',
)}>
<Tooltip
position='right'
noDecoration
popupContent={
<LinkedAppsPanel
relatedApps={relatedApps?.data || []}
isMobile={isMobile}
/>
}
>
<div className='system-xs-medium-uppercase inline-flex cursor-pointer items-center space-x-1 whitespace-nowrap text-text-secondary'>
<span>{relatedAppsTotal || '--'} {t('common.datasetMenus.relatedApp')}</span>
<RiInformation2Line className='h-4 w-4' />
</div>
</Tooltip>
</div>
{/* Related apps for mobile */}
<div className={classNames(
'transition-all duration-200 ease-in-out',
(hasRelatedApps && isMobile)
? 'w-auto opacity-100'
: 'pointer-events-none h-0 w-0 overflow-hidden opacity-0',
)}>
<div className={classNames('pb-2 pt-4 text-xs font-medium uppercase text-text-tertiary', 'flex items-center justify-center gap-1 whitespace-nowrap !px-0')}>
{relatedAppsTotal || '--'}
<PaperClipIcon className='h-4 w-4 text-text-secondary' />
</div>
</div>
{/* No related apps tooltip */}
<div className={classNames(
'transition-all duration-200 ease-in-out',
(!hasRelatedApps && !expand)
? 'w-auto opacity-100'
: 'pointer-events-none h-0 w-0 overflow-hidden opacity-0',
)}>
<Tooltip
position='right'
noDecoration
popupContent={
<div className='w-[240px] rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-4'>
<div className='inline-flex rounded-lg border-[0.5px] border-components-panel-border-subtle bg-background-default-subtle p-2'>
<RiApps2AddLine className='h-4 w-4 text-text-tertiary' />
</div>
<div className='my-2 text-xs text-text-tertiary'>{t('common.datasetMenus.emptyTip')}</div>
<a
className='mt-2 inline-flex cursor-pointer items-center text-xs text-text-accent'
href={docLink('/guides/knowledge-base/integrate-knowledge-within-application')}
target='_blank' rel='noopener noreferrer'
>
<RiBookOpenLine className='mr-1 text-text-accent' />
{t('common.datasetMenus.viewDoc')}
</a>
</div>
}
>
<div className='system-xs-medium-uppercase inline-flex cursor-pointer items-center space-x-1 whitespace-nowrap text-text-secondary'>
<span>{t('common.datasetMenus.noRelatedApp')}</span>
<RiInformation2Line className='h-4 w-4' />
</div>
</Tooltip>
</div>
</div>
}
const DatasetDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
const {
children,
params: { datasetId },
} = props
const pathname = usePathname()
const hideSideBar = pathname.endsWith('documents/create')
const { t } = useTranslation()
const pathname = usePathname()
const hideSideBar = pathname.endsWith('documents/create') || pathname.endsWith('documents/create-from-pipeline')
const isPipelineCanvas = pathname.endsWith('/pipeline')
const workflowCanvasMaximize = localStorage.getItem('workflow-canvas-maximize') === 'true'
const [hideHeader, setHideHeader] = useState(workflowCanvasMaximize)
const { eventEmitter } = useEventEmitterContextContext()
eventEmitter?.useSubscription((v: any) => {
if (v?.type === 'workflow-canvas-maximize')
setHideHeader(v.payload)
})
const { isCurrentWorkspaceDatasetOperator } = useAppContext()
const media = useBreakpoints()
const isMobile = media === MediaType.mobile
const { data: datasetRes, error, mutate: mutateDatasetRes } = useSWR({
url: 'fetchDatasetDetail',
datasetId,
}, apiParams => fetchDatasetDetail(apiParams.datasetId))
const { data: datasetRes, error, refetch: mutateDatasetRes } = useDatasetDetail(datasetId)
const { data: relatedApps } = useSWR({
action: 'fetchDatasetRelatedApps',
datasetId,
}, apiParams => fetchDatasetRelatedApps(apiParams.datasetId))
const { data: relatedApps } = useDatasetRelatedApps(datasetId)
const isButtonDisabledWithPipeline = useMemo(() => {
if (!datasetRes)
return true
if (datasetRes.provider === 'external')
return false
if (datasetRes.runtime_mode === 'general')
return false
return !datasetRes.is_published
}, [datasetRes])
const navigation = useMemo(() => {
const baseNavigation = [
{ name: t('common.datasetMenus.hitTesting'), href: `/datasets/${datasetId}/hitTesting`, icon: RiFocus2Line, selectedIcon: RiFocus2Fill },
{ name: t('common.datasetMenus.settings'), href: `/datasets/${datasetId}/settings`, icon: RiEqualizer2Line, selectedIcon: RiEqualizer2Fill },
{
name: t('common.datasetMenus.hitTesting'),
href: `/datasets/${datasetId}/hitTesting`,
icon: RiFocus2Line,
selectedIcon: RiFocus2Fill,
disabled: isButtonDisabledWithPipeline,
},
{
name: t('common.datasetMenus.settings'),
href: `/datasets/${datasetId}/settings`,
icon: RiEqualizer2Line,
selectedIcon: RiEqualizer2Fill,
disabled: false,
},
]
if (datasetRes?.provider !== 'external') {
baseNavigation.unshift({
name: t('common.datasetMenus.pipeline'),
href: `/datasets/${datasetId}/pipeline`,
icon: PipelineLine as RemixiconComponentType,
selectedIcon: PipelineFill as RemixiconComponentType,
disabled: false,
})
baseNavigation.unshift({
name: t('common.datasetMenus.documents'),
href: `/datasets/${datasetId}/documents`,
icon: RiFileTextLine,
selectedIcon: RiFileTextFill,
disabled: isButtonDisabledWithPipeline,
})
}
return baseNavigation
}, [datasetRes?.provider, datasetId, t])
}, [t, datasetId, isButtonDisabledWithPipeline, datasetRes?.provider])
useDocumentTitle(datasetRes?.name || t('common.menus.datasets'))
const setAppSiderbarExpand = useStore(state => state.setAppSiderbarExpand)
const setAppSidebarExpand = useStore(state => state.setAppSidebarExpand)
useEffect(() => {
const localeMode = localStorage.getItem('app-detail-collapse-or-expand') || 'expand'
const mode = isMobile ? 'collapse' : 'expand'
setAppSiderbarExpand(isMobile ? mode : localeMode)
}, [isMobile, setAppSiderbarExpand])
setAppSidebarExpand(isMobile ? mode : localeMode)
}, [isMobile, setAppSidebarExpand])
if (!datasetRes && !error)
return <Loading type='app' />
return (
<div className='flex grow overflow-hidden'>
{!hideSideBar && <AppSideBar
title={datasetRes?.name || '--'}
icon={datasetRes?.icon || 'https://static.dify.ai/images/dataset-default-icon.png'}
icon_background={datasetRes?.icon_background || '#F5F5F5'}
desc={datasetRes?.description || '--'}
isExternal={datasetRes?.provider === 'external'}
navigation={navigation}
extraInfo={!isCurrentWorkspaceDatasetOperator ? mode => <ExtraInfo isMobile={mode === 'collapse'} relatedApps={relatedApps} expand={mode === 'collapse'} /> : undefined}
iconType={datasetRes?.data_source_type === DataSourceType.NOTION ? 'notion' : 'dataset'}
/>}
<div
className={cn(
'flex grow overflow-hidden',
hideHeader && isPipelineCanvas ? '' : 'rounded-t-2xl border-t border-effects-highlight',
)}
>
<DatasetDetailContext.Provider value={{
indexingTechnique: datasetRes?.indexing_technique,
dataset: datasetRes,
mutateDatasetRes: () => mutateDatasetRes(),
mutateDatasetRes,
}}>
<div className="grow overflow-hidden bg-background-default-subtle">{children}</div>
{!hideSideBar && (
<AppSideBar
navigation={navigation}
extraInfo={
!isCurrentWorkspaceDatasetOperator
? mode => <ExtraInfo relatedApps={relatedApps} expand={mode === 'expand'} documentCount={datasetRes?.document_count} />
: undefined
}
iconType='dataset'
/>
)}
<div className='grow overflow-hidden bg-background-default-subtle'>{children}</div>
</DatasetDetailContext.Provider>
</div>
)

View File

@@ -0,0 +1,11 @@
'use client'
import RagPipeline from '@/app/components/rag-pipeline'
const PipelinePage = () => {
return (
<div className='h-full w-full overflow-x-auto'>
<RagPipeline />
</div>
)
}
export default PipelinePage

View File

@@ -8,8 +8,8 @@ const Settings = async () => {
return (
<div className='h-full overflow-y-auto'>
<div className='px-6 py-3'>
<div className='system-xl-semibold mb-1 text-text-primary'>{t('title')}</div>
<div className='flex flex-col gap-y-0.5 px-6 pb-2 pt-3'>
<div className='system-xl-semibold text-text-primary'>{t('title')}</div>
<div className='system-sm-regular text-text-tertiary'>{t('desc')}</div>
</div>
<Form />

View File

@@ -1,143 +0,0 @@
'use client'
// Libraries
import { useEffect, useMemo, useRef, useState } from 'react'
import { useRouter } from 'next/navigation'
import { useTranslation } from 'react-i18next'
import { useBoolean, useDebounceFn } from 'ahooks'
import { useQuery } from '@tanstack/react-query'
// Components
import ExternalAPIPanel from '../../components/datasets/external-api/external-api-panel'
import Datasets from './datasets'
import DatasetFooter from './dataset-footer'
import ApiServer from '../../components/develop/ApiServer'
import Doc from './doc'
import TabSliderNew from '@/app/components/base/tab-slider-new'
import TagManagementModal from '@/app/components/base/tag-management'
import TagFilter from '@/app/components/base/tag-management/filter'
import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
import CheckboxWithLabel from '@/app/components/datasets/create/website/base/checkbox-with-label'
// Services
import { fetchDatasetApiBaseUrl } from '@/service/datasets'
// Hooks
import { useTabSearchParams } from '@/hooks/use-tab-searchparams'
import { useStore as useTagStore } from '@/app/components/base/tag-management/store'
import { useAppContext } from '@/context/app-context'
import { useExternalApiPanel } from '@/context/external-api-panel-context'
import { useGlobalPublicStore } from '@/context/global-public-context'
import useDocumentTitle from '@/hooks/use-document-title'
const Container = () => {
const { t } = useTranslation()
const { systemFeatures } = useGlobalPublicStore()
const router = useRouter()
const { currentWorkspace, isCurrentWorkspaceOwner } = useAppContext()
const showTagManagementModal = useTagStore(s => s.showTagManagementModal)
const { showExternalApiPanel, setShowExternalApiPanel } = useExternalApiPanel()
const [includeAll, { toggle: toggleIncludeAll }] = useBoolean(false)
useDocumentTitle(t('dataset.knowledge'))
const options = useMemo(() => {
return [
{ value: 'dataset', text: t('dataset.datasets') },
...(currentWorkspace.role === 'dataset_operator' ? [] : [{ value: 'api', text: t('dataset.datasetsApi') }]),
]
}, [currentWorkspace.role, t])
const [activeTab, setActiveTab] = useTabSearchParams({
defaultTab: 'dataset',
})
const containerRef = useRef<HTMLDivElement>(null)
const { data } = useQuery(
{
queryKey: ['datasetApiBaseInfo'],
queryFn: () => fetchDatasetApiBaseUrl('/datasets/api-base-info'),
enabled: activeTab !== 'dataset',
},
)
const [keywords, setKeywords] = useState('')
const [searchKeywords, setSearchKeywords] = useState('')
const { run: handleSearch } = useDebounceFn(() => {
setSearchKeywords(keywords)
}, { wait: 500 })
const handleKeywordsChange = (value: string) => {
setKeywords(value)
handleSearch()
}
const [tagFilterValue, setTagFilterValue] = useState<string[]>([])
const [tagIDs, setTagIDs] = useState<string[]>([])
const { run: handleTagsUpdate } = useDebounceFn(() => {
setTagIDs(tagFilterValue)
}, { wait: 500 })
const handleTagsChange = (value: string[]) => {
setTagFilterValue(value)
handleTagsUpdate()
}
useEffect(() => {
if (currentWorkspace.role === 'normal')
return router.replace('/apps')
}, [currentWorkspace, router])
return (
<div ref={containerRef} className={`scroll-container relative flex grow flex-col overflow-y-auto rounded-t-xl outline-none ${activeTab === 'dataset' ? 'bg-background-body' : 'bg-components-panel-bg'}`}>
<div className={`sticky top-0 z-10 flex shrink-0 flex-wrap items-center justify-between gap-y-2 rounded-t-xl px-6 py-2 ${activeTab === 'api' ? 'border-b border-solid border-b-divider-regular' : ''} ${activeTab === 'dataset' ? 'bg-background-body' : 'bg-components-panel-bg'}`}>
<TabSliderNew
value={activeTab}
onChange={newActiveTab => setActiveTab(newActiveTab)}
options={options}
/>
{activeTab === 'dataset' && (
<div className='flex items-center justify-center gap-2'>
{isCurrentWorkspaceOwner && <CheckboxWithLabel
isChecked={includeAll}
onChange={toggleIncludeAll}
label={t('dataset.allKnowledge')}
labelClassName='system-md-regular text-text-secondary'
className='mr-2'
tooltip={t('dataset.allKnowledgeDescription') as string}
/>}
<TagFilter type='knowledge' value={tagFilterValue} onChange={handleTagsChange} />
<Input
showLeftIcon
showClearIcon
wrapperClassName='w-[200px]'
value={keywords}
onChange={e => handleKeywordsChange(e.target.value)}
onClear={() => handleKeywordsChange('')}
/>
<div className="h-4 w-[1px] bg-divider-regular" />
<Button
className='shadows-shadow-xs gap-0.5'
onClick={() => setShowExternalApiPanel(true)}
>
<ApiConnectionMod className='h-4 w-4 text-components-button-secondary-text' />
<div className='system-sm-medium flex items-center justify-center gap-1 px-0.5 text-components-button-secondary-text'>{t('dataset.externalAPIPanelTitle')}</div>
</Button>
</div>
)}
{activeTab === 'api' && data && <ApiServer apiBaseUrl={data.api_base_url || ''} />}
</div>
{activeTab === 'dataset' && (
<>
<Datasets containerRef={containerRef} tags={tagIDs} keywords={searchKeywords} includeAll={includeAll} />
{!systemFeatures.branding.enabled && <DatasetFooter />}
{showTagManagementModal && (
<TagManagementModal type='knowledge' show={showTagManagementModal} />
)}
</>
)}
{activeTab === 'api' && data && <Doc apiBaseUrl={data.api_base_url || ''} />}
{showExternalApiPanel && <ExternalAPIPanel onClose={() => setShowExternalApiPanel(false)} />}
</div>
)
}
export default Container

View File

@@ -0,0 +1,10 @@
import React from 'react'
import CreateFromPipeline from '@/app/components/datasets/create-from-pipeline'
const DatasetCreation = async () => {
return (
<CreateFromPipeline />
)
}
export default DatasetCreation

View File

@@ -1,249 +0,0 @@
'use client'
import { useContext } from 'use-context-selector'
import { useRouter } from 'next/navigation'
import { useCallback, useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { RiMoreFill } from '@remixicon/react'
import { mutate } from 'swr'
import cn from '@/utils/classnames'
import Confirm from '@/app/components/base/confirm'
import { ToastContext } from '@/app/components/base/toast'
import { checkIsUsedInApp, deleteDataset } from '@/service/datasets'
import type { DataSet } from '@/models/datasets'
import Tooltip from '@/app/components/base/tooltip'
import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
import type { HtmlContentProps } from '@/app/components/base/popover'
import CustomPopover from '@/app/components/base/popover'
import Divider from '@/app/components/base/divider'
import RenameDatasetModal from '@/app/components/datasets/rename-modal'
import type { Tag } from '@/app/components/base/tag-management/constant'
import TagSelector from '@/app/components/base/tag-management/selector'
import CornerLabel from '@/app/components/base/corner-label'
import { useAppContext } from '@/context/app-context'
export type DatasetCardProps = {
dataset: DataSet
onSuccess?: () => void
}
const DatasetCard = ({
dataset,
onSuccess,
}: DatasetCardProps) => {
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const { push } = useRouter()
const EXTERNAL_PROVIDER = 'external' as const
const { isCurrentWorkspaceDatasetOperator } = useAppContext()
const [tags, setTags] = useState<Tag[]>(dataset.tags)
const [showRenameModal, setShowRenameModal] = useState(false)
const [showConfirmDelete, setShowConfirmDelete] = useState(false)
const [confirmMessage, setConfirmMessage] = useState<string>('')
const isExternalProvider = (provider: string): boolean => provider === EXTERNAL_PROVIDER
const detectIsUsedByApp = useCallback(async () => {
try {
const { is_using: isUsedByApp } = await checkIsUsedInApp(dataset.id)
setConfirmMessage(isUsedByApp ? t('dataset.datasetUsedByApp')! : t('dataset.deleteDatasetConfirmContent')!)
}
catch (e: any) {
const res = await e.json()
notify({ type: 'error', message: res?.message || 'Unknown error' })
}
setShowConfirmDelete(true)
}, [dataset.id, notify, t])
const onConfirmDelete = useCallback(async () => {
try {
await deleteDataset(dataset.id)
// Clear SWR cache to prevent stale data in knowledge retrieval nodes
mutate(
(key) => {
if (typeof key === 'string') return key.includes('/datasets')
if (typeof key === 'object' && key !== null)
return key.url === '/datasets' || key.url?.includes('/datasets')
return false
},
undefined,
{ revalidate: true },
)
notify({ type: 'success', message: t('dataset.datasetDeleted') })
if (onSuccess)
onSuccess()
}
catch {
}
setShowConfirmDelete(false)
}, [dataset.id, notify, onSuccess, t])
const Operations = (props: HtmlContentProps & { showDelete: boolean }) => {
const onMouseLeave = async () => {
props.onClose?.()
}
const onClickRename = async (e: React.MouseEvent<HTMLDivElement>) => {
e.stopPropagation()
props.onClick?.()
e.preventDefault()
setShowRenameModal(true)
}
const onClickDelete = async (e: React.MouseEvent<HTMLDivElement>) => {
e.stopPropagation()
props.onClick?.()
e.preventDefault()
detectIsUsedByApp()
}
return (
<div className="relative w-full py-1" onMouseLeave={onMouseLeave}>
<div className='mx-1 flex h-8 cursor-pointer items-center gap-2 rounded-lg px-3 py-[6px] hover:bg-state-base-hover' onClick={onClickRename}>
<span className='text-sm text-text-secondary'>{t('common.operation.settings')}</span>
</div>
{props.showDelete && (
<>
<Divider className="!my-1" />
<div
className='group mx-1 flex h-8 cursor-pointer items-center gap-2 rounded-lg px-3 py-[6px] hover:bg-state-destructive-hover'
onClick={onClickDelete}
>
<span className={cn('text-sm text-text-secondary', 'group-hover:text-text-destructive')}>
{t('common.operation.delete')}
</span>
</div>
</>
)}
</div>
)
}
useEffect(() => {
setTags(dataset.tags)
}, [dataset])
return (
<>
<div
className='group relative col-span-1 flex min-h-[171px] cursor-pointer flex-col rounded-xl border-[0.5px] border-solid border-components-card-border bg-components-card-bg shadow-sm transition-all duration-200 ease-in-out hover:shadow-lg'
data-disable-nprogress={true}
onClick={(e) => {
e.preventDefault()
isExternalProvider(dataset.provider)
? push(`/datasets/${dataset.id}/hitTesting`)
: push(`/datasets/${dataset.id}/documents`)
}}
>
{isExternalProvider(dataset.provider) && <CornerLabel label='External' className='absolute right-0' labelClassName='rounded-tr-xl' />}
<div className='flex h-[66px] shrink-0 grow-0 items-center gap-3 px-[14px] pb-3 pt-[14px]'>
<div className={cn(
'flex shrink-0 items-center justify-center rounded-md border-[0.5px] border-[#E0EAFF] bg-[#F5F8FF] p-2.5',
!dataset.embedding_available && 'opacity-50 hover:opacity-100',
)}>
<Folder className='h-5 w-5 text-[#444CE7]' />
</div>
<div className='w-0 grow py-[1px]'>
<div className='flex items-center text-sm font-semibold leading-5 text-text-secondary'>
<div className={cn('truncate', !dataset.embedding_available && 'text-text-tertiary opacity-50 hover:opacity-100')} title={dataset.name}>{dataset.name}</div>
{!dataset.embedding_available && (
<Tooltip
popupContent={t('dataset.unavailableTip')}
>
<span className='ml-1 inline-flex w-max shrink-0 rounded-md border border-divider-regular px-1 text-xs font-normal leading-[18px] text-text-tertiary'>{t('dataset.unavailable')}</span>
</Tooltip>
)}
</div>
<div className='mt-[1px] flex items-center text-xs leading-[18px] text-text-tertiary'>
<div
className={cn('truncate', (!dataset.embedding_available || !dataset.document_count) && 'opacity-50')}
title={dataset.provider === 'external' ? `${dataset.app_count}${t('dataset.appCount')}` : `${dataset.document_count}${t('dataset.documentCount')} · ${Math.round(dataset.word_count / 1000)}${t('dataset.wordCount')} · ${dataset.app_count}${t('dataset.appCount')}`}
>
{dataset.provider === 'external'
? <>
<span>{dataset.app_count}{t('dataset.appCount')}</span>
</>
: <>
<span>{dataset.document_count}{t('dataset.documentCount')}</span>
<span className='mx-0.5 w-1 shrink-0 text-text-tertiary'>·</span>
<span>{Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}</span>
<span className='mx-0.5 w-1 shrink-0 text-text-tertiary'>·</span>
<span>{dataset.app_count}{t('dataset.appCount')}</span>
</>
}
</div>
</div>
</div>
</div>
<div
className={cn(
'mb-2 line-clamp-2 max-h-[36px] grow px-[14px] text-xs leading-normal text-text-tertiary',
!dataset.embedding_available && 'opacity-50 hover:opacity-100',
)}
title={dataset.description}>
{dataset.description}
</div>
<div className='mt-4 flex h-[42px] shrink-0 items-center pb-[6px] pl-[14px] pr-[6px] pt-1'>
<div className={cn('flex w-0 grow items-center gap-1', !dataset.embedding_available && 'opacity-50 hover:opacity-100')} onClick={(e) => {
e.stopPropagation()
e.preventDefault()
}}>
<div className={cn(
'mr-[41px] w-full grow group-hover:!mr-0',
)}>
<TagSelector
position='bl'
type='knowledge'
targetID={dataset.id}
value={tags.map(tag => tag.id)}
selectedTags={tags}
onCacheUpdate={setTags}
onChange={onSuccess}
/>
</div>
</div>
<div className='mx-1 !hidden h-[14px] w-[1px] shrink-0 bg-divider-regular group-hover:!flex' />
<div className='!hidden shrink-0 group-hover:!flex'>
<CustomPopover
htmlContent={<Operations showDelete={!isCurrentWorkspaceDatasetOperator} />}
position="br"
trigger="click"
btnElement={
<div
className='flex h-8 w-8 cursor-pointer items-center justify-center rounded-md'
>
<RiMoreFill className='h-4 w-4 text-text-secondary' />
</div>
}
btnClassName={open =>
cn(
open ? '!bg-state-base-hover !shadow-none' : '!bg-transparent',
'h-8 w-8 rounded-md border-none !p-2 hover:!bg-state-base-hover',
)
}
className={'!z-20 h-fit !w-[128px]'}
/>
</div>
</div>
</div>
{showRenameModal && (
<RenameDatasetModal
show={showRenameModal}
dataset={dataset}
onClose={() => setShowRenameModal(false)}
onSuccess={onSuccess}
/>
)}
{showConfirmDelete && (
<Confirm
title={t('dataset.deleteDatasetConfirmTitle')}
content={confirmMessage}
isShow={showConfirmDelete}
onConfirm={onConfirmDelete}
onCancel={() => setShowConfirmDelete(false)}
/>
)}
</>
)
}
export default DatasetCard

View File

@@ -1,19 +0,0 @@
'use client'
import { useTranslation } from 'react-i18next'
const DatasetFooter = () => {
const { t } = useTranslation()
return (
<footer className='shrink-0 grow-0 px-12 py-6'>
<h3 className='text-gradient text-xl font-semibold leading-tight'>{t('dataset.didYouKnow')}</h3>
<p className='mt-1 text-sm font-normal leading-tight text-text-secondary'>
{t('dataset.intro1')}<span className='inline-flex items-center gap-1 text-text-accent'>{t('dataset.intro2')}</span>{t('dataset.intro3')}<br />
{t('dataset.intro4')}<span className='inline-flex items-center gap-1 text-text-accent'>{t('dataset.intro5')}</span>{t('dataset.intro6')}
</p>
</footer>
)
}
export default DatasetFooter

View File

@@ -1,96 +0,0 @@
'use client'
import { useCallback, useEffect, useRef } from 'react'
import useSWRInfinite from 'swr/infinite'
import { debounce } from 'lodash-es'
import NewDatasetCard from './new-dataset-card'
import DatasetCard from './dataset-card'
import type { DataSetListResponse, FetchDatasetsParams } from '@/models/datasets'
import { fetchDatasets } from '@/service/datasets'
import { useAppContext } from '@/context/app-context'
import { useTranslation } from 'react-i18next'
const getKey = (
pageIndex: number,
previousPageData: DataSetListResponse,
tags: string[],
keyword: string,
includeAll: boolean,
) => {
if (!pageIndex || previousPageData.has_more) {
const params: FetchDatasetsParams = {
url: 'datasets',
params: {
page: pageIndex + 1,
limit: 30,
include_all: includeAll,
},
}
if (tags.length)
params.params.tag_ids = tags
if (keyword)
params.params.keyword = keyword
return params
}
return null
}
type Props = {
containerRef: React.RefObject<HTMLDivElement | null>
tags: string[]
keywords: string
includeAll: boolean
}
const Datasets = ({
containerRef,
tags,
keywords,
includeAll,
}: Props) => {
const { t } = useTranslation()
const { isCurrentWorkspaceEditor } = useAppContext()
const { data, isLoading, setSize, mutate } = useSWRInfinite(
(pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords, includeAll),
fetchDatasets,
{ revalidateFirstPage: false, revalidateAll: true },
)
const loadingStateRef = useRef(false)
const anchorRef = useRef<HTMLAnchorElement>(null)
useEffect(() => {
loadingStateRef.current = isLoading
}, [isLoading, t])
const onScroll = useCallback(
debounce(() => {
if (!loadingStateRef.current && containerRef.current && anchorRef.current) {
const { scrollTop, clientHeight } = containerRef.current
const anchorOffset = anchorRef.current.offsetTop
if (anchorOffset - scrollTop - clientHeight < 100)
setSize(size => size + 1)
}
}, 50),
[setSize],
)
useEffect(() => {
const currentContainer = containerRef.current
currentContainer?.addEventListener('scroll', onScroll)
return () => {
currentContainer?.removeEventListener('scroll', onScroll)
onScroll.cancel()
}
}, [containerRef, onScroll])
return (
<nav className='grid shrink-0 grow grid-cols-1 content-start gap-4 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4'>
{isCurrentWorkspaceEditor && <NewDatasetCard ref={anchorRef} />}
{data?.map(({ data: datasets }) => datasets.map(dataset => (
<DatasetCard key={dataset.id} dataset={dataset} onSuccess={mutate} />),
))}
</nav>
)
}
export default Datasets

View File

@@ -1,203 +0,0 @@
'use client'
import { useEffect, useMemo, useState } from 'react'
import { useContext } from 'use-context-selector'
import { useTranslation } from 'react-i18next'
import { RiCloseLine, RiListUnordered } from '@remixicon/react'
import TemplateEn from './template/template.en.mdx'
import TemplateZh from './template/template.zh.mdx'
import TemplateJa from './template/template.ja.mdx'
import I18n from '@/context/i18n'
import { LanguagesSupported } from '@/i18n-config/language'
import useTheme from '@/hooks/use-theme'
import { Theme } from '@/types/app'
import cn from '@/utils/classnames'
type DocProps = {
apiBaseUrl: string
}
const Doc = ({ apiBaseUrl }: DocProps) => {
const { locale } = useContext(I18n)
const { t } = useTranslation()
const [toc, setToc] = useState<Array<{ href: string; text: string }>>([])
const [isTocExpanded, setIsTocExpanded] = useState(false)
const [activeSection, setActiveSection] = useState<string>('')
const { theme } = useTheme()
// Set initial TOC expanded state based on screen width
useEffect(() => {
const mediaQuery = window.matchMedia('(min-width: 1280px)')
setIsTocExpanded(mediaQuery.matches)
}, [])
// Extract TOC from article content
useEffect(() => {
const extractTOC = () => {
const article = document.querySelector('article')
if (article) {
const headings = article.querySelectorAll('h2')
const tocItems = Array.from(headings).map((heading) => {
const anchor = heading.querySelector('a')
if (anchor) {
return {
href: anchor.getAttribute('href') || '',
text: anchor.textContent || '',
}
}
return null
}).filter((item): item is { href: string; text: string } => item !== null)
setToc(tocItems)
// Set initial active section
if (tocItems.length > 0)
setActiveSection(tocItems[0].href.replace('#', ''))
}
}
setTimeout(extractTOC, 0)
}, [locale])
// Track scroll position for active section highlighting
useEffect(() => {
const handleScroll = () => {
const scrollContainer = document.querySelector('.scroll-container')
if (!scrollContainer || toc.length === 0)
return
// Find active section based on scroll position
let currentSection = ''
toc.forEach((item) => {
const targetId = item.href.replace('#', '')
const element = document.getElementById(targetId)
if (element) {
const rect = element.getBoundingClientRect()
// Consider section active if its top is above the middle of viewport
if (rect.top <= window.innerHeight / 2)
currentSection = targetId
}
})
if (currentSection && currentSection !== activeSection)
setActiveSection(currentSection)
}
const scrollContainer = document.querySelector('.scroll-container')
if (scrollContainer) {
scrollContainer.addEventListener('scroll', handleScroll)
handleScroll() // Initial check
return () => scrollContainer.removeEventListener('scroll', handleScroll)
}
}, [toc, activeSection])
// Handle TOC item click
const handleTocClick = (e: React.MouseEvent<HTMLAnchorElement>, item: { href: string; text: string }) => {
e.preventDefault()
const targetId = item.href.replace('#', '')
const element = document.getElementById(targetId)
if (element) {
const scrollContainer = document.querySelector('.scroll-container')
if (scrollContainer) {
const headerOffset = -40
const elementTop = element.offsetTop - headerOffset
scrollContainer.scrollTo({
top: elementTop,
behavior: 'smooth',
})
}
}
}
const Template = useMemo(() => {
switch (locale) {
case LanguagesSupported[1]:
return <TemplateZh apiBaseUrl={apiBaseUrl} />
case LanguagesSupported[7]:
return <TemplateJa apiBaseUrl={apiBaseUrl} />
default:
return <TemplateEn apiBaseUrl={apiBaseUrl} />
}
}, [apiBaseUrl, locale])
return (
<div className="flex">
<div className={`fixed right-20 top-32 z-10 transition-all duration-150 ease-out ${isTocExpanded ? 'w-[280px]' : 'w-11'}`}>
{isTocExpanded
? (
<nav className="toc flex max-h-[calc(100vh-150px)] w-full flex-col overflow-hidden rounded-xl border-[0.5px] border-components-panel-border bg-background-default-hover shadow-xl">
<div className="relative z-10 flex items-center justify-between border-b border-components-panel-border-subtle bg-background-default-hover px-4 py-2.5">
<span className="text-xs font-medium uppercase tracking-wide text-text-tertiary">
{t('appApi.develop.toc')}
</span>
<button
onClick={() => setIsTocExpanded(false)}
className="group flex h-6 w-6 items-center justify-center rounded-md transition-colors hover:bg-state-base-hover"
aria-label="Close"
>
<RiCloseLine className="h-3 w-3 text-text-quaternary transition-colors group-hover:text-text-secondary" />
</button>
</div>
<div className="from-components-panel-border-subtle/20 pointer-events-none absolute left-0 right-0 top-[41px] z-10 h-2 bg-gradient-to-b to-transparent"></div>
<div className="pointer-events-none absolute left-0 right-0 top-[43px] z-10 h-3 bg-gradient-to-b from-background-default-hover to-transparent"></div>
<div className="relative flex-1 overflow-y-auto px-3 py-3 pt-1">
{toc.length === 0 ? (
<div className="px-2 py-8 text-center text-xs text-text-quaternary">
{t('appApi.develop.noContent')}
</div>
) : (
<ul className="space-y-0.5">
{toc.map((item, index) => {
const isActive = activeSection === item.href.replace('#', '')
return (
<li key={index}>
<a
href={item.href}
onClick={e => handleTocClick(e, item)}
className={cn(
'group relative flex items-center rounded-md px-3 py-2 text-[13px] transition-all duration-200',
isActive
? 'bg-state-base-hover font-medium text-text-primary'
: 'text-text-tertiary hover:bg-state-base-hover hover:text-text-secondary',
)}
>
<span
className={cn(
'mr-2 h-1.5 w-1.5 rounded-full transition-all duration-200',
isActive
? 'scale-100 bg-text-accent'
: 'scale-75 bg-components-panel-border',
)}
/>
<span className="flex-1 truncate">
{item.text}
</span>
</a>
</li>
)
})}
</ul>
)}
</div>
<div className="pointer-events-none absolute bottom-0 left-0 right-0 z-10 h-4 rounded-b-xl bg-gradient-to-t from-background-default-hover to-transparent"></div>
</nav>
)
: (
<button
onClick={() => setIsTocExpanded(true)}
className="group flex h-11 w-11 items-center justify-center rounded-full border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg transition-all duration-150 hover:bg-background-default-hover hover:shadow-xl"
aria-label="Open table of contents"
>
<RiListUnordered className="h-5 w-5 text-text-tertiary transition-colors group-hover:text-text-secondary" />
</button>
)}
</div>
<article className={cn('prose-xl prose', theme === Theme.dark && 'prose-invert')}>
{Template}
</article>
</div>
)
}
export default Doc

View File

@@ -1,41 +0,0 @@
'use client'
import { useTranslation } from 'react-i18next'
import {
RiAddLine,
RiArrowRightLine,
} from '@remixicon/react'
import Link from 'next/link'
type CreateAppCardProps = {
ref?: React.Ref<HTMLAnchorElement>
}
const CreateAppCard = ({ ref }: CreateAppCardProps) => {
const { t } = useTranslation()
return (
<div className='bg-background-default-dimm flex min-h-[160px] flex-col rounded-xl border-[0.5px]
border-components-panel-border transition-all duration-200 ease-in-out'
>
<Link ref={ref} className='group flex grow cursor-pointer items-start p-4' href='/datasets/create'>
<div className='flex items-center gap-3'>
<div className='flex h-10 w-10 items-center justify-center rounded-lg border border-dashed border-divider-regular bg-background-default-lighter
p-2 group-hover:border-solid group-hover:border-effects-highlight group-hover:bg-background-default-dodge'
>
<RiAddLine className='h-4 w-4 text-text-tertiary group-hover:text-text-accent' />
</div>
<div className='system-md-semibold text-text-secondary group-hover:text-text-accent'>{t('dataset.createDataset')}</div>
</div>
</Link>
<div className='system-xs-regular p-4 pt-0 text-text-tertiary'>{t('dataset.createDatasetIntro')}</div>
<Link className='group flex cursor-pointer items-center gap-1 rounded-b-xl border-t-[0.5px] border-divider-subtle p-4' href='/datasets/connect'>
<div className='system-xs-medium text-text-tertiary group-hover:text-text-accent'>{t('dataset.connectDataset')}</div>
<RiArrowRightLine className='h-3.5 w-3.5 text-text-tertiary group-hover:text-text-accent' />
</Link>
</div>
)
}
CreateAppCard.displayName = 'CreateAppCard'
export default CreateAppCard

View File

@@ -1,12 +1,7 @@
'use client'
import { useTranslation } from 'react-i18next'
import Container from './container'
import useDocumentTitle from '@/hooks/use-document-title'
import List from '../../components/datasets/list'
const AppList = () => {
const { t } = useTranslation()
useDocumentTitle(t('common.menus.datasets'))
return <Container />
const DatasetList = async () => {
return <List />
}
export default AppList
export default DatasetList

View File

@@ -1,11 +0,0 @@
import { create } from 'zustand'
type DatasetStore = {
showExternalApiPanel: boolean
setShowExternalApiPanel: (show: boolean) => void
}
export const useDatasetStore = create<DatasetStore>(set => ({
showExternalApiPanel: false,
setShowExternalApiPanel: show => set({ showExternalApiPanel: show }),
}))

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff