feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -1,13 +1,15 @@
import { useCallback, useEffect, useMemo, useState } from 'react'
import useSWR from 'swr'
import { RiEqualizer2Line } from '@remixicon/react'
import WorkspaceSelector from './workspace-selector'
import type { NotionCredential } from './credential-selector'
import WorkspaceSelector from './credential-selector'
import SearchInput from './search-input'
import PageSelector from './page-selector'
import { preImportNotionPages } from '@/service/datasets'
import { NotionConnector } from '@/app/components/datasets/create/step-one'
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
import { useModalContext } from '@/context/modal-context'
import { useModalContextSelector } from '@/context/modal-context'
import NotionConnector from '../notion-connector'
import { useInvalidPreImportNotionPages, usePreImportNotionPages } from '@/service/knowledge/use-import'
import Header from '../../datasets/create/website/base/header'
import type { DataSourceCredential } from '../../header/account-setting/data-source-page-new/types'
import Loading from '../loading'
type NotionPageSelectorProps = {
value?: string[]
@@ -16,6 +18,8 @@ type NotionPageSelectorProps = {
previewPageId?: string
onPreview?: (selectedPage: NotionPage) => void
datasetId?: string
credentialList: DataSourceCredential[]
onSelectCredential?: (credentialId: string) => void
}
const NotionPageSelector = ({
@@ -25,110 +29,155 @@ const NotionPageSelector = ({
previewPageId,
onPreview,
datasetId = '',
credentialList,
onSelectCredential,
}: NotionPageSelectorProps) => {
const { data, mutate } = useSWR({ url: '/notion/pre-import/pages', datasetId }, preImportNotionPages)
const [prevData, setPrevData] = useState(data)
const [searchValue, setSearchValue] = useState('')
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
const { setShowAccountSettingModal } = useModalContext()
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
const notionWorkspaces = useMemo(() => {
return data?.notion_info || []
}, [data?.notion_info])
const firstWorkspaceId = notionWorkspaces[0]?.workspace_id
const currentWorkspace = notionWorkspaces.find(workspace => workspace.workspace_id === currentWorkspaceId)
const invalidPreImportNotionPages = useInvalidPreImportNotionPages()
const getPagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
const notionCredentials = useMemo((): NotionCredential[] => {
return credentialList.map((item) => {
return {
credentialId: item.id,
credentialName: item.name,
workspaceIcon: item.credential.workspace_icon,
workspaceName: item.credential.workspace_name,
}
})
}, [credentialList])
const [currentCredential, setCurrentCredential] = useState(notionCredentials[0])
useEffect(() => {
const credential = notionCredentials.find(item => item.credentialId === currentCredential?.credentialId)
if (!credential) {
const firstCredential = notionCredentials[0]
invalidPreImportNotionPages({ datasetId, credentialId: firstCredential.credentialId })
setCurrentCredential(notionCredentials[0])
onSelect([]) // Clear selected pages when changing credential
onSelectCredential?.(firstCredential.credentialId)
}
else {
onSelectCredential?.(credential?.credentialId || '')
}
}, [notionCredentials])
const {
data: notionsPages,
isFetching: isFetchingNotionPages,
isError: isFetchingNotionPagesError,
} = usePreImportNotionPages({ datasetId, credentialId: currentCredential.credentialId || '' })
const pagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
const selectedPagesId = new Set<string>()
const boundPagesId = new Set<string>()
const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
next.pages.forEach((page) => {
const notionWorkspaces = notionsPages?.notion_info || []
const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, cur: DataSourceNotionWorkspace) => {
cur.pages.forEach((page) => {
if (page.is_bound) {
selectedPagesId.add(page.page_id)
boundPagesId.add(page.page_id)
}
prev[page.page_id] = {
...page,
workspace_id: next.workspace_id,
workspace_id: cur.workspace_id,
}
})
return prev
}, {})
return [pagesMap, selectedPagesId, boundPagesId]
}, [notionWorkspaces])
const defaultSelectedPagesId = [...Array.from(getPagesMapAndSelectedPagesId[1]), ...(value || [])]
}, [notionsPages?.notion_info])
const defaultSelectedPagesId = useMemo(() => {
return [...Array.from(pagesMapAndSelectedPagesId[1]), ...(value || [])]
}, [pagesMapAndSelectedPagesId, value])
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
if (prevData !== data) {
setPrevData(data)
useEffect(() => {
setSelectedPagesId(new Set(defaultSelectedPagesId))
}
}, [defaultSelectedPagesId])
const handleSearchValueChange = useCallback((value: string) => {
setSearchValue(value)
}, [])
const handleSelectWorkspace = useCallback((workspaceId: string) => {
setCurrentWorkspaceId(workspaceId)
}, [])
const handleSelectPages = (newSelectedPagesId: Set<string>) => {
const selectedPages = Array.from(newSelectedPagesId).map(pageId => getPagesMapAndSelectedPagesId[0][pageId])
const handleSelectCredential = useCallback((credentialId: string) => {
const credential = notionCredentials.find(item => item.credentialId === credentialId)!
invalidPreImportNotionPages({ datasetId, credentialId: credential.credentialId })
setCurrentCredential(credential)
onSelect([]) // Clear selected pages when changing credential
onSelectCredential?.(credential.credentialId)
}, [invalidPreImportNotionPages, onSelect, onSelectCredential])
const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
const selectedPages = Array.from(newSelectedPagesId).map(pageId => pagesMapAndSelectedPagesId[0][pageId])
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
onSelect(selectedPages)
}
const handlePreviewPage = (previewPageId: string) => {
if (onPreview)
onPreview(getPagesMapAndSelectedPagesId[0][previewPageId])
}
}, [pagesMapAndSelectedPagesId, onSelect])
useEffect(() => {
setCurrentWorkspaceId(firstWorkspaceId)
}, [firstWorkspaceId])
const handlePreviewPage = useCallback((previewPageId: string) => {
if (onPreview)
onPreview(pagesMapAndSelectedPagesId[0][previewPageId])
}, [pagesMapAndSelectedPagesId, onPreview])
const handleConfigureNotion = useCallback(() => {
setShowAccountSettingModal({ payload: 'data-source' })
}, [setShowAccountSettingModal])
if (isFetchingNotionPagesError) {
return (
<NotionConnector
onSetting={handleConfigureNotion}
/>
)
}
return (
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
{
data?.notion_info?.length
? (
<>
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
<div className='flex grow items-center gap-x-1'>
<WorkspaceSelector
value={currentWorkspaceId || firstWorkspaceId}
items={notionWorkspaces}
onSelect={handleSelectWorkspace}
/>
<div className='mx-1 h-3 w-[1px] bg-divider-regular' />
<RiEqualizer2Line
className='h-4 w-4 cursor-pointer text-text-tertiary'
onClick={() => setShowAccountSettingModal({ payload: 'data-source', onCancelCallback: mutate })}
/>
</div>
<SearchInput
value={searchValue}
onChange={handleSearchValueChange}
/>
</div>
<div className='overflow-hidden rounded-b-xl'>
<PageSelector
value={selectedPagesId}
disabledValue={getPagesMapAndSelectedPagesId[2]}
searchValue={searchValue}
list={currentWorkspace?.pages || []}
pagesMap={getPagesMapAndSelectedPagesId[0]}
onSelect={handleSelectPages}
canPreview={canPreview}
previewPageId={previewPageId}
onPreview={handlePreviewPage}
/>
</div>
</>
)
: (
<NotionConnector onSetting={() => setShowAccountSettingModal({ payload: 'data-source', onCancelCallback: mutate })} />
)
}
<div className='flex flex-col gap-y-2'>
<Header
onClickConfiguration={handleConfigureNotion}
title={'Choose notion pages'}
buttonText={'Configure Notion'}
docTitle={'Notion docs'}
docLink={'https://www.notion.so/docs'}
/>
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
<div className='flex grow items-center gap-x-1'>
<WorkspaceSelector
value={currentCredential.credentialId}
items={notionCredentials}
onSelect={handleSelectCredential}
/>
</div>
<SearchInput
value={searchValue}
onChange={handleSearchValueChange}
/>
</div>
<div className='overflow-hidden rounded-b-xl'>
{isFetchingNotionPages ? (
<div className='flex h-[296px] items-center justify-center'>
<Loading />
</div>
) : (
<PageSelector
value={selectedPagesId}
disabledValue={pagesMapAndSelectedPagesId[2]}
searchValue={searchValue}
list={notionsPages!.notion_info?.[0].pages || []}
pagesMap={pagesMapAndSelectedPagesId[0]}
onSelect={handleSelectPages}
canPreview={canPreview}
previewPageId={previewPageId}
onPreview={handlePreviewPage}
/>
)}
</div>
</div>
</div>
)
}

View File

@@ -0,0 +1,113 @@
'use client'
import { useTranslation } from 'react-i18next'
import React, { Fragment, useMemo } from 'react'
import { Menu, MenuButton, MenuItem, MenuItems, Transition } from '@headlessui/react'
import { RiArrowDownSLine } from '@remixicon/react'
import NotionIcon from '../../notion-icon'
export type NotionCredential = {
credentialId: string
credentialName: string
workspaceIcon?: string
workspaceName?: string
}
type CredentialSelectorProps = {
value: string
items: NotionCredential[]
onSelect: (v: string) => void
}
const CredentialSelector = ({
value,
items,
onSelect,
}: CredentialSelectorProps) => {
const { t } = useTranslation()
const currentCredential = items.find(item => item.credentialId === value)!
const getDisplayName = (item: NotionCredential) => {
return item.workspaceName || t('datasetPipeline.credentialSelector.name', {
credentialName: item.credentialName,
pluginName: 'Notion',
})
}
const currentDisplayName = useMemo(() => {
return getDisplayName(currentCredential)
}, [currentCredential])
return (
<Menu as='div' className='relative inline-block text-left'>
{
({ open }) => (
<>
<MenuButton className={`flex h-7 items-center justify-center rounded-md p-1 pr-2 hover:bg-state-base-hover ${open && 'bg-state-base-hover'} cursor-pointer`}>
<NotionIcon
className='mr-2'
src={currentCredential?.workspaceIcon}
name={currentDisplayName}
/>
<div
className='mr-1 w-[90px] truncate text-left text-sm font-medium text-text-secondary'
title={currentDisplayName}
>
{currentDisplayName}
</div>
<RiArrowDownSLine className='h-4 w-4 text-text-secondary' />
</MenuButton>
<Transition
as={Fragment}
enter='transition ease-out duration-100'
enterFrom='transform opacity-0 scale-95'
enterTo='transform opacity-100 scale-100'
leave='transition ease-in duration-75'
leaveFrom='transform opacity-100 scale-100'
leaveTo='transform opacity-0 scale-95'
>
<MenuItems
className='absolute left-0 top-8 z-10 w-80
origin-top-right rounded-lg border-[0.5px]
border-components-panel-border bg-components-panel-bg-blur shadow-lg shadow-shadow-shadow-5'
>
<div className='max-h-50 overflow-auto p-1'>
{
items.map((item) => {
const displayName = getDisplayName(item)
return (
<MenuItem key={item.credentialId}>
<div
className='flex h-9 cursor-pointer items-center rounded-lg px-3 hover:bg-state-base-hover'
onClick={() => onSelect(item.credentialId)}
>
<NotionIcon
className='mr-2 shrink-0'
src={item.workspaceIcon}
name={displayName}
/>
<div
className='system-sm-medium mr-2 grow truncate text-text-secondary'
title={displayName}
>
{displayName}
</div>
{/* // ?Cannot get page length with new auth system */}
{/* <div className='system-xs-medium shrink-0 text-text-accent'>
{item.pages.length} {t('common.dataSource.notion.selector.pageSelected')}
</div> */}
</div>
</MenuItem>
)
})
}
</div>
</MenuItems>
</Transition>
</>
)
}
</Menu>
)
}
export default React.memo(CredentialSelector)

View File

@@ -1,2 +1 @@
export { default as NotionPageSelectorModal } from './notion-page-selector-modal'
export { default as NotionPageSelector } from './base'

View File

@@ -1,28 +0,0 @@
.modal {
width: 600px !important;
max-width: 600px !important;
padding: 24px 32px !important;
}
.operate {
padding: 0 8px;
min-width: 96px;
height: 36px;
line-height: 36px;
text-align: center;
background-color: #ffffff;
box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
border-radius: 8px;
border: 0.5px solid #eaecf0;
font-size: 14px;
font-weight: 500;
color: #667085;
cursor: pointer;
}
.operate-save {
margin-left: 8px;
border-color: #155eef;
background-color: #155eef;
color: #ffffff;
}

View File

@@ -1,63 +0,0 @@
import { useState } from 'react'
import { useTranslation } from 'react-i18next'
import { XMarkIcon } from '@heroicons/react/24/outline'
import NotionPageSelector from '../base'
import s from './index.module.css'
import type { NotionPage } from '@/models/common'
import cn from '@/utils/classnames'
import Modal from '@/app/components/base/modal'
import { noop } from 'lodash-es'
type NotionPageSelectorModalProps = {
isShow: boolean
onClose: () => void
onSave: (selectedPages: NotionPage[]) => void
datasetId: string
}
const NotionPageSelectorModal = ({
isShow,
onClose,
onSave,
datasetId,
}: NotionPageSelectorModalProps) => {
const { t } = useTranslation()
const [selectedPages, setSelectedPages] = useState<NotionPage[]>([])
const handleClose = () => {
onClose()
}
const handleSelectPage = (newSelectedPages: NotionPage[]) => {
setSelectedPages(newSelectedPages)
}
const handleSave = () => {
onSave(selectedPages)
}
return (
<Modal
className={s.modal}
isShow={isShow}
onClose={noop}
>
<div className='mb-6 flex h-8 items-center justify-between'>
<div className='text-xl font-semibold text-gray-900'>{t('common.dataSource.notion.selector.addPages')}</div>
<div
className='-mr-2 flex h-8 w-8 cursor-pointer items-center justify-center'
onClick={handleClose}>
<XMarkIcon className='h-4 w-4' />
</div>
</div>
<NotionPageSelector
onSelect={handleSelectPage}
canPreview={false}
datasetId={datasetId}
/>
<div className='mt-8 flex justify-end'>
<div className={s.operate} onClick={handleClose}>{t('common.operation.cancel')}</div>
<div className={cn(s.operate, s['operate-save'])} onClick={handleSave}>{t('common.operation.save')}</div>
</div>
</Modal>
)
}
export default NotionPageSelectorModal

View File

@@ -1,4 +1,4 @@
import { memo, useMemo, useState } from 'react'
import { memo, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { FixedSizeList as List, areEqual } from 'react-window'
import type { ListChildComponentProps } from 'react-window'
@@ -82,7 +82,19 @@ const ItemComponent = ({ index, style, data }: ListChildComponentProps<{
pagesMap: DataSourceNotionPageMap
}>) => {
const { t } = useTranslation()
const { dataList, handleToggle, checkedIds, disabledCheckedIds, handleCheck, canPreview, handlePreview, listMapWithChildrenAndDescendants, searchValue, previewPageId, pagesMap } = data
const {
dataList,
handleToggle,
checkedIds,
disabledCheckedIds,
handleCheck,
canPreview,
handlePreview,
listMapWithChildrenAndDescendants,
searchValue,
previewPageId,
pagesMap,
} = data
const current = dataList[index]
const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
@@ -182,11 +194,10 @@ const PageSelector = ({
onPreview,
}: PageSelectorProps) => {
const { t } = useTranslation()
const [prevDataList, setPrevDataList] = useState(list)
const [dataList, setDataList] = useState<NotionPageItem[]>([])
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
if (prevDataList !== list) {
setPrevDataList(list)
useEffect(() => {
setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
return {
...item,
@@ -194,7 +205,8 @@ const PageSelector = ({
depth: 0,
}
}))
}
}, [list])
const searchDataList = list.filter((item) => {
return item.page_name.includes(searchValue)
}).map((item) => {

View File

@@ -1,80 +0,0 @@
'use client'
import { useTranslation } from 'react-i18next'
import { Fragment } from 'react'
import { Menu, MenuButton, MenuItem, MenuItems, Transition } from '@headlessui/react'
import { RiArrowDownSLine } from '@remixicon/react'
import NotionIcon from '../../notion-icon'
import type { DataSourceNotionWorkspace } from '@/models/common'
type WorkspaceSelectorProps = {
value: string
items: Omit<DataSourceNotionWorkspace, 'total'>[]
onSelect: (v: string) => void
}
export default function WorkspaceSelector({
value,
items,
onSelect,
}: WorkspaceSelectorProps) {
const { t } = useTranslation()
const currentWorkspace = items.find(item => item.workspace_id === value)
return (
<Menu as="div" className="relative inline-block text-left">
{
({ open }) => (
<>
<MenuButton className={`flex h-7 items-center justify-center rounded-md p-1 pr-2 hover:bg-state-base-hover ${open && 'bg-state-base-hover'} cursor-pointer`}>
<NotionIcon
className='mr-2'
src={currentWorkspace?.workspace_icon}
name={currentWorkspace?.workspace_name}
/>
<div className='mr-1 w-[90px] truncate text-left text-sm font-medium text-text-secondary' title={currentWorkspace?.workspace_name}>{currentWorkspace?.workspace_name}</div>
{/* <div className='mr-1 px-1 h-[18px] bg-primary-50 rounded-lg text-xs font-medium text-text-accent'>{currentWorkspace?.pages.length}</div> */}
<RiArrowDownSLine className='h-4 w-4 text-text-secondary' />
</MenuButton>
<Transition
as={Fragment}
enter="transition ease-out duration-100"
enterFrom="transform opacity-0 scale-95"
enterTo="transform opacity-100 scale-100"
leave="transition ease-in duration-75"
leaveFrom="transform opacity-100 scale-100"
leaveTo="transform opacity-0 scale-95"
>
<MenuItems
className='absolute left-0 top-8 z-10 w-80
origin-top-right rounded-lg border-[0.5px]
border-components-panel-border bg-components-panel-bg-blur shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'
>
<div className="max-h-50 overflow-auto p-1">
{
items.map(item => (
<MenuItem key={item.workspace_id}>
<div
className='flex h-9 cursor-pointer items-center rounded-lg px-3 hover:bg-state-base-hover'
onClick={() => onSelect(item.workspace_id)}
>
<NotionIcon
className='mr-2 shrink-0'
src={item.workspace_icon}
name={item.workspace_name}
/>
<div className='system-sm-medium mr-2 grow truncate text-text-secondary' title={item.workspace_name}>{item.workspace_name}</div>
<div className='system-xs-medium shrink-0 text-text-accent'>
{item.pages.length} {t('common.dataSource.notion.selector.pageSelected')}
</div>
</div>
</MenuItem>
))
}
</div>
</MenuItems>
</Transition>
</>
)
}
</Menu>
)
}