feat: knowledge pipeline (#25360)
Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
@@ -15,24 +15,26 @@ import type { DataSourceProvider, NotionPage } from '@/models/common'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
|
||||
import { useDatasetDetailContext } from '@/context/dataset-detail'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useProviderContext } from '@/context/provider-context'
|
||||
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
||||
import classNames from '@/utils/classnames'
|
||||
import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
|
||||
import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
|
||||
import NotionConnector from '@/app/components/base/notion-connector'
|
||||
import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'
|
||||
|
||||
type IStepOneProps = {
|
||||
datasetId?: string
|
||||
dataSourceType?: DataSourceType
|
||||
dataSourceTypeDisable: boolean
|
||||
hasConnection: boolean
|
||||
onSetting: () => void
|
||||
files: FileItem[]
|
||||
updateFileList: (files: FileItem[]) => void
|
||||
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
notionPages?: NotionPage[]
|
||||
notionCredentialId: string
|
||||
updateNotionPages: (value: NotionPage[]) => void
|
||||
updateNotionCredentialId: (credentialId: string) => void
|
||||
onStepChange: () => void
|
||||
changeType: (type: DataSourceType) => void
|
||||
websitePages?: CrawlResultItem[]
|
||||
@@ -41,28 +43,7 @@ type IStepOneProps = {
|
||||
onWebsiteCrawlJobIdChange: (jobId: string) => void
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
type NotionConnectorProps = {
|
||||
onSetting: () => void
|
||||
}
|
||||
export const NotionConnector = (props: NotionConnectorProps) => {
|
||||
const { onSetting } = props
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex w-[640px] flex-col items-start rounded-2xl bg-workflow-process-bg p-6'>
|
||||
<span className={cn(s.notionIcon, 'mb-2 h-12 w-12 rounded-[10px] border-[0.5px] border-components-card-border p-3 shadow-lg shadow-shadow-shadow-5')} />
|
||||
<div className='mb-1 flex flex-col gap-y-1 pb-3 pt-1'>
|
||||
<span className='system-md-semibold text-text-secondary'>
|
||||
{t('datasetCreation.stepOne.notionSyncTitle')}
|
||||
<Icon3Dots className='relative -left-1.5 -top-2.5 inline h-4 w-4 text-text-secondary' />
|
||||
</span>
|
||||
<div className='system-sm-regular text-text-tertiary'>{t('datasetCreation.stepOne.notionSyncTip')}</div>
|
||||
</div>
|
||||
<Button className='h-8' variant='primary' onClick={onSetting}>{t('datasetCreation.stepOne.connect')}</Button>
|
||||
</div>
|
||||
)
|
||||
authedDataSourceList: DataSourceAuth[]
|
||||
}
|
||||
|
||||
const StepOne = ({
|
||||
@@ -70,22 +51,24 @@ const StepOne = ({
|
||||
dataSourceType: inCreatePageDataSourceType,
|
||||
dataSourceTypeDisable,
|
||||
changeType,
|
||||
hasConnection,
|
||||
onSetting,
|
||||
onStepChange,
|
||||
files,
|
||||
updateFileList,
|
||||
updateFile,
|
||||
notionPages = [],
|
||||
notionCredentialId,
|
||||
updateNotionPages,
|
||||
updateNotionCredentialId,
|
||||
websitePages = [],
|
||||
updateWebsitePages,
|
||||
onWebsiteCrawlProviderChange,
|
||||
onWebsiteCrawlJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
authedDataSourceList,
|
||||
}: IStepOneProps) => {
|
||||
const { dataset } = useDatasetDetailContext()
|
||||
const dataset = useDatasetDetailContextWithSelector(state => state.dataset)
|
||||
const [showModal, setShowModal] = useState(false)
|
||||
const [currentFile, setCurrentFile] = useState<File | undefined>()
|
||||
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
|
||||
@@ -136,6 +119,17 @@ const StepOne = ({
|
||||
return isShowVectorSpaceFull
|
||||
}, [files, isShowVectorSpaceFull])
|
||||
|
||||
const isNotionAuthed = useMemo(() => {
|
||||
if (!authedDataSourceList) return false
|
||||
const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource')
|
||||
if (!notionSource) return false
|
||||
return notionSource.credentials_list.length > 0
|
||||
}, [authedDataSourceList])
|
||||
|
||||
const notionCredentialList = useMemo(() => {
|
||||
return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || []
|
||||
}, [authedDataSourceList])
|
||||
|
||||
return (
|
||||
<div className='h-full w-full overflow-x-auto'>
|
||||
<div className='flex h-full w-full min-w-[1440px]'>
|
||||
@@ -243,7 +237,6 @@ const StepOne = ({
|
||||
</div>
|
||||
)}
|
||||
<div className="flex max-w-[640px] justify-end gap-2">
|
||||
{/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
|
||||
<Button disabled={nextDisabled} variant='primary' onClick={onStepChange}>
|
||||
<span className="flex gap-0.5 px-[10px]">
|
||||
<span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
|
||||
@@ -255,14 +248,17 @@ const StepOne = ({
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<>
|
||||
{!hasConnection && <NotionConnector onSetting={onSetting} />}
|
||||
{hasConnection && (
|
||||
{!isNotionAuthed && <NotionConnector onSetting={onSetting} />}
|
||||
{isNotionAuthed && (
|
||||
<>
|
||||
<div className='mb-8 w-[640px]'>
|
||||
<NotionPageSelector
|
||||
value={notionPages.map(page => page.page_id)}
|
||||
onSelect={updateNotionPages}
|
||||
onPreview={updateCurrentPage}
|
||||
credentialList={notionCredentialList}
|
||||
onSelectCredential={updateNotionCredentialId}
|
||||
datasetId={datasetId}
|
||||
/>
|
||||
</div>
|
||||
{isShowVectorSpaceFull && (
|
||||
@@ -271,7 +267,6 @@ const StepOne = ({
|
||||
</div>
|
||||
)}
|
||||
<div className="flex max-w-[640px] justify-end gap-2">
|
||||
{/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
|
||||
<Button disabled={isShowVectorSpaceFull || !notionPages.length} variant='primary' onClick={onStepChange}>
|
||||
<span className="flex gap-0.5 px-[10px]">
|
||||
<span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
|
||||
@@ -294,6 +289,7 @@ const StepOne = ({
|
||||
onJobIdChange={onWebsiteCrawlJobIdChange}
|
||||
crawlOptions={crawlOptions}
|
||||
onCrawlOptionsChange={onCrawlOptionsChange}
|
||||
authedDataSourceList={authedDataSourceList}
|
||||
/>
|
||||
</div>
|
||||
{isShowVectorSpaceFull && (
|
||||
@@ -302,7 +298,6 @@ const StepOne = ({
|
||||
</div>
|
||||
)}
|
||||
<div className="flex max-w-[640px] justify-end gap-2">
|
||||
{/* <Button>{t('datasetCreation.stepOne.cancel')}</Button> */}
|
||||
<Button disabled={isShowVectorSpaceFull || !websitePages.length} variant='primary' onClick={onStepChange}>
|
||||
<span className="flex gap-0.5 px-[10px]">
|
||||
<span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
|
||||
@@ -327,7 +322,13 @@ const StepOne = ({
|
||||
</div>
|
||||
<div className='h-full w-1/2 overflow-y-auto'>
|
||||
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
|
||||
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
|
||||
{currentNotionPage && (
|
||||
<NotionPagePreview
|
||||
currentPage={currentNotionPage}
|
||||
hidePreview={hideNotionPagePreview}
|
||||
notionCredentialId={notionCredentialId}
|
||||
/>
|
||||
)}
|
||||
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user