admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Feat/dataset notion import (#392)

Browse Source 
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: JzoNg <jzongcode@gmail.com>
This commit is contained in:
Jyong
2023-06-16 21:47:51 +08:00
committed by GitHub
parent f350948bde
commit 9253f72dea
96 changed files with 4479 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
web/app/components/datasets/documents/detail/index.tsx
View File

@@ -8,15 +8,16 @@ import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { omit } from 'lodash-es'
import cn from 'classnames'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import { fetchDocumentDetail, MetadataType } from '@/service/datasets'
import { OperationAction, StatusItem } from '../list'
import s from '../style.module.css'
import Completed from './completed'
import Embedding from './embedding'
import Metadata from './metadata'
import s from '../style.module.css'
import style from './style.module.css'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import type { MetadataType } from '@/service/datasets'
import { fetchDocumentDetail } from '@/service/datasets'
export const BackCircleBtn: FC<{ onClick: () => void }> = ({ onClick }) => {
return (
@@ -29,11 +30,11 @@ export const BackCircleBtn: FC<{ onClick: () => void }> = ({ onClick }) => {
export const DocumentContext = createContext<{ datasetId?: string; documentId?: string }>({})
type DocumentTitleProps = {
extension?: string;
name?: string;
iconCls?: string;
textCls?: string;
wrapperCls?: string;
extension?: string
name?: string
iconCls?: string
textCls?: string
wrapperCls?: string
}
export const DocumentTitle: FC<DocumentTitleProps> = ({ extension, name, iconCls, textCls, wrapperCls }) => {
@@ -58,15 +59,16 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
action: 'fetchDocumentDetail',
datasetId,
documentId,
params: { metadata: 'without' as MetadataType }
params: { metadata: 'without' as MetadataType },
}, apiParams => fetchDocumentDetail(omit(apiParams, 'action')))
const { data: documentMetadata, error: metadataErr, mutate: metadataMutate } = useSWR({
action: 'fetchDocumentDetail',
datasetId,
documentId,
params: { metadata: 'only' as MetadataType }
}, apiParams => fetchDocumentDetail(omit(apiParams, 'action')))
params: { metadata: 'only' as MetadataType },
}, apiParams => fetchDocumentDetail(omit(apiParams, 'action')),
)
const backToPrev = () => {
router.push(`/datasets/${datasetId}/documents`)
@@ -77,6 +79,13 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
const embedding = ['queuing', 'indexing', 'paused'].includes((documentDetail?.display_status || '').toLowerCase())
const handleOperate = (operateName?: string) => {
if (operateName === 'delete')
backToPrev()
else
detailMutate()
}
return (
<DocumentContext.Provider value={{ datasetId, documentId }}>
<div className='flex flex-col h-full'>
@@ -90,10 +99,10 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
detail={{
enabled: documentDetail?.enabled || false,
archived: documentDetail?.archived || false,
id: documentId
id: documentId,
}}
datasetId={datasetId}
onUpdate={detailMutate}
onUpdate={handleOperate}
className='!w-[216px]'
/>
<button
@@ -102,8 +111,9 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
/>
</div>
<div className='flex flex-row flex-1' style={{ height: 'calc(100% - 4rem)' }}>
{isDetailLoading ? <Loading type='app' /> :
<div className={`box-border h-full w-full overflow-y-scroll ${embedding ? 'py-12 px-16' : 'pb-[30px] pt-3 px-6'}`}>
{isDetailLoading
? <Loading type='app' />
: <div className={`box-border h-full w-full overflow-y-scroll ${embedding ? 'py-12 px-16' : 'pb-[30px] pt-3 px-6'}`}>
{embedding ? <Embedding detail={documentDetail} /> : <Completed />}
</div>
}

13
web/app/components/datasets/documents/detail/settings/index.tsx
View File

@@ -1,5 +1,5 @@
'use client'
import React, { useEffect, useState } from 'react'
import React, { useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import { useContext } from 'use-context-selector'
@@ -43,6 +43,15 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
}, [])
const [documentDetail, setDocumentDetail] = useState<FullDocumentDetail | null>(null)
const currentPage = useMemo(() => {
return {
workspace_id: documentDetail?.data_source_info.notion_workspace_id,
page_id: documentDetail?.data_source_info.notion_page_id,
page_name: documentDetail?.name,
page_icon: documentDetail?.data_source_info.notion_page_icon,
type: documentDetail?.data_source_info.type,
}
}, [documentDetail])
useEffect(() => {
(async () => {
try {
@@ -71,6 +80,8 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
hasSetAPIKEY={hasSetAPIKEY}
onSetting={showSetAPIKey}
datasetId={datasetId}
dataSourceType={documentDetail.data_source_type}
notionPages={[currentPage]}
indexingType={indexingTechnique || ''}
isSetting
documentDetail={documentDetail}

128
web/app/components/datasets/documents/index.tsx
View File

@@ -4,7 +4,7 @@ import React, { useMemo, useState } from 'react'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { debounce, omit } from 'lodash-es'
import { debounce, groupBy, omit } from 'lodash-es'
// import Link from 'next/link'
import { PlusIcon } from '@heroicons/react/24/solid'
import List from './list'
@@ -14,7 +14,12 @@ import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import Pagination from '@/app/components/base/pagination'
import { get } from '@/service/base'
import { fetchDocuments } from '@/service/datasets'
import { createDocument, fetchDocuments } from '@/service/datasets'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import { NotionPageSelectorModal } from '@/app/components/base/notion-page-selector'
import type { DataSourceNotionPage } from '@/models/common'
import type { CreateDocumentReq } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
// Custom page count is not currently supported.
const limit = 15
@@ -75,20 +80,63 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
const [searchValue, setSearchValue] = useState<string>('')
const [currPage, setCurrPage] = React.useState<number>(0)
const router = useRouter()
const { dataset } = useDatasetDetailContext()
const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false)
const [timerCanRun, setTimerCanRun] = useState(true)
const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
const query = useMemo(() => {
return { page: currPage + 1, limit, keyword: searchValue }
}, [searchValue, currPage])
return { page: currPage + 1, limit, keyword: searchValue, fetch: isDataSourceNotion ? true : '' }
}, [searchValue, currPage, isDataSourceNotion])
const { data: documentsRes, error, mutate } = useSWR({
action: 'fetchDocuments',
datasetId,
params: query,
}, apiParams => fetchDocuments(omit(apiParams, 'action')))
const { data: documentsRes, error, mutate } = useSWR(
{
action: 'fetchDocuments',
datasetId,
params: query,
},
apiParams => fetchDocuments(omit(apiParams, 'action')),
{ refreshInterval: (isDataSourceNotion && timerCanRun) ? 2500 : 0 },
)
const documentsWithProgress = useMemo(() => {
let completedNum = 0
let percent = 0
const documentsData = documentsRes?.data?.map((documentItem) => {
const { indexing_status, completed_segments, total_segments } = documentItem
const isEmbeddinged = indexing_status === 'completed' || indexing_status === 'paused' || indexing_status === 'error'
if (isEmbeddinged)
completedNum++
const completedCount = completed_segments || 0
const totalCount = total_segments || 0
if (totalCount === 0 && completedCount === 0) {
percent = isEmbeddinged ? 100 : 0
}
else {
const per = Math.round(completedCount * 100 / totalCount)
percent = per > 100 ? 100 : per
}
return {
...documentItem,
percent,
}
})
if (completedNum === documentsRes?.data?.length)
setTimerCanRun(false)
return {
...documentsRes,
data: documentsData,
}
}, [documentsRes])
const total = documentsRes?.total || 0
const routeToDocCreate = () => {
if (isDataSourceNotion) {
setNotionPageSelectorModalVisible(true)
return
}
router.push(`/datasets/${datasetId}/documents/create`)
}
@@ -96,6 +144,54 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
const isLoading = !documentsRes && !error
const handleSaveNotionPageSelected = async (selectedPages: (DataSourceNotionPage & { workspace_id: string })[]) => {
const workspacesMap = groupBy(selectedPages, 'workspace_id')
const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
return {
workspaceId,
pages: workspacesMap[workspaceId],
}
})
const params = {
data_source: {
type: dataset?.data_source_type,
info_list: {
data_source_type: dataset?.data_source_type,
notion_info_list: workspaces.map((workspace) => {
return {
workspace_id: workspace.workspaceId,
pages: workspace.pages.map((page) => {
const { page_id, page_name, page_icon, type } = page
return {
page_id,
page_name,
page_icon,
type,
}
}),
}
}),
},
},
indexing_technique: dataset?.indexing_technique,
process_rule: {
rules: {},
mode: 'automatic',
},
} as CreateDocumentReq
await createDocument({
datasetId,
body: params,
})
mutate()
setTimerCanRun(true)
// mutateDatasetIndexingStatus(undefined, { revalidate: true })
setNotionPageSelectorModalVisible(false)
}
const documentsList = isDataSourceNotion ? documentsWithProgress?.data : documentsRes?.data
return (
<div className='flex flex-col h-full overflow-y-auto'>
<div className='flex flex-col justify-center gap-1 px-6 pt-4'>
@@ -113,19 +209,29 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
/>
<Button type='primary' onClick={routeToDocCreate} className='!h-8 !text-[13px]'>
<PlusIcon className='h-4 w-4 mr-2 stroke-current' />
{t('datasetDocuments.list.addFile')}
{
isDataSourceNotion
? t('datasetDocuments.list.addPages')
: t('datasetDocuments.list.addFile')
}
</Button>
</div>
{isLoading
? <Loading type='app' />
: total > 0
? <List documents={documentsRes?.data || []} datasetId={datasetId} onUpdate={mutate} />
? <List documents={documentsList || []} datasetId={datasetId} onUpdate={mutate} />
: <EmptyElement onClick={routeToDocCreate} />
}
{/* Show Pagination only if the total is more than the limit */}
{(total && total > limit)
? <Pagination current={currPage} onChange={setCurrPage} total={total} limit={limit} />
: null}
<NotionPageSelectorModal
isShow={notionPageSelectorModalVisible}
onClose={() => setNotionPageSelectorModalVisible(false)}
onSave={handleSaveNotionPageSelected}
datasetId={dataset?.id || ''}
/>
</div>
</div>
)

61
web/app/components/datasets/documents/list.tsx
View File

@@ -22,8 +22,10 @@ import type { IndicatorProps } from '@/app/components/header/indicator'
import Indicator from '@/app/components/header/indicator'
import { asyncRunSafe } from '@/utils'
import { formatNumber } from '@/utils/format'
import { archiveDocument, deleteDocument, disableDocument, enableDocument } from '@/service/datasets'
import type { DocumentDisplayStatus, DocumentListResponse } from '@/models/datasets'
import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument } from '@/service/datasets'
import NotionIcon from '@/app/components/base/notion-icon'
import ProgressBar from '@/app/components/base/progress-bar'
import { DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
import type { CommonResponse } from '@/models/common'
export const SettingsIcon: FC<{ className?: string }> = ({ className }) => {
@@ -32,6 +34,12 @@ export const SettingsIcon: FC<{ className?: string }> = ({ className }) => {
</svg>
}
export const SyncIcon: FC<{ className?: string }> = () => {
return <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M5.69773 13.1783C7.29715 13.8879 9.20212 13.8494 10.8334 12.9075C13.5438 11.3427 14.4724 7.87704 12.9076 5.16672L12.7409 4.87804M3.09233 10.8335C1.52752 8.12314 2.45615 4.65746 5.16647 3.09265C6.7978 2.15081 8.70277 2.11227 10.3022 2.82185M1.66226 10.8892L3.48363 11.3773L3.97166 9.5559M12.0284 6.44393L12.5164 4.62256L14.3378 5.1106" stroke="#667085" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
}
export const FilePlusIcon: FC<{ className?: string }> = ({ className }) => {
return <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
<path d="M13.3332 6.99992V4.53325C13.3332 3.41315 13.3332 2.85309 13.1152 2.42527C12.9234 2.04895 12.6175 1.74299 12.2412 1.55124C11.8133 1.33325 11.2533 1.33325 10.1332 1.33325H5.8665C4.7464 1.33325 4.18635 1.33325 3.75852 1.55124C3.3822 1.74299 3.07624 2.04895 2.88449 2.42527C2.6665 2.85309 2.6665 3.41315 2.6665 4.53325V11.4666C2.6665 12.5867 2.6665 13.1467 2.88449 13.5746C3.07624 13.9509 3.3822 14.2569 3.75852 14.4486C4.18635 14.6666 4.7464 14.6666 5.8665 14.6666H7.99984M9.33317 7.33325H5.33317M6.6665 9.99992H5.33317M10.6665 4.66659H5.33317M11.9998 13.9999V9.99992M9.99984 11.9999H13.9998" stroke="#667085" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
@@ -77,7 +85,7 @@ export const StatusItem: FC<{
</div>
}
type OperationName = 'delete' | 'archive' | 'enable' | 'disable'
type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync'
// operation action for list and detail
export const OperationAction: FC<{
@@ -85,13 +93,14 @@ export const OperationAction: FC<{
enabled: boolean
archived: boolean
id: string
data_source_type: string
}
datasetId: string
onUpdate: () => void
onUpdate: (operationName?: string) => void
scene?: 'list' | 'detail'
className?: string
}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '' }) => {
const { id, enabled = false, archived = false } = detail || {}
const { id, enabled = false, archived = false, data_source_type } = detail || {}
const [showModal, setShowModal] = useState(false)
const { notify } = useContext(ToastContext)
const { t } = useTranslation()
@@ -111,6 +120,9 @@ export const OperationAction: FC<{
case 'disable':
opApi = disableDocument
break
case 'sync':
opApi = syncDocument
break
default:
opApi = deleteDocument
break
@@ -120,7 +132,7 @@ export const OperationAction: FC<{
notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
else
notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
onUpdate()
onUpdate(operationName)
}
return <div
@@ -173,10 +185,14 @@ export const OperationAction: FC<{
<SettingsIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.settings')}</span>
</div>
{/* <div className={s.actionItem} onClick={() => router.push(`/datasets/${datasetId}/documents/create`)}>
<FilePlusIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.uploadFile')}</span>
</div> */}
{
data_source_type === 'notion_import' && (
<div className={s.actionItem} onClick={() => onOperate('sync')}>
<SyncIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.sync')}</span>
</div>
)
}
<Divider className='my-1' />
</>
)}
@@ -236,8 +252,9 @@ const renderCount = (count: number | undefined) => {
return `${formatNumber((count / 1000).toFixed(1))}k`
}
type LocalDoc = SimpleDocumentDetail & { percent?: number }
type IDocumentListProps = {
documents: DocumentListResponse['data']
documents: LocalDoc[]
datasetId: string
onUpdate: () => void
}
@@ -248,7 +265,7 @@ type IDocumentListProps = {
const DocumentList: FC<IDocumentListProps> = ({ documents = [], datasetId, onUpdate }) => {
const { t } = useTranslation()
const router = useRouter()
const [localDocs, setLocalDocs] = useState<DocumentListResponse['data']>(documents)
const [localDocs, setLocalDocs] = useState<LocalDoc[]>(documents)
const [enableSort, setEnableSort] = useState(false)
useEffect(() => {
@@ -296,8 +313,16 @@ const DocumentList: FC<IDocumentListProps> = ({ documents = [], datasetId, onUpd
}}>
<td className='text-left align-middle text-gray-500 text-xs'>{doc.position}</td>
<td className={s.tdValue}>
<div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? suffix}Icon`], s.commonIcon, 'mr-1.5')}></div>
<span>{doc?.name?.replace(/\.[^/.]+$/, '')}<span className='text-gray-500'>.{suffix}</span></span>
{
doc?.data_source_type === DataSourceType.NOTION
? <NotionIcon className='inline-flex -mt-[3px] mr-1.5 align-middle' type='page' src={doc.data_source_info.notion_page_icon} />
: <div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? suffix}Icon`], s.commonIcon, 'mr-1.5')}></div>
}
{
doc.data_source_type === DataSourceType.NOTION
? <span>{doc.name}</span>
: <span>{doc?.name?.replace(/\.[^/.]+$/, '')}<span className='text-gray-500'>.{suffix}</span></span>
}
</td>
<td>{renderCount(doc.word_count)}</td>
<td>{renderCount(doc.hit_count)}</td>
@@ -305,12 +330,16 @@ const DocumentList: FC<IDocumentListProps> = ({ documents = [], datasetId, onUpd
{dayjs.unix(doc.created_at).format(t('datasetHitTesting.dateTimeFormat') as string)}
</td>
<td>
<StatusItem status={doc.display_status} />
{
(['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION)
? <ProgressBar percent={doc.percent || 0} />
: <StatusItem status={doc.display_status} />
}
</td>
<td>
<OperationAction
datasetId={datasetId}
detail={pick(doc, ['enabled', 'archived', 'id'])}
detail={pick(doc, ['enabled', 'archived', 'id', 'data_source_type'])}
onUpdate={onUpdate}
/>
</td>

3
web/app/components/datasets/documents/style.module.css
View File

@@ -75,6 +75,9 @@
.markdownIcon {
background-image: url(./assets/md.svg);
}
.mdIcon {
background-image: url(./assets/md.svg);
}
.xlsIcon {
background-image: url(./assets/xlsx.svg);
}