feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/app/components/header/account-setting/data-source-page-new/card.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/card.tsx
@@ -0,0 +1,195 @@
+import {
+  memo,
+  useCallback,
+  useRef,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import Item from './item'
+import Configure from './configure'
+import type {
+  DataSourceAuth,
+  DataSourceCredential,
+} from './types'
+import { useRenderI18nObject } from '@/hooks/use-i18n'
+import { AuthCategory } from '@/app/components/plugins/plugin-auth/types'
+import {
+  ApiKeyModal,
+  usePluginAuthAction,
+} from '@/app/components/plugins/plugin-auth'
+import { useDataSourceAuthUpdate } from './hooks'
+import Confirm from '@/app/components/base/confirm'
+import { useGetDataSourceOAuthUrl } from '@/service/use-datasource'
+import { openOAuthPopup } from '@/hooks/use-oauth'
+
+type CardProps = {
+  item: DataSourceAuth
+  disabled?: boolean
+}
+const Card = ({
+  item,
+  disabled,
+}: CardProps) => {
+  const { t } = useTranslation()
+  const renderI18nObject = useRenderI18nObject()
+  const {
+    icon,
+    label,
+    author,
+    name,
+    credentials_list,
+    credential_schema,
+  } = item
+  const pluginPayload = {
+    category: AuthCategory.datasource,
+    provider: `${item.plugin_id}/${item.name}`,
+  }
+  const { handleAuthUpdate } = useDataSourceAuthUpdate({
+    pluginId: item.plugin_id,
+    provider: item.name,
+  })
+  const {
+    deleteCredentialId,
+    doingAction,
+    handleConfirm,
+    handleEdit,
+    handleRemove,
+    handleRename,
+    handleSetDefault,
+    editValues,
+    setEditValues,
+    openConfirm,
+    closeConfirm,
+    pendingOperationCredentialId,
+  } = usePluginAuthAction(pluginPayload, handleAuthUpdate)
+  const changeCredentialIdRef = useRef<string | undefined>(undefined)
+  const {
+    mutateAsync: getPluginOAuthUrl,
+  } = useGetDataSourceOAuthUrl(pluginPayload.provider)
+  const handleOAuth = useCallback(async () => {
+    const { authorization_url } = await getPluginOAuthUrl(changeCredentialIdRef.current)
+
+    if (authorization_url) {
+      openOAuthPopup(
+        authorization_url,
+        handleAuthUpdate,
+      )
+    }
+  }, [getPluginOAuthUrl, handleAuthUpdate])
+  const handleAction = useCallback((
+    action: string,
+    credentialItem: DataSourceCredential,
+    renamePayload?: Record<string, any>,
+  ) => {
+    if (action === 'edit') {
+      handleEdit(
+        credentialItem.id,
+        {
+          ...credentialItem.credential,
+          __name__: credentialItem.name,
+          __credential_id__: credentialItem.id,
+        },
+      )
+    }
+    if (action === 'delete')
+      openConfirm(credentialItem.id)
+
+    if (action === 'setDefault')
+      handleSetDefault(credentialItem.id)
+
+    if (action === 'rename')
+      handleRename(renamePayload as any)
+
+    if (action === 'change') {
+      changeCredentialIdRef.current = credentialItem.id
+      handleOAuth()
+    }
+  }, [
+    openConfirm,
+    handleEdit,
+    handleSetDefault,
+    handleRename,
+  ])
+
+  return (
+    <div className='rounded-xl bg-background-section-burn'>
+      <div className='flex items-center p-3 pb-2'>
+        <img
+          src={icon}
+          className='mr-3 flex h-10 w-10 shrink-0 items-center justify-center'
+        />
+        <div className='grow'>
+          <div className='system-md-semibold text-text-primary'>
+            {renderI18nObject(label)}
+          </div>
+          <div className='system-xs-regular flex h-4 items-center text-text-tertiary'>
+            {author}
+            <div className='mx-0.5 text-text-quaternary'>/</div>
+            {name}
+          </div>
+        </div>
+        <Configure
+          pluginPayload={pluginPayload}
+          item={item}
+          onUpdate={handleAuthUpdate}
+        />
+      </div>
+      <div className='system-xs-medium flex h-4 items-center pl-3 text-text-tertiary'>
+        {t('plugin.auth.connectedWorkspace')}
+        <div className='ml-3 h-[1px] grow bg-divider-subtle'></div>
+      </div>
+      {
+        !!credentials_list.length && (
+          <div className='space-y-1 p-3 pt-2'>
+            {
+              credentials_list.map(credentialItem => (
+                <Item
+                  key={credentialItem.id}
+                  credentialItem={credentialItem}
+                  onAction={handleAction}
+                />
+              ))
+            }
+          </div>
+        )
+      }
+      {
+        !credentials_list.length && (
+          <div className='p-3 pt-1'>
+            <div className='system-xs-regular flex h-10 items-center justify-center rounded-[10px] bg-background-section text-text-tertiary'>
+              {t('plugin.auth.emptyAuth')}
+            </div>
+          </div>
+        )
+      }
+      {
+        deleteCredentialId && (
+          <Confirm
+            isShow
+            title={t('datasetDocuments.list.delete.title')}
+            isDisabled={doingAction}
+            onCancel={closeConfirm}
+            onConfirm={handleConfirm}
+          />
+        )
+      }
+      {
+        !!editValues && (
+          <ApiKeyModal
+            pluginPayload={pluginPayload}
+            onClose={() => {
+              setEditValues(null)
+              pendingOperationCredentialId.current = null
+            }}
+            onUpdate={handleAuthUpdate}
+            formSchemas={credential_schema}
+            editValues={editValues}
+            onRemove={handleRemove}
+            disabled={disabled || doingAction}
+          />
+        )
+      }
+    </div>
+  )
+}
+
+export default memo(Card)
--- a/web/app/components/header/account-setting/data-source-page-new/configure.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/configure.tsx
@@ -0,0 +1,117 @@
+import {
+  memo,
+  useMemo,
+} from 'react'
+import {
+  RiAddLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+import Button from '@/app/components/base/button'
+import {
+  AddApiKeyButton,
+  AddOAuthButton,
+} from '@/app/components/plugins/plugin-auth'
+import type { DataSourceAuth } from './types'
+import type {
+  AddApiKeyButtonProps,
+  AddOAuthButtonProps,
+  PluginPayload,
+} from '@/app/components/plugins/plugin-auth/types'
+
+type ConfigureProps = {
+  item: DataSourceAuth
+  pluginPayload: PluginPayload
+  onUpdate?: () => void
+  disabled?: boolean
+}
+const Configure = ({
+  item,
+  pluginPayload,
+  onUpdate,
+  disabled,
+}: ConfigureProps) => {
+  const { t } = useTranslation()
+  const canApiKey = item.credential_schema?.length
+  const oAuthData = item.oauth_schema || {}
+  const canOAuth = oAuthData.client_schema?.length
+  const oAuthButtonProps: AddOAuthButtonProps = useMemo(() => {
+    return {
+      buttonText: t('plugin.auth.addOAuth'),
+      pluginPayload,
+    }
+  }, [pluginPayload, t])
+
+  const apiKeyButtonProps: AddApiKeyButtonProps = useMemo(() => {
+    return {
+      pluginPayload,
+      buttonText: t('plugin.auth.addApi'),
+    }
+  }, [pluginPayload, t])
+
+  return (
+    <>
+      <PortalToFollowElem
+        placement='bottom-end'
+        offset={{
+          mainAxis: 4,
+          crossAxis: -4,
+        }}
+      >
+        <PortalToFollowElemTrigger>
+          <Button
+            variant='secondary-accent'
+          >
+            <RiAddLine className='h-4 w-4' />
+            {t('common.dataSource.configure')}
+          </Button>
+        </PortalToFollowElemTrigger>
+        <PortalToFollowElemContent className='z-[61]'>
+          <div className='w-[240px] space-y-1.5 rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-2 shadow-lg'>
+            {
+              !!canOAuth && (
+                <AddOAuthButton
+                  {...oAuthButtonProps}
+                  onUpdate={onUpdate}
+                  oAuthData={{
+                    schema: oAuthData.client_schema || [],
+                    is_oauth_custom_client_enabled: oAuthData.is_oauth_custom_client_enabled,
+                    is_system_oauth_params_exists: oAuthData.is_system_oauth_params_exists,
+                    client_params: oAuthData.oauth_custom_client_params,
+                    redirect_uri: oAuthData.redirect_uri,
+                  }}
+                  disabled={disabled}
+                />
+              )
+            }
+            {
+              !!canApiKey && !!canOAuth && (
+                <div className='system-2xs-medium-uppercase flex h-4 items-center p-2 text-text-quaternary'>
+                  <div className='mr-2 h-[1px] grow bg-gradient-to-l from-[rgba(16,24,40,0.08)]' />
+                  OR
+                  <div className='ml-2 h-[1px] grow bg-gradient-to-r from-[rgba(16,24,40,0.08)]' />
+                </div>
+              )
+            }
+            {
+              !!canApiKey && (
+                <AddApiKeyButton
+                  {...apiKeyButtonProps}
+                  formSchemas={item.credential_schema}
+                  onUpdate={onUpdate}
+                  disabled={disabled}
+                />
+              )
+            }
+          </div>
+        </PortalToFollowElemContent>
+      </PortalToFollowElem>
+    </>
+  )
+}
+
+export default memo(Configure)
--- a/web/app/components/header/account-setting/data-source-page-new/hooks/index.ts
+++ b/web/app/components/header/account-setting/data-source-page-new/hooks/index.ts
@@ -0,0 +1,2 @@
+export * from './use-marketplace-all-plugins'
+export * from './use-data-source-auth-update'
--- a/web/app/components/header/account-setting/data-source-page-new/hooks/use-data-source-auth-update.ts
+++ b/web/app/components/header/account-setting/data-source-page-new/hooks/use-data-source-auth-update.ts
@@ -0,0 +1,30 @@
+import { useCallback } from 'react'
+import { useInvalidDataSourceAuth, useInvalidDataSourceListAuth } from '@/service/use-datasource'
+import { useInvalidDefaultDataSourceListAuth } from '@/service/use-datasource'
+import { useInvalidDataSourceList } from '@/service/use-pipeline'
+
+export const useDataSourceAuthUpdate = ({
+  pluginId,
+  provider,
+}: {
+  pluginId: string
+  provider: string
+}) => {
+  const invalidateDataSourceListAuth = useInvalidDataSourceListAuth()
+  const invalidDefaultDataSourceListAuth = useInvalidDefaultDataSourceListAuth()
+  const invalidateDataSourceList = useInvalidDataSourceList()
+  const invalidateDataSourceAuth = useInvalidDataSourceAuth({
+    pluginId,
+    provider,
+  })
+  const handleAuthUpdate = useCallback(() => {
+    invalidateDataSourceListAuth()
+    invalidDefaultDataSourceListAuth()
+    invalidateDataSourceList()
+    invalidateDataSourceAuth()
+  }, [invalidateDataSourceListAuth, invalidateDataSourceList, invalidateDataSourceAuth, invalidDefaultDataSourceListAuth])
+
+  return {
+    handleAuthUpdate,
+  }
+}
--- a/web/app/components/header/account-setting/data-source-page-new/hooks/use-marketplace-all-plugins.ts
+++ b/web/app/components/header/account-setting/data-source-page-new/hooks/use-marketplace-all-plugins.ts
@@ -0,0 +1,80 @@
+import {
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react'
+import {
+  useMarketplacePlugins,
+} from '@/app/components/plugins/marketplace/hooks'
+import type { Plugin } from '@/app/components/plugins/types'
+import { PluginType } from '@/app/components/plugins/types'
+import { getMarketplacePluginsByCollectionId } from '@/app/components/plugins/marketplace/utils'
+
+export const useMarketplaceAllPlugins = (providers: any[], searchText: string) => {
+  const exclude = useMemo(() => {
+    return providers.map(provider => provider.plugin_id)
+  }, [providers])
+  const [collectionPlugins, setCollectionPlugins] = useState<Plugin[]>([])
+
+  const {
+    plugins,
+    queryPlugins,
+    queryPluginsWithDebounced,
+    isLoading,
+  } = useMarketplacePlugins()
+
+  const getCollectionPlugins = useCallback(async () => {
+    const collectionPlugins = await getMarketplacePluginsByCollectionId('__model-settings-pinned-models')
+
+    setCollectionPlugins(collectionPlugins)
+  }, [])
+
+  useEffect(() => {
+    getCollectionPlugins()
+  }, [getCollectionPlugins])
+
+  useEffect(() => {
+    if (searchText) {
+      queryPluginsWithDebounced({
+        query: searchText,
+        category: PluginType.datasource,
+        exclude,
+        type: 'plugin',
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
+      })
+    }
+    else {
+      queryPlugins({
+        query: '',
+        category: PluginType.datasource,
+        type: 'plugin',
+        pageSize: 1000,
+        exclude,
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
+      })
+    }
+  }, [queryPlugins, queryPluginsWithDebounced, searchText, exclude])
+
+  const allPlugins = useMemo(() => {
+    const allPlugins = collectionPlugins.filter(plugin => !exclude.includes(plugin.plugin_id))
+
+    if (plugins?.length) {
+      for (let i = 0; i < plugins.length; i++) {
+        const plugin = plugins[i]
+
+        if (plugin.type !== 'bundle' && !allPlugins.find(p => p.plugin_id === plugin.plugin_id))
+          allPlugins.push(plugin)
+      }
+    }
+
+    return allPlugins
+  }, [plugins, collectionPlugins, exclude])
+
+  return {
+    plugins: allPlugins,
+    isLoading,
+  }
+}
--- a/web/app/components/header/account-setting/data-source-page-new/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/index.tsx
@@ -0,0 +1,35 @@
+import { memo } from 'react'
+import Card from './card'
+import InstallFromMarketplace from './install-from-marketplace'
+import { useGlobalPublicStore } from '@/context/global-public-context'
+import { useGetDataSourceListAuth } from '@/service/use-datasource'
+
+const DataSourcePage = () => {
+  const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures)
+  const { data } = useGetDataSourceListAuth()
+
+  return (
+    <div>
+      <div className='space-y-2'>
+        {
+          data?.result.map(item => (
+            <Card
+              key={item.plugin_unique_identifier}
+              item={item}
+            />
+          ))
+        }
+      </div>
+      {
+        enable_marketplace && (
+          <InstallFromMarketplace
+            providers={data?.result || []}
+            searchText={''}
+          />
+        )
+      }
+    </div>
+  )
+}
+
+export default memo(DataSourcePage)
--- a/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/install-from-marketplace.tsx
@@ -0,0 +1,84 @@
+import {
+  memo,
+  useCallback,
+  useState,
+} from 'react'
+import { useTheme } from 'next-themes'
+import { useTranslation } from 'react-i18next'
+import Link from 'next/link'
+import {
+  RiArrowDownSLine,
+  RiArrowRightUpLine,
+} from '@remixicon/react'
+import {
+  useMarketplaceAllPlugins,
+} from './hooks'
+import Divider from '@/app/components/base/divider'
+import Loading from '@/app/components/base/loading'
+import ProviderCard from '@/app/components/plugins/provider-card'
+import List from '@/app/components/plugins/marketplace/list'
+import type { Plugin } from '@/app/components/plugins/types'
+import cn from '@/utils/classnames'
+import { getLocaleOnClient } from '@/i18n-config'
+import { getMarketplaceUrl } from '@/utils/var'
+
+type InstallFromMarketplaceProps = {
+  providers: any[]
+  searchText: string
+}
+const InstallFromMarketplace = ({
+  providers,
+  searchText,
+}: InstallFromMarketplaceProps) => {
+  const { t } = useTranslation()
+  const { theme } = useTheme()
+  const [collapse, setCollapse] = useState(false)
+  const locale = getLocaleOnClient()
+  const {
+    plugins: allPlugins,
+    isLoading: isAllPluginsLoading,
+  } = useMarketplaceAllPlugins(providers, searchText)
+
+  const cardRender = useCallback((plugin: Plugin) => {
+    if (plugin.type === 'bundle')
+      return null
+
+    return <ProviderCard key={plugin.plugin_id} payload={plugin} />
+  }, [])
+
+  return (
+    <div className='mb-2'>
+      <Divider className='!mt-4 h-px' />
+      <div className='flex items-center justify-between'>
+        <div className='system-md-semibold flex cursor-pointer items-center gap-1 text-text-primary' onClick={() => setCollapse(!collapse)}>
+          <RiArrowDownSLine className={cn('h-4 w-4', collapse && '-rotate-90')} />
+          {t('common.modelProvider.installProvider')}
+        </div>
+        <div className='mb-2 flex items-center pt-2'>
+          <span className='system-sm-regular pr-1 text-text-tertiary'>{t('common.modelProvider.discoverMore')}</span>
+          <Link target="_blank" href={getMarketplaceUrl('', { theme })} className='system-sm-medium inline-flex items-center text-text-accent'>
+            {t('plugin.marketplace.difyMarketplace')}
+            <RiArrowRightUpLine className='h-4 w-4' />
+          </Link>
+        </div>
+      </div>
+      {!collapse && isAllPluginsLoading && <Loading type='area' />}
+      {
+        !isAllPluginsLoading && !collapse && (
+          <List
+            marketplaceCollections={[]}
+            marketplaceCollectionPluginsMap={{}}
+            plugins={allPlugins}
+            showInstallButton
+            locale={locale}
+            cardContainerClassName='grid grid-cols-2 gap-2'
+            cardRender={cardRender}
+            emptyClassName='h-auto'
+          />
+        )
+      }
+    </div>
+  )
+}
+
+export default memo(InstallFromMarketplace)
--- a/web/app/components/header/account-setting/data-source-page-new/item.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/item.tsx
@@ -0,0 +1,95 @@
+import {
+  memo,
+  useState,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import Indicator from '@/app/components/header/indicator'
+import Operator from './operator'
+import type {
+  DataSourceCredential,
+} from './types'
+import Input from '@/app/components/base/input'
+import Button from '@/app/components/base/button'
+
+type ItemProps = {
+  credentialItem: DataSourceCredential
+  onAction: (action: string, credentialItem: DataSourceCredential, renamePayload?: Record<string, any>) => void
+}
+const Item = ({
+  credentialItem,
+  onAction,
+}: ItemProps) => {
+  const { t } = useTranslation()
+  const [renaming, setRenaming] = useState(false)
+  const [renameValue, setRenameValue] = useState(credentialItem.name)
+
+  return (
+    <div className='flex h-10 items-center rounded-lg bg-components-panel-on-panel-item-bg pl-3 pr-1'>
+      {/* <div className='mr-2 h-5 w-5 shrink-0'></div> */}
+      {
+        renaming && (
+          <div className='flex w-full items-center space-x-1'>
+            <Input
+              wrapperClassName='grow rounded-[6px]'
+              className='h-6'
+              value={renameValue}
+              onChange={e => setRenameValue(e.target.value)}
+              placeholder={t('common.placeholder.input')}
+              onClick={e => e.stopPropagation()}
+            />
+            <Button
+              size='small'
+              variant='primary'
+              onClick={(e) => {
+                e.stopPropagation()
+                onAction?.(
+                  'rename',
+                  credentialItem,
+                  {
+                    credential_id: credentialItem.id,
+                    name: renameValue,
+                  },
+                )
+                setRenaming(false)
+              }}
+            >
+              {t('common.operation.save')}
+            </Button>
+            <Button
+              size='small'
+              onClick={(e) => {
+                e.stopPropagation()
+                setRenaming(false)
+              }}
+            >
+              {t('common.operation.cancel')}
+            </Button>
+          </div>
+        )
+      }
+      {
+        !renaming && (
+          <div className='system-sm-medium grow text-text-secondary'>
+            {credentialItem.name}
+          </div>
+        )
+      }
+      <div className='flex shrink-0 items-center'>
+        <div className='mr-1 flex h-3 w-3 items-center justify-center'>
+          <Indicator color='green' />
+        </div>
+        <div className='system-xs-semibold-uppercase text-util-colors-green-green-600'>
+          connected
+        </div>
+      </div>
+      <div className='ml-3 mr-2 h-3 w-[1px] bg-divider-regular'></div>
+      <Operator
+        credentialItem={credentialItem}
+        onAction={onAction}
+        onRename={() => setRenaming(true)}
+      />
+    </div>
+  )
+}
+
+export default memo(Item)
--- a/web/app/components/header/account-setting/data-source-page-new/operator.tsx
+++ b/web/app/components/header/account-setting/data-source-page-new/operator.tsx
@@ -0,0 +1,135 @@
+import {
+  memo,
+  useCallback,
+  useMemo,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  RiDeleteBinLine,
+  RiEditLine,
+  RiEqualizer2Line,
+  RiHome9Line,
+  RiStickyNoteAddLine,
+} from '@remixicon/react'
+import Dropdown from '@/app/components/base/dropdown'
+import type { Item } from '@/app/components/base/dropdown'
+import type {
+  DataSourceCredential,
+} from './types'
+import { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth/types'
+
+type OperatorProps = {
+  credentialItem: DataSourceCredential
+  onAction: (action: string, credentialItem: DataSourceCredential) => void
+  onRename?: () => void
+}
+const Operator = ({
+  credentialItem,
+  onAction,
+  onRename,
+}: OperatorProps) => {
+  const { t } = useTranslation()
+  const {
+    type,
+  } = credentialItem
+  const items = useMemo(() => {
+    const commonItems = [
+      {
+        value: 'setDefault',
+        text: (
+          <div className='flex items-center'>
+            <RiHome9Line className='mr-2 h-4 w-4 text-text-tertiary' />
+            <div className='system-sm-semibold text-text-secondary'>{t('plugin.auth.setDefault')}</div>
+          </div>
+        ),
+      },
+      ...(
+        type === CredentialTypeEnum.OAUTH2
+          ? [
+            {
+              value: 'rename',
+              text: (
+                <div className='flex items-center'>
+                  <RiEditLine className='mr-2 h-4 w-4 text-text-tertiary' />
+                  <div className='system-sm-semibold text-text-secondary'>{t('common.operation.rename')}</div>
+                </div>
+              ),
+            },
+          ]
+          : []
+      ),
+      ...(
+        type === CredentialTypeEnum.API_KEY
+          ? [
+            {
+              value: 'edit',
+              text: (
+                <div className='flex items-center'>
+                  <RiEqualizer2Line className='mr-2 h-4 w-4 text-text-tertiary' />
+                  <div className='system-sm-semibold text-text-secondary'>{t('common.operation.edit')}</div>
+                </div>
+              ),
+            },
+          ]
+          : []
+      ),
+    ]
+    if (type === CredentialTypeEnum.OAUTH2) {
+      const oAuthItems = [
+        {
+          value: 'change',
+          text: (
+            <div className='flex items-center'>
+              <RiStickyNoteAddLine className='mr-2 h-4 w-4 text-text-tertiary' />
+              <div className='system-sm-semibold mb-1 text-text-secondary'>{t('common.dataSource.notion.changeAuthorizedPages')}</div>
+            </div>
+          ),
+        },
+      ]
+      commonItems.push(...oAuthItems)
+    }
+    return commonItems
+  }, [t, type])
+
+  const secondItems = useMemo(() => {
+    return [
+      {
+        value: 'delete',
+        text: (
+          <div className='flex items-center'>
+            <RiDeleteBinLine className='mr-2 h-4 w-4 text-text-tertiary' />
+            <div className='system-sm-semibold text-text-secondary'>
+              {t('common.operation.remove')}
+            </div>
+          </div>
+        ),
+      },
+    ]
+  }, [])
+  const handleSelect = useCallback((item: Item) => {
+    if (item.value === 'rename') {
+      onRename?.()
+      return
+    }
+    onAction(
+      item.value as string,
+      credentialItem,
+    )
+  }, [onAction, credentialItem, onRename])
+
+  return (
+    <Dropdown
+      items={items}
+      secondItems={secondItems}
+      onSelect={handleSelect}
+      popupClassName='z-[61]'
+      triggerProps={{
+        size: 'l',
+      }}
+      itemClassName='py-2 h-auto hover:bg-state-base-hover'
+      secondItemClassName='py-2 h-auto hover:bg-state-base-hover'
+    />
+  )
+}
+
+export default memo(Operator)
--- a/web/app/components/header/account-setting/data-source-page-new/types.ts
+++ b/web/app/components/header/account-setting/data-source-page-new/types.ts
@@ -0,0 +1,34 @@
+import type {
+  FormSchema,
+  TypeWithI18N,
+} from '@/app/components/base/form/types'
+import type { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth/types'
+
+export type DataSourceCredential = {
+  credential: Record<string, any>
+  type: CredentialTypeEnum
+  name: string
+  id: string
+  is_default: boolean
+  avatar_url: string
+}
+export type DataSourceAuth = {
+  author: string
+  provider: string
+  plugin_id: string
+  plugin_unique_identifier: string
+  icon: string
+  name: string
+  label: TypeWithI18N
+  description: TypeWithI18N
+  credential_schema?: FormSchema[]
+  oauth_schema?: {
+    client_schema?: FormSchema[]
+    credentials_schema?: FormSchema[]
+    is_oauth_custom_client_enabled?: boolean
+    is_system_oauth_params_exists?: boolean
+    oauth_custom_client_params?: Record<string, any>
+    redirect_uri?: string
+  }
+  credentials_list: DataSourceCredential[]
+}
--- a/web/app/components/header/account-setting/data-source-page/index.module.css
+++ b/web/app/components/header/account-setting/data-source-page/index.module.css
--- a/web/app/components/header/account-setting/data-source-page/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/index.tsx
@@ -1,20 +0,0 @@
-import useSWR from 'swr'
-import DataSourceNotion from './data-source-notion'
-import DataSourceWebsite from './data-source-website'
-import { fetchDataSource } from '@/service/common'
-import { DataSourceProvider } from '@/models/common'
-import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
-
-export default function DataSourcePage() {
-  const { data } = useSWR({ url: 'data-source/integrates' }, fetchDataSource)
-  const notionWorkspaces = data?.data.filter(item => item.provider === 'notion') || []
-
-  return (
-    <div className='mb-8'>
-      <DataSourceNotion workspaces={notionWorkspaces} />
-      {ENABLE_WEBSITE_JINAREADER && <DataSourceWebsite provider={DataSourceProvider.jinaReader} />}
-      {ENABLE_WEBSITE_FIRECRAWL && <DataSourceWebsite provider={DataSourceProvider.fireCrawl} />}
-      {ENABLE_WEBSITE_WATERCRAWL && <DataSourceWebsite provider={DataSourceProvider.waterCrawl} />}
-    </div>
-  )
-}
--- a/web/app/components/header/account-setting/index.tsx
+++ b/web/app/components/header/account-setting/index.tsx
@@ -21,7 +21,7 @@ import Button from '../../base/button'
 import MembersPage from './members-page'
 import LanguagePage from './language-page'
 import ApiBasedExtensionPage from './api-based-extension-page'
-import DataSourcePage from './data-source-page'
+import DataSourcePage from './data-source-page-new'
 import ModelProviderPage from './model-provider-page'
 import cn from '@/utils/classnames'
 import BillingPage from '@/app/components/billing/billing-page'
--- a/web/app/components/header/app-nav/index.tsx
+++ b/web/app/components/header/app-nav/index.tsx
@@ -57,7 +57,7 @@ const AppNav = () => {
    { revalidateFirstPage: false },
  )

-  const handleLoadmore = useCallback(() => {
+  const handleLoadMore = useCallback(() => {
    setSize(size => size + 1)
  }, [setSize])

@@ -122,11 +122,11 @@ const AppNav = () => {
        text={t('common.menus.apps')}
        activeSegment={['apps', 'app']}
        link='/apps'
-        curNav={appDetail as any}
-        navs={navItems}
+        curNav={appDetail}
+        navigationItems={navItems}
        createText={t('common.menus.newApp')}
        onCreate={openModal}
-        onLoadmore={handleLoadmore}
+        onLoadMore={handleLoadMore}
      />
      <CreateAppModal
        show={showNewAppDialog}
--- a/web/app/components/header/dataset-nav/index.tsx
+++ b/web/app/components/header/dataset-nav/index.tsx
@@ -1,64 +1,97 @@
 'use client'

-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useParams, useRouter } from 'next/navigation'
 import {
  RiBook2Fill,
  RiBook2Line,
 } from '@remixicon/react'
-import useSWR from 'swr'
-import useSWRInfinite from 'swr/infinite'
 import { flatten } from 'lodash-es'
 import Nav from '../nav'
 import type { NavItem } from '../nav/nav-selector'
-import { fetchDatasetDetail, fetchDatasets } from '@/service/datasets'
-import type { DataSetListResponse } from '@/models/datasets'
-
-const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
-  if (!pageIndex || previousPageData.has_more)
-    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
-  return null
-}
+import { basePath } from '@/utils/var'
+import { useDatasetDetail, useDatasetList } from '@/service/knowledge/use-dataset'
+import type { DataSet } from '@/models/datasets'

 const DatasetNav = () => {
  const { t } = useTranslation()
  const router = useRouter()
  const { datasetId } = useParams()
-  const { data: currentDataset } = useSWR(
-    datasetId
-      ? {
-        url: 'fetchDatasetDetail',
-        datasetId,
-      }
-      : null,
-    apiParams => fetchDatasetDetail(apiParams.datasetId as string))
-  const { data: datasetsData, setSize } = useSWRInfinite(datasetId ? getKey : () => null, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true })
-  const datasetItems = flatten(datasetsData?.map(datasetData => datasetData.data))
+  const { data: currentDataset } = useDatasetDetail(datasetId as string)
+  const {
+    data: datasetList,
+    fetchNextPage,
+    hasNextPage,
+  } = useDatasetList({
+    initialPage: 1,
+    limit: 30,
+  })
+  const datasetItems = flatten(datasetList?.pages.map(datasetData => datasetData.data))

-  const handleLoadmore = useCallback(() => {
-    setSize(size => size + 1)
-  }, [setSize])
+  const curNav = useMemo(() => {
+    if (!currentDataset) return
+    return {
+      id: currentDataset.id,
+      name: currentDataset.name,
+      icon: currentDataset.icon_info.icon,
+      icon_type: currentDataset.icon_info.icon_type,
+      icon_background: currentDataset.icon_info.icon_background,
+      icon_url: currentDataset.icon_info.icon_url,
+    } as Omit<NavItem, 'link'>
+  }, [currentDataset?.id, currentDataset?.name, currentDataset?.icon_info])
+
+  const getDatasetLink = useCallback((dataset: DataSet) => {
+    const isPipelineUnpublished = dataset.runtime_mode === 'rag_pipeline' && !dataset.is_published
+    const link = isPipelineUnpublished
+      ? `/datasets/${dataset.id}/pipeline`
+      : `/datasets/${dataset.id}/documents`
+    return dataset.provider === 'external'
+      ? `/datasets/${dataset.id}/hitTesting`
+      : link
+  }, [])
+
+  const navigationItems = useMemo(() => {
+    return datasetItems.map((dataset) => {
+      const link = getDatasetLink(dataset)
+      return {
+        id: dataset.id,
+        name: dataset.name,
+        link,
+        icon: dataset.icon_info.icon,
+        icon_type: dataset.icon_info.icon_type,
+        icon_background: dataset.icon_info.icon_background,
+        icon_url: dataset.icon_info.icon_url,
+      }
+    }) as NavItem[]
+  }, [datasetItems, getDatasetLink])
+
+  const createRoute = useMemo(() => {
+    const runtimeMode = currentDataset?.runtime_mode
+    if (runtimeMode === 'rag_pipeline')
+      return `${basePath}/datasets/create-from-pipeline`
+    else
+      return `${basePath}/datasets/create`
+  }, [currentDataset?.runtime_mode])
+
+  const handleLoadMore = useCallback(() => {
+    if (hasNextPage)
+      fetchNextPage()
+  }, [hasNextPage, fetchNextPage])

  return (
    <Nav
+      isApp={false}
      icon={<RiBook2Line className='h-4 w-4' />}
      activeIcon={<RiBook2Fill className='h-4 w-4' />}
      text={t('common.menus.datasets')}
      activeSegment='datasets'
      link='/datasets'
-      curNav={currentDataset as any}
-      navs={datasetItems.map(dataset => ({
-        id: dataset.id,
-        name: dataset.name,
-        link: dataset.provider === 'external' ? `/datasets/${dataset.id}/hitTesting` : `/datasets/${dataset.id}/documents`,
-        icon: dataset.icon,
-        icon_background: dataset.icon_background,
-      })) as NavItem[]}
+      curNav={curNav}
+      navigationItems={navigationItems}
      createText={t('common.menus.newDataset')}
-      onCreate={() => router.push('/datasets/create')}
-      onLoadmore={handleLoadmore}
-      isApp={false}
+      onCreate={() => router.push(createRoute)}
+      onLoadMore={handleLoadMore}
    />
  )
 }
--- a/web/app/components/header/header-wrapper.tsx
+++ b/web/app/components/header/header-wrapper.tsx
@@ -16,6 +16,7 @@ const HeaderWrapper = ({
  const isBordered = ['/apps', '/datasets/create', '/tools'].includes(pathname)
  // Check if the current path is a workflow canvas & fullscreen
  const inWorkflowCanvas = pathname.endsWith('/workflow')
+  const isPipelineCanvas = pathname.endsWith('/pipeline')
  const workflowCanvasMaximize = localStorage.getItem('workflow-canvas-maximize') === 'true'
  const [hideHeader, setHideHeader] = useState(workflowCanvasMaximize)
  const { eventEmitter } = useEventEmitterContextContext()
@@ -30,7 +31,7 @@ const HeaderWrapper = ({
      'sticky left-0 right-0 top-0 z-[15] flex min-h-[56px] shrink-0 grow-0 basis-auto flex-col',
      s.header,
      isBordered ? 'border-b border-divider-regular' : '',
-      hideHeader && inWorkflowCanvas && 'hidden',
+      hideHeader && (inWorkflowCanvas || isPipelineCanvas) && 'hidden',
    )}
    >
      {children}
--- a/web/app/components/header/indicator/index.tsx
+++ b/web/app/components/header/indicator/index.tsx
@@ -46,14 +46,14 @@ export default function Indicator({
  className = '',
 }: IndicatorProps) {
  return (
-    <div className={classNames(
-      'h-2 w-2 rounded-[3px] border border-solid',
-      BACKGROUND_MAP[color],
-      BORDER_MAP[color],
-      SHADOW_MAP[color],
-      className,
-    )}>
-
-    </div>
+    <div
+      className={classNames(
+        'h-2 w-2 rounded-[3px] border border-solid',
+        BACKGROUND_MAP[color],
+        BORDER_MAP[color],
+        SHADOW_MAP[color],
+        className,
+      )}
+    />
  )
 }
--- a/web/app/components/header/nav/index.tsx
+++ b/web/app/components/header/nav/index.tsx
@@ -25,10 +25,10 @@ const Nav = ({
  activeSegment,
  link,
  curNav,
-  navs,
+  navigationItems,
  createText,
  onCreate,
-  onLoadmore,
+  onLoadMore,
  isApp,
 }: INavProps) => {
  const setAppDetail = useAppStore(state => state.setAppDetail)
@@ -53,11 +53,11 @@ const Nav = ({
      <Link href={link + (linkLastSearchParams && `?${linkLastSearchParams}`)}>
        <div
          onClick={() => setAppDetail()}
-          className={classNames(`
-            flex h-7 cursor-pointer items-center rounded-[10px] px-2.5
-            ${isActivated ? 'text-components-main-nav-nav-button-text-active' : 'text-components-main-nav-nav-button-text'}
-            ${curNav && isActivated && 'hover:bg-components-main-nav-nav-button-bg-active-hover'}
-          `)}
+          className={classNames(
+            'flex h-7 cursor-pointer items-center rounded-[10px] px-2.5',
+            isActivated ? 'text-components-main-nav-nav-button-text-active' : 'text-components-main-nav-nav-button-text',
+            curNav && isActivated && 'hover:bg-components-main-nav-nav-button-bg-active-hover',
+          )}
          onMouseEnter={() => setHovered(true)}
          onMouseLeave={() => setHovered(false)}
        >
@@ -82,10 +82,10 @@ const Nav = ({
            <NavSelector
              isApp={isApp}
              curNav={curNav}
-              navs={navs}
+              navigationItems={navigationItems}
              createText={createText}
              onCreate={onCreate}
-              onLoadmore={onLoadmore}
+              onLoadMore={onLoadMore}
            />
          </>
        )
--- a/web/app/components/header/nav/nav-selector/index.tsx
+++ b/web/app/components/header/nav/nav-selector/index.tsx
@@ -24,31 +24,31 @@ export type NavItem = {
  link: string
  icon_type: AppIconType | null
  icon: string
-  icon_background: string
+  icon_background: string | null
  icon_url: string | null
  mode?: string
 }
 export type INavSelectorProps = {
-  navs: NavItem[]
+  navigationItems: NavItem[]
  curNav?: Omit<NavItem, 'link'>
  createText: string
  isApp?: boolean
  onCreate: (state: string) => void
-  onLoadmore?: () => void
+  onLoadMore?: () => void
 }

-const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }: INavSelectorProps) => {
+const NavSelector = ({ curNav, navigationItems, createText, isApp, onCreate, onLoadMore }: INavSelectorProps) => {
  const { t } = useTranslation()
  const router = useRouter()
  const { isCurrentWorkspaceEditor } = useAppContext()
  const setAppDetail = useAppStore(state => state.setAppDetail)

  const handleScroll = useCallback(debounce((e) => {
-    if (typeof onLoadmore === 'function') {
+    if (typeof onLoadMore === 'function') {
      const { clientHeight, scrollHeight, scrollTop } = e.target

      if (clientHeight + scrollTop > scrollHeight - 50)
-        onLoadmore()
+        onLoadMore()
    }
  }, 50), [])

@@ -75,7 +75,7 @@ const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }:
          >
            <div className="overflow-auto px-1 py-1" style={{ maxHeight: '50vh' }} onScroll={handleScroll}>
              {
-                navs.map(nav => (
+                navigationItems.map(nav => (
                  <MenuItem key={nav.id}>
                    <div className='flex w-full cursor-pointer items-center truncate rounded-lg px-3 py-[6px] text-[14px] font-normal text-text-secondary hover:bg-state-base-hover' onClick={() => {
                      if (curNav?.id === nav.id)
@@ -84,7 +84,13 @@ const NavSelector = ({ curNav, navs, createText, isApp, onCreate, onLoadmore }:
                      router.push(nav.link)
                    }} title={nav.name}>
                      <div className='relative mr-2 h-6 w-6 rounded-md'>
-                        <AppIcon size='tiny' iconType={nav.icon_type} icon={nav.icon} background={nav.icon_background} imageUrl={nav.icon_url} />
+                        <AppIcon
+                          size='tiny'
+                          iconType={nav.icon_type}
+                          icon={nav.icon}
+                          background={nav.icon_background}
+                          imageUrl={nav.icon_url}
+                        />
                        {!!nav.mode && (
                          <span className={cn(
                            'absolute -bottom-0.5 -right-0.5 h-3.5 w-3.5 rounded border-[0.5px] border-[rgba(0,0,0,0.02)] bg-white p-0.5 shadow-sm',