feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/plugins/plugin-auth/authorize/add-api-key-button.tsx

@@ -6,6 +6,7 @@ import Button from '@/app/components/base/button'
 import type { ButtonProps } from '@/app/components/base/button'
 import ApiKeyModal from './api-key-modal'
 import type { PluginPayload } from '../types'
+import type { FormSchema } from '@/app/components/base/form/types'
 
 export type AddApiKeyButtonProps = {
   pluginPayload: PluginPayload
@@ -13,13 +14,15 @@ export type AddApiKeyButtonProps = {
   buttonText?: string
   disabled?: boolean
   onUpdate?: () => void
+  formSchemas?: FormSchema[]
 }
 const AddApiKeyButton = ({
   pluginPayload,
   buttonVariant = 'secondary-accent',
-  buttonText = 'use api key',
+  buttonText = 'Use Api Key',
   disabled,
   onUpdate,
+  formSchemas = [],
 }: AddApiKeyButtonProps) => {
   const [isApiKeyModalOpen, setIsApiKeyModalOpen] = useState(false)
 
@@ -39,6 +42,7 @@ const AddApiKeyButton = ({
             pluginPayload={pluginPayload}
             onClose={() => setIsApiKeyModalOpen(false)}
             onUpdate={onUpdate}
+            formSchemas={formSchemas}
           />
         )
       }

web/app/components/plugins/plugin-auth/authorize/add-oauth-button.tsx

@@ -36,6 +36,13 @@ export type AddOAuthButtonProps = {
   dividerClassName?: string
   disabled?: boolean
   onUpdate?: () => void
+  oAuthData?: {
+    schema?: FormSchema[]
+    is_oauth_custom_client_enabled?: boolean
+    is_system_oauth_params_exists?: boolean
+    client_params?: Record<string, any>
+    redirect_uri?: string
+  }
 }
 const AddOAuthButton = ({
   pluginPayload,
@@ -47,19 +54,26 @@ const AddOAuthButton = ({
   dividerClassName,
   disabled,
   onUpdate,
+  oAuthData,
 }: AddOAuthButtonProps) => {
   const { t } = useTranslation()
   const renderI18nObject = useRenderI18nObject()
   const [isOAuthSettingsOpen, setIsOAuthSettingsOpen] = useState(false)
   const { mutateAsync: getPluginOAuthUrl } = useGetPluginOAuthUrlHook(pluginPayload)
   const { data, isLoading } = useGetPluginOAuthClientSchemaHook(pluginPayload)
+  const mergedOAuthData = useMemo(() => {
+    if (oAuthData)
+      return oAuthData
+
+    return data
+  }, [oAuthData, data])
   const {
     schema = [],
     is_oauth_custom_client_enabled,
     is_system_oauth_params_exists,
     client_params,
     redirect_uri,
-  } = data || {}
+  } = mergedOAuthData as any || {}
   const isConfigured = is_system_oauth_params_exists || is_oauth_custom_client_enabled
   const handleOAuth = useCallback(async () => {
     const { authorization_url } = await getPluginOAuthUrl()
@@ -86,7 +100,7 @@ const AddOAuthButton = ({
             {
               redirect_uri && (
                 <div className='system-sm-medium flex w-full py-0.5'>
-                  <div className='w-0 grow break-words'>{redirect_uri}</div>
+                  <div className='w-0 grow break-words break-all'>{redirect_uri}</div>
                   <ActionButton
                     className='shrink-0'
                     onClick={() => {
@@ -112,7 +126,7 @@ const AddOAuthButton = ({
     )
   }, [t, redirect_uri, renderI18nObject])
   const memorizedSchemas = useMemo(() => {
-    const result: FormSchema[] = schema.map((item, index) => {
+    const result: FormSchema[] = (schema as FormSchema[]).map((item, index) => {
       return {
         ...item,
         label: index === 0 ? renderCustomLabel(item) : item.label,

web/app/components/plugins/plugin-auth/authorize/api-key-modal.tsx

@@ -10,7 +10,10 @@ import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
 import Modal from '@/app/components/base/modal/modal'
 import { CredentialTypeEnum } from '../types'
 import AuthForm from '@/app/components/base/form/form-scenarios/auth'
-import type { FormRefObject } from '@/app/components/base/form/types'
+import type {
+  FormRefObject,
+  FormSchema,
+} from '@/app/components/base/form/types'
 import { FormTypeEnum } from '@/app/components/base/form/types'
 import { useToastContext } from '@/app/components/base/toast'
 import Loading from '@/app/components/base/loading'
@@ -28,6 +31,7 @@ export type ApiKeyModalProps = {
   onRemove?: () => void
   disabled?: boolean
   onUpdate?: () => void
+  formSchemas?: FormSchema[]
 }
 const ApiKeyModal = ({
   pluginPayload,
@@ -36,6 +40,7 @@ const ApiKeyModal = ({
   onRemove,
   disabled,
   onUpdate,
+  formSchemas: formSchemasFromProps = [],
 }: ApiKeyModalProps) => {
   const { t } = useTranslation()
   const { notify } = useToastContext()
@@ -46,6 +51,12 @@ const ApiKeyModal = ({
     setDoingAction(value)
   }, [])
   const { data = [], isLoading } = useGetPluginCredentialSchemaHook(pluginPayload, CredentialTypeEnum.API_KEY)
+  const mergedData = useMemo(() => {
+    if (formSchemasFromProps?.length)
+      return formSchemasFromProps
+
+    return data
+  }, [formSchemasFromProps, data])
   const formSchemas = useMemo(() => {
     return [
       {
@@ -54,9 +65,9 @@ const ApiKeyModal = ({
         label: t('plugin.auth.authorizationName'),
         required: false,
       },
-      ...data,
+      ...mergedData,
     ]
-  }, [data, t])
+  }, [mergedData, t])
   const defaultValues = formSchemas.reduce((acc, schema) => {
     if (schema.default)
       acc[schema.name] = schema.default
@@ -150,7 +161,7 @@ const ApiKeyModal = ({
         )
       }
       {
-        !isLoading && !!data.length && (
+        !isLoading && !!mergedData.length && (
           <AuthForm
             ref={formRef}
             formSchemas={formSchemas}

web/app/components/plugins/plugin-auth/authorized-in-data-source-node.tsx

@@ -0,0 +1,43 @@
+import {
+  memo,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiEqualizer2Line } from '@remixicon/react'
+import Button from '@/app/components/base/button'
+import Indicator from '@/app/components/header/indicator'
+import cn from '@/utils/classnames'
+
+type AuthorizedInDataSourceNodeProps = {
+  authorizationsNum: number
+  onJumpToDataSourcePage: () => void
+}
+const AuthorizedInDataSourceNode = ({
+  authorizationsNum,
+  onJumpToDataSourcePage,
+}: AuthorizedInDataSourceNodeProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <Button
+      size='small'
+      onClick={onJumpToDataSourcePage}
+    >
+      <Indicator
+        className='mr-1.5'
+        color='green'
+      />
+      {
+        authorizationsNum > 1
+          ? t('plugin.auth.authorizations')
+          : t('plugin.auth.authorization')
+      }
+      <RiEqualizer2Line
+        className={cn(
+          'h-3.5 w-3.5 text-components-button-ghost-text',
+        )}
+      />
+    </Button>
+  )
+}
+
+export default memo(AuthorizedInDataSourceNode)

web/app/components/plugins/plugin-auth/hooks/use-get-api.ts

@@ -24,6 +24,22 @@ export const useGetApi = ({ category = AuthCategory.tool, provider }: PluginPayl
     }
   }
 
+  if (category === AuthCategory.datasource) {
+    return {
+      getCredentialInfo: '',
+      setDefaultCredential: `/auth/plugin/datasource/${provider}/default`,
+      getCredentials: `/auth/plugin/datasource/${provider}`,
+      addCredential: `/auth/plugin/datasource/${provider}`,
+      updateCredential: `/auth/plugin/datasource/${provider}/update`,
+      deleteCredential: `/auth/plugin/datasource/${provider}/delete`,
+      getCredentialSchema: () => '',
+      getOauthUrl: `/oauth/plugin/${provider}/datasource/get-authorization-url`,
+      getOauthClientSchema: '',
+      setCustomOauthClient: `/auth/plugin/datasource/${provider}/custom-client`,
+      deleteCustomOAuthClient: `/auth/plugin/datasource/${provider}/custom-client`,
+    }
+  }
+
   return {
     getCredentialInfo: '',
     setDefaultCredential: '',

web/app/components/plugins/plugin-auth/hooks/use-plugin-auth-action.ts

@@ -5,7 +5,7 @@ import {
 } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useToastContext } from '@/app/components/base/toast'
-import type { PluginPayload } from '@/app/components/plugins/plugin-auth/types'
+import type { PluginPayload } from '../types'
 import {
   useDeletePluginCredentialHook,
   useSetPluginDefaultCredentialHook,

web/app/components/plugins/plugin-auth/index.tsx

@@ -3,4 +3,10 @@ export { default as Authorized } from './authorized'
 export { default as AuthorizedInNode } from './authorized-in-node'
 export { default as PluginAuthInAgent } from './plugin-auth-in-agent'
 export { usePluginAuth } from './hooks/use-plugin-auth'
+export { default as PluginAuthInDataSourceNode } from './plugin-auth-in-datasource-node'
+export { default as AuthorizedInDataSourceNode } from './authorized-in-data-source-node'
+export { default as AddOAuthButton } from './authorize/add-oauth-button'
+export { default as AddApiKeyButton } from './authorize/add-api-key-button'
+export { default as ApiKeyModal } from './authorize/api-key-modal'
+export * from './hooks/use-plugin-auth-action'
 export * from './types'

web/app/components/plugins/plugin-auth/plugin-auth-in-datasource-node.tsx

@@ -0,0 +1,39 @@
+import { memo } from 'react'
+import type { ReactNode } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiAddLine } from '@remixicon/react'
+import Button from '@/app/components/base/button'
+
+type PluginAuthInDataSourceNodeProps = {
+  children?: ReactNode
+  isAuthorized?: boolean
+  onJumpToDataSourcePage: () => void
+}
+const PluginAuthInDataSourceNode = ({
+  children,
+  isAuthorized,
+  onJumpToDataSourcePage,
+}: PluginAuthInDataSourceNodeProps) => {
+  const { t } = useTranslation()
+  return (
+    <>
+      {
+        !isAuthorized && (
+          <div className='px-4 pb-2'>
+            <Button
+              className='w-full'
+              variant='primary'
+              onClick={onJumpToDataSourcePage}
+            >
+              <RiAddLine className='mr-1 h-4 w-4' />
+              {t('common.integrations.connect')}
+            </Button>
+          </div>
+        )
+      }
+      {isAuthorized && children}
+    </>
+  )
+}
+
+export default memo(PluginAuthInDataSourceNode)

web/app/components/plugins/plugin-auth/types.ts

@@ -1,3 +1,6 @@
+export type { AddApiKeyButtonProps } from './authorize/add-api-key-button'
+export type { AddOAuthButtonProps } from './authorize/add-oauth-button'
+
 export enum AuthCategory {
   tool = 'tool',
   datasource = 'datasource',