feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -26,7 +26,7 @@ const CopyFeedbackNew = ({ content }: Props) => {
}, 100)
return (
<div className='inline-flex pb-0.5' onClick={e => e.stopPropagation()} onMouseLeave={onMouseLeave}>
<div className='inline-flex w-full pb-0.5' onClick={e => e.stopPropagation()} onMouseLeave={onMouseLeave}>
<Tooltip
popupContent={
(isCopied
@@ -35,13 +35,13 @@ const CopyFeedbackNew = ({ content }: Props) => {
}
>
<div
className='group/copy flex items-center gap-0.5 '
className='group/copy flex w-full items-center gap-0.5 '
onClick={onClickCopy}
>
<div
className='system-2xs-regular cursor-pointer text-text-quaternary group-hover:text-text-tertiary'
className='system-2xs-regular w-0 grow cursor-pointer truncate text-text-quaternary group-hover:text-text-tertiary'
>{content}</div>
<RiFileCopyLine className='h-3 w-3 text-text-tertiary opacity-0 group-hover/copy:opacity-100' />
<RiFileCopyLine className='h-3 w-3 shrink-0 text-text-tertiary opacity-0 group-hover/copy:opacity-100' />
</div>
</Tooltip>
</div>

View File

@@ -181,7 +181,7 @@ const InputVarList: FC<Props> = ({
return (
<div key={variable} className='space-y-1'>
<div className='flex h-[18px] items-center space-x-2'>
<div className='flex items-center space-x-2 leading-[18px]'>
<span className='code-sm-semibold text-text-secondary'>{label[language] || label.en_US}</span>
<span className='system-xs-regular text-text-tertiary'>{paramType(type)}</span>
{required && <span className='system-xs-regular text-util-colors-orange-dark-orange-dark-600'>Required</span>}

View File

@@ -10,6 +10,7 @@ import type {
} from '@/app/components/workflow/types'
import { BlockEnum } from '@/app/components/workflow/types'
import cn from '@/utils/classnames'
import { useStore } from '@/app/components/workflow/store'
type MixedVariableTextInputProps = {
readOnly?: boolean
@@ -17,6 +18,8 @@ type MixedVariableTextInputProps = {
availableNodes?: Node[]
value?: string
onChange?: (text: string) => void
showManageInputField?: boolean
onManageInputField?: () => void
}
const MixedVariableTextInput = ({
readOnly = false,
@@ -24,10 +27,15 @@ const MixedVariableTextInput = ({
availableNodes = [],
value = '',
onChange,
showManageInputField,
onManageInputField,
}: MixedVariableTextInputProps) => {
const { t } = useTranslation()
const controlPromptEditorRerenderKey = useStore(s => s.controlPromptEditorRerenderKey)
return (
<PromptEditor
key={controlPromptEditorRerenderKey}
wrapperClassName={cn(
'w-full rounded-lg border border-transparent bg-components-input-bg-normal px-2 py-1',
'hover:border-components-input-border-hover hover:bg-components-input-bg-hover',
@@ -52,6 +60,8 @@ const MixedVariableTextInput = ({
}
return acc
}, {} as any),
showManageInputField,
onManageInputField,
}}
placeholder={<Placeholder />}
onChange={onChange}

View File

@@ -16,6 +16,8 @@ type Props = {
inPanel?: boolean
currentTool?: Tool
currentProvider?: ToolWithProvider
showManageInputField?: boolean
onManageInputField?: () => void
}
const ToolForm: FC<Props> = ({
@@ -27,6 +29,8 @@ const ToolForm: FC<Props> = ({
inPanel,
currentTool,
currentProvider,
showManageInputField,
onManageInputField,
}) => {
return (
<div className='space-y-1'>
@@ -42,6 +46,8 @@ const ToolForm: FC<Props> = ({
inPanel={inPanel}
currentTool={currentTool}
currentProvider={currentProvider}
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
/>
))
}

View File

@@ -24,6 +24,8 @@ type Props = {
inPanel?: boolean
currentTool?: Tool
currentProvider?: ToolWithProvider
showManageInputField?: boolean
onManageInputField?: () => void
}
const ToolFormItem: FC<Props> = ({
@@ -35,6 +37,8 @@ const ToolFormItem: FC<Props> = ({
inPanel,
currentTool,
currentProvider,
showManageInputField,
onManageInputField,
}) => {
const language = useLanguage()
const { name, label, type, required, tooltip, input_schema } = schema
@@ -89,6 +93,8 @@ const ToolFormItem: FC<Props> = ({
inPanel={inPanel}
currentTool={currentTool}
currentProvider={currentProvider}
showManageInputField={showManageInputField}
onManageInputField={onManageInputField}
/>
{isShowSchema && (

View File

@@ -1,27 +1,27 @@
import { BlockEnum } from '../../types'
import type { NodeDefault } from '../../types'
import { genNodeMetaData } from '@/app/components/workflow/utils'
import { BlockEnum, VarType } from '@/app/components/workflow/types'
import type { NodeDefault, ToolWithProvider } from '../../types'
import type { ToolNodeType } from './types'
import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types'
import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/app/components/workflow/blocks'
import { TOOL_OUTPUT_STRUCT } from '../../constants'
import { CollectionType } from '@/app/components/tools/types'
import { canFindTool } from '@/utils'
import { getMatchedSchemaType } from '../_base/components/variable/use-match-schema-type'
const i18nPrefix = 'workflow.errorMsg'
const metaData = genNodeMetaData({
sort: -1,
type: BlockEnum.Tool,
helpLinkUri: 'tools',
})
const nodeDefault: NodeDefault<ToolNodeType> = {
metaData,
defaultValue: {
tool_parameters: {},
tool_configurations: {},
tool_node_version: '2',
},
getAvailablePrevNodes(isChatMode: boolean) {
const nodes = isChatMode
? ALL_CHAT_AVAILABLE_BLOCKS
: ALL_COMPLETION_AVAILABLE_BLOCKS.filter(type => type !== BlockEnum.End)
return nodes
},
getAvailableNextNodes(isChatMode: boolean) {
const nodes = isChatMode ? ALL_CHAT_AVAILABLE_BLOCKS : ALL_COMPLETION_AVAILABLE_BLOCKS
return nodes
},
checkValid(payload: ToolNodeType, t: any, moreDataForCheckValid: any) {
const { toolInputsSchema, toolSettingSchema, language, notAuthed } = moreDataForCheckValid
let errorMessages = ''
@@ -66,6 +66,65 @@ const nodeDefault: NodeDefault<ToolNodeType> = {
errorMessage: errorMessages,
}
},
getOutputVars(payload: ToolNodeType, allPluginInfoList: Record<string, ToolWithProvider[]>, _ragVars: any, { schemaTypeDefinitions } = { schemaTypeDefinitions: [] }) {
const { provider_id, provider_type } = payload
let currentTools: ToolWithProvider[] = []
switch (provider_type) {
case CollectionType.builtIn:
currentTools = allPluginInfoList.buildInTools ?? []
break
case CollectionType.custom:
currentTools = allPluginInfoList.customTools ?? []
break
case CollectionType.workflow:
currentTools = allPluginInfoList.workflowTools ?? []
break
case CollectionType.mcp:
currentTools = allPluginInfoList.mcpTools ?? []
break
default:
currentTools = []
}
const currCollection = currentTools.find(item => canFindTool(item.id, provider_id))
const currTool = currCollection?.tools.find(tool => tool.name === payload.tool_name)
const output_schema = currTool?.output_schema
let res: any[] = []
if (!output_schema || !output_schema.properties) {
res = TOOL_OUTPUT_STRUCT
}
else {
const outputSchema: any[] = []
Object.keys(output_schema.properties).forEach((outputKey) => {
const output = output_schema.properties[outputKey]
const dataType = output.type
const schemaType = getMatchedSchemaType(output, schemaTypeDefinitions)
let type = dataType === 'array'
? `Array[${output.items?.type ? output.items.type.slice(0, 1).toLocaleLowerCase() + output.items.type.slice(1) : 'Unknown'}]`
: `${output.type ? output.type.slice(0, 1).toLocaleLowerCase() + output.type.slice(1) : 'Unknown'}`
if (type === VarType.object && schemaType === 'file')
type = VarType.file
outputSchema.push({
variable: outputKey,
type,
description: output.description,
schemaType,
children: output.type === 'object' ? {
schema: {
type: 'object',
properties: output.properties,
},
} : undefined,
})
})
res = [
...TOOL_OUTPUT_STRUCT,
...outputSchema,
]
}
return res
},
}
export default nodeDefault

View File

@@ -10,7 +10,9 @@ import type { NodePanelProps } from '@/app/components/workflow/types'
import Loading from '@/app/components/base/loading'
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
import { Type } from '../llm/types'
import { useStore } from '@/app/components/workflow/store'
import { wrapStructuredVarItem } from '@/app/components/workflow/utils/tool'
import useMatchSchemaType, { getMatchedSchemaType } from '../_base/components/variable/use-match-schema-type'
const i18nPrefix = 'workflow.nodes.tool'
@@ -19,7 +21,6 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
data,
}) => {
const { t } = useTranslation()
const {
readOnly,
inputs,
@@ -37,11 +38,16 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
} = useConfig(id, data)
const [collapsed, setCollapsed] = React.useState(false)
const pipelineId = useStore(s => s.pipelineId)
const setShowInputFieldPanel = useStore(s => s.setShowInputFieldPanel)
const { schemaTypeDefinitions } = useMatchSchemaType()
if (isLoading) {
return <div className='flex h-[200px] items-center justify-center'>
<Loading />
</div>
return (
<div className='flex h-[200px] items-center justify-center'>
<Loading />
</div>
)
}
return (
@@ -61,6 +67,8 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
onChange={setInputVar}
currentProvider={currCollection}
currentTool={currTool}
showManageInputField={!!pipelineId}
onManageInputField={() => setShowInputFieldPanel?.(true)}
/>
</Field>
)}
@@ -111,30 +119,27 @@ const Panel: FC<NodePanelProps<ToolNodeType>> = ({
description={t(`${i18nPrefix}.outputVars.json`)}
isIndent={hasObjectOutput}
/>
{outputSchema.map(outputItem => (
<div key={outputItem.name}>
{outputItem.value?.type === 'object' ? (
<StructureOutputItem
rootClassName='code-sm-semibold text-text-secondary'
payload={{
schema: {
type: Type.object,
properties: {
[outputItem.name]: outputItem.value,
},
additionalProperties: false,
},
}} />
) : (
<VarItem
name={outputItem.name}
type={outputItem.type.toLocaleLowerCase()}
description={outputItem.description}
isIndent={hasObjectOutput}
/>
)}
</div>
))}
{outputSchema.map((outputItem) => {
const schemaType = getMatchedSchemaType(outputItem.value, schemaTypeDefinitions)
return (
<div key={outputItem.name}>
{outputItem.value?.type === 'object' ? (
<StructureOutputItem
rootClassName='code-sm-semibold text-text-secondary'
payload={wrapStructuredVarItem(outputItem, schemaType)}
/>
) : (
<VarItem
name={outputItem.name}
// eslint-disable-next-line sonarjs/no-nested-template-literals
type={`${outputItem.type.toLocaleLowerCase()}${schemaType ? ` (${schemaType})` : ''}`}
description={outputItem.description}
isIndent={hasObjectOutput}
/>
)}
</div>
)
})}
</>
</OutputVars>
</div>

View File

@@ -20,8 +20,10 @@ export type ToolNodeType = CommonNodeType & {
tool_label: string
tool_parameters: ToolVarInputs
tool_configurations: Record<string, any>
output_schema: Record<string, any>
paramSchemas?: Record<string, any>[]
version?: string
tool_node_version?: string
tool_description?: string
is_team_authorization?: boolean
params?: Record<string, any>
}

View File

@@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import produce from 'immer'
import { useBoolean } from 'ahooks'
import { useStore } from '../../store'
import { useStore, useWorkflowStore } from '../../store'
import type { ToolNodeType, ToolVarInputs } from './types'
import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks'
import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud'
@@ -21,18 +21,27 @@ import {
import { canFindTool } from '@/utils'
const useConfig = (id: string, payload: ToolNodeType) => {
const workflowStore = useWorkflowStore()
const { nodesReadOnly: readOnly } = useNodesReadOnly()
const { handleFetchAllTools } = useFetchToolsData()
const { t } = useTranslation()
const language = useLanguage()
const { inputs, setInputs: doSetInputs } = useNodeCrud<ToolNodeType>(id, payload)
const { inputs, setInputs: doSetInputs } = useNodeCrud<ToolNodeType>(
id,
payload,
)
/*
* tool_configurations: tool setting, not dynamic setting (form type = form)
* tool_parameters: tool dynamic setting(form type = llm)
* output_schema: tool dynamic output
*/
const { provider_id, provider_type, tool_name, tool_configurations, output_schema, tool_parameters } = inputs
* tool_configurations: tool setting, not dynamic setting (form type = form)
* tool_parameters: tool dynamic setting(form type = llm)
*/
const {
provider_id,
provider_type,
tool_name,
tool_configurations,
tool_parameters,
} = inputs
const isBuiltIn = provider_type === CollectionType.builtIn
const buildInTools = useStore(s => s.buildInTools)
const customTools = useStore(s => s.customTools)
@@ -53,102 +62,143 @@ const useConfig = (id: string, payload: ToolNodeType) => {
return []
}
}, [buildInTools, customTools, mcpTools, provider_type, workflowTools])
const currCollection = currentTools.find(item => canFindTool(item.id, provider_id))
const currCollection = useMemo(() => {
return currentTools.find(item => canFindTool(item.id, provider_id))
}, [currentTools, provider_id])
// Auth
const needAuth = !!currCollection?.allow_delete
const isAuthed = !!currCollection?.is_team_authorization
const isShowAuthBtn = isBuiltIn && needAuth && !isAuthed
const [showSetAuth, {
setTrue: showSetAuthModal,
setFalse: hideSetAuthModal,
}] = useBoolean(false)
const [
showSetAuth,
{ setTrue: showSetAuthModal, setFalse: hideSetAuthModal },
] = useBoolean(false)
const handleSaveAuth = useCallback(async (value: any) => {
await updateBuiltInToolCredential(currCollection?.name as string, value)
const handleSaveAuth = useCallback(
async (value: any) => {
await updateBuiltInToolCredential(currCollection?.name as string, value)
Toast.notify({
type: 'success',
message: t('common.api.actionSuccess'),
})
handleFetchAllTools(provider_type)
hideSetAuthModal()
}, [currCollection?.name, hideSetAuthModal, t, handleFetchAllTools, provider_type])
Toast.notify({
type: 'success',
message: t('common.api.actionSuccess'),
})
handleFetchAllTools(provider_type)
hideSetAuthModal()
},
[
currCollection?.name,
hideSetAuthModal,
t,
handleFetchAllTools,
provider_type,
],
)
const currTool = currCollection?.tools.find(tool => tool.name === tool_name)
const currTool = useMemo(() => {
return currCollection?.tools.find(tool => tool.name === tool_name)
}, [currCollection, tool_name])
const formSchemas = useMemo(() => {
return currTool ? toolParametersToFormSchemas(currTool.parameters) : []
}, [currTool])
const toolInputVarSchema = formSchemas.filter((item: any) => item.form === 'llm')
const toolInputVarSchema = useMemo(() => {
return formSchemas.filter((item: any) => item.form === 'llm')
}, [formSchemas])
// use setting
const toolSettingSchema = formSchemas.filter((item: any) => item.form !== 'llm')
const hasShouldTransferTypeSettingInput = toolSettingSchema.some(item => item.type === 'boolean' || item.type === 'number-input')
const toolSettingSchema = useMemo(() => {
return formSchemas.filter((item: any) => item.form !== 'llm')
}, [formSchemas])
const hasShouldTransferTypeSettingInput = toolSettingSchema.some(
item => item.type === 'boolean' || item.type === 'number-input',
)
const setInputs = useCallback((value: ToolNodeType) => {
if (!hasShouldTransferTypeSettingInput) {
doSetInputs(value)
return
}
const newInputs = produce(value, (draft) => {
const newConfig = { ...draft.tool_configurations }
Object.keys(draft.tool_configurations).forEach((key) => {
const schema = formSchemas.find(item => item.variable === key)
const value = newConfig[key]
if (schema?.type === 'boolean') {
if (typeof value === 'string')
newConfig[key] = value === 'true' || value === '1'
const setInputs = useCallback(
(value: ToolNodeType) => {
if (!hasShouldTransferTypeSettingInput) {
doSetInputs(value)
return
}
const newInputs = produce(value, (draft) => {
const newConfig = { ...draft.tool_configurations }
Object.keys(draft.tool_configurations).forEach((key) => {
const schema = formSchemas.find(item => item.variable === key)
const value = newConfig[key]
if (schema?.type === 'boolean') {
if (typeof value === 'string')
newConfig[key] = value === 'true' || value === '1'
if (typeof value === 'number')
newConfig[key] = value === 1
}
if (typeof value === 'number') newConfig[key] = value === 1
}
if (schema?.type === 'number-input') {
if (typeof value === 'string' && value !== '')
newConfig[key] = Number.parseFloat(value)
}
if (schema?.type === 'number-input') {
if (typeof value === 'string' && value !== '')
newConfig[key] = Number.parseFloat(value)
}
})
draft.tool_configurations = newConfig
})
draft.tool_configurations = newConfig
})
doSetInputs(newInputs)
}, [doSetInputs, formSchemas, hasShouldTransferTypeSettingInput])
doSetInputs(newInputs)
},
[doSetInputs, formSchemas, hasShouldTransferTypeSettingInput],
)
const [notSetDefaultValue, setNotSetDefaultValue] = useState(false)
const toolSettingValue = useMemo(() => {
if (notSetDefaultValue)
return tool_configurations
if (notSetDefaultValue) return tool_configurations
return getConfiguredValue(tool_configurations, toolSettingSchema)
}, [notSetDefaultValue, toolSettingSchema, tool_configurations])
const setToolSettingValue = useCallback((value: Record<string, any>) => {
setNotSetDefaultValue(true)
setInputs({
...inputs,
tool_configurations: value,
})
}, [inputs, setInputs])
const setToolSettingValue = useCallback(
(value: Record<string, any>) => {
setNotSetDefaultValue(true)
setInputs({
...inputs,
tool_configurations: value,
})
},
[inputs, setInputs],
)
const formattingParameters = () => {
const inputsWithDefaultValue = produce(inputs, (draft) => {
if (!draft.tool_configurations || Object.keys(draft.tool_configurations).length === 0)
draft.tool_configurations = getConfiguredValue(tool_configurations, toolSettingSchema)
if (!draft.tool_parameters || Object.keys(draft.tool_parameters).length === 0)
draft.tool_parameters = getConfiguredValue(tool_parameters, toolInputVarSchema)
if (
!draft.tool_configurations
|| Object.keys(draft.tool_configurations).length === 0
) {
draft.tool_configurations = getConfiguredValue(
tool_configurations,
toolSettingSchema,
)
}
if (
!draft.tool_parameters
|| Object.keys(draft.tool_parameters).length === 0
) {
draft.tool_parameters = getConfiguredValue(
tool_parameters,
toolInputVarSchema,
)
}
})
return inputsWithDefaultValue
}
useEffect(() => {
if (!currTool)
return
if (!currTool) return
const inputsWithDefaultValue = formattingParameters()
const { setControlPromptEditorRerenderKey } = workflowStore.getState()
setInputs(inputsWithDefaultValue)
setTimeout(() => setControlPromptEditorRerenderKey(Date.now()))
}, [currTool])
// setting when call
const setInputVar = useCallback((value: ToolVarInputs) => {
setInputs({
...inputs,
tool_parameters: value,
})
}, [inputs, setInputs])
const setInputVar = useCallback(
(value: ToolVarInputs) => {
setInputs({
...inputs,
tool_parameters: value,
})
},
[inputs, setInputs],
)
const isLoading = currTool && (isBuiltIn ? !currCollection : false)
@@ -174,8 +224,9 @@ const useConfig = (id: string, payload: ToolNodeType) => {
const outputSchema = useMemo(() => {
const res: any[] = []
if (!output_schema)
return []
const output_schema = currTool?.output_schema
if (!output_schema || !output_schema.properties) return res
Object.keys(output_schema.properties).forEach((outputKey) => {
const output = output_schema.properties[outputKey]
const type = output.type
@@ -188,22 +239,35 @@ const useConfig = (id: string, payload: ToolNodeType) => {
else {
res.push({
name: outputKey,
type: output.type === 'array'
? `Array[${output.items?.type ? output.items.type.slice(0, 1).toLocaleUpperCase() + output.items.type.slice(1) : 'Unknown'}]`
: `${output.type ? output.type.slice(0, 1).toLocaleUpperCase() + output.type.slice(1) : 'Unknown'}`,
type:
output.type === 'array'
? `Array[${
output.items?.type
? output.items.type.slice(0, 1).toLocaleUpperCase()
+ output.items.type.slice(1)
: 'Unknown'
}]`
: `${
output.type
? output.type.slice(0, 1).toLocaleUpperCase()
+ output.type.slice(1)
: 'Unknown'
}`,
description: output.description,
})
}
})
return res
}, [output_schema])
}, [currTool])
const hasObjectOutput = useMemo(() => {
if (!output_schema)
return false
const output_schema = currTool?.output_schema
if (!output_schema || !output_schema.properties) return false
const properties = output_schema.properties
return Object.keys(properties).some(key => properties[key].type === 'object')
}, [output_schema])
return Object.keys(properties).some(
key => properties[key].type === 'object',
)
}, [currTool])
return {
readOnly,