admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: knowledge pipeline (#25360)

Browse Source 
Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts
View File

@@ -4,7 +4,11 @@ import {
useWorkflow,
useWorkflowVariables,
} from '@/app/components/workflow/hooks'
import type { Node, ValueSelector, Var } from '@/app/components/workflow/types'
import type { NodeOutPutVar } from '@/app/components/workflow/types'
import { BlockEnum, type Node, type ValueSelector, type Var } from '@/app/components/workflow/types'
import { useStore as useWorkflowStore } from '@/app/components/workflow/store'
import { inputVarTypeToVarType } from '../../data-source/utils'
type Params = {
onlyLeafNodeVar?: boolean
hideEnv?: boolean
@@ -24,29 +28,57 @@ const useAvailableVarList = (nodeId: string, {
onlyLeafNodeVar: false,
filterVar: () => true,
}) => {
const { getTreeLeafNodes, getBeforeNodesInSameBranchIncludeParent } = useWorkflow()
const { getTreeLeafNodes, getNodeById, getBeforeNodesInSameBranchIncludeParent } = useWorkflow()
const { getNodeAvailableVars } = useWorkflowVariables()
const isChatMode = useIsChatMode()
const availableNodes = passedInAvailableNodes || (onlyLeafNodeVar ? getTreeLeafNodes(nodeId) : getBeforeNodesInSameBranchIncludeParent(nodeId))
const {
parentNode: iterationNode,
} = useNodeInfo(nodeId)
const availableVars = getNodeAvailableVars({
const currNode = getNodeById(nodeId)
const ragPipelineVariables = useWorkflowStore(s => s.ragPipelineVariables)
const isDataSourceNode = currNode?.data?.type === BlockEnum.DataSource
const dataSourceRagVars: NodeOutPutVar[] = []
if (isDataSourceNode) {
const ragVariablesInDataSource = ragPipelineVariables?.filter(ragVariable => ragVariable.belong_to_node_id === nodeId)
const filterVars = ragVariablesInDataSource?.filter(v => filterVar({
variable: v.variable,
type: inputVarTypeToVarType(v.type),
nodeId,
isRagVariable: true,
}, ['rag', nodeId, v.variable]))
if (filterVars?.length) {
dataSourceRagVars.push({
nodeId,
title: currNode.data?.title,
vars: filterVars.map((v) => {
return {
variable: `rag.${nodeId}.${v.variable}`,
type: inputVarTypeToVarType(v.type),
description: v.label,
isRagVariable: true,
} as Var
}),
})
}
}
const availableVars = [...getNodeAvailableVars({
parentNode: iterationNode,
beforeNodes: availableNodes,
isChatMode,
filterVar,
hideEnv,
hideChatVar,
})
}), ...dataSourceRagVars]
return {
availableVars,
availableNodes,
availableNodesWithParent: availableNodes,
availableNodesWithParent: [
...availableNodes,
...(isDataSourceNode ? [currNode] : []),
],
}
}

68
web/app/components/workflow/nodes/_base/hooks/use-node-help-link.ts
View File

@@ -1,67 +1,15 @@
import { useMemo } from 'react'
import { useDocLink, useGetLanguage } from '@/context/i18n'
import { BlockEnum } from '@/app/components/workflow/types'
import type { BlockEnum } from '@/app/components/workflow/types'
import { useNodesMetaData } from '@/app/components/workflow/hooks'
export const useNodeHelpLink = (nodeType: BlockEnum) => {
const language = useGetLanguage()
const docLink = useDocLink()
const prefixLink = useMemo(() => {
return docLink('/guides/workflow/node/')
}, [language])
const linkMap = useMemo(() => {
if (language === 'zh_Hans') {
return {
[BlockEnum.Start]: 'start',
[BlockEnum.End]: 'end',
[BlockEnum.Answer]: 'answer',
[BlockEnum.LLM]: 'llm',
[BlockEnum.KnowledgeRetrieval]: 'knowledge-retrieval',
[BlockEnum.QuestionClassifier]: 'question-classifier',
[BlockEnum.IfElse]: 'ifelse',
[BlockEnum.Code]: 'code',
[BlockEnum.TemplateTransform]: 'template',
[BlockEnum.VariableAssigner]: 'variable-assigner',
[BlockEnum.VariableAggregator]: 'variable-aggregator',
[BlockEnum.Assigner]: 'variable-assigner',
[BlockEnum.Iteration]: 'iteration',
[BlockEnum.Loop]: 'loop',
[BlockEnum.ParameterExtractor]: 'parameter-extractor',
[BlockEnum.HttpRequest]: 'http-request',
[BlockEnum.Tool]: 'tools',
[BlockEnum.DocExtractor]: 'doc-extractor',
[BlockEnum.ListFilter]: 'list-operator',
[BlockEnum.Agent]: 'agent',
}
}
const availableNodesMetaData = useNodesMetaData()
return {
[BlockEnum.Start]: 'start',
[BlockEnum.End]: 'end',
[BlockEnum.Answer]: 'answer',
[BlockEnum.LLM]: 'llm',
[BlockEnum.KnowledgeRetrieval]: 'knowledge-retrieval',
[BlockEnum.QuestionClassifier]: 'question-classifier',
[BlockEnum.IfElse]: 'ifelse',
[BlockEnum.Code]: 'code',
[BlockEnum.TemplateTransform]: 'template',
[BlockEnum.VariableAssigner]: 'variable-assigner',
[BlockEnum.VariableAggregator]: 'variable-aggregator',
[BlockEnum.Assigner]: 'variable-assigner',
[BlockEnum.Iteration]: 'iteration',
[BlockEnum.Loop]: 'loop',
[BlockEnum.ParameterExtractor]: 'parameter-extractor',
[BlockEnum.HttpRequest]: 'http-request',
[BlockEnum.Tool]: 'tools',
[BlockEnum.DocExtractor]: 'doc-extractor',
[BlockEnum.ListFilter]: 'list-operator',
[BlockEnum.Agent]: 'agent',
}
}, [language]) as Record<string, string>
const link = useMemo(() => {
const result = availableNodesMetaData?.nodesMap?.[nodeType]?.metaData.helpLinkUri || ''
const link = linkMap[nodeType]
return result
}, [availableNodesMetaData, nodeType])
if (!link)
return ''
return `${prefixLink}${link}`
return link
}

42
web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts
View File

@@ -11,7 +11,6 @@ import { getNodeInfoById, isConversationVar, isENV, isSystemVar, toNodeOutputVar
import type { CommonNodeType, InputVar, ValueSelector, Var, Variable } from '@/app/components/workflow/types'
import { BlockEnum, InputVarType, NodeRunningStatus, VarType } from '@/app/components/workflow/types'
import { useStore as useAppStore } from '@/app/components/app/store'
import { useStore, useWorkflowStore } from '@/app/components/workflow/store'
import { fetchNodeInspectVars, getIterationSingleNodeRunUrl, getLoopSingleNodeRunUrl, singleNodeRun } from '@/service/workflow'
import Toast from '@/app/components/base/toast'
@@ -52,6 +51,8 @@ import {
} from 'reactflow'
import { useInvalidLastRun } from '@/service/use-workflow'
import useInspectVarsCrud from '../../../hooks/use-inspect-vars-crud'
import type { FlowType } from '@/types/common'
import useMatchSchemaType from '../components/variable/use-match-schema-type'
// eslint-disable-next-line ts/no-unsafe-function-type
const checkValidFns: Record<BlockEnum, Function> = {
[BlockEnum.LLM]: checkLLMValid,
@@ -72,6 +73,8 @@ const checkValidFns: Record<BlockEnum, Function> = {
export type Params<T> = {
id: string
flowId: string
flowType: FlowType
data: CommonNodeType<T>
defaultRunInputData: Record<string, any>
moreDataForCheckValid?: any
@@ -108,6 +111,8 @@ const varTypeToInputVarType = (type: VarType, {
const useOneStepRun = <T>({
id,
flowId,
flowType,
data,
defaultRunInputData,
moreDataForCheckValid,
@@ -126,9 +131,26 @@ const useOneStepRun = <T>({
const availableNodes = getBeforeNodesInSameBranch(id)
const availableNodesIncludeParent = getBeforeNodesInSameBranchIncludeParent(id)
const allOutputVars = toNodeOutputVars(availableNodes, isChatMode, undefined, undefined, conversationVariables)
const workflowStore = useWorkflowStore()
const { schemaTypeDefinitions } = useMatchSchemaType()
const getVar = (valueSelector: ValueSelector): Var | undefined => {
const isSystem = valueSelector[0] === 'sys'
const {
buildInTools,
customTools,
workflowTools,
mcpTools,
dataSourceList,
} = workflowStore.getState()
const allPluginInfoList = {
buildInTools,
customTools,
workflowTools,
mcpTools,
dataSourceList: dataSourceList ?? [],
}
const allOutputVars = toNodeOutputVars(availableNodes, isChatMode, undefined, undefined, conversationVariables, [], allPluginInfoList, schemaTypeDefinitions)
const targetVar = allOutputVars.find(item => isSystem ? !!item.isStartNode : item.nodeId === valueSelector[0])
if (!targetVar)
return undefined
@@ -155,7 +177,6 @@ const useOneStepRun = <T>({
const checkValid = checkValidFns[data.type]
const appId = useAppStore.getState().appDetail?.id
const [runInputData, setRunInputData] = useState<Record<string, any>>(defaultRunInputData || {})
const runInputDataRef = useRef(runInputData)
const handleSetRunInputData = useCallback((data: Record<string, any>) => {
@@ -166,11 +187,10 @@ const useOneStepRun = <T>({
const loopTimes = loopInputKey ? runInputData[loopInputKey]?.length : 0
const store = useStoreApi()
const workflowStore = useWorkflowStore()
const {
setShowSingleRunPanel,
} = workflowStore.getState()
const invalidLastRun = useInvalidLastRun(appId!, id)
const invalidLastRun = useInvalidLastRun(flowType, flowId!, id)
const [runResult, doSetRunResult] = useState<NodeRunResult | null>(null)
const {
appendNodeInspectVars,
@@ -197,7 +217,7 @@ const useOneStepRun = <T>({
}
// run fail may also update the inspect vars when the node set the error default output.
const vars = await fetchNodeInspectVars(appId!, id)
const vars = await fetchNodeInspectVars(flowType, flowId!, id)
const { getNodes } = store.getState()
const nodes = getNodes()
appendNodeInspectVars(id, vars, nodes)
@@ -207,7 +227,7 @@ const useOneStepRun = <T>({
invalidateSysVarValues()
invalidateConversationVarValues() // loop, iteration, variable assigner node can update the conversation variables, but to simple the logic(some nodes may also can update in the future), all nodes refresh.
}
}, [isRunAfterSingleRun, runningStatus, appId, id, store, appendNodeInspectVars, invalidLastRun, isStartNode, invalidateSysVarValues, invalidateConversationVarValues])
}, [isRunAfterSingleRun, runningStatus, flowId, id, store, appendNodeInspectVars, invalidLastRun, isStartNode, invalidateSysVarValues, invalidateConversationVarValues])
const { handleNodeDataUpdate }: { handleNodeDataUpdate: (data: any) => void } = useNodeDataUpdate()
const setNodeRunning = () => {
@@ -306,14 +326,14 @@ const useOneStepRun = <T>({
else {
postData.inputs = submitData
}
res = await singleNodeRun(appId!, id, postData) as any
res = await singleNodeRun(flowType, flowId!, id, postData) as any
}
else if (isIteration) {
setIterationRunResult([])
let _iterationResult: NodeTracing[] = []
let _runResult: any = null
ssePost(
getIterationSingleNodeRunUrl(isChatMode, appId!, id),
getIterationSingleNodeRunUrl(flowType, isChatMode, flowId!, id),
{ body: { inputs: submitData } },
{
onWorkflowStarted: noop,
@@ -416,7 +436,7 @@ const useOneStepRun = <T>({
let _loopResult: NodeTracing[] = []
let _runResult: any = null
ssePost(
getLoopSingleNodeRunUrl(isChatMode, appId!, id),
getLoopSingleNodeRunUrl(flowType, isChatMode, flowId!, id),
{ body: { inputs: submitData } },
{
onWorkflowStarted: noop,
@@ -634,7 +654,6 @@ const useOneStepRun = <T>({
}
return {
appId,
isShowSingleRun,
hideSingleRun,
showSingleRun,
@@ -649,6 +668,7 @@ const useOneStepRun = <T>({
runInputDataRef,
setRunInputData: handleSetRunInputData,
runResult,
setRunResult: doSetRunResult,
iterationRunResult,
loopRunResult,
setNodeRunning,