admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: knowledge pipeline (#25360)

Browse Source 
Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
web/app/components/workflow/hooks/use-checklist.ts
View File

@@ -12,41 +12,49 @@ import type {
ValueSelector,
} from '../types'
import { BlockEnum } from '../types'
import { useStore } from '../store'
import {
useStore,
useWorkflowStore,
} from '../store'
import {
getDataSourceCheckParams,
getToolCheckParams,
getValidTreeNodes,
} from '../utils'
import {
CUSTOM_NODE,
} from '../constants'
import {
useGetToolIcon,
useWorkflow,
} from '../hooks'
import type { ToolNodeType } from '../nodes/tool/types'
import { useIsChatMode } from './use-workflow'
import { useNodesExtraData } from './use-nodes-data'
import type { DataSourceNodeType } from '../nodes/data-source/types'
import { useNodesMetaData } from './use-nodes-meta-data'
import { useToastContext } from '@/app/components/base/toast'
import { CollectionType } from '@/app/components/tools/types'
import { useGetLanguage } from '@/context/i18n'
import type { AgentNodeType } from '../nodes/agent/types'
import { useStrategyProviders } from '@/service/use-strategy'
import { canFindTool } from '@/utils'
import { useDatasetsDetailStore } from '../datasets-detail-store/store'
import type { KnowledgeRetrievalNodeType } from '../nodes/knowledge-retrieval/types'
import type { DataSet } from '@/models/datasets'
import { fetchDatasets } from '@/service/datasets'
import { MAX_TREE_DEPTH } from '@/config'
import useNodesAvailableVarList from './use-nodes-available-var-list'
import { getNodeUsedVars, isConversationVar, isENV, isSystemVar } from '../nodes/_base/components/variable/utils'
import useNodesAvailableVarList, { useGetNodesAvailableVarList } from './use-nodes-available-var-list'
import { getNodeUsedVars, isSpecialVar } from '../nodes/_base/components/variable/utils'
export const useChecklist = (nodes: Node[], edges: Edge[]) => {
const { t } = useTranslation()
const language = useGetLanguage()
const nodesExtraData = useNodesExtraData()
const isChatMode = useIsChatMode()
const { nodesMap: nodesExtraData } = useNodesMetaData()
const buildInTools = useStore(s => s.buildInTools)
const customTools = useStore(s => s.customTools)
const workflowTools = useStore(s => s.workflowTools)
const dataSourceList = useStore(s => s.dataSourceList)
const { data: strategyProviders } = useStrategyProviders()
const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail)
const { getStartNodes } = useWorkflow()
const getToolIcon = useGetToolIcon()
const map = useNodesAvailableVarList(nodes)
@@ -69,28 +77,28 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
const needWarningNodes = useMemo(() => {
const list = []
const { validNodes } = getValidTreeNodes(nodes.filter(node => node.type === CUSTOM_NODE), edges)
const filteredNodes = nodes.filter(node => node.type === CUSTOM_NODE)
const startNodes = getStartNodes(filteredNodes)
const validNodesFlattened = startNodes.map(startNode => getValidTreeNodes(startNode, filteredNodes, edges))
const validNodes = validNodesFlattened.reduce((acc, curr) => {
if (curr.validNodes)
acc.push(...curr.validNodes)
return acc
}, [] as Node[])
for (let i = 0; i < nodes.length; i++) {
const node = nodes[i]
let toolIcon
for (let i = 0; i < filteredNodes.length; i++) {
const node = filteredNodes[i]
let moreDataForCheckValid
let usedVars: ValueSelector[] = []
if (node.data.type === BlockEnum.Tool) {
const { provider_type } = node.data
if (node.data.type === BlockEnum.Tool)
moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language)
if (provider_type === CollectionType.builtIn)
toolIcon = buildInTools.find(tool => canFindTool(tool.id, node.data.provider_id || ''))?.icon
if (provider_type === CollectionType.custom)
toolIcon = customTools.find(tool => tool.id === node.data.provider_id)?.icon
if (node.data.type === BlockEnum.DataSource)
moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language)
if (provider_type === CollectionType.workflow)
toolIcon = workflowTools.find(tool => tool.id === node.data.provider_id)?.icon
}
else if (node.data.type === BlockEnum.Agent) {
const toolIcon = getToolIcon(node.data)
if (node.data.type === BlockEnum.Agent) {
const data = node.data as AgentNodeType
const isReadyForCheckValid = !!strategyProviders
const provider = strategyProviders?.find(provider => provider.declaration.identity.name === data.agent_strategy_provider_name)
@@ -108,16 +116,14 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
if (node.type === CUSTOM_NODE) {
const checkData = getCheckData(node.data)
let { errorMessage } = nodesExtraData[node.data.type].checkValid(checkData, t, moreDataForCheckValid)
let { errorMessage } = nodesExtraData![node.data.type].checkValid(checkData, t, moreDataForCheckValid)
if (!errorMessage) {
const availableVars = map[node.id].availableVars
for (const variable of usedVars) {
const isEnv = isENV(variable)
const isConvVar = isConversationVar(variable)
const isSysVar = isSystemVar(variable)
if (!isEnv && !isConvVar && !isSysVar) {
const isSpecialVars = isSpecialVar(variable[0])
if (!isSpecialVars) {
const usedNode = availableVars.find(v => v.nodeId === variable?.[0])
if (usedNode) {
const usedVar = usedNode.vars.find(v => v.variable === variable?.[1])
@@ -130,7 +136,6 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
}
}
}
if (errorMessage || !validNodes.find(n => n.id === node.id)) {
list.push({
id: node.id,
@@ -144,26 +149,21 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
}
}
if (isChatMode && !nodes.find(node => node.data.type === BlockEnum.Answer)) {
list.push({
id: 'answer-need-added',
type: BlockEnum.Answer,
title: t('workflow.blocks.answer'),
errorMessage: t('workflow.common.needAnswerNode'),
})
}
const isRequiredNodesType = Object.keys(nodesExtraData!).filter((key: any) => (nodesExtraData as any)[key].metaData.isRequired)
if (!isChatMode && !nodes.find(node => node.data.type === BlockEnum.End)) {
list.push({
id: 'end-need-added',
type: BlockEnum.End,
title: t('workflow.blocks.end'),
errorMessage: t('workflow.common.needEndNode'),
})
}
isRequiredNodesType.forEach((type: string) => {
if (!filteredNodes.find(node => node.data.type === type)) {
list.push({
id: `${type}-need-added`,
type,
title: t(`workflow.blocks.${type}`),
errorMessage: t('workflow.common.needAdd', { node: t(`workflow.blocks.${type}`) }),
})
}
})
return list
}, [nodes, edges, isChatMode, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData])
}, [nodes, getStartNodes, nodesExtraData, edges, buildInTools, customTools, workflowTools, language, dataSourceList, getToolIcon, strategyProviders, getCheckData, t, map])
return needWarningNodes
}
@@ -171,16 +171,15 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
export const useChecklistBeforePublish = () => {
const { t } = useTranslation()
const language = useGetLanguage()
const buildInTools = useStore(s => s.buildInTools)
const customTools = useStore(s => s.customTools)
const workflowTools = useStore(s => s.workflowTools)
const { notify } = useToastContext()
const isChatMode = useIsChatMode()
const store = useStoreApi()
const nodesExtraData = useNodesExtraData()
const { nodesMap: nodesExtraData } = useNodesMetaData()
const { data: strategyProviders } = useStrategyProviders()
const updateDatasetsDetail = useDatasetsDetailStore(s => s.updateDatasetsDetail)
const updateTime = useRef(0)
const { getStartNodes } = useWorkflow()
const workflowStore = useWorkflowStore()
const { getNodesAvailableVarList } = useGetNodesAvailableVarList()
const getCheckData = useCallback((data: CommonNodeType<{}>, datasets: DataSet[]) => {
let checkData = data
@@ -208,18 +207,31 @@ export const useChecklistBeforePublish = () => {
getNodes,
edges,
} = store.getState()
const nodes = getNodes().filter(node => node.type === CUSTOM_NODE)
const {
validNodes,
maxDepth,
} = getValidTreeNodes(nodes.filter(node => node.type === CUSTOM_NODE), edges)
dataSourceList,
buildInTools,
customTools,
workflowTools,
} = workflowStore.getState()
const nodes = getNodes()
const filteredNodes = nodes.filter(node => node.type === CUSTOM_NODE)
const startNodes = getStartNodes(filteredNodes)
const validNodesFlattened = startNodes.map(startNode => getValidTreeNodes(startNode, filteredNodes, edges))
const validNodes = validNodesFlattened.reduce((acc, curr) => {
if (curr.validNodes)
acc.push(...curr.validNodes)
return acc
}, [] as Node[])
const maxDepthArr = validNodesFlattened.map(item => item.maxDepth)
if (maxDepth > MAX_TREE_DEPTH) {
notify({ type: 'error', message: t('workflow.common.maxTreeDepth', { depth: MAX_TREE_DEPTH }) })
return false
for (let i = 0; i < maxDepthArr.length; i++) {
if (maxDepthArr[i] > MAX_TREE_DEPTH) {
notify({ type: 'error', message: t('workflow.common.maxTreeDepth', { depth: MAX_TREE_DEPTH }) })
return false
}
}
// Before publish, we need to fetch datasets detail, in case of the settings of datasets have been changed
const knowledgeRetrievalNodes = nodes.filter(node => node.data.type === BlockEnum.KnowledgeRetrieval)
const knowledgeRetrievalNodes = filteredNodes.filter(node => node.data.type === BlockEnum.KnowledgeRetrieval)
const allDatasetIds = knowledgeRetrievalNodes.reduce<string[]>((acc, node) => {
return Array.from(new Set([...acc, ...(node.data as CommonNodeType<KnowledgeRetrievalNodeType>).dataset_ids]))
}, [])
@@ -236,13 +248,17 @@ export const useChecklistBeforePublish = () => {
updateDatasetsDetail(datasetsDetail)
}
}
for (let i = 0; i < nodes.length; i++) {
const node = nodes[i]
const map = getNodesAvailableVarList(nodes)
for (let i = 0; i < filteredNodes.length; i++) {
const node = filteredNodes[i]
let moreDataForCheckValid
let usedVars: ValueSelector[] = []
if (node.data.type === BlockEnum.Tool)
moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language)
if (node.data.type === BlockEnum.DataSource)
moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language)
if (node.data.type === BlockEnum.Agent) {
const data = node.data as AgentNodeType
const isReadyForCheckValid = !!strategyProviders
@@ -255,33 +271,55 @@ export const useChecklistBeforePublish = () => {
isReadyForCheckValid,
}
}
else {
usedVars = getNodeUsedVars(node).filter(v => v.length > 0)
}
const checkData = getCheckData(node.data, datasets)
const { errorMessage } = nodesExtraData[node.data.type as BlockEnum].checkValid(checkData, t, moreDataForCheckValid)
const { errorMessage } = nodesExtraData![node.data.type as BlockEnum].checkValid(checkData, t, moreDataForCheckValid)
if (errorMessage) {
notify({ type: 'error', message: `[${node.data.title}] ${errorMessage}` })
return false
}
const availableVars = map[node.id].availableVars
for (const variable of usedVars) {
const isSpecialVars = isSpecialVar(variable[0])
if (!isSpecialVars) {
const usedNode = availableVars.find(v => v.nodeId === variable?.[0])
if (usedNode) {
const usedVar = usedNode.vars.find(v => v.variable === variable?.[1])
if (!usedVar) {
notify({ type: 'error', message: `[${node.data.title}] ${t('workflow.errorMsg.invalidVariable')}` })
return false
}
}
else {
notify({ type: 'error', message: `[${node.data.title}] ${t('workflow.errorMsg.invalidVariable')}` })
return false
}
}
}
if (!validNodes.find(n => n.id === node.id)) {
notify({ type: 'error', message: `[${node.data.title}] ${t('workflow.common.needConnectTip')}` })
return false
}
}
if (isChatMode && !nodes.find(node => node.data.type === BlockEnum.Answer)) {
notify({ type: 'error', message: t('workflow.common.needAnswerNode') })
return false
}
const isRequiredNodesType = Object.keys(nodesExtraData!).filter((key: any) => (nodesExtraData as any)[key].metaData.isRequired)
if (!isChatMode && !nodes.find(node => node.data.type === BlockEnum.End)) {
notify({ type: 'error', message: t('workflow.common.needEndNode') })
return false
for (let i = 0; i < isRequiredNodesType.length; i++) {
const type = isRequiredNodesType[i]
if (!filteredNodes.find(node => node.data.type === type)) {
notify({ type: 'error', message: t('workflow.common.needAdd', { node: t(`workflow.blocks.${type}`) }) })
return false
}
}
return true
}, [store, isChatMode, notify, t, buildInTools, customTools, workflowTools, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData])
}, [store, notify, t, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData, getStartNodes, workflowStore])
return {
handleCheckBeforePublish,