feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/app/components/workflow/nodes/data-source/hooks/use-config.ts
+++ b/web/app/components/workflow/nodes/data-source/hooks/use-config.ts
@@ -0,0 +1,117 @@
+import {
+  useCallback,
+  useEffect,
+  useMemo,
+} from 'react'
+import { useStoreApi } from 'reactflow'
+import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
+import type {
+  DataSourceNodeType,
+  ToolVarInputs,
+} from '../types'
+
+export const useConfig = (id: string, dataSourceList?: any[]) => {
+  const store = useStoreApi()
+  const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
+
+  const getNodeData = useCallback(() => {
+    const { getNodes } = store.getState()
+    const nodes = getNodes()
+
+    return nodes.find(node => node.id === id)
+  }, [store, id])
+
+  const handleNodeDataUpdate = useCallback((data: Partial<DataSourceNodeType>) => {
+    handleNodeDataUpdateWithSyncDraft({
+      id,
+      data,
+    })
+  }, [id, handleNodeDataUpdateWithSyncDraft])
+
+  const handleLocalFileDataSourceInit = useCallback(() => {
+    const nodeData = getNodeData()
+
+    if (nodeData?.data._dataSourceStartToAdd && nodeData?.data.provider_type === 'local_file') {
+      handleNodeDataUpdate({
+        ...nodeData.data,
+        _dataSourceStartToAdd: false,
+      })
+    }
+  }, [getNodeData, handleNodeDataUpdate])
+
+  useEffect(() => {
+    handleLocalFileDataSourceInit()
+  }, [handleLocalFileDataSourceInit])
+
+  const handleFileExtensionsChange = useCallback((fileExtensions: string[]) => {
+    const nodeData = getNodeData()
+    handleNodeDataUpdate({
+      ...nodeData?.data,
+      fileExtensions,
+    })
+  }, [handleNodeDataUpdate, getNodeData])
+
+  const handleParametersChange = useCallback((datasource_parameters: ToolVarInputs) => {
+    const nodeData = getNodeData()
+    handleNodeDataUpdate({
+      ...nodeData?.data,
+      datasource_parameters,
+    })
+  }, [handleNodeDataUpdate, getNodeData])
+
+  const outputSchema = useMemo(() => {
+    const nodeData = getNodeData()
+    if (!nodeData?.data || !dataSourceList) return []
+
+    const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
+    const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
+    const output_schema = currentDataSourceItem?.output_schema
+
+    const res: any[] = []
+    if (!output_schema || !output_schema.properties)
+      return res
+
+    Object.keys(output_schema.properties).forEach((outputKey) => {
+      const output = output_schema.properties[outputKey]
+      const type = output.type
+      if (type === 'object') {
+        res.push({
+          name: outputKey,
+          value: output,
+        })
+      }
+      else {
+        res.push({
+          name: outputKey,
+          type: output.type === 'array'
+            ? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]`
+            : `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`,
+          description: output.description,
+        })
+      }
+    })
+    return res
+  }, [getNodeData, dataSourceList])
+
+  const hasObjectOutput = useMemo(() => {
+    const nodeData = getNodeData()
+    if (!nodeData?.data || !dataSourceList) return false
+
+    const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
+    const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
+    const output_schema = currentDataSourceItem?.output_schema
+
+    if (!output_schema || !output_schema.properties)
+      return false
+
+    const properties = output_schema.properties
+    return Object.keys(properties).some(key => properties[key].type === 'object')
+  }, [getNodeData, dataSourceList])
+
+  return {
+    handleFileExtensionsChange,
+    handleParametersChange,
+    outputSchema,
+    hasObjectOutput,
+  }
+}