feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/workflow/nodes/llm/components/json-schema-config-modal/code-editor.tsx

@@ -15,6 +15,9 @@ type CodeEditorProps = {
   editorWrapperClassName?: string
   readOnly?: boolean
   hideTopMenu?: boolean
+  onFocus?: () => void
+  onBlur?: () => void
+  topContent?: React.ReactNode
 } & React.HTMLAttributes<HTMLDivElement>
 
 const CodeEditor: FC<CodeEditorProps> = ({
@@ -24,7 +27,10 @@ const CodeEditor: FC<CodeEditorProps> = ({
   editorWrapperClassName,
   readOnly = false,
   hideTopMenu = false,
+  topContent,
   className,
+  onFocus,
+  onBlur,
 }) => {
   const { t } = useTranslation()
   const { theme } = useTheme()
@@ -45,6 +51,14 @@ const CodeEditor: FC<CodeEditorProps> = ({
   const handleEditorDidMount = useCallback((editor: any, monaco: any) => {
     editorRef.current = editor
     monacoRef.current = monaco
+
+    editor.onDidFocusEditorText(() => {
+      onFocus?.()
+    })
+    editor.onDidBlurEditorText(() => {
+      onBlur?.()
+    })
+
     monaco.editor.defineTheme('light-theme', {
       base: 'vs',
       inherit: true,
@@ -127,6 +141,7 @@ const CodeEditor: FC<CodeEditorProps> = ({
           </div>
         </div>
       )}
+      {topContent}
       <div className={classNames('relative overflow-hidden', editorWrapperClassName)}>
         <Editor
           defaultLanguage='json'

web/app/components/workflow/nodes/llm/components/json-schema-config-modal/schema-editor.tsx

@@ -1,6 +1,7 @@
 import React, { type FC } from 'react'
 import CodeEditor from './code-editor'
 import cn from '@/utils/classnames'
+import LargeDataAlert from '@/app/components/workflow/variable-inspect/large-data-alert'
 
 type SchemaEditorProps = {
   schema: string
@@ -8,6 +9,9 @@ type SchemaEditorProps = {
   hideTopMenu?: boolean
   className?: string
   readonly?: boolean
+  onFocus?: () => void
+  onBlur?: () => void
+  isTruncated?: boolean
 }
 
 const SchemaEditor: FC<SchemaEditorProps> = ({
@@ -16,6 +20,9 @@ const SchemaEditor: FC<SchemaEditorProps> = ({
   hideTopMenu,
   className,
   readonly = false,
+  onFocus,
+  onBlur,
+  isTruncated,
 }) => {
   return (
     <CodeEditor
@@ -25,6 +32,9 @@ const SchemaEditor: FC<SchemaEditorProps> = ({
       value={schema}
       onUpdate={onUpdate}
       hideTopMenu={hideTopMenu}
+      onFocus={onFocus}
+      onBlur={onBlur}
+      topContent={isTruncated && <LargeDataAlert className='mx-1 mb-3 mt-[-4px]' />}
     />
   )
 }

web/app/components/workflow/nodes/llm/default.ts

@@ -2,7 +2,7 @@
 import { BlockEnum, EditionType } from '../../types'
 import { type NodeDefault, type PromptItem, PromptRole } from '../../types'
 import type { LLMNodeType } from './types'
-import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/app/components/workflow/blocks'
+import { genNodeMetaData } from '@/app/components/workflow/utils'
 
 const RETRIEVAL_OUTPUT_STRUCT = `{
   "content": "",
@@ -26,7 +26,12 @@ const RETRIEVAL_OUTPUT_STRUCT = `{
 
 const i18nPrefix = 'workflow.errorMsg'
 
+const metaData = genNodeMetaData({
+  sort: 1,
+  type: BlockEnum.LLM,
+})
 const nodeDefault: NodeDefault<LLMNodeType> = {
+  metaData,
   defaultValue: {
     model: {
       provider: '',
@@ -52,16 +57,6 @@ const nodeDefault: NodeDefault<LLMNodeType> = {
     '#context#': [RETRIEVAL_OUTPUT_STRUCT],
     '#files#': [],
   },
-  getAvailablePrevNodes(isChatMode: boolean) {
-    const nodes = isChatMode
-      ? ALL_CHAT_AVAILABLE_BLOCKS
-      : ALL_COMPLETION_AVAILABLE_BLOCKS.filter(type => type !== BlockEnum.End)
-    return nodes
-  },
-  getAvailableNextNodes(isChatMode: boolean) {
-    const nodes = isChatMode ? ALL_CHAT_AVAILABLE_BLOCKS : ALL_COMPLETION_AVAILABLE_BLOCKS
-    return nodes
-  },
   checkValid(payload: LLMNodeType, t: any) {
     let errorMessages = ''
     if (!errorMessages && !payload.model.provider)

web/app/components/workflow/nodes/llm/types.ts

@@ -29,6 +29,7 @@ export enum Type {
   arrayString = 'array[string]',
   arrayNumber = 'array[number]',
   arrayObject = 'array[object]',
+  file = 'file',
 }
 
 export enum ArrayType {
@@ -55,6 +56,7 @@ export type Field = {
   items?: ArrayItems // Array has items. Define the item type
   enum?: SchemaEnumType // Enum values
   additionalProperties?: false // Required in object by api. Just set false
+  schemaType?: string // an another type defined in backend schemas
 }
 
 export type StructuredOutput = {

web/app/components/workflow/nodes/llm/utils.ts

@@ -10,6 +10,7 @@ export const checkNodeValid = (_payload: LLMNodeType) => {
 
 export const getFieldType = (field: Field) => {
   const { type, items } = field
+  if(field.schemaType === 'file') return 'file'
   if (type !== Type.array || !items)
     return type