feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions

View File

@@ -0,0 +1,116 @@
import React, { useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { RiBracesLine, RiEyeLine } from '@remixicon/react'
import Textarea from '@/app/components/base/textarea'
import { Markdown } from '@/app/components/base/markdown'
import SchemaEditor from '@/app/components/workflow/nodes/llm/components/json-schema-config-modal/schema-editor'
import { SegmentedControl } from '@/app/components/base/segmented-control'
import cn from '@/utils/classnames'
import { ChunkCardList } from '@/app/components/rag-pipeline/components/chunk-card-list'
import type { ChunkInfo } from '@/app/components/rag-pipeline/components/chunk-card-list/types'
import type { ParentMode } from '@/models/datasets'
import { ChunkingMode } from '@/models/datasets'
import { PreviewType, ViewMode } from './types'
import type { VarType } from '../types'
type DisplayContentProps = {
previewType: PreviewType
varType: VarType
schemaType?: string
mdString?: string
jsonString?: string
readonly: boolean
handleTextChange?: (value: string) => void
handleEditorChange?: (value: string) => void
className?: string
}
const DisplayContent = (props: DisplayContentProps) => {
const { previewType, varType, schemaType, mdString, jsonString, readonly, handleTextChange, handleEditorChange, className } = props
const [viewMode, setViewMode] = useState<ViewMode>(ViewMode.Code)
const [isFocused, setIsFocused] = useState(false)
const { t } = useTranslation()
const chunkType = useMemo(() => {
if (previewType !== PreviewType.Chunks || !schemaType)
return undefined
if (schemaType === 'general_structure')
return ChunkingMode.text
if (schemaType === 'parent_child_structure')
return ChunkingMode.parentChild
if (schemaType === 'qa_structure')
return ChunkingMode.qa
}, [previewType, schemaType])
const parentMode = useMemo(() => {
if (previewType !== PreviewType.Chunks || !schemaType || !jsonString)
return undefined
if (schemaType === 'parent_child_structure')
return JSON.parse(jsonString!)?.parent_mode as ParentMode
return undefined
}, [previewType, schemaType, jsonString])
return (
<div className={cn('flex h-full flex-col rounded-[10px] bg-components-input-bg-normal', isFocused && 'bg-components-input-bg-active outline outline-1 outline-components-input-border-active', className)}>
<div className='flex shrink-0 items-center justify-end p-1'>
{previewType === PreviewType.Markdown && (
<div className='system-xs-semibold-uppercase flex grow items-center px-2 py-0.5 text-text-secondary'>
{previewType.toUpperCase()}
</div>
)}
{previewType === PreviewType.Chunks && (
<div className='system-xs-semibold-uppercase flex grow items-center px-2 py-0.5 text-text-secondary'>
{varType.toUpperCase()}{schemaType ? `(${schemaType})` : ''}
</div>
)}
<SegmentedControl
options={[
{ value: ViewMode.Code, text: t('workflow.nodes.templateTransform.code'), Icon: RiBracesLine },
{ value: ViewMode.Preview, text: t('workflow.common.preview'), Icon: RiEyeLine },
]}
value={viewMode}
onChange={setViewMode}
size='small'
padding='with'
activeClassName='!text-text-accent-light-mode-only'
btnClassName='!pl-1.5 !pr-0.5 gap-[3px]'
className='shrink-0'
/>
</div>
<div className='flex flex-1 overflow-auto rounded-b-[10px] pl-3 pr-1'>
{viewMode === ViewMode.Code && (
previewType === PreviewType.Markdown
? <Textarea
readOnly={readonly}
disabled={readonly}
className='h-full border-none bg-transparent p-0 text-text-secondary hover:bg-transparent focus:bg-transparent focus:shadow-none'
value={mdString as any}
onChange={e => handleTextChange?.(e.target.value)}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
/>
: <SchemaEditor
readonly={readonly}
className='overflow-y-auto bg-transparent'
hideTopMenu
schema={jsonString!}
onUpdate={handleEditorChange!}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
/>
)}
{viewMode === ViewMode.Preview && (
previewType === PreviewType.Markdown
? <Markdown className='grow overflow-auto rounded-lg px-4 py-3' content={(mdString ?? '') as string} />
: <ChunkCardList
chunkType={chunkType!}
parentMode={parentMode}
chunkInfo={JSON.parse(jsonString!) as ChunkInfo}
/>
)}
</div>
</div>
)
}
export default React.memo(DisplayContent)

View File

@@ -40,7 +40,7 @@ const Group = ({
const { t } = useTranslation()
const [isCollapsed, setIsCollapsed] = useState(false)
const toolIcon = useToolIcon(nodeData?.nodePayload as any)
const toolIcon = useToolIcon(nodeData?.nodePayload)
const isEnv = varType === VarInInspectType.environment
const isChatVar = varType === VarInInspectType.conversation

View File

@@ -0,0 +1,33 @@
'use client'
import { RiInformation2Fill } from '@remixicon/react'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
import { useTranslation } from 'react-i18next'
type Props = {
textHasNoExport?: boolean
downloadUrl?: string
className?: string
}
const LargeDataAlert: FC<Props> = ({
textHasNoExport,
downloadUrl,
className,
}) => {
const { t } = useTranslation()
const text = textHasNoExport ? t('workflow.debug.variableInspect.largeDataNoExport') : t('workflow.debug.variableInspect.largeData')
return (
<div className={cn('flex h-8 items-center justify-between rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-bg-blur px-2 shadow-xs', className)}>
<div className='flex h-full w-0 grow items-center space-x-1'>
<RiInformation2Fill className='size-4 shrink-0 text-text-accent' />
<div className='system-xs-regular w-0 grow truncate text-text-primary'>{text}</div>
</div>
{downloadUrl && (
<div className='system-xs-medium-uppercase ml-1 shrink-0 cursor-pointer text-text-accent'>{t('workflow.debug.variableInspect.export')}</div>
)}
</div>
)
}
export default React.memo(LargeDataAlert)

View File

@@ -14,6 +14,8 @@ import type { VarInInspect } from '@/types/workflow'
import { VarInInspectType } from '@/types/workflow'
import cn from '@/utils/classnames'
import type { NodeProps } from '../types'
import useMatchSchemaType from '../nodes/_base/components/variable/use-match-schema-type'
export type currentVarType = {
nodeId: string
@@ -21,6 +23,7 @@ export type currentVarType = {
title: string
isValueFetched?: boolean
var: VarInInspect
nodeData: NodeProps['data']
}
const Panel: FC = () => {
@@ -114,6 +117,7 @@ const Panel: FC = () => {
title: targetNode.title,
isSingRunRunning: targetNode.isSingRunRunning,
isValueFetched: targetNode.isValueFetched,
nodeData: targetNode.nodePayload,
...(currentVar ? { var: currentVar } : {}),
}
}, [currentFocusNodeId, currentVarId, environmentVariables, conversationVars, systemVars, nodesWithInspectVars])
@@ -130,13 +134,15 @@ const Panel: FC = () => {
setCurrentVarId(node.var.id)
}, [setCurrentFocusNodeId, setCurrentVarId])
const { isLoading, schemaTypeDefinitions } = useMatchSchemaType()
useEffect(() => {
if (currentFocusNodeId && currentVarId) {
if (currentFocusNodeId && currentVarId && !isLoading) {
const targetNode = nodesWithInspectVars.find(node => node.nodeId === currentFocusNodeId)
if (targetNode && !targetNode.isValueFetched)
fetchInspectVarValue([currentFocusNodeId])
fetchInspectVarValue([currentFocusNodeId], schemaTypeDefinitions!)
}
}, [currentFocusNodeId, currentVarId, nodesWithInspectVars, fetchInspectVarValue])
}, [currentFocusNodeId, currentVarId, nodesWithInspectVars, fetchInspectVarValue, schemaTypeDefinitions, isLoading])
if (isEmpty) {
return (

View File

@@ -2,6 +2,7 @@ import { useTranslation } from 'react-i18next'
import {
RiArrowGoBackLine,
RiCloseLine,
RiFileDownloadFill,
RiMenuLine,
RiSparklingFill,
} from '@remixicon/react'
@@ -26,7 +27,7 @@ import GetCodeGeneratorResModal from '../../app/configuration/config/code-genera
import { AppType } from '@/types/app'
import { useHooksStore } from '../hooks-store'
import { useCallback, useMemo } from 'react'
import { useNodesInteractions } from '../hooks'
import { useNodesInteractions, useToolIcon } from '../hooks'
import { CodeLanguage } from '../nodes/code/types'
import useNodeCrud from '../nodes/_base/hooks/use-node-crud'
import type { GenRes } from '@/service/debug'
@@ -52,6 +53,9 @@ const Right = ({
const bottomPanelWidth = useStore(s => s.bottomPanelWidth)
const setShowVariableInspectPanel = useStore(s => s.setShowVariableInspectPanel)
const setCurrentFocusNodeId = useStore(s => s.setCurrentFocusNodeId)
const toolIcon = useToolIcon(currentNodeVar?.nodeData)
const isTruncated = currentNodeVar?.var.is_truncated
const fullContent = currentNodeVar?.var.full_content
const {
resetConversationVar,
@@ -151,6 +155,9 @@ const Right = ({
} as any)
handleHidePromptGenerator()
}, [setInputs, blockType, nodeId, node?.data, handleHidePromptGenerator])
const displaySchemaType = currentNodeVar?.var.schemaType ? (`(${currentNodeVar.var.schemaType})`) : ''
return (
<div className={cn('flex h-full flex-col')}>
{/* header */}
@@ -171,19 +178,32 @@ const Right = ({
/>
)
}
{currentNodeVar.nodeType !== VarInInspectType.environment && currentNodeVar.nodeType !== VarInInspectType.conversation && currentNodeVar.nodeType !== VarInInspectType.system && (
<>
<BlockIcon
className='shrink-0'
type={currentNodeVar.nodeType as BlockEnum}
size='xs'
/>
<div className='system-sm-regular shrink-0 text-text-secondary'>{currentNodeVar.title}</div>
<div className='system-sm-regular shrink-0 text-text-quaternary'>/</div>
</>
)}
{currentNodeVar.nodeType !== VarInInspectType.environment
&& currentNodeVar.nodeType !== VarInInspectType.conversation
&& currentNodeVar.nodeType !== VarInInspectType.system
&& (
<>
<BlockIcon
className='shrink-0'
type={currentNodeVar.nodeType as BlockEnum}
size='xs'
toolIcon={toolIcon}
/>
<div className='system-sm-regular shrink-0 text-text-secondary'>{currentNodeVar.title}</div>
<div className='system-sm-regular shrink-0 text-text-quaternary'>/</div>
</>
)}
<div title={currentNodeVar.var.name} className='system-sm-semibold truncate text-text-secondary'>{currentNodeVar.var.name}</div>
<div className='system-xs-medium ml-1 shrink-0 text-text-tertiary'>{currentNodeVar.var.value_type}</div>
<div className='system-xs-medium ml-1 shrink-0 space-x-2 text-text-tertiary'>
<span>{`${currentNodeVar.var.value_type}${displaySchemaType}`}</span>
{isTruncated && (
<>
<span>·</span>
<span>{((fullContent?.size_bytes || 0) / 1024 / 1024).toFixed(1)}MB</span>
</>
)}
</div>
</>
)}
</div>
@@ -200,20 +220,32 @@ const Right = ({
</div>
</Tooltip>
)}
{currentNodeVar.var.edited && (
{isTruncated && (
<Tooltip popupContent={t('workflow.debug.variableInspect.exportToolTip')}>
<ActionButton>
<a
href={fullContent?.download_url}
target='_blank'
>
<RiFileDownloadFill className='size-4' />
</a>
</ActionButton>
</Tooltip>
)}
{!isTruncated && currentNodeVar.var.edited && (
<Badge>
<span className='ml-[2.5px] mr-[4.5px] h-[3px] w-[3px] rounded bg-text-accent-secondary'></span>
<span className='system-2xs-semibold-uupercase'>{t('workflow.debug.variableInspect.edited')}</span>
</Badge>
)}
{currentNodeVar.var.edited && currentNodeVar.var.type !== VarInInspectType.conversation && (
{!isTruncated && currentNodeVar.var.edited && currentNodeVar.var.type !== VarInInspectType.conversation && (
<Tooltip popupContent={t('workflow.debug.variableInspect.reset')}>
<ActionButton onClick={resetValue}>
<RiArrowGoBackLine className='h-4 w-4' />
</ActionButton>
</Tooltip>
)}
{currentNodeVar.var.edited && currentNodeVar.var.type === VarInInspectType.conversation && (
{!isTruncated && currentNodeVar.var.edited && currentNodeVar.var.type === VarInInspectType.conversation && (
<Tooltip popupContent={t('workflow.debug.variableInspect.resetConversationVar')}>
<ActionButton onClick={handleClear}>
<RiArrowGoBackLine className='h-4 w-4' />
@@ -238,7 +270,13 @@ const Right = ({
<Loading />
</div>
)}
{currentNodeVar && !isValueFetching && <ValueContent currentVar={currentNodeVar.var} handleValueChange={handleValueChange} />}
{currentNodeVar && !isValueFetching && (
<ValueContent
currentVar={currentNodeVar.var}
handleValueChange={handleValueChange}
isTruncated={!!isTruncated}
/>
)}
</div>
{isShowPromptGenerator && (
isCodeBlock

View File

@@ -107,7 +107,7 @@ const VariableInspectTrigger: FC = () => {
className='system-xs-medium flex h-6 cursor-pointer items-center gap-1 rounded-md border-[0.5px] border-effects-highlight bg-components-actionbar-bg px-2 text-text-accent shadow-lg backdrop-blur-sm hover:bg-components-actionbar-bg-accent'
onClick={() => setShowVariableInspectPanel(true)}
>
<RiLoader2Line className='h-4 w-4' />
<RiLoader2Line className='h-4 w-4 animate-spin' />
<span className='text-text-accent'>{t('workflow.debug.variableInspect.trigger.running')}</span>
</div>
{isPreviewRunning && (

View File

@@ -1 +1,13 @@
export const EVENT_WORKFLOW_STOP = 'WORKFLOW_STOP'
export const CHUNK_SCHEMA_TYPES = ['general_structure', 'parent_child_structure', 'qa_structure']
export enum ViewMode {
Code = 'code',
Preview = 'preview',
}
export enum PreviewType {
Markdown = 'markdown',
Chunks = 'chunks',
}

View File

@@ -1,4 +1,4 @@
import { useEffect, useRef, useState } from 'react'
import React, { useEffect, useMemo, useRef, useState } from 'react'
import { useDebounceFn } from 'ahooks'
import Textarea from '@/app/components/base/textarea'
import SchemaEditor from '@/app/components/workflow/nodes/llm/components/json-schema-config-modal/schema-editor'
@@ -12,7 +12,6 @@ import {
import {
validateJSONSchema,
} from '@/app/components/workflow/variable-inspect/utils'
import { useFeatures } from '@/app/components/base/features/hooks'
import { getProcessedFiles, getProcessedFilesFromResponse } from '@/app/components/base/file-uploader/utils'
import { JSON_SCHEMA_MAX_DEPTH } from '@/config'
import { TransferMethod } from '@/types/app'
@@ -21,16 +20,23 @@ import { SupportUploadFileTypes } from '@/app/components/workflow/types'
import type { VarInInspect } from '@/types/workflow'
import { VarInInspectType } from '@/types/workflow'
import cn from '@/utils/classnames'
import LargeDataAlert from './large-data-alert'
import BoolValue from '../panel/chat-variable-panel/components/bool-value'
import { useStore } from '@/app/components/workflow/store'
import { PreviewMode } from '../../base/features/types'
import DisplayContent from './display-content'
import { CHUNK_SCHEMA_TYPES, PreviewType } from './types'
type Props = {
currentVar: VarInInspect
handleValueChange: (varId: string, value: any) => void
isTruncated: boolean
}
const ValueContent = ({
currentVar,
handleValueChange,
isTruncated,
}: Props) => {
const contentContainerRef = useRef<HTMLDivElement>(null)
const errorMessageRef = useRef<HTMLDivElement>(null)
@@ -43,6 +49,13 @@ const ValueContent = ({
const showFileEditor = isSysFiles || currentVar.value_type === 'file' || currentVar.value_type === 'array[file]'
const textEditorDisabled = currentVar.type === VarInInspectType.environment || (currentVar.type === VarInInspectType.system && currentVar.name !== 'query' && currentVar.name !== 'files')
const JSONEditorDisabled = currentVar.value_type === 'array[any]'
const fileUploadConfig = useStore(s => s.fileUploadConfig)
const hasChunks = useMemo(() => {
if (!currentVar.schemaType)
return false
return CHUNK_SCHEMA_TYPES.includes(currentVar.schemaType)
}, [currentVar.schemaType])
const formatFileValue = (value: VarInInspect) => {
if (value.value_type === 'file')
@@ -56,7 +69,6 @@ const ValueContent = ({
const [json, setJson] = useState('')
const [parseError, setParseError] = useState<Error | null>(null)
const [validationError, setValidationError] = useState<string>('')
const fileFeature = useFeatures(s => s.features.file)
const [fileValue, setFileValue] = useState<any>(formatFileValue(currentVar))
const { run: debounceValueChange } = useDebounceFn(handleValueChange, { wait: 500 })
@@ -78,6 +90,8 @@ const ValueContent = ({
}, [currentVar.id, currentVar.value])
const handleTextChange = (value: string) => {
if (isTruncated)
return
if (currentVar.value_type === 'string')
setValue(value)
@@ -127,6 +141,8 @@ const ValueContent = ({
}
const handleEditorChange = (value: string) => {
if (isTruncated)
return
setJson(value)
if (jsonValueValidate(value, currentVar.value_type)) {
const parsed = JSON.parse(value)
@@ -170,15 +186,31 @@ const ValueContent = ({
ref={contentContainerRef}
className='flex h-full flex-col'
>
<div className={cn('grow')} style={{ height: `${editorHeight}px` }}>
<div className={cn('relative grow')} style={{ height: `${editorHeight}px` }}>
{showTextEditor && (
<Textarea
readOnly={textEditorDisabled}
disabled={textEditorDisabled}
className='h-full'
value={value as any}
onChange={e => handleTextChange(e.target.value)}
/>
<>
{isTruncated && <LargeDataAlert className='absolute left-3 right-3 top-1' />}
{
currentVar.value_type === 'string' ? (
<DisplayContent
previewType={PreviewType.Markdown}
varType={currentVar.value_type}
mdString={value as any}
readonly={textEditorDisabled}
handleTextChange={handleTextChange}
className={cn(isTruncated && 'pt-[36px]')}
/>
) : (
<Textarea
readOnly={textEditorDisabled}
disabled={textEditorDisabled || isTruncated}
className={cn('h-full', isTruncated && 'pt-[48px]')}
value={value as any}
onChange={e => handleTextChange(e.target.value)}
/>
)
}
</>
)}
{showBoolEditor && (
<div className='w-[295px]'>
@@ -210,13 +242,27 @@ const ValueContent = ({
)
}
{showJSONEditor && (
<SchemaEditor
readonly={JSONEditorDisabled}
className='overflow-y-auto'
hideTopMenu
schema={json}
onUpdate={handleEditorChange}
/>
hasChunks
? (
<DisplayContent
previewType={PreviewType.Chunks}
varType={currentVar.value_type}
schemaType={currentVar.schemaType ?? ''}
jsonString={json ?? '{}'}
readonly={JSONEditorDisabled}
handleEditorChange={handleEditorChange}
/>
)
: (
<SchemaEditor
readonly={JSONEditorDisabled || isTruncated}
className='overflow-y-auto'
hideTopMenu
schema={json}
onUpdate={handleEditorChange}
isTruncated={isTruncated}
/>
)
)}
{showFileEditor && (
<div className='max-w-[460px]'>
@@ -237,8 +283,12 @@ const ValueContent = ({
...FILE_EXTS[SupportUploadFileTypes.video],
],
allowed_file_upload_methods: [TransferMethod.local_file, TransferMethod.remote_url],
number_limits: currentVar.value_type === 'file' ? 1 : (fileFeature as any).fileUploadConfig?.workflow_file_upload_limit || 5,
fileUploadConfig: (fileFeature as any).fileUploadConfig,
number_limits: currentVar.value_type === 'file' ? 1 : fileUploadConfig?.workflow_file_upload_limit || 5,
fileUploadConfig,
preview_config: {
mode: PreviewMode.NewPage,
file_type_list: ['application/pdf'],
},
}}
isDisabled={textEditorDisabled}
/>
@@ -249,8 +299,8 @@ const ValueContent = ({
{parseError && <ErrorMessage className='mt-1' message={parseError.message} />}
{validationError && <ErrorMessage className='mt-1' message={validationError} />}
</div>
</div>
</div >
)
}
export default ValueContent
export default React.memo(ValueContent)