admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: knowledge pipeline (#25360)

Browse Source 
Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>
This commit is contained in:
-LAN-
2025-09-18 12:49:10 +08:00
committed by GitHub
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
web/app/components/datasets/create/website/jina-reader/base/checkbox-with-label.tsx
View File

@@ -1,40 +0,0 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
import Checkbox from '@/app/components/base/checkbox'
import Tooltip from '@/app/components/base/tooltip'
type Props = {
className?: string
isChecked: boolean
onChange: (isChecked: boolean) => void
label: string
labelClassName?: string
tooltip?: string
}
const CheckboxWithLabel: FC<Props> = ({
className = '',
isChecked,
onChange,
label,
labelClassName,
tooltip,
}) => {
return (
<label className={cn(className, 'flex h-7 items-center space-x-2')}>
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
<div className={cn('text-sm font-normal text-gray-800', labelClassName)}>{label}</div>
{tooltip && (
<Tooltip
popupContent={
<div className='w-[200px]'>{tooltip}</div>
}
triggerClassName='ml-0.5 w-4 h-4'
/>
)}
</label>
)
}
export default React.memo(CheckboxWithLabel)

30
web/app/components/datasets/create/website/jina-reader/base/error-message.tsx
View File

@@ -1,30 +0,0 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
type Props = {
className?: string
title: string
errorMsg?: string
}
const ErrorMessage: FC<Props> = ({
className,
title,
errorMsg,
}) => {
return (
<div className={cn(className, 'border-t border-gray-200 bg-[#FFFAEB] px-4 py-2')}>
<div className='flex h-5 items-center'>
<AlertTriangle className='mr-2 h-4 w-4 text-[#F79009]' />
<div className='text-sm font-medium text-[#DC6803]'>{title}</div>
</div>
{errorMsg && (
<div className='mt-1 pl-6 text-xs font-normal leading-[18px] text-gray-700'>{errorMsg}</div>
)}
</div>
)
}
export default React.memo(ErrorMessage)

54
web/app/components/datasets/create/website/jina-reader/base/field.tsx
View File

@@ -1,54 +0,0 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import Input from './input'
import cn from '@/utils/classnames'
import Tooltip from '@/app/components/base/tooltip'
type Props = {
className?: string
label: string
labelClassName?: string
value: string | number
onChange: (value: string | number) => void
isRequired?: boolean
placeholder?: string
isNumber?: boolean
tooltip?: string
}
const Field: FC<Props> = ({
className,
label,
labelClassName,
value,
onChange,
isRequired = false,
placeholder = '',
isNumber = false,
tooltip,
}) => {
return (
<div className={cn(className)}>
<div className='flex py-[7px]'>
<div className={cn(labelClassName, 'flex h-[18px] items-center text-[13px] font-medium text-gray-900')}>{label} </div>
{isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
{tooltip && (
<Tooltip
popupContent={
<div className='w-[200px]'>{tooltip}</div>
}
triggerClassName='ml-0.5 w-4 h-4'
/>
)}
</div>
<Input
value={value}
onChange={onChange}
placeholder={placeholder}
isNumber={isNumber}
/>
</div>
)
}
export default React.memo(Field)

58
web/app/components/datasets/create/website/jina-reader/base/input.tsx
View File

@@ -1,58 +0,0 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
type Props = {
value: string | number
onChange: (value: string | number) => void
placeholder?: string
isNumber?: boolean
}
const MIN_VALUE = 0
const Input: FC<Props> = ({
value,
onChange,
placeholder = '',
isNumber = false,
}) => {
const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const value = e.target.value
if (isNumber) {
let numberValue = Number.parseInt(value, 10) // integer only
if (isNaN(numberValue)) {
onChange('')
return
}
if (numberValue < MIN_VALUE)
numberValue = MIN_VALUE
onChange(numberValue)
return
}
onChange(value)
}, [isNumber, onChange])
const otherOption = (() => {
if (isNumber) {
return {
min: MIN_VALUE,
}
}
return {
}
})()
return (
<input
type={isNumber ? 'number' : 'text'}
{...otherOption}
value={value}
onChange={handleChange}
className='flex h-9 w-full rounded-lg bg-gray-100 px-2 py-1 text-xs leading-normal caret-primary-600 placeholder:text-gray-400 hover:bg-gray-100 focus:bg-gray-50 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none'
placeholder={placeholder}
/>
)
}
export default React.memo(Input)

54
web/app/components/datasets/create/website/jina-reader/base/options-wrap.tsx
View File

@@ -1,54 +0,0 @@
'use client'
import { useBoolean } from 'ahooks'
import type { FC } from 'react'
import React, { useEffect } from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type Props = {
className?: string
children: React.ReactNode
controlFoldOptions?: number
}
const OptionsWrap: FC<Props> = ({
className = '',
children,
controlFoldOptions,
}) => {
const { t } = useTranslation()
const [fold, {
toggle: foldToggle,
setTrue: foldHide,
}] = useBoolean(false)
useEffect(() => {
if (controlFoldOptions)
foldHide()
}, [controlFoldOptions])
return (
<div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
<div
className='flex h-[26px] cursor-pointer select-none items-center justify-between py-1'
onClick={foldToggle}
>
<div className='flex items-center text-gray-700'>
<Settings04 className='mr-1 h-4 w-4' />
<div className='text-[13px] font-semibold uppercase text-gray-800'>{t(`${I18N_PREFIX}.options`)}</div>
</div>
<ChevronRight className={cn(!fold && 'rotate-90', 'h-4 w-4 text-gray-500')} />
</div>
{!fold && (
<div className='mb-4'>
{children}
</div>
)}
</div>
)
}
export default React.memo(OptionsWrap)

2
web/app/components/datasets/create/website/jina-reader/base/url-input.tsx
View File

@@ -2,7 +2,7 @@
import type { FC } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Input from './input'
import Input from '../../base/input'
import Button from '@/app/components/base/button'
import { useDocLink } from '@/context/i18n'

43
web/app/components/datasets/create/website/jina-reader/header.tsx
View File

@@ -1,43 +0,0 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
import Button from '@/app/components/base/button'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type Props = {
onSetting: () => void
}
const Header: FC<Props> = ({
onSetting,
}) => {
const { t } = useTranslation()
return (
<div className='flex h-6 items-center justify-between'>
<div className='flex items-center'>
<div className='text-sm font-semibold text-text-secondary'>{t(`${I18N_PREFIX}.jinaReaderTitle`)}</div>
<div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
<Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
<RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
<span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
{t(`${I18N_PREFIX}.configureJinaReader`)}
</span>
</Button>
</div>
<a
href='https://jina.ai/reader'
target='_blank'
rel='noopener noreferrer'
className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
>
<RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
<span>{t(`${I18N_PREFIX}.jinaReaderDoc`)}</span>
</a>
</div>
)
}
export default React.memo(Header)

19
web/app/components/datasets/create/website/jina-reader/index.tsx
View File

@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
import CrawledResult from '../base/crawled-result'
import Crawling from '../base/crawling'
import ErrorMessage from '../base/error-message'
import Header from './header'
import Options from './options'
import { useModalContext } from '@/context/modal-context'
import Toast from '@/app/components/base/toast'
import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
import { sleep } from '@/utils'
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
import Header from '../base/header'
const ERROR_I18N_PREFIX = 'common.errorMsg'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -150,14 +150,15 @@ const JinaReader: FC<Props> = ({
}) as any
if (res.data) {
const { title, content, description, url } = res.data
const data = {
current: 1,
total: 1,
data: [{
title: res.data.title,
markdown: res.data.content,
description: res.data.description,
source_url: res.data.url,
title,
content,
description,
source_url: url,
}],
time_consuming: (Date.now() - startTime) / 1000,
}
@@ -190,7 +191,13 @@ const JinaReader: FC<Props> = ({
return (
<div>
<Header onSetting={handleSetting} />
<Header
onClickConfiguration={handleSetting}
title={t(`${I18N_PREFIX}.jinaReaderTitle`)}
buttonText={t(`${I18N_PREFIX}.configureJinaReader`)}
docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`)}
docLink={'https://jina.ai/reader'}
/>
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
<UrlInput onRun={handleRun} isRunning={isRunning} />
<OptionsWrap