feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/app/components/plugins/marketplace/description/index.tsx
+++ b/web/app/components/plugins/marketplace/description/index.tsx
@@ -44,6 +44,10 @@ const Description = async ({
          <span className='relative z-[2] lowercase'>{t('category.tools')}</span>
        </span>
        ,
+        <span className="body-md-medium relative z-[1] ml-1 text-text-secondary after:absolute after:bottom-[1.5px] after:left-0 after:h-2 after:w-full after:bg-text-text-selected after:content-['']">
+          <span className='relative z-[2] lowercase'>{t('category.datasources')}</span>
+        </span>
+        ,
        <span className="body-md-medium relative z-[1] ml-1 text-text-secondary after:absolute after:bottom-[1.5px] after:left-0 after:h-2 after:w-full after:bg-text-text-selected after:content-['']">
          <span className='relative z-[2] lowercase'>{t('category.agents')}</span>
        </span>
--- a/web/app/components/plugins/marketplace/hooks.ts
+++ b/web/app/components/plugins/marketplace/hooks.ts
@@ -106,10 +106,13 @@ export const useMarketplacePlugins = () => {
  }
 }

+/**
+ * ! Support zh-Hans, pt-BR, ja-JP and en-US for Marketplace page
+ * ! For other languages, use en-US as fallback
+ */
 export const useMixedTranslation = (localeFromOuter?: string) => {
  let t = useTranslation().t

-  // !localeFromOuter only support zh-Hans and en-US for now
  if (localeFromOuter)
    t = i18n.getFixedT(localeFromOuter)

--- a/web/app/components/plugins/marketplace/list/card-wrapper.tsx
+++ b/web/app/components/plugins/marketplace/list/card-wrapper.tsx
@@ -29,7 +29,7 @@ const CardWrapper = ({
    setFalse: hideInstallFromMarketplace,
  }] = useBoolean(false)
  const { locale: localeFromLocale } = useI18N()
-  const { tagsMap } = useTags(t)
+  const { getTagLabel } = useTags(t)

  if (showInstallButton) {
    return (
@@ -43,7 +43,7 @@ const CardWrapper = ({
          footer={
            <CardMoreInfo
              downloadCount={plugin.install_count}
-              tags={plugin.tags.map(tag => tagsMap[tag.name].label)}
+              tags={plugin.tags.map(tag => getTagLabel(tag.name))}
            />
          }
        />
@@ -92,7 +92,7 @@ const CardWrapper = ({
        footer={
          <CardMoreInfo
            downloadCount={plugin.install_count}
-            tags={plugin.tags.map(tag => tagsMap[tag.name].label)}
+            tags={plugin.tags.map(tag => getTagLabel(tag.name))}
          />
        }
      />
--- a/web/app/components/plugins/marketplace/plugin-type-switch.tsx
+++ b/web/app/components/plugins/marketplace/plugin-type-switch.tsx
@@ -2,6 +2,7 @@
 import {
  RiArchive2Line,
  RiBrain2Line,
+  RiDatabase2Line,
  RiHammerLine,
  RiPuzzle2Line,
  RiSpeakAiLine,
@@ -21,6 +22,7 @@ export const PLUGIN_TYPE_SEARCH_MAP = {
  tool: PluginType.tool,
  agent: PluginType.agent,
  extension: PluginType.extension,
+  datasource: PluginType.datasource,
  bundle: 'bundle',
 }
 type PluginTypeSwitchProps = {
@@ -56,6 +58,11 @@ const PluginTypeSwitch = ({
      text: t('plugin.category.tools'),
      icon: <RiHammerLine className='mr-1.5 h-4 w-4' />,
    },
+    {
+      value: PLUGIN_TYPE_SEARCH_MAP.datasource,
+      text: t('plugin.category.datasources'),
+      icon: <RiDatabase2Line className='mr-1.5 h-4 w-4' />,
+    },
    {
      value: PLUGIN_TYPE_SEARCH_MAP.agent,
      text: t('plugin.category.agents'),
--- a/web/app/components/plugins/marketplace/search-box/index.tsx
+++ b/web/app/components/plugins/marketplace/search-box/index.tsx
@@ -4,6 +4,7 @@ import TagsFilter from './tags-filter'
 import ActionButton from '@/app/components/base/action-button'
 import cn from '@/utils/classnames'
 import { RiAddLine } from '@remixicon/react'
+import Divider from '@/app/components/base/divider'

 type SearchBoxProps = {
  search: string
@@ -12,10 +13,10 @@ type SearchBoxProps = {
  inputClassName?: string
  tags: string[]
  onTagsChange: (tags: string[]) => void
-  size?: 'small' | 'large'
  placeholder?: string
  locale?: string
  supportAddCustomTool?: boolean
+  usedInMarketplace?: boolean
  onShowAddCustomCollectionModal?: () => void
  onAddedCustomTool?: () => void
 }
@@ -26,9 +27,9 @@ const SearchBox = ({
  inputClassName,
  tags,
  onTagsChange,
-  size = 'small',
  placeholder = '',
  locale,
+  usedInMarketplace = false,
  supportAddCustomTool,
  onShowAddCustomCollectionModal,
 }: SearchBoxProps) => {
@@ -38,42 +39,82 @@ const SearchBox = ({
    >
      <div className={
        cn('flex items-center',
-          size === 'large' && 'rounded-xl border border-components-chat-input-border bg-components-panel-bg-blur p-1.5 shadow-md',
-          size === 'small' && 'rounded-lg bg-components-input-bg-normal p-0.5',
+          usedInMarketplace && 'rounded-xl border border-components-chat-input-border bg-components-panel-bg-blur p-1.5 shadow-md',
+          !usedInMarketplace && 'rounded-lg bg-components-input-bg-normal p-0.5',
          inputClassName,
        )
      }>
-        <div className='relative flex grow items-center p-1 pl-2'>
-          <div className='mr-2 flex w-full items-center'>
-            <RiSearchLine className='mr-1.5 size-4 text-text-placeholder' />
-            <input
-              className={cn(
-                'body-md-medium block grow appearance-none bg-transparent text-text-secondary outline-none',
-              )}
-              value={search}
-              onChange={(e) => {
-                onSearchChange(e.target.value)
-              }}
-              placeholder={placeholder}
-            />
-            {
-              search && (
-                <div className='absolute right-2 top-1/2 -translate-y-1/2'>
-                  <ActionButton onClick={() => onSearchChange('')}>
-                    <RiCloseLine className='h-4 w-4' />
-                  </ActionButton>
-                </div>
-              )
-            }
-          </div>
-        </div>
-        <div className='mx-1 h-3.5 w-[1px] bg-divider-regular'></div>
-        <TagsFilter
-          tags={tags}
-          onTagsChange={onTagsChange}
-          size={size}
-          locale={locale}
-        />
+        {
+          usedInMarketplace && (
+            <>
+              <TagsFilter
+                tags={tags}
+                onTagsChange={onTagsChange}
+                usedInMarketplace
+                locale={locale}
+              />
+              <Divider type='vertical' className='mx-1 h-3.5' />
+              <div className='flex grow items-center gap-x-2 p-1'>
+                <input
+                  className={cn(
+                    'body-md-medium inline-block grow appearance-none bg-transparent text-text-secondary outline-none',
+                  )}
+                  value={search}
+                  onChange={(e) => {
+                    onSearchChange(e.target.value)
+                  }}
+                  placeholder={placeholder}
+                />
+                {
+                  search && (
+                    <ActionButton
+                      onClick={() => onSearchChange('')}
+                      className='shrink-0'
+                    >
+                      <RiCloseLine className='size-4' />
+                    </ActionButton>
+                  )
+                }
+              </div>
+            </>
+          )
+        }
+        {
+          !usedInMarketplace && (
+            <>
+              <div className='flex grow items-center p-2'>
+                <RiSearchLine className='size-4 text-components-input-text-placeholder' />
+                <input
+                  className={cn(
+                    'body-md-medium ml-1.5 mr-1 inline-block grow appearance-none bg-transparent text-text-secondary outline-none',
+                    search && 'mr-2',
+                  )}
+                  value={search}
+                  onChange={(e) => {
+                    onSearchChange(e.target.value)
+                  }}
+                  placeholder={placeholder}
+                />
+                {
+                  search && (
+                    <ActionButton
+                      onClick={() => onSearchChange('')}
+                      className='shrink-0'
+                    >
+                      <RiCloseLine className='size-4' />
+                    </ActionButton>
+                  )
+                }
+              </div>
+              <Divider type='vertical' className='mx-0 mr-0.5 h-3.5' />
+              <TagsFilter
+                tags={tags}
+                onTagsChange={onTagsChange}
+                locale={locale}
+              />
+            </>
+          )
+        }
      </div>
      {supportAddCustomTool && (
        <div className='flex shrink-0 items-center'>
--- a/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx
+++ b/web/app/components/plugins/marketplace/search-box/search-box-wrapper.tsx
@@ -36,9 +36,9 @@ const SearchBoxWrapper = ({
      onSearchChange={handleSearchPluginTextChange}
      tags={filterPluginTags}
      onTagsChange={handleFilterPluginTagsChange}
-      size='large'
      locale={locale}
      placeholder={t('plugin.searchPlugins')}
+      usedInMarketplace
    />
  )
 }
--- a/web/app/components/plugins/marketplace/search-box/tags-filter.tsx
+++ b/web/app/components/plugins/marketplace/search-box/tags-filter.tsx
@@ -1,36 +1,34 @@
 'use client'

 import { useState } from 'react'
-import {
-  RiPriceTag3Line,
-} from '@remixicon/react'
 import {
  PortalToFollowElem,
  PortalToFollowElemContent,
  PortalToFollowElemTrigger,
 } from '@/app/components/base/portal-to-follow-elem'
 import Checkbox from '@/app/components/base/checkbox'
-import cn from '@/utils/classnames'
 import Input from '@/app/components/base/input'
 import { useTags } from '@/app/components/plugins/hooks'
 import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks'
+import MarketplaceTrigger from './trigger/marketplace'
+import ToolSelectorTrigger from './trigger/tool-selector'

 type TagsFilterProps = {
  tags: string[]
  onTagsChange: (tags: string[]) => void
-  size: 'small' | 'large'
+  usedInMarketplace?: boolean
  locale?: string
 }
 const TagsFilter = ({
  tags,
  onTagsChange,
-  size,
+  usedInMarketplace = false,
  locale,
 }: TagsFilterProps) => {
  const { t } = useMixedTranslation(locale)
  const [open, setOpen] = useState(false)
  const [searchText, setSearchText] = useState('')
-  const { tags: options } = useTags(t)
+  const { tags: options, tagsMap } = useTags(t)
  const filteredOptions = options.filter(option => option.label.toLowerCase().includes(searchText.toLowerCase()))
  const handleCheck = (id: string) => {
    if (tags.includes(id))
@@ -38,6 +36,7 @@ const TagsFilter = ({
    else
      onTagsChange([...tags, id])
  }
+  const selectedTagsLength = tags.length

  return (
    <PortalToFollowElem
@@ -53,17 +52,29 @@ const TagsFilter = ({
        className='shrink-0'
        onClick={() => setOpen(v => !v)}
      >
-        <div className={cn(
-          'ml-0.5 mr-1.5 flex  select-none items-center text-text-tertiary',
-          size === 'large' && 'h-8 py-1',
-          size === 'small' && 'h-7 py-0.5 ',
-          // selectedTagsLength && 'text-text-secondary',
-          // open && 'bg-state-base-hover',
-        )}>
-          <div className='cursor-pointer rounded-md p-0.5 hover:bg-state-base-hover'>
-            <RiPriceTag3Line className='h-4 w-4 text-text-tertiary' />
-          </div>
-        </div>
+        {
+          usedInMarketplace && (
+            <MarketplaceTrigger
+              selectedTagsLength={selectedTagsLength}
+              open={open}
+              tags={tags}
+              tagsMap={tagsMap}
+              locale={locale}
+              onTagsChange={onTagsChange}
+            />
+          )
+        }
+        {
+          !usedInMarketplace && (
+            <ToolSelectorTrigger
+              selectedTagsLength={selectedTagsLength}
+              open={open}
+              tags={tags}
+              tagsMap={tagsMap}
+              onTagsChange={onTagsChange}
+            />
+          )
+        }
      </PortalToFollowElemTrigger>
      <PortalToFollowElemContent className='z-[1000]'>
        <div className='w-[240px] rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur shadow-lg backdrop-blur-sm'>
--- a/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx
+++ b/web/app/components/plugins/marketplace/search-box/trigger/marketplace.tsx
@@ -0,0 +1,75 @@
+import React from 'react'
+import { RiArrowDownSLine, RiCloseCircleFill, RiFilter3Line } from '@remixicon/react'
+import type { Tag } from '../../../hooks'
+import cn from '@/utils/classnames'
+import { useMixedTranslation } from '../../hooks'
+
+type MarketplaceTriggerProps = {
+  selectedTagsLength: number
+  open: boolean
+  tags: string[]
+  tagsMap: Record<string, Tag>
+  locale?: string
+  onTagsChange: (tags: string[]) => void
+}
+
+const MarketplaceTrigger = ({
+  selectedTagsLength,
+  open,
+  tags,
+  tagsMap,
+  locale,
+  onTagsChange,
+}: MarketplaceTriggerProps) => {
+  const { t } = useMixedTranslation(locale)
+
+  return (
+    <div
+      className={cn(
+        'flex h-8 cursor-pointer select-none items-center rounded-lg px-2 py-1 text-text-tertiary',
+        !!selectedTagsLength && 'border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg shadow-xs shadow-shadow-shadow-3',
+        open && !selectedTagsLength && 'bg-state-base-hover',
+      )}
+    >
+      <div className='p-0.5'>
+        <RiFilter3Line className={cn('size-4', !!selectedTagsLength && 'text-text-secondary')} />
+      </div>
+      <div className='system-sm-medium flex items-center gap-x-1 p-1'>
+        {
+          !selectedTagsLength && <span>{t('pluginTags.allTags')}</span>
+        }
+        {
+          !!selectedTagsLength && (
+            <span className='text-text-secondary'>
+              {tags.map(tag => tagsMap[tag].label).slice(0, 2).join(',')}
+            </span>
+          )
+        }
+        {
+          selectedTagsLength > 2 && (
+            <div className='system-xs-medium text-text-tertiary'>
+              +{selectedTagsLength - 2}
+            </div>
+          )
+        }
+      </div>
+      {
+        !!selectedTagsLength && (
+          <RiCloseCircleFill
+            className='size-4 text-text-quaternary'
+            onClick={() => onTagsChange([])}
+          />
+        )
+      }
+      {
+        !selectedTagsLength && (
+          <div className='p-0.5'>
+            <RiArrowDownSLine className='size-4 text-text-tertiary' />
+          </div>
+        )
+      }
+    </div>
+  )
+}
+
+export default React.memo(MarketplaceTrigger)
--- a/web/app/components/plugins/marketplace/search-box/trigger/tool-selector.tsx
+++ b/web/app/components/plugins/marketplace/search-box/trigger/tool-selector.tsx
@@ -0,0 +1,63 @@
+import React from 'react'
+import type { Tag } from '../../../hooks'
+import cn from '@/utils/classnames'
+import { RiCloseCircleFill, RiPriceTag3Line } from '@remixicon/react'
+
+type ToolSelectorTriggerProps = {
+  selectedTagsLength: number
+  open: boolean
+  tags: string[]
+  tagsMap: Record<string, Tag>
+  onTagsChange: (tags: string[]) => void
+}
+
+const ToolSelectorTrigger = ({
+  selectedTagsLength,
+  open,
+  tags,
+  tagsMap,
+  onTagsChange,
+}: ToolSelectorTriggerProps) => {
+  return (
+    <div className={cn(
+      'flex h-7 cursor-pointer select-none items-center rounded-md p-0.5 text-text-tertiary',
+      !selectedTagsLength && 'py-1 pl-1.5 pr-2',
+      !!selectedTagsLength && 'border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg py-0.5 pl-1 pr-1.5 shadow-xs shadow-shadow-shadow-3',
+      open && !selectedTagsLength && 'bg-state-base-hover',
+    )}
+    >
+      <div className='p-0.5'>
+        <RiPriceTag3Line className={cn('size-4', !!selectedTagsLength && 'text-text-secondary')} />
+      </div>
+      {
+        !!selectedTagsLength && (
+          <div className='system-sm-medium flex items-center gap-x-0.5 px-0.5 py-1'>
+            <span className='text-text-secondary'>
+              {tags.map(tag => tagsMap[tag].label).slice(0, 2).join(',')}
+            </span>
+            {
+              selectedTagsLength > 2 && (
+                <div className='system-xs-medium text-text-tertiary'>
+                  +{selectedTagsLength - 2}
+                </div>
+              )
+            }
+          </div>
+        )
+      }
+      {
+        !!selectedTagsLength && (
+          <RiCloseCircleFill
+            className='size-4 text-text-quaternary'
+            onClick={(e) => {
+              e.stopPropagation()
+              onTagsChange([])
+            }}
+          />
+        )
+      }
+    </div>
+  )
+}
+
+export default React.memo(ToolSelectorTrigger)
--- a/web/app/components/plugins/marketplace/utils.ts
+++ b/web/app/components/plugins/marketplace/utils.ts
@@ -7,6 +7,7 @@ import type {
  PluginsSearchParams,
 } from '@/app/components/plugins/marketplace/types'
 import {
+  APP_VERSION,
  MARKETPLACE_API_PREFIX,
 } from '@/config'
 import { getMarketplaceUrl } from '@/utils/var'
@@ -49,11 +50,15 @@ export const getMarketplacePluginsByCollectionId = async (collectionId: string,

  try {
    const url = `${MARKETPLACE_API_PREFIX}/collections/${collectionId}/plugins`
+    const headers = new Headers({
+      'X-Dify-Version': APP_VERSION,
+    })
    const marketplaceCollectionPluginsData = await globalThis.fetch(
      url,
      {
        cache: 'no-store',
        method: 'POST',
+        headers,
        body: JSON.stringify({
          category: query?.category,
          exclude: query?.exclude,
@@ -83,7 +88,10 @@ export const getMarketplaceCollectionsAndPlugins = async (query?: CollectionsAnd
      marketplaceUrl += `&condition=${query.condition}`
    if (query?.type)
      marketplaceUrl += `&type=${query.type}`
-    const marketplaceCollectionsData = await globalThis.fetch(marketplaceUrl, { cache: 'no-store' })
+    const headers = new Headers({
+      'X-Dify-Version': APP_VERSION,
+    })
+    const marketplaceCollectionsData = await globalThis.fetch(marketplaceUrl, { headers, cache: 'no-store' })
    const marketplaceCollectionsDataJson = await marketplaceCollectionsData.json()
    marketplaceCollections = marketplaceCollectionsDataJson.data.collections
    await Promise.all(marketplaceCollections.map(async (collection: MarketplaceCollection) => {
@@ -117,6 +125,9 @@ export const getMarketplaceListCondition = (pluginType: string) => {
  if (pluginType === PluginType.extension)
    return 'category=endpoint'

+  if (pluginType === PluginType.datasource)
+    return 'category=datasource'
+
  if (pluginType === 'bundle')
    return 'type=bundle'