feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

api/core/helper/code_executor/code_node_provider.py

@@ -1,9 +1,33 @@
-from abc import abstractmethod
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from typing import TypedDict
 
 from pydantic import BaseModel
 
 
-class CodeNodeProvider(BaseModel):
+class VariableConfig(TypedDict):
+    variable: str
+    value_selector: Sequence[str | int]
+
+
+class OutputConfig(TypedDict):
+    type: str
+    children: None
+
+
+class CodeConfig(TypedDict):
+    variables: Sequence[VariableConfig]
+    code_language: str
+    code: str
+    outputs: Mapping[str, OutputConfig]
+
+
+class DefaultConfig(TypedDict):
+    type: str
+    config: CodeConfig
+
+
+class CodeNodeProvider(BaseModel, ABC):
     @staticmethod
     @abstractmethod
     def get_language() -> str:
@@ -22,11 +46,14 @@ class CodeNodeProvider(BaseModel):
         pass
 
     @classmethod
-    def get_default_config(cls):
+    def get_default_config(cls) -> DefaultConfig:
         return {
             "type": "code",
             "config": {
-                "variables": [{"variable": "arg1", "value_selector": []}, {"variable": "arg2", "value_selector": []}],
+                "variables": [
+                    {"variable": "arg1", "value_selector": []},
+                    {"variable": "arg2", "value_selector": []},
+                ],
                 "code_language": cls.get_language(),
                 "code": cls.get_default_code(),
                 "outputs": {"result": {"type": "string", "children": None}},

api/core/helper/encrypter.py

@@ -16,8 +16,8 @@ def full_mask_token(token_length=20):
 
 
 def encrypt_token(tenant_id: str, token: str):
+    from extensions.ext_database import db
     from models.account import Tenant
-    from models.engine import db
 
     if not (tenant := db.session.query(Tenant).where(Tenant.id == tenant_id).first()):
         raise ValueError(f"Tenant with id {tenant_id} not found")

api/core/helper/name_generator.py

@@ -0,0 +1,42 @@
+import logging
+import re
+from collections.abc import Sequence
+from typing import Any
+
+from core.tools.entities.tool_entities import CredentialType
+
+logger = logging.getLogger(__name__)
+
+
+def generate_provider_name(
+    providers: Sequence[Any], credential_type: CredentialType, fallback_context: str = "provider"
+) -> str:
+    try:
+        return generate_incremental_name(
+            [provider.name for provider in providers],
+            f"{credential_type.get_name()}",
+        )
+    except Exception as e:
+        logger.warning("Error generating next provider name for %r: %r", fallback_context, e)
+        return f"{credential_type.get_name()} 1"
+
+
+def generate_incremental_name(
+    names: Sequence[str],
+    default_pattern: str,
+) -> str:
+    pattern = rf"^{re.escape(default_pattern)}\s+(\d+)$"
+    numbers = []
+
+    for name in names:
+        if not name:
+            continue
+        match = re.match(pattern, name.strip())
+        if match:
+            numbers.append(int(match.group(1)))
+
+    if not numbers:
+        return f"{default_pattern} 1"
+
+    max_number = max(numbers)
+    return f"{default_pattern} {max_number + 1}"