feat(stress-test): add comprehensive stress testing suite using Locust (#25617)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/core/helper/position_helper.py

@@ -1,12 +1,14 @@
 import os
 from collections import OrderedDict
 from collections.abc import Callable
+from functools import lru_cache
 from typing import TypeVar
 
 from configs import dify_config
-from core.tools.utils.yaml_utils import load_yaml_file
+from core.tools.utils.yaml_utils import load_yaml_file_cached
 
 
+@lru_cache(maxsize=128)
 def get_position_map(folder_path: str, *, file_name: str = "_position.yaml") -> dict[str, int]:
     """
     Get the mapping from name to index from a YAML file
@@ -14,12 +16,17 @@ def get_position_map(folder_path: str, *, file_name: str = "_position.yaml") ->
     :param file_name: the YAML file name, default to '_position.yaml'
     :return: a dict with name as key and index as value
     """
+    # FIXME(-LAN-): Cache position maps to prevent file descriptor exhaustion during high-load benchmarks
     position_file_path = os.path.join(folder_path, file_name)
-    yaml_content = load_yaml_file(file_path=position_file_path, default_value=[])
+    try:
+        yaml_content = load_yaml_file_cached(file_path=position_file_path)
+    except Exception:
+        yaml_content = []
     positions = [item.strip() for item in yaml_content if item and isinstance(item, str) and item.strip()]
     return {name: index for index, name in enumerate(positions)}
 
 
+@lru_cache(maxsize=128)
 def get_tool_position_map(folder_path: str, file_name: str = "_position.yaml") -> dict[str, int]:
     """
     Get the mapping for tools from name to index from a YAML file.
@@ -35,20 +42,6 @@ def get_tool_position_map(folder_path: str, file_name: str = "_position.yaml") -
     )
 
 
-def get_provider_position_map(folder_path: str, file_name: str = "_position.yaml") -> dict[str, int]:
-    """
-    Get the mapping for providers from name to index from a YAML file.
-    :param folder_path:
-    :param file_name: the YAML file name, default to '_position.yaml'
-    :return: a dict with name as key and index as value
-    """
-    position_map = get_position_map(folder_path, file_name=file_name)
-    return pin_position_map(
-        position_map,
-        pin_list=dify_config.POSITION_PROVIDER_PINS_LIST,
-    )
-
-
 def pin_position_map(original_position_map: dict[str, int], pin_list: list[str]) -> dict[str, int]:
     """
     Pin the items in the pin list to the beginning of the position map.

api/core/model_runtime/model_providers/model_provider_factory.py

@@ -1,14 +1,10 @@
 import hashlib
 import logging
-import os
 from collections.abc import Sequence
 from threading import Lock
 from typing import Optional
 
-from pydantic import BaseModel
-
 import contexts
-from core.helper.position_helper import get_provider_position_map, sort_to_dict_by_position_map
 from core.model_runtime.entities.model_entities import AIModelEntity, ModelType
 from core.model_runtime.entities.provider_entities import ProviderConfig, ProviderEntity, SimpleProviderEntity
 from core.model_runtime.model_providers.__base.ai_model import AIModel
@@ -28,48 +24,20 @@ from core.plugin.impl.model import PluginModelClient
 logger = logging.getLogger(__name__)
 
 
-class ModelProviderExtension(BaseModel):
-    plugin_model_provider_entity: PluginModelProviderEntity
-    position: Optional[int] = None
-
-
 class ModelProviderFactory:
-    provider_position_map: dict[str, int]
-
     def __init__(self, tenant_id: str):
-        self.provider_position_map = {}
-
         self.tenant_id = tenant_id
         self.plugin_model_manager = PluginModelClient()
 
-        if not self.provider_position_map:
-            # get the path of current classes
-            current_path = os.path.abspath(__file__)
-            model_providers_path = os.path.dirname(current_path)
-
-            # get _position.yaml file path
-            self.provider_position_map = get_provider_position_map(model_providers_path)
-
     def get_providers(self) -> Sequence[ProviderEntity]:
         """
         Get all providers
         :return: list of providers
         """
-        # Fetch plugin model providers
+        # FIXME(-LAN-): Removed position map sorting since providers are fetched from plugin server
+        # The plugin server should return providers in the desired order
         plugin_providers = self.get_plugin_model_providers()
-
-        # Convert PluginModelProviderEntity to ModelProviderExtension
-        model_provider_extensions = []
-        for provider in plugin_providers:
-            model_provider_extensions.append(ModelProviderExtension(plugin_model_provider_entity=provider))
-
-        sorted_extensions = sort_to_dict_by_position_map(
-            position_map=self.provider_position_map,
-            data=model_provider_extensions,
-            name_func=lambda x: x.plugin_model_provider_entity.declaration.provider,
-        )
-
-        return [extension.plugin_model_provider_entity.declaration for extension in sorted_extensions.values()]
+        return [provider.declaration for provider in plugin_providers]
 
     def get_plugin_model_providers(self) -> Sequence[PluginModelProviderEntity]:
         """

api/core/tools/builtin_tool/provider.py

@@ -18,7 +18,7 @@ from core.tools.entities.values import ToolLabelEnum, default_tool_label_dict
 from core.tools.errors import (
     ToolProviderNotFoundError,
 )
-from core.tools.utils.yaml_utils import load_yaml_file
+from core.tools.utils.yaml_utils import load_yaml_file_cached
 
 
 class BuiltinToolProviderController(ToolProviderController):
@@ -31,7 +31,7 @@ class BuiltinToolProviderController(ToolProviderController):
         provider = self.__class__.__module__.split(".")[-1]
         yaml_path = path.join(path.dirname(path.realpath(__file__)), "providers", provider, f"{provider}.yaml")
         try:
-            provider_yaml = load_yaml_file(yaml_path, ignore_error=False)
+            provider_yaml = load_yaml_file_cached(yaml_path)
         except Exception as e:
             raise ToolProviderNotFoundError(f"can not load provider yaml for {provider}: {e}")
 
@@ -71,7 +71,7 @@ class BuiltinToolProviderController(ToolProviderController):
         for tool_file in tool_files:
             # get tool name
             tool_name = tool_file.split(".")[0]
-            tool = load_yaml_file(path.join(tool_path, tool_file), ignore_error=False)
+            tool = load_yaml_file_cached(path.join(tool_path, tool_file))
 
             # get tool class, import the module
             assistant_tool_class: type = load_single_subclass_from_source(

api/core/tools/utils/yaml_utils.py

@@ -1,4 +1,5 @@
 import logging
+from functools import lru_cache
 from pathlib import Path
 from typing import Any
 
@@ -8,28 +9,25 @@ from yaml import YAMLError
 logger = logging.getLogger(__name__)
 
 
-def load_yaml_file(file_path: str, ignore_error: bool = True, default_value: Any = {}):
-    """
-    Safe loading a YAML file
-    :param file_path: the path of the YAML file
-    :param ignore_error:
-        if True, return default_value if error occurs and the error will be logged in debug level
-        if False, raise error if error occurs
-    :param default_value: the value returned when errors ignored
-    :return: an object of the YAML content
-    """
+def _load_yaml_file(*, file_path: str):
     if not file_path or not Path(file_path).exists():
-        if ignore_error:
-            return default_value
-        else:
-            raise FileNotFoundError(f"File not found: {file_path}")
+        raise FileNotFoundError(f"File not found: {file_path}")
 
     with open(file_path, encoding="utf-8") as yaml_file:
         try:
             yaml_content = yaml.safe_load(yaml_file)
-            return yaml_content or default_value
+            return yaml_content
         except Exception as e:
-            if ignore_error:
-                return default_value
-            else:
-                raise YAMLError(f"Failed to load YAML file {file_path}: {e}") from e
+            raise YAMLError(f"Failed to load YAML file {file_path}: {e}") from e
+
+
+@lru_cache(maxsize=128)
+def load_yaml_file_cached(file_path: str) -> Any:
+    """
+    Cached version of load_yaml_file for static configuration files.
+    Only use for files that don't change during runtime (e.g., position files)
+
+    :param file_path: the path of the YAML file
+    :return: an object of the YAML content
+    """
+    return _load_yaml_file(file_path=file_path)