add service layer OTel Span (#28582)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/extensions/otel/decorators/base.py

@@ -0,0 +1,61 @@
+import functools
+import os
+from collections.abc import Callable
+from typing import Any, TypeVar, cast
+
+from opentelemetry.trace import get_tracer
+
+from configs import dify_config
+from extensions.otel.decorators.handler import SpanHandler
+
+T = TypeVar("T", bound=Callable[..., Any])
+
+_HANDLER_INSTANCES: dict[type[SpanHandler], SpanHandler] = {SpanHandler: SpanHandler()}
+
+
+def _is_instrument_flag_enabled() -> bool:
+    """
+    Check if external instrumentation is enabled via environment variable.
+
+    Third-party non-invasive instrumentation agents set this flag to coordinate
+    with Dify's manual OpenTelemetry instrumentation.
+    """
+    return os.getenv("ENABLE_OTEL_FOR_INSTRUMENT", "").strip().lower() == "true"
+
+
+def _get_handler_instance(handler_class: type[SpanHandler]) -> SpanHandler:
+    """Get or create a singleton instance of the handler class."""
+    if handler_class not in _HANDLER_INSTANCES:
+        _HANDLER_INSTANCES[handler_class] = handler_class()
+    return _HANDLER_INSTANCES[handler_class]
+
+
+def trace_span(handler_class: type[SpanHandler] | None = None) -> Callable[[T], T]:
+    """
+    Decorator that traces a function with an OpenTelemetry span.
+
+    The decorator uses the provided handler class to create a singleton handler instance
+    and delegates the wrapper implementation to that handler.
+
+    :param handler_class: Optional handler class to use for this span. If None, uses the default SpanHandler.
+    """
+
+    def decorator(func: T) -> T:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            if not (dify_config.ENABLE_OTEL or _is_instrument_flag_enabled()):
+                return func(*args, **kwargs)
+
+            handler = _get_handler_instance(handler_class or SpanHandler)
+            tracer = get_tracer(__name__)
+
+            return handler.wrapper(
+                tracer=tracer,
+                wrapped=func,
+                args=args,
+                kwargs=kwargs,
+            )
+
+        return cast(T, wrapper)
+
+    return decorator

api/extensions/otel/decorators/handler.py

@@ -0,0 +1,95 @@
+import inspect
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from opentelemetry.trace import SpanKind, Status, StatusCode
+
+
+class SpanHandler:
+    """
+    Base class for all span handlers.
+
+    Each instrumentation point provides a handler implementation that fully controls
+    how spans are created, annotated, and finalized through the wrapper method.
+
+    This class provides a default implementation that creates a basic span and handles
+    exceptions. Handlers can override the wrapper method to customize behavior.
+    """
+
+    _signature_cache: dict[Callable[..., Any], inspect.Signature] = {}
+
+    def _build_span_name(self, wrapped: Callable[..., Any]) -> str:
+        """
+        Build the span name from the wrapped function.
+
+        Handlers can override this method to customize span name generation.
+
+        :param wrapped: The original function being traced
+        :return: The span name
+        """
+        return f"{wrapped.__module__}.{wrapped.__qualname__}"
+
+    def _extract_arguments(
+        self,
+        wrapped: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: Mapping[str, Any],
+    ) -> dict[str, Any] | None:
+        """
+        Extract function arguments using inspect.signature.
+
+        Returns a dictionary of bound arguments, or None if extraction fails.
+        Handlers can use this to safely extract parameters from args/kwargs.
+
+        The function signature is cached to improve performance on repeated calls.
+
+        :param wrapped: The function being traced
+        :param args: Positional arguments
+        :param kwargs: Keyword arguments
+        :return: Dictionary of bound arguments, or None if extraction fails
+        """
+        try:
+            if wrapped not in self._signature_cache:
+                self._signature_cache[wrapped] = inspect.signature(wrapped)
+
+            sig = self._signature_cache[wrapped]
+            bound = sig.bind(*args, **kwargs)
+            bound.apply_defaults()
+            return bound.arguments
+        except Exception:
+            return None
+
+    def wrapper(
+        self,
+        tracer: Any,
+        wrapped: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: Mapping[str, Any],
+    ) -> Any:
+        """
+        Fully control the wrapper behavior.
+
+        Default implementation creates a basic span and handles exceptions.
+        Handlers can override this method to provide complete control over:
+        - Span creation and configuration
+        - Attribute extraction
+        - Function invocation
+        - Exception handling
+        - Status setting
+
+        :param tracer: OpenTelemetry tracer instance
+        :param wrapped: The original function being traced
+        :param args: Positional arguments (including self/cls if applicable)
+        :param kwargs: Keyword arguments
+        :return: Result of calling wrapped function
+        """
+        span_name = self._build_span_name(wrapped)
+        with tracer.start_as_current_span(span_name, kind=SpanKind.INTERNAL) as span:
+            try:
+                result = wrapped(*args, **kwargs)
+                span.set_status(Status(StatusCode.OK))
+                return result
+            except Exception as exc:
+                span.record_exception(exc)
+                span.set_status(Status(StatusCode.ERROR, str(exc)))
+                raise

api/extensions/otel/decorators/handlers/__init__.py

@@ -0,0 +1 @@
+

api/extensions/otel/decorators/handlers/generate_handler.py

@@ -0,0 +1,64 @@
+import logging
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.util.types import AttributeValue
+
+from extensions.otel.decorators.handler import SpanHandler
+from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes
+from models.model import Account
+
+logger = logging.getLogger(__name__)
+
+
+class AppGenerateHandler(SpanHandler):
+    """Span handler for ``AppGenerateService.generate``."""
+
+    def wrapper(
+        self,
+        tracer: Any,
+        wrapped: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: Mapping[str, Any],
+    ) -> Any:
+        try:
+            arguments = self._extract_arguments(wrapped, args, kwargs)
+            if not arguments:
+                return wrapped(*args, **kwargs)
+
+            app_model = arguments.get("app_model")
+            user = arguments.get("user")
+            args_dict = arguments.get("args", {})
+            streaming = arguments.get("streaming", True)
+
+            if not app_model or not user or not isinstance(args_dict, dict):
+                return wrapped(*args, **kwargs)
+            app_id = getattr(app_model, "id", None) or "unknown"
+            tenant_id = getattr(app_model, "tenant_id", None) or "unknown"
+            user_id = getattr(user, "id", None) or "unknown"
+            workflow_id = args_dict.get("workflow_id") or "unknown"
+
+            attributes: dict[str, AttributeValue] = {
+                DifySpanAttributes.APP_ID: app_id,
+                DifySpanAttributes.TENANT_ID: tenant_id,
+                GenAIAttributes.USER_ID: user_id,
+                DifySpanAttributes.USER_TYPE: "Account" if isinstance(user, Account) else "EndUser",
+                DifySpanAttributes.STREAMING: streaming,
+                DifySpanAttributes.WORKFLOW_ID: workflow_id,
+            }
+
+            span_name = self._build_span_name(wrapped)
+        except Exception as exc:
+            logger.warning("Failed to prepare span attributes for AppGenerateService.generate: %s", exc, exc_info=True)
+            return wrapped(*args, **kwargs)
+
+        with tracer.start_as_current_span(span_name, kind=SpanKind.INTERNAL, attributes=attributes) as span:
+            try:
+                result = wrapped(*args, **kwargs)
+                span.set_status(Status(StatusCode.OK))
+                return result
+            except Exception as exc:
+                span.record_exception(exc)
+                span.set_status(Status(StatusCode.ERROR, str(exc)))
+                raise

api/extensions/otel/decorators/handlers/workflow_app_runner_handler.py

@@ -0,0 +1,65 @@
+import logging
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.util.types import AttributeValue
+
+from extensions.otel.decorators.handler import SpanHandler
+from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowAppRunnerHandler(SpanHandler):
+    """Span handler for ``WorkflowAppRunner.run``."""
+
+    def wrapper(
+        self,
+        tracer: Any,
+        wrapped: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: Mapping[str, Any],
+    ) -> Any:
+        try:
+            arguments = self._extract_arguments(wrapped, args, kwargs)
+            if not arguments:
+                return wrapped(*args, **kwargs)
+
+            runner = arguments.get("self")
+            if runner is None or not hasattr(runner, "application_generate_entity"):
+                return wrapped(*args, **kwargs)
+
+            entity = runner.application_generate_entity
+            app_config = getattr(entity, "app_config", None)
+            if app_config is None:
+                return wrapped(*args, **kwargs)
+
+            user_id: AttributeValue = getattr(entity, "user_id", None) or "unknown"
+            app_id: AttributeValue = getattr(app_config, "app_id", None) or "unknown"
+            tenant_id: AttributeValue = getattr(app_config, "tenant_id", None) or "unknown"
+            workflow_id: AttributeValue = getattr(app_config, "workflow_id", None) or "unknown"
+            streaming = getattr(entity, "stream", True)
+
+            attributes: dict[str, AttributeValue] = {
+                DifySpanAttributes.APP_ID: app_id,
+                DifySpanAttributes.TENANT_ID: tenant_id,
+                GenAIAttributes.USER_ID: user_id,
+                DifySpanAttributes.STREAMING: streaming,
+                DifySpanAttributes.WORKFLOW_ID: workflow_id,
+            }
+
+            span_name = self._build_span_name(wrapped)
+        except Exception as exc:
+            logger.warning("Failed to prepare span attributes for WorkflowAppRunner.run: %s", exc, exc_info=True)
+            return wrapped(*args, **kwargs)
+
+        with tracer.start_as_current_span(span_name, kind=SpanKind.INTERNAL, attributes=attributes) as span:
+            try:
+                result = wrapped(*args, **kwargs)
+                span.set_status(Status(StatusCode.OK))
+                return result
+            except Exception as exc:
+                span.record_exception(exc)
+                span.set_status(Status(StatusCode.ERROR, str(exc)))
+                raise