Feat/environment variables in workflow (#6515)

Co-authored-by: JzoNg <jzongcode@gmail.com>

api/core/app/segments/__init__.py

@@ -0,0 +1,27 @@
+from .segment_group import SegmentGroup
+from .segments import Segment
+from .types import SegmentType
+from .variables import (
+    ArrayVariable,
+    FileVariable,
+    FloatVariable,
+    IntegerVariable,
+    ObjectVariable,
+    SecretVariable,
+    StringVariable,
+    Variable,
+)
+
+__all__ = [
+    'IntegerVariable',
+    'FloatVariable',
+    'ObjectVariable',
+    'SecretVariable',
+    'FileVariable',
+    'StringVariable',
+    'ArrayVariable',
+    'Variable',
+    'SegmentType',
+    'SegmentGroup',
+    'Segment'
+]

api/core/app/segments/factory.py

@@ -0,0 +1,64 @@
+from collections.abc import Mapping
+from typing import Any
+
+from core.file.file_obj import FileVar
+
+from .segments import Segment, StringSegment
+from .types import SegmentType
+from .variables import (
+    ArrayVariable,
+    FileVariable,
+    FloatVariable,
+    IntegerVariable,
+    ObjectVariable,
+    SecretVariable,
+    StringVariable,
+    Variable,
+)
+
+
+def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable:
+    if (value_type := m.get('value_type')) is None:
+        raise ValueError('missing value type')
+    if not m.get('name'):
+        raise ValueError('missing name')
+    if (value := m.get('value')) is None:
+        raise ValueError('missing value')
+    match value_type:
+        case SegmentType.STRING:
+            return StringVariable.model_validate(m)
+        case SegmentType.NUMBER if isinstance(value, int):
+            return IntegerVariable.model_validate(m)
+        case SegmentType.NUMBER if isinstance(value, float):
+            return FloatVariable.model_validate(m)
+        case SegmentType.SECRET:
+            return SecretVariable.model_validate(m)
+        case SegmentType.NUMBER if not isinstance(value, float | int):
+            raise ValueError(f'invalid number value {value}')
+    raise ValueError(f'not supported value type {value_type}')
+
+
+def build_anonymous_variable(value: Any, /) -> Variable:
+    if isinstance(value, str):
+        return StringVariable(name='anonymous', value=value)
+    if isinstance(value, int):
+        return IntegerVariable(name='anonymous', value=value)
+    if isinstance(value, float):
+        return FloatVariable(name='anonymous', value=value)
+    if isinstance(value, dict):
+        # TODO: Limit the depth of the object
+        obj = {k: build_anonymous_variable(v) for k, v in value.items()}
+        return ObjectVariable(name='anonymous', value=obj)
+    if isinstance(value, list):
+        # TODO: Limit the depth of the array
+        elements = [build_anonymous_variable(v) for v in value]
+        return ArrayVariable(name='anonymous', value=elements)
+    if isinstance(value, FileVar):
+        return FileVariable(name='anonymous', value=value)
+    raise ValueError(f'not supported value {value}')
+
+
+def build_segment(value: Any, /) -> Segment:
+    if isinstance(value, str):
+        return StringSegment(value=value)
+    raise ValueError(f'not supported value {value}')

api/core/app/segments/parser.py

@@ -0,0 +1,17 @@
+import re
+
+from core.app.segments import SegmentGroup, factory
+from core.workflow.entities.variable_pool import VariablePool
+
+VARIABLE_PATTERN = re.compile(r'\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}')
+
+
+def convert_template(*, template: str, variable_pool: VariablePool):
+    parts = re.split(VARIABLE_PATTERN, template)
+    segments = []
+    for part in parts:
+        if '.' in part and (value := variable_pool.get(part.split('.'))):
+            segments.append(value)
+        else:
+            segments.append(factory.build_segment(part))
+    return SegmentGroup(segments=segments)

api/core/app/segments/segment_group.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+from .segments import Segment
+
+
+class SegmentGroup(BaseModel):
+    segments: list[Segment]
+
+    @property
+    def text(self):
+        return ''.join([segment.text for segment in self.segments])
+
+    @property
+    def log(self):
+        return ''.join([segment.log for segment in self.segments])
+
+    @property
+    def markdown(self):
+        return ''.join([segment.markdown for segment in self.segments])

api/core/app/segments/segments.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, field_validator
+
+from .types import SegmentType
+
+
+class Segment(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    value_type: SegmentType
+    value: Any
+
+    @field_validator('value_type')
+    def validate_value_type(cls, value):
+        """
+        This validator checks if the provided value is equal to the default value of the 'value_type' field.
+        If the value is different, a ValueError is raised.
+        """
+        if value != cls.model_fields['value_type'].default:
+            raise ValueError("Cannot modify 'value_type'")
+        return value
+
+    @property
+    def text(self) -> str:
+        return str(self.value)
+
+    @property
+    def log(self) -> str:
+        return str(self.value)
+
+    @property
+    def markdown(self) -> str:
+        return str(self.value)
+
+
+class StringSegment(Segment):
+    value_type: SegmentType = SegmentType.STRING
+    value: str

api/core/app/segments/types.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class SegmentType(str, Enum):
+    STRING = 'string'
+    NUMBER = 'number'
+    FILE = 'file'
+
+    SECRET = 'secret'
+
+    OBJECT = 'object'
+
+    ARRAY = 'array'
+    ARRAY_STRING = 'array[string]'
+    ARRAY_NUMBER = 'array[number]'
+    ARRAY_OBJECT = 'array[object]'
+    ARRAY_FILE = 'array[file]'

api/core/app/segments/variables.py

@@ -0,0 +1,83 @@
+import json
+from collections.abc import Mapping, Sequence
+
+from pydantic import Field
+
+from core.file.file_obj import FileVar
+from core.helper import encrypter
+
+from .segments import Segment, StringSegment
+from .types import SegmentType
+
+
+class Variable(Segment):
+    """
+    A variable is a segment that has a name.
+    """
+
+    id: str = Field(
+        default='',
+        description="Unique identity for variable. It's only used by environment variables now.",
+    )
+    name: str
+
+
+class StringVariable(StringSegment, Variable):
+    pass
+
+
+class FloatVariable(Variable):
+    value_type: SegmentType = SegmentType.NUMBER
+    value: float
+
+
+class IntegerVariable(Variable):
+    value_type: SegmentType = SegmentType.NUMBER
+    value: int
+
+
+class ObjectVariable(Variable):
+    value_type: SegmentType = SegmentType.OBJECT
+    value: Mapping[str, Variable]
+
+    @property
+    def text(self) -> str:
+        # TODO: Process variables.
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False)
+
+    @property
+    def log(self) -> str:
+        # TODO: Process variables.
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2)
+
+    @property
+    def markdown(self) -> str:
+        # TODO: Use markdown code block
+        return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2)
+
+
+class ArrayVariable(Variable):
+    value_type: SegmentType = SegmentType.ARRAY
+    value: Sequence[Variable]
+
+    @property
+    def markdown(self) -> str:
+        return '\n'.join(['- ' + item.markdown for item in self.value])
+
+
+class FileVariable(Variable):
+    value_type: SegmentType = SegmentType.FILE
+    # TODO: embed FileVar in this model.
+    value: FileVar
+
+    @property
+    def markdown(self) -> str:
+        return self.value.to_markdown()
+
+
+class SecretVariable(StringVariable):
+    value_type: SegmentType = SegmentType.SECRET
+
+    @property
+    def log(self) -> str:
+        return encrypter.obfuscated_token(self.value)