feat: introduce pydantic-settings for config definition and validation (#5202)

Co-authored-by: -LAN- <laipz8200@outlook.com>

api/configs/app_configs.py

@@ -0,0 +1,42 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from configs.deploy import DeploymentConfigs
+from configs.enterprise import EnterpriseFeatureConfigs
+from configs.extra import ExtraServiceConfigs
+from configs.feature import FeatureConfigs
+from configs.middleware import MiddlewareConfigs
+from configs.packaging import PackagingInfo
+
+
+class DifyConfigs(
+    # based on pydantic-settings
+    BaseSettings,
+
+    # Packaging info
+    PackagingInfo,
+
+    # Deployment configs
+    DeploymentConfigs,
+
+    # Feature configs
+    FeatureConfigs,
+
+    # Middleware configs
+    MiddlewareConfigs,
+
+    # Extra service configs
+    ExtraServiceConfigs,
+
+    # Enterprise feature configs
+    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    EnterpriseFeatureConfigs,
+):
+
+    model_config = SettingsConfigDict(
+        # read from dotenv format config file
+        env_file='.env',
+        env_file_encoding='utf-8',
+
+        # ignore extra attributes
+        extra='ignore',
+    )

api/configs/deploy/__init__.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel, Field
+
+
+class DeploymentConfigs(BaseModel):
+    """
+    Deployment configs
+    """
+    EDITION: str = Field(
+        description='deployment edition',
+        default='SELF_HOSTED',
+    )
+
+    DEPLOY_ENV: str = Field(
+        description='deployment environment, default to PRODUCTION.',
+        default='PRODUCTION',
+    )

api/configs/enterprise/__init__.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+
+
+class EnterpriseFeatureConfigs(BaseModel):
+    """
+    Enterprise feature configs.
+    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    """
+    ENTERPRISE_ENABLED: bool = Field(
+        description='whether to enable enterprise features.'
+                    'Before using, please contact business@dify.ai by email to inquire about licensing matters.',
+        default=False,
+    )
+
+    CAN_REPLACE_LOGO: bool = Field(
+        description='whether to allow replacing enterprise logo.',
+        default=False,
+    )

api/configs/extra/__init__.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel
+
+from configs.extra.notion_configs import NotionConfigs
+from configs.extra.sentry_configs import SentryConfigs
+
+
+class ExtraServiceConfigs(
+    # place the configs in alphabet order
+    NotionConfigs,
+    SentryConfigs,
+):
+    pass

api/configs/extra/notion_configs.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class NotionConfigs(BaseModel):
+    """
+    Notion integration configs
+    """
+    NOTION_CLIENT_ID: Optional[str] = Field(
+        description='Notion client ID',
+        default=None,
+    )
+
+    NOTION_CLIENT_SECRET: Optional[str] = Field(
+        description='Notion client secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+        description='Notion integration type, default to None, available values: internal.',
+        default=None,
+    )
+
+    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+        description='Notion internal secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+        description='Notion integration token',
+        default=None,
+    )

api/configs/extra/sentry_configs.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveFloat
+
+
+class SentryConfigs(BaseModel):
+    """
+    Sentry configs
+    """
+    SENTRY_DSN: Optional[str] = Field(
+        description='Sentry DSN',
+        default=None,
+    )
+
+    SENTRY_TRACES_SAMPLE_RATE: PositiveFloat = Field(
+        description='Sentry trace sample rate',
+        default=1.0,
+    )
+
+    SENTRY_PROFILES_SAMPLE_RATE: PositiveFloat = Field(
+        description='Sentry profiles sample rate',
+        default=1.0,
+    )

api/configs/feature/__init__.py

@@ -0,0 +1,420 @@
+from typing import Optional
+
+from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt
+
+
+class SecurityConfigs(BaseModel):
+    """
+    Secret Key configs
+    """
+    SECRET_KEY: Optional[str] = Field(
+        description='Your App secret key will be used for securely signing the session cookie'
+                    'Make sure you are changing this key for your deployment with a strong key.'
+                    'You can generate a strong key using `openssl rand -base64 42`.'
+                    'Alternatively you can set it with `SECRET_KEY` environment variable.',
+        default=None,
+    )
+
+
+class AppExecutionConfigs(BaseModel):
+    """
+    App Execution configs
+    """
+    APP_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='execution timeout in seconds for app execution',
+        default=1200,
+    )
+
+
+class CodeExecutionSandboxConfigs(BaseModel):
+    """
+    Code Execution Sandbox configs
+    """
+    CODE_EXECUTION_ENDPOINT: str = Field(
+        description='whether to enable HTTP response compression of gzip',
+        default='http://sandbox:8194',
+    )
+
+    CODE_EXECUTION_API_KEY: str = Field(
+        description='API key for code execution service',
+        default='dify-sandbox',
+    )
+
+
+class EndpointConfigs(BaseModel):
+    """
+    Module URL configs
+    """
+    CONSOLE_API_URL: str = Field(
+        description='The backend URL prefix of the console API.'
+                    'used to concatenate the login authorization callback or notion integration callback.',
+        default='https://cloud.dify.ai',
+    )
+
+    CONSOLE_WEB_URL: str = Field(
+        description='The front-end URL prefix of the console web.'
+                    'used to concatenate some front-end addresses and for CORS configuration use.',
+        default='https://cloud.dify.ai',
+    )
+
+    SERVICE_API_URL: str = Field(
+        description='Service API Url prefix.'
+                    'used to display Service API Base Url to the front-end.',
+        default='https://api.dify.ai',
+    )
+
+    APP_WEB_URL: str = Field(
+        description='WebApp Url prefix.'
+                    'used to display WebAPP API Base Url to the front-end.',
+        default='https://udify.app',
+    )
+
+
+class FileAccessConfigs(BaseModel):
+    """
+    File Access configs
+    """
+    FILES_URL: str = Field(
+        description='File preview or download Url prefix.'
+                    ' used to display File preview or download Url to the front-end or as Multi-model inputs;'
+                    'Url is signed and has expiration time.',
+        validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'),
+        alias_priority=1,
+        default='https://cloud.dify.ai',
+    )
+
+    FILES_ACCESS_TIMEOUT: int = Field(
+        description='timeout in seconds for file accessing',
+        default=300,
+    )
+
+
+class FileUploadConfigs(BaseModel):
+    """
+    File Uploading configs
+    """
+    UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='size limit in Megabytes for uploading files',
+        default=15,
+    )
+
+    UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
+        description='batch size limit for uploading files',
+        default=5,
+    )
+
+    UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='image file size limit in Megabytes for uploading files',
+        default=10,
+    )
+
+    BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
+        description='',  # todo: to be clarified
+        default=20,
+    )
+
+
+class HttpConfigs(BaseModel):
+    """
+    HTTP configs
+    """
+    API_COMPRESSION_ENABLED: bool = Field(
+        description='whether to enable HTTP response compression of gzip',
+        default=False,
+    )
+
+
+class InnerAPIConfigs(BaseModel):
+    """
+    Inner API configs
+    """
+    INNER_API: bool = Field(
+        description='whether to enable the inner API',
+        default=False,
+    )
+
+    INNER_API_KEY: Optional[str] = Field(
+        description='The inner API key is used to authenticate the inner API',
+        default=None,
+    )
+
+
+class LoggingConfigs(BaseModel):
+    """
+    Logging configs
+    """
+
+    LOG_LEVEL: str = Field(
+        description='Log output level, default to INFO.'
+                    'It is recommended to set it to ERROR for production.',
+        default='INFO',
+    )
+
+    LOG_FILE: Optional[str] = Field(
+        description='logging output file path',
+        default=None,
+    )
+
+    LOG_FORMAT: str = Field(
+        description='log format',
+        default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
+    )
+
+    LOG_DATEFORMAT: Optional[str] = Field(
+        description='log date format',
+        default=None,
+    )
+
+
+class ModelLoadBalanceConfigs(BaseModel):
+    """
+    Model load balance configs
+    """
+    MODEL_LB_ENABLED: bool = Field(
+        description='whether to enable model load balancing',
+        default=False,
+    )
+
+
+class BillingConfigs(BaseModel):
+    """
+    Platform Billing Configurations
+    """
+    BILLING_ENABLED: bool = Field(
+        description='whether to enable billing',
+        default=False,
+    )
+
+
+class UpdateConfigs(BaseModel):
+    """
+    Update configs
+    """
+    CHECK_UPDATE_URL: str = Field(
+        description='url for checking updates',
+        default='https://updates.dify.ai',
+    )
+
+
+class WorkflowConfigs(BaseModel):
+    """
+    Workflow feature configs
+    """
+
+    WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
+        description='max execution steps in single workflow execution',
+        default=500,
+    )
+
+    WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='max execution time in seconds in single workflow execution',
+        default=1200,
+    )
+
+    WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
+        description='max depth of calling in single workflow execution',
+        default=5,
+    )
+
+
+class OAuthConfigs(BaseModel):
+    """
+    oauth configs
+    """
+    OAUTH_REDIRECT_PATH: str = Field(
+        description='redirect path for OAuth',
+        default='/console/api/oauth/authorize',
+    )
+
+    GITHUB_CLIENT_ID: Optional[str] = Field(
+        description='GitHub client id for OAuth',
+        default=None,
+    )
+
+    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+        description='GitHub client secret key for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_ID: Optional[str] = Field(
+        description='Google client id for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+        description='Google client secret key for OAuth',
+        default=None,
+    )
+
+
+class ModerationConfigs(BaseModel):
+    """
+    Moderation in app configs.
+    """
+
+    # todo: to be clarified in usage and unit
+    OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field(
+        description='buffer size for moderation',
+        default=300,
+    )
+
+
+class ToolConfigs(BaseModel):
+    """
+    Tool configs
+    """
+
+    TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
+        description='max age in seconds for tool icon caching',
+        default=3600,
+    )
+
+
+class MailConfigs(BaseModel):
+    """
+    Mail Configurations
+    """
+
+    MAIL_TYPE: Optional[str] = Field(
+        description='Mail provider type name, default to None, availabile values are `smtp` and `resend`.',
+        default=None,
+    )
+
+    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+        description='default email address for sending from ',
+        default=None,
+    )
+
+    RESEND_API_KEY: Optional[str] = Field(
+        description='API key for Resend',
+        default=None,
+    )
+
+    RESEND_API_URL: Optional[str] = Field(
+        description='API URL for Resend',
+        default=None,
+    )
+
+    SMTP_SERVER: Optional[str] = Field(
+        description='smtp server host',
+        default=None,
+    )
+
+    SMTP_PORT: Optional[int] = Field(
+        description='smtp server port',
+        default=None,
+    )
+
+    SMTP_USERNAME: Optional[str] = Field(
+        description='smtp server username',
+        default=None,
+    )
+
+    SMTP_PASSWORD: Optional[str] = Field(
+        description='smtp server password',
+        default=None,
+    )
+
+    SMTP_USE_TLS: bool = Field(
+        description='whether to use TLS connection to smtp server',
+        default=False,
+    )
+
+    SMTP_OPPORTUNISTIC_TLS: bool = Field(
+        description='whether to use opportunistic TLS connection to smtp server',
+        default=False,
+    )
+
+
+class RagEtlConfigs(BaseModel):
+    """
+    RAG ETL Configurations.
+    """
+
+    ETL_TYPE: str = Field(
+        description='RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ',
+        default='dify',
+    )
+
+    KEYWORD_DATA_SOURCE_TYPE: str = Field(
+        description='source type for keyword data, default to `database`, available values are `database` .',
+        default='database',
+    )
+
+    UNSTRUCTURED_API_URL: Optional[str] = Field(
+        description='API URL for Unstructured',
+        default=None,
+    )
+
+    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+        description='API key for Unstructured',
+        default=None,
+    )
+
+
+class DataSetConfigs(BaseModel):
+    """
+    Dataset configs
+    """
+
+    CLEAN_DAY_SETTING: PositiveInt = Field(
+        description='interval in days for cleaning up dataset',
+        default=30,
+    )
+
+
+class WorkspaceConfigs(BaseModel):
+    """
+    Workspace configs
+    """
+
+    INVITE_EXPIRY_HOURS: PositiveInt = Field(
+        description='workspaces invitation expiration in hours',
+        default=72,
+    )
+
+
+class IndexingConfigs(BaseModel):
+    """
+    Indexing configs.
+    """
+
+    INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
+        description='max segmentation token length for indexing',
+        default=1000,
+    )
+
+
+class ImageFormatConfigs(BaseModel):
+    MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
+        description='multi model send image format, support base64, url, default is base64',
+        default='base64',
+    )
+
+
+class FeatureConfigs(
+    # place the configs in alphabet order
+    AppExecutionConfigs,
+    BillingConfigs,
+    CodeExecutionSandboxConfigs,
+    DataSetConfigs,
+    EndpointConfigs,
+    FileAccessConfigs,
+    FileUploadConfigs,
+    HttpConfigs,
+    ImageFormatConfigs,
+    InnerAPIConfigs,
+    IndexingConfigs,
+    LoggingConfigs,
+    MailConfigs,
+    ModelLoadBalanceConfigs,
+    ModerationConfigs,
+    OAuthConfigs,
+    RagEtlConfigs,
+    SecurityConfigs,
+    ToolConfigs,
+    UpdateConfigs,
+    WorkflowConfigs,
+    WorkspaceConfigs,
+):
+    pass

api/configs/middleware/__init__.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from configs.middleware.redis_configs import RedisConfigs
+
+
+class StorageConfigs(BaseModel):
+    STORAGE_TYPE: str = Field(
+        description='storage type,'
+                    ' default to `local`,'
+                    ' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.',
+        default='local',
+    )
+
+    STORAGE_LOCAL_PATH: str = Field(
+        description='local storage path',
+        default='storage',
+    )
+
+
+class VectorStoreConfigs(BaseModel):
+    VECTOR_STORE: Optional[str] = Field(
+        description='vector store type',
+        default=None,
+    )
+
+
+class KeywordStoreConfigs(BaseModel):
+    KEYWORD_STORE: str = Field(
+        description='keyword store type',
+        default='jieba',
+    )
+
+
+class MiddlewareConfigs(
+    # place the configs in alphabet order
+    KeywordStoreConfigs,
+    RedisConfigs,
+    StorageConfigs,
+    VectorStoreConfigs,
+):
+    pass

api/configs/middleware/redis_configs.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
+
+
+class RedisConfigs(BaseModel):
+    """
+    Redis configs
+    """
+    REDIS_HOST: str = Field(
+        description='Redis host',
+        default='localhost',
+    )
+
+    REDIS_PORT: PositiveInt = Field(
+        description='Redis port',
+        default=6379,
+    )
+
+    REDIS_USERNAME: Optional[str] = Field(
+        description='Redis username',
+        default=None,
+    )
+
+    REDIS_PASSWORD: Optional[str] = Field(
+        description='Redis password',
+        default=None,
+    )
+
+    REDIS_DB: NonNegativeInt = Field(
+        description='Redis database id, default to 0',
+        default=0,
+    )
+
+    REDIS_USE_SSL: bool = Field(
+        description='whether to use SSL for Redis connection',
+        default=False,
+    )

api/configs/packaging/__init__.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+
+
+class PackagingInfo(BaseModel):
+    """
+    Packaging build information
+    """
+
+    CURRENT_VERSION: str = Field(
+        description='Dify version',
+        default='0.6.11',
+    )
+
+    COMMIT_SHA: str = Field(
+        description="SHA-1 checksum of the git commit used to build the app",
+        default='',
+    )