refactor: extract vdb configs into pydantic-setting based dify configs (#5426)

api/configs/middleware/__init__.py

@@ -3,6 +3,16 @@ from typing import Optional
 from pydantic import BaseModel, Field
 
 from configs.middleware.redis_configs import RedisConfigs
+from configs.middleware.vdb.chroma_configs import ChromaConfigs
+from configs.middleware.vdb.milvus_configs import MilvusConfigs
+from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs
+from configs.middleware.vdb.pgvector_configs import PGVectorConfigs
+from configs.middleware.vdb.pgvectors_configs import PGVectoRSConfigs
+from configs.middleware.vdb.qdrant_configs import QdrantConfigs
+from configs.middleware.vdb.relyt_configs import RelytConfigs
+from configs.middleware.vdb.tencent_vector_configs import TencentVectorDBConfigs
+from configs.middleware.vdb.tidb_vector_configs import TiDBVectorConfigs
+from configs.middleware.vdb.weaviate_configs import WeaviateConfigs
 
 
 class StorageConfigs(BaseModel):
@@ -38,6 +48,18 @@ class MiddlewareConfigs(
     KeywordStoreConfigs,
     RedisConfigs,
     StorageConfigs,
+
+    # configs of vdb and vdb providers
     VectorStoreConfigs,
+    ChromaConfigs,
+    MilvusConfigs,
+    OpenSearchConfigs,
+    PGVectorConfigs,
+    PGVectoRSConfigs,
+    QdrantConfigs,
+    RelytConfigs,
+    TencentVectorDBConfigs,
+    TiDBVectorConfigs,
+    WeaviateConfigs,
 ):
     pass

api/configs/middleware/vdb/chroma_configs.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class ChromaConfigs(BaseModel):
+    """
+    Chroma configs
+    """
+
+    CHROMA_HOST: Optional[str] = Field(
+        description='Chroma host',
+        default=None,
+    )
+
+    CHROMA_PORT: PositiveInt = Field(
+        description='Chroma port',
+        default=8000,
+    )
+
+    CHROMA_TENANT: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_DATABASE: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+        description='Chroma authentication provider',
+        default=None,
+    )
+
+    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+        description='Chroma authentication credentials',
+        default=None,
+    )

api/configs/middleware/vdb/milvus_configs.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class MilvusConfigs(BaseModel):
+    """
+    Milvus configs
+    """
+
+    MILVUS_HOST: Optional[str] = Field(
+        description='Milvus host',
+        default=None,
+    )
+
+    MILVUS_PORT: PositiveInt = Field(
+        description='Milvus RestFul API port',
+        default=9091,
+    )
+
+    MILVUS_USER: Optional[str] = Field(
+        description='Milvus user',
+        default=None,
+    )
+
+    MILVUS_PASSWORD: Optional[str] = Field(
+        description='Milvus password',
+        default=None,
+    )
+
+    MILVUS_SECURE: bool = Field(
+        description='wheter to use SSL connection for Milvus',
+        default=False,
+    )
+
+    MILVUS_DATABASE: str = Field(
+        description='Milvus database',
+        default='default',
+    )

api/configs/middleware/vdb/opensearch_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class OpenSearchConfigs(BaseModel):
+    """
+    OpenSearch configs
+    """
+
+    OPENSEARCH_HOST: Optional[str] = Field(
+        description='OpenSearch host',
+        default=None,
+    )
+
+    OPENSEARCH_PORT: PositiveInt = Field(
+        description='OpenSearch port',
+        default=9200,
+    )
+
+    OPENSEARCH_USER: Optional[str] = Field(
+        description='OpenSearch user',
+        default=None,
+    )
+
+    OPENSEARCH_PASSWORD: Optional[str] = Field(
+        description='OpenSearch password',
+        default=None,
+    )
+
+    OPENSEARCH_SECURE: bool = Field(
+        description='whether to use SSL connection for OpenSearch',
+        default=False,
+    )

api/configs/middleware/vdb/pgvector_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class PGVectorConfigs(BaseModel):
+    """
+    PGVector configs
+    """
+
+    PGVECTOR_HOST: Optional[str] = Field(
+        description='PGVector host',
+        default=None,
+    )
+
+    PGVECTOR_PORT: Optional[PositiveInt] = Field(
+        description='PGVector port',
+        default=None,
+    )
+
+    PGVECTOR_USER: Optional[str] = Field(
+        description='PGVector user',
+        default=None,
+    )
+
+    PGVECTOR_PASSWORD: Optional[str] = Field(
+        description='PGVector password',
+        default=None,
+    )
+
+    PGVECTOR_DATABASE: Optional[str] = Field(
+        description='PGVector database',
+        default=None,
+    )

api/configs/middleware/vdb/pgvectors_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class PGVectoRSConfigs(BaseModel):
+    """
+    PGVectoRS configs
+    """
+
+    PGVECTO_RS_HOST: Optional[str] = Field(
+        description='PGVectoRS host',
+        default=None,
+    )
+
+    PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
+        description='PGVectoRS port',
+        default=None,
+    )
+
+    PGVECTO_RS_USER: Optional[str] = Field(
+        description='PGVectoRS user',
+        default=None,
+    )
+
+    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+        description='PGVectoRS password',
+        default=None,
+    )
+
+    PGVECTO_RS_DATABASE: Optional[str] = Field(
+        description='PGVectoRS database',
+        default=None,
+    )

api/configs/middleware/vdb/qdrant_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
+
+
+class QdrantConfigs(BaseModel):
+    """
+    Qdrant configs
+    """
+
+    QDRANT_URL: Optional[str] = Field(
+        description='Qdrant url',
+        default=None,
+    )
+
+    QDRANT_API_KEY: Optional[str] = Field(
+        description='Qdrant api key',
+        default=None,
+    )
+
+    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description='Qdrant client timeout in seconds',
+        default=20,
+    )
+
+    QDRANT_GRPC_ENABLED: bool = Field(
+        description='whether enable grpc support for Qdrant connection',
+        default=False,
+    )
+
+    QDRANT_GRPC_PORT: PositiveInt = Field(
+        description='Qdrant grpc port',
+        default=6334,
+    )

api/configs/middleware/vdb/relyt_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class RelytConfigs(BaseModel):
+    """
+    Relyt configs
+    """
+
+    RELYT_HOST: Optional[str] = Field(
+        description='Relyt host',
+        default=None,
+    )
+
+    RELYT_PORT: PositiveInt = Field(
+        description='Relyt port',
+        default=9200,
+    )
+
+    RELYT_USER: Optional[str] = Field(
+        description='Relyt user',
+        default=None,
+    )
+
+    RELYT_PASSWORD: Optional[str] = Field(
+        description='Relyt password',
+        default=None,
+    )
+
+    RELYT_DATABASE: Optional[str] = Field(
+        description='Relyt database',
+        default='default',
+    )

api/configs/middleware/vdb/tencent_vector_configs.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class TencentVectorDBConfigs(BaseModel):
+    """
+    Tencent Vector configs
+    """
+
+    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+        description='Tencent Vector URL',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+        description='Tencent Vector api key',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
+        description='Tencent Vector timeout',
+        default=30,
+    )
+
+    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+        description='Tencent Vector password',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+        description='Tencent Vector password',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
+        description='Tencent Vector sharding number',
+        default=1,
+    )
+
+    TENCENT_VECTOR_DB_REPLICAS: PositiveInt = Field(
+        description='Tencent Vector replicas',
+        default=2,
+    )

api/configs/middleware/vdb/tidb_vector_configs.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class TiDBVectorConfigs(BaseModel):
+    """
+    TiDB Vector configs
+    """
+
+    TIDB_VECTOR_HOST: Optional[str] = Field(
+        description='TiDB Vector host',
+        default=None,
+    )
+
+    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+        description='TiDB Vector port',
+        default=None,
+    )
+
+    TIDB_VECTOR_USER: Optional[str] = Field(
+        description='TiDB Vector user',
+        default=None,
+    )
+
+    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+        description='TiDB Vector password',
+        default=None,
+    )
+
+    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+        description='TiDB Vector database',
+        default=None,
+    )

api/configs/middleware/vdb/weaviate_configs.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class WeaviateConfigs(BaseModel):
+    """
+    Weaviate configs
+    """
+
+    WEAVIATE_ENDPOINT: Optional[str] = Field(
+        description='Weaviate endpoint URL',
+        default=None,
+    )
+
+    WEAVIATE_API_KEY: Optional[str] = Field(
+        description='Weaviate API key',
+        default=None,
+    )
+
+    WEAVIATE_GRPC_ENABLED: bool = Field(
+        description='whether to enable gRPC for Weaviate connection',
+        default=True,
+    )
+
+    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
+        description='Weaviate batch size',
+        default=100,
+    )