chore: refine python dependency list and check dependencies in order (#9061)

api/pyproject.toml

@@ -104,25 +104,23 @@ name = "dify-api"
 package-mode = false
 
 ############################################################
-# Main dependencies
+# [ Main ] Dependency group
 ############################################################
 
 [tool.poetry.dependencies]
 anthropic = "~0.23.1"
 authlib = "1.3.1"
+azure-ai-inference = "~1.0.0b3"
+azure-ai-ml = "~1.20.0"
 azure-identity = "1.16.1"
 azure-storage-blob = "12.13.0"
 beautifulsoup4 = "4.12.2"
 boto3 = "1.35.17"
-sagemaker = "2.231.0"
 bs4 = "~0.0.1"
 cachetools = "~5.3.0"
 celery = "~5.3.6"
 chardet = "~5.1.0"
 cohere = "~5.2.4"
-cos-python-sdk-v5 = "1.9.30"
-esdk-obs-python = "3.24.6.1"
-bce-python-sdk = "~0.9.23"
 dashscope = { version = "~1.17.0", extras = ["tokenizer"] }
 flask = "~3.0.1"
 flask-compress = "~1.14"
@@ -130,7 +128,7 @@ flask-cors = "~4.0.0"
 flask-login = "~0.6.3"
 flask-migrate = "~4.0.5"
 flask-restful = "~0.3.10"
-Flask-SQLAlchemy = "~3.1.1"
+flask-sqlalchemy = "~3.1.1"
 gevent = "~23.9.1"
 gmpy2 = "~2.2.1"
 google-ai-generativelanguage = "0.6.9"
@@ -139,22 +137,22 @@ google-api-python-client = "2.90.0"
 google-auth = "2.29.0"
 google-auth-httplib2 = "0.2.0"
 google-cloud-aiplatform = "1.49.0"
-google-cloud-storage = "2.16.0"
 google-generativeai = "0.8.1"
 googleapis-common-protos = "1.63.0"
 gunicorn = "~22.0.0"
 httpx = { version = "~0.27.0", extras = ["socks"] }
 huggingface-hub = "~0.16.4"
 jieba = "0.42.1"
-langfuse = "^2.48.0"
-langsmith = "^0.1.77"
+langfuse = "~2.51.3"
+langsmith = "~0.1.77"
 mailchimp-transactional = "~1.0.50"
 markdown = "~3.5.1"
-novita-client = "^0.5.7"
+nomic = "~3.1.2"
+novita-client = "~0.5.7"
 numpy = "~1.26.4"
+oci = "~2.135.1"
 openai = "~1.29.0"
 openpyxl = "~3.1.5"
-oss2 = "2.18.5"
 pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
 psycopg2-binary = "~2.9.6"
 pycryptodome = "3.19.1"
@@ -171,7 +169,8 @@ readabilipy = "0.2.0"
 redis = { version = "~5.0.3", extras = ["hiredis"] }
 replicate = "~0.22.0"
 resend = "~0.7.0"
-scikit-learn = "^1.5.1"
+sagemaker = "2.231.0"
+scikit-learn = "~1.5.1"
 sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
 sqlalchemy = "~2.0.29"
 tencentcloud-sdk-python-hunyuan = "~3.0.1158"
@@ -179,6 +178,8 @@ tiktoken = "~0.7.0"
 tokenizers = "~0.15.0"
 transformers = "~4.35.0"
 unstructured = { version = "~0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
+validators = "0.21.0"
+volcengine-python-sdk = {extras = ["ark"], version = "~1.0.98"}
 websocket-client = "~1.7.0"
 werkzeug = "~3.0.1"
 xinference-client = "0.15.2"
@@ -187,32 +188,26 @@ zhipuai = "1.0.7"
 # Before adding new dependency, consider place it in alphabet order (a-z) and suitable group.
 
 ############################################################
+# [ Indirect ] dependency group
 # Related transparent dependencies with pinned version
 # required by main implementations
 ############################################################
-azure-ai-ml = "^1.19.0"
-azure-ai-inference = "^1.0.0b3"
-volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"}
-oci = "^2.133.0"
-tos = "^2.7.1"
-nomic = "^3.1.2"
-validators = "0.21.0"
-[tool.poetry.group.indriect.dependencies]
+[tool.poetry.group.indirect.dependencies]
 kaleido = "0.2.1"
 rank-bm25 = "~0.2.2"
 safetensors = "~0.4.3"
 
 ############################################################
-# Tool dependencies required by tool implementations
+# [ Tools ] dependency group
 ############################################################
-
-[tool.poetry.group.tool.dependencies]
+[tool.poetry.group.tools.dependencies]
 arxiv = "2.1.0"
 cloudscraper = "1.2.71"
-matplotlib = "~3.8.2"
-newspaper3k = "0.2.8"
 duckduckgo-search = "~6.3.0"
 jsonpath-ng = "1.6.1"
+matplotlib = "~3.8.2"
+newspaper3k = "0.2.8"
+nltk = "3.8.1"
 numexpr = "~2.9.0"
 opensearch-py = "2.4.0"
 qrcode = "~7.4.2"
@@ -220,11 +215,23 @@ twilio = "~9.0.4"
 vanna = { version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"] }
 wikipedia = "1.4.0"
 yfinance = "~0.2.40"
-nltk = "3.8.1"
-############################################################
-# VDB dependencies required by vector store clients
-############################################################
 
+############################################################
+# [ Storage ] dependency group
+# Required for storage clients
+############################################################
+[tool.poetry.group.storage.dependencies]
+bce-python-sdk = "~0.9.23"
+cos-python-sdk-v5 = "1.9.30"
+esdk-obs-python = "3.24.6.1"
+google-cloud-storage = "2.16.0"
+oss2 = "2.18.5"
+tos = "~2.7.1"
+
+############################################################
+# [ VDB ] dependency group
+# Required by vector store clients
+############################################################
 [tool.poetry.group.vdb.dependencies]
 alibabacloud_gpdb20160503 = "~3.8.0"
 alibabacloud_tea_openapi = "~0.3.9"
@@ -235,18 +242,17 @@ oracledb = "~2.2.1"
 pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] }
 pgvector = "0.2.5"
 pymilvus = "~2.4.4"
+qdrant-client = "1.7.3"
 tcvectordb = "1.3.2"
 tidb-vector = "0.0.9"
-qdrant-client = "1.7.3"
 weaviate-client = "~3.21.0"
 
 ############################################################
-# Dev dependencies for running tests
+# [ Dev ] dependency group
+# Required for development and running tests
 ############################################################
-
 [tool.poetry.group.dev]
 optional = true
-
 [tool.poetry.group.dev.dependencies]
 coverage = "~7.2.4"
 pytest = "~8.3.2"
@@ -255,12 +261,11 @@ pytest-env = "~1.1.3"
 pytest-mock = "~3.14.0"
 
 ############################################################
-# Lint dependencies for code style linting
+# [ Lint ] dependency group
+# Required for code style linting
 ############################################################
-
 [tool.poetry.group.lint]
 optional = true
-
 [tool.poetry.group.lint.dependencies]
 dotenv-linter = "~0.5.0"
 ruff = "~0.6.9"