admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

chore: apply ruff's pyupgrade linter rules to modernize Python code with targeted version (#2419)

Browse Source
This commit is contained in:
Bowen Liang
2024-02-09 15:21:33 +08:00
committed by GitHub
parent 589099a005
commit 063191889d
246 changed files with 912 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/core/tools/entities/tool_bundle.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from pydantic import BaseModel
@@ -18,7 +18,7 @@ class ApiBasedToolBundle(BaseModel):
# operation_id
operation_id: str = None
# parameters
parameters: Optional[List[ToolParameter]] = None
parameters: Optional[list[ToolParameter]] = None
# author
author: str
# icon
@@ -31,6 +31,6 @@ class AppToolBundle(BaseModel):
This class is used to store the schema information of an tool for an app.
"""
type: ToolProviderType
credential: Optional[Dict[str, Any]] = None
credential: Optional[dict[str, Any]] = None
provider_id: str
tool_name: str

12
api/core/tools/entities/tool_entities.py
View File

@@ -1,5 +1,5 @@
from enum import Enum
from typing import Any, Dict, List, Optional, Union, cast
from typing import Any, Optional, Union, cast
from pydantic import BaseModel, Field
@@ -82,7 +82,7 @@ class ToolInvokeMessage(BaseModel):
plain text, image url or link url
"""
message: Union[str, bytes] = None
meta: Dict[str, Any] = None
meta: dict[str, Any] = None
save_as: str = ''
class ToolInvokeMessageBinary(BaseModel):
@@ -116,12 +116,12 @@ class ToolParameter(BaseModel):
default: Optional[str] = None
min: Optional[Union[float, int]] = None
max: Optional[Union[float, int]] = None
options: Optional[List[ToolParameterOption]] = None
options: Optional[list[ToolParameterOption]] = None
@classmethod
def get_simple_instance(cls,
name: str, llm_description: str, type: ToolParameterType,
required: bool, options: Optional[List[str]] = None) -> 'ToolParameter':
required: bool, options: Optional[list[str]] = None) -> 'ToolParameter':
"""
get a simple tool parameter
@@ -192,7 +192,7 @@ class ToolProviderCredentials(BaseModel):
type: CredentialsType = Field(..., description="The type of the credentials")
required: bool = False
default: Optional[str] = None
options: Optional[List[ToolCredentialsOption]] = None
options: Optional[list[ToolCredentialsOption]] = None
label: Optional[I18nObject] = None
help: Optional[I18nObject] = None
url: Optional[str] = None
@@ -232,7 +232,7 @@ class ToolRuntimeVariablePool(BaseModel):
user_id: str = Field(..., description="The user id")
tenant_id: str = Field(..., description="The tenant id of assistant")
pool: List[ToolRuntimeVariable] = Field(..., description="The pool of variables")
pool: list[ToolRuntimeVariable] = Field(..., description="The pool of variables")
def __init__(self, **data: Any):
pool = data.get('pool', [])

6
api/core/tools/entities/user_entities.py
View File

@@ -1,5 +1,5 @@
from enum import Enum
from typing import Dict, List, Optional
from typing import Optional
from pydantic import BaseModel
@@ -40,11 +40,11 @@ class UserToolProvider(BaseModel):
}
class UserToolProviderCredentials(BaseModel):
credentials: Dict[str, ToolProviderCredentials]
credentials: dict[str, ToolProviderCredentials]
class UserTool(BaseModel):
author: str
name: str # identifier
label: I18nObject # label
description: I18nObject
parameters: Optional[List[ToolParameter]]
parameters: Optional[list[ToolParameter]]

6
api/core/tools/model/tool_model_manager.py
View File

@@ -5,7 +5,7 @@
"""
import json
from typing import List, cast
from typing import cast
from core.model_manager import ModelManager
from core.model_runtime.entities.llm_entities import LLMResult
@@ -56,7 +56,7 @@ class ToolModelManager:
@staticmethod
def calculate_tokens(
tenant_id: str,
prompt_messages: List[PromptMessage]
prompt_messages: list[PromptMessage]
) -> int:
"""
calculate tokens from prompt messages and model parameters
@@ -82,7 +82,7 @@ class ToolModelManager:
def invoke(
user_id: str, tenant_id: str,
tool_type: str, tool_name: str,
prompt_messages: List[PromptMessage]
prompt_messages: list[PromptMessage]
) -> LLMResult:
"""
invoke model with parameters in user's own context

14
api/core/tools/provider/api_tool_provider.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List
from typing import Any
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiBasedToolBundle
@@ -83,10 +83,10 @@ class ApiBasedToolProviderController(ToolProviderController):
def app_type(self) -> ToolProviderType:
return ToolProviderType.API_BASED
def _validate_credentials(self, tool_name: str, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, tool_name: str, credentials: dict[str, Any]) -> None:
pass
def validate_parameters(self, tool_name: str, tool_parameters: Dict[str, Any]) -> None:
def validate_parameters(self, tool_name: str, tool_parameters: dict[str, Any]) -> None:
pass
def _parse_tool_bundle(self, tool_bundle: ApiBasedToolBundle) -> ApiTool:
@@ -117,7 +117,7 @@ class ApiBasedToolProviderController(ToolProviderController):
'parameters' : tool_bundle.parameters if tool_bundle.parameters else [],
})
def load_bundled_tools(self, tools: List[ApiBasedToolBundle]) -> List[ApiTool]:
def load_bundled_tools(self, tools: list[ApiBasedToolBundle]) -> list[ApiTool]:
"""
load bundled tools
@@ -128,7 +128,7 @@ class ApiBasedToolProviderController(ToolProviderController):
return self.tools
def get_tools(self, user_id: str, tenant_id: str) -> List[ApiTool]:
def get_tools(self, user_id: str, tenant_id: str) -> list[ApiTool]:
"""
fetch tools from database
@@ -139,10 +139,10 @@ class ApiBasedToolProviderController(ToolProviderController):
if self.tools is not None:
return self.tools
tools: List[Tool] = []
tools: list[Tool] = []
# get tenant api providers
db_providers: List[ApiToolProvider] = db.session.query(ApiToolProvider).filter(
db_providers: list[ApiToolProvider] = db.session.query(ApiToolProvider).filter(
ApiToolProvider.tenant_id == tenant_id,
ApiToolProvider.name == self.identity.name
).all()

12
api/core/tools/provider/app_tool_provider.py
View File

@@ -1,5 +1,5 @@
import logging
from typing import Any, Dict, List
from typing import Any
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolParameter, ToolParameterOption, ToolProviderType
@@ -16,21 +16,21 @@ class AppBasedToolProviderEntity(ToolProviderController):
def app_type(self) -> ToolProviderType:
return ToolProviderType.APP_BASED
def _validate_credentials(self, tool_name: str, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, tool_name: str, credentials: dict[str, Any]) -> None:
pass
def validate_parameters(self, tool_name: str, tool_parameters: Dict[str, Any]) -> None:
def validate_parameters(self, tool_name: str, tool_parameters: dict[str, Any]) -> None:
pass
def get_tools(self, user_id: str) -> List[Tool]:
db_tools: List[PublishedAppTool] = db.session.query(PublishedAppTool).filter(
def get_tools(self, user_id: str) -> list[Tool]:
db_tools: list[PublishedAppTool] = db.session.query(PublishedAppTool).filter(
PublishedAppTool.user_id == user_id,
).all()
if not db_tools or len(db_tools) == 0:
return []
tools: List[Tool] = []
tools: list[Tool] = []
for db_tool in db_tools:
tool = {

5
api/core/tools/provider/builtin/_positions.py
View File

@@ -1,5 +1,4 @@
import os.path
from typing import List
from yaml import FullLoader, load
@@ -9,12 +8,12 @@ position = {}
class BuiltinToolProviderSort:
@staticmethod
def sort(providers: List[UserToolProvider]) -> List[UserToolProvider]:
def sort(providers: list[UserToolProvider]) -> list[UserToolProvider]:
global position
if not position:
tmp_position = {}
file_path = os.path.join(os.path.dirname(__file__), '..', '_position.yaml')
with open(file_path, 'r') as f:
with open(file_path) as f:
for pos, val in enumerate(load(f, Loader=FullLoader)):
tmp_position[val] = pos
position = tmp_position

4
api/core/tools/provider/builtin/azuredalle/azuredalle.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.azuredalle.tools.dalle3 import DallE3Tool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class AzureDALLEProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
DallE3Tool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/azuredalle/tools/dalle3.py
View File

@@ -1,5 +1,5 @@
from base64 import b64decode
from typing import Any, Dict, List, Union
from typing import Any, Union
from openai import AzureOpenAI
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class DallE3Tool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/bing/bing.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.bing.tools.bing_web_search import BingSearchTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class BingProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
BingSearchTool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/bing/tools/bing_web_search.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
from requests import get
@@ -11,8 +11,8 @@ class BingSearchTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

6
api/core/tools/provider/builtin/chart/tools/bar.py
View File

@@ -1,5 +1,5 @@
import io
from typing import Any, Dict, List, Union
from typing import Any, Union
import matplotlib.pyplot as plt
@@ -8,8 +8,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class BarChartTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
data = tool_parameters.get('data', '')
if not data:
return self.create_text_message('Please input data')

6
api/core/tools/provider/builtin/chart/tools/line.py
View File

@@ -1,5 +1,5 @@
import io
from typing import Any, Dict, List, Union
from typing import Any, Union
import matplotlib.pyplot as plt
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class LinearChartTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
data = tool_parameters.get('data', '')
if not data:
return self.create_text_message('Please input data')

6
api/core/tools/provider/builtin/chart/tools/pie.py
View File

@@ -1,5 +1,5 @@
import io
from typing import Any, Dict, List, Union
from typing import Any, Union
import matplotlib.pyplot as plt
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class PieChartTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
data = tool_parameters.get('data', '')
if not data:
return self.create_text_message('Please input data')

4
api/core/tools/provider/builtin/dalle/dalle.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class DALLEProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
DallE2Tool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/dalle/tools/dalle2.py
View File

@@ -1,6 +1,6 @@
from base64 import b64decode
from os.path import join
from typing import Any, Dict, List, Union
from typing import Any, Union
from openai import OpenAI
@@ -11,8 +11,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class DallE2Tool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

6
api/core/tools/provider/builtin/dalle/tools/dalle3.py
View File

@@ -1,6 +1,6 @@
from base64 import b64decode
from os.path import join
from typing import Any, Dict, List, Union
from typing import Any, Union
from openai import OpenAI
@@ -11,8 +11,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class DallE3Tool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/gaode/tools/gaode_weather.py
View File

@@ -1,5 +1,5 @@
import json
from typing import Any, Dict, List, Union
from typing import Any, Union
import requests
@@ -8,7 +8,7 @@ from core.tools.tool.builtin_tool import BuiltinTool
class GaodeRepositoriesTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/github/tools/github_repositories.py
View File

@@ -1,6 +1,6 @@
import json
from datetime import datetime
from typing import Any, Dict, List, Union
from typing import Any, Union
from urllib.parse import quote
import requests
@@ -10,7 +10,7 @@ from core.tools.tool.builtin_tool import BuiltinTool
class GihubRepositoriesTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/google/google.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.google.tools.google_search import GoogleSearchTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class GoogleProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
GoogleSearchTool().fork_tool_runtime(
meta={

8
api/core/tools/provider/builtin/google/tools/google_search.py
View File

@@ -1,6 +1,6 @@
import os
import sys
from typing import Any, Dict, List, Union
from typing import Any, Union
from serpapi import GoogleSearch
@@ -48,7 +48,7 @@ class SerpAPI:
res = search.get_dict()
return res
def get_params(self, query: str) -> Dict[str, str]:
def get_params(self, query: str) -> dict[str, str]:
"""Get parameters for SerpAPI."""
_params = {
"api_key": self.serpapi_api_key,
@@ -148,8 +148,8 @@ class SerpAPI:
class GoogleSearchTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/maths/maths.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.maths.tools.eval_expression import EvaluateExpressionTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class MathsProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
EvaluateExpressionTool().invoke(
user_id='',

6
api/core/tools/provider/builtin/maths/tools/eval_expression.py
View File

@@ -1,5 +1,5 @@
import logging
from typing import Any, Dict, List, Union
from typing import Any, Union
import numexpr as ne
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class EvaluateExpressionTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/stablediffusion/stablediffusion.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.stablediffusion.tools.stable_diffusion import StableDiffusionTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class StableDiffusionProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
StableDiffusionTool().fork_tool_runtime(
meta={

14
api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py
View File

@@ -3,7 +3,7 @@ import json
from base64 import b64decode, b64encode
from copy import deepcopy
from os.path import join
from typing import Any, Dict, List, Union
from typing import Any, Union
from httpx import get, post
from PIL import Image
@@ -60,8 +60,8 @@ DRAW_TEXT_OPTIONS = {
class StableDiffusionTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""
@@ -141,7 +141,7 @@ class StableDiffusionTool(BuiltinTool):
height=height,
steps=steps)
def validate_models(self) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def validate_models(self) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
validate models
"""
@@ -168,7 +168,7 @@ class StableDiffusionTool(BuiltinTool):
def img2img(self, base_url: str, lora: str, image_binary: bytes,
prompt: str, negative_prompt: str,
width: int, height: int, steps: int) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
generate image
"""
@@ -207,7 +207,7 @@ class StableDiffusionTool(BuiltinTool):
return self.create_text_message('Failed to generate image')
def text2img(self, base_url: str, lora: str, prompt: str, negative_prompt: str, width: int, height: int, steps: int) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
generate image
"""
@@ -239,7 +239,7 @@ class StableDiffusionTool(BuiltinTool):
except Exception as e:
return self.create_text_message('Failed to generate image')
def get_runtime_parameters(self) -> List[ToolParameter]:
def get_runtime_parameters(self) -> list[ToolParameter]:
parameters = [
ToolParameter(name='prompt',
label=I18nObject(en_US='Prompt', zh_Hans='Prompt'),

4
api/core/tools/provider/builtin/time/time.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.time.tools.current_time import CurrentTimeTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class WikiPediaProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
CurrentTimeTool().invoke(
user_id='',

6
api/core/tools/provider/builtin/time/tools/current_time.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime, timezone
from typing import Any, Dict, List, Union
from typing import Any, Union
from pytz import timezone as pytz_timezone
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class CurrentTimeTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

8
api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py
View File

@@ -1,5 +1,5 @@
from base64 import b64decode
from typing import Any, Dict, List, Union
from typing import Any, Union
from httpx import post
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class VectorizerTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""
@@ -56,7 +56,7 @@ class VectorizerTool(BuiltinTool):
meta={'mime_type': 'image/svg+xml'})
]
def get_runtime_parameters(self) -> List[ToolParameter]:
def get_runtime_parameters(self) -> list[ToolParameter]:
"""
override the runtime parameters
"""

4
api/core/tools/provider/builtin/vectorizer/vectorizer.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.vectorizer.tools.vectorizer import VectorizerTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class VectorizerProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
VectorizerTool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/webscraper/tools/webscraper.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.errors import ToolInvokeError
@@ -8,8 +8,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class WebscraperTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/webscraper/webscraper.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.webscraper.tools.webscraper import WebscraperTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class WebscraperProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
WebscraperTool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
from langchain import WikipediaAPIWrapper
from langchain.tools import WikipediaQueryRun
@@ -14,8 +14,8 @@ class WikipediaInput(BaseModel):
class WikiPediaSearchTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

6
api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
from httpx import get
@@ -12,8 +12,8 @@ class WolframAlphaTool(BuiltinTool):
def _invoke(self,
user_id: str,
tool_parameters: Dict[str, Any],
) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
tool_parameters: dict[str, Any],
) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

4
api/core/tools/provider/builtin/wolframalpha/wolframalpha.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.wolframalpha.tools.wolframalpha import WolframAlphaTool
@@ -6,7 +6,7 @@ from core.tools.provider.builtin_tool_provider import BuiltinToolProviderControl
class GoogleProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
try:
WolframAlphaTool().fork_tool_runtime(
meta={

6
api/core/tools/provider/builtin/yahoo/tools/analytics.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from typing import Any, Dict, List, Union
from typing import Any, Union
import pandas as pd
from requests.exceptions import HTTPError, ReadTimeout
@@ -10,8 +10,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class YahooFinanceAnalyticsTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

6
api/core/tools/provider/builtin/yahoo/tools/news.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
import yfinance
from requests.exceptions import HTTPError, ReadTimeout
@@ -8,8 +8,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class YahooFinanceSearchTickerTool(BuiltinTool):
def _invoke(self,user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self,user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
'''
invoke tools
'''

6
api/core/tools/provider/builtin/yahoo/tools/ticker.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Union
from typing import Any, Union
from requests.exceptions import HTTPError, ReadTimeout
from yfinance import Ticker
@@ -8,8 +8,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class YahooFinanceSearchTickerTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

6
api/core/tools/provider/builtin/youtube/tools/videos.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from typing import Any, Dict, List, Union
from typing import Any, Union
from googleapiclient.discovery import build
@@ -8,8 +8,8 @@ from core.tools.tool.builtin_tool import BuiltinTool
class YoutubeVideosAnalyticsTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \
-> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \
-> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools
"""

28
api/core/tools/provider/builtin_tool_provider.py
View File

@@ -1,7 +1,7 @@
import importlib
from abc import abstractmethod
from os import listdir, path
from typing import Any, Dict, List
from typing import Any
from yaml import FullLoader, load
@@ -28,7 +28,7 @@ class BuiltinToolProviderController(ToolProviderController):
provider = self.__class__.__module__.split('.')[-1]
yaml_path = path.join(path.dirname(path.realpath(__file__)), 'builtin', provider, f'{provider}.yaml')
try:
with open(yaml_path, 'r') as f:
with open(yaml_path) as f:
provider_yaml = load(f.read(), FullLoader)
except:
raise ToolProviderNotFoundError(f'can not load provider yaml for {provider}')
@@ -43,7 +43,7 @@ class BuiltinToolProviderController(ToolProviderController):
'credentials_schema': provider_yaml['credentials_for_provider'] if 'credentials_for_provider' in provider_yaml else None,
})
def _get_builtin_tools(self) -> List[Tool]:
def _get_builtin_tools(self) -> list[Tool]:
"""
returns a list of tools that the provider can provide
@@ -58,7 +58,7 @@ class BuiltinToolProviderController(ToolProviderController):
tool_files = list(filter(lambda x: x.endswith(".yaml") and not x.startswith("__"), listdir(tool_path)))
tools = []
for tool_file in tool_files:
with open(path.join(tool_path, tool_file), "r") as f:
with open(path.join(tool_path, tool_file)) as f:
# get tool name
tool_name = tool_file.split(".")[0]
tool = load(f.read(), FullLoader)
@@ -78,7 +78,7 @@ class BuiltinToolProviderController(ToolProviderController):
self.tools = tools
return tools
def get_credentials_schema(self) -> Dict[str, ToolProviderCredentials]:
def get_credentials_schema(self) -> dict[str, ToolProviderCredentials]:
"""
returns the credentials schema of the provider
@@ -98,7 +98,7 @@ class BuiltinToolProviderController(ToolProviderController):
credentials = self.credentials_schema.copy()
return UserToolProviderCredentials(credentials=credentials)
def get_tools(self) -> List[Tool]:
def get_tools(self) -> list[Tool]:
"""
returns a list of tools that the provider can provide
@@ -112,7 +112,7 @@ class BuiltinToolProviderController(ToolProviderController):
"""
return next(filter(lambda x: x.identity.name == tool_name, self.get_tools()), None)
def get_parameters(self, tool_name: str) -> List[ToolParameter]:
def get_parameters(self, tool_name: str) -> list[ToolParameter]:
"""
returns the parameters of the tool
@@ -142,7 +142,7 @@ class BuiltinToolProviderController(ToolProviderController):
"""
return ToolProviderType.BUILT_IN
def validate_parameters(self, tool_id: int, tool_name: str, tool_parameters: Dict[str, Any]) -> None:
def validate_parameters(self, tool_id: int, tool_name: str, tool_parameters: dict[str, Any]) -> None:
"""
validate the parameters of the tool and set the default value if needed
@@ -151,7 +151,7 @@ class BuiltinToolProviderController(ToolProviderController):
"""
tool_parameters_schema = self.get_parameters(tool_name)
tool_parameters_need_to_validate: Dict[str, ToolParameter] = {}
tool_parameters_need_to_validate: dict[str, ToolParameter] = {}
for parameter in tool_parameters_schema:
tool_parameters_need_to_validate[parameter.name] = parameter
@@ -166,7 +166,7 @@ class BuiltinToolProviderController(ToolProviderController):
raise ToolParameterValidationError(f'parameter {parameter} should be string')
elif parameter_schema.type == ToolParameter.ToolParameterType.NUMBER:
if not isinstance(tool_parameters[parameter], (int, float)):
if not isinstance(tool_parameters[parameter], int | float):
raise ToolParameterValidationError(f'parameter {parameter} should be number')
if parameter_schema.min is not None and tool_parameters[parameter] < parameter_schema.min:
@@ -211,7 +211,7 @@ class BuiltinToolProviderController(ToolProviderController):
tool_parameters[parameter] = default_value
def validate_credentials_format(self, credentials: Dict[str, Any]) -> None:
def validate_credentials_format(self, credentials: dict[str, Any]) -> None:
"""
validate the format of the credentials of the provider and set the default value if needed
@@ -221,7 +221,7 @@ class BuiltinToolProviderController(ToolProviderController):
if credentials_schema is None:
return
credentials_need_to_validate: Dict[str, ToolProviderCredentials] = {}
credentials_need_to_validate: dict[str, ToolProviderCredentials] = {}
for credential_name in credentials_schema:
credentials_need_to_validate[credential_name] = credentials_schema[credential_name]
@@ -266,7 +266,7 @@ class BuiltinToolProviderController(ToolProviderController):
credentials[credential_name] = default_value
def validate_credentials(self, credentials: Dict[str, Any]) -> None:
def validate_credentials(self, credentials: dict[str, Any]) -> None:
"""
validate the credentials of the provider
@@ -280,7 +280,7 @@ class BuiltinToolProviderController(ToolProviderController):
self._validate_credentials(credentials)
@abstractmethod
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
"""
validate the credentials of the provider

26
api/core/tools/provider/tool_provider.py
View File

@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from pydantic import BaseModel
@@ -16,10 +16,10 @@ from core.tools.tool.tool import Tool
class ToolProviderController(BaseModel, ABC):
identity: Optional[ToolProviderIdentity] = None
tools: Optional[List[Tool]] = None
credentials_schema: Optional[Dict[str, ToolProviderCredentials]] = None
tools: Optional[list[Tool]] = None
credentials_schema: Optional[dict[str, ToolProviderCredentials]] = None
def get_credentials_schema(self) -> Dict[str, ToolProviderCredentials]:
def get_credentials_schema(self) -> dict[str, ToolProviderCredentials]:
"""
returns the credentials schema of the provider
@@ -37,7 +37,7 @@ class ToolProviderController(BaseModel, ABC):
return UserToolProviderCredentials(credentials=credentials)
@abstractmethod
def get_tools(self) -> List[Tool]:
def get_tools(self) -> list[Tool]:
"""
returns a list of tools that the provider can provide
@@ -54,7 +54,7 @@ class ToolProviderController(BaseModel, ABC):
"""
pass
def get_parameters(self, tool_name: str) -> List[ToolParameter]:
def get_parameters(self, tool_name: str) -> list[ToolParameter]:
"""
returns the parameters of the tool
@@ -75,7 +75,7 @@ class ToolProviderController(BaseModel, ABC):
"""
return ToolProviderType.BUILT_IN
def validate_parameters(self, tool_id: int, tool_name: str, tool_parameters: Dict[str, Any]) -> None:
def validate_parameters(self, tool_id: int, tool_name: str, tool_parameters: dict[str, Any]) -> None:
"""
validate the parameters of the tool and set the default value if needed
@@ -84,7 +84,7 @@ class ToolProviderController(BaseModel, ABC):
"""
tool_parameters_schema = self.get_parameters(tool_name)
tool_parameters_need_to_validate: Dict[str, ToolParameter] = {}
tool_parameters_need_to_validate: dict[str, ToolParameter] = {}
for parameter in tool_parameters_schema:
tool_parameters_need_to_validate[parameter.name] = parameter
@@ -99,7 +99,7 @@ class ToolProviderController(BaseModel, ABC):
raise ToolParameterValidationError(f'parameter {parameter} should be string')
elif parameter_schema.type == ToolParameter.ToolParameterType.NUMBER:
if not isinstance(tool_parameters[parameter], (int, float)):
if not isinstance(tool_parameters[parameter], int | float):
raise ToolParameterValidationError(f'parameter {parameter} should be number')
if parameter_schema.min is not None and tool_parameters[parameter] < parameter_schema.min:
@@ -144,7 +144,7 @@ class ToolProviderController(BaseModel, ABC):
tool_parameters[parameter] = default_value
def validate_credentials_format(self, credentials: Dict[str, Any]) -> None:
def validate_credentials_format(self, credentials: dict[str, Any]) -> None:
"""
validate the format of the credentials of the provider and set the default value if needed
@@ -154,7 +154,7 @@ class ToolProviderController(BaseModel, ABC):
if credentials_schema is None:
return
credentials_need_to_validate: Dict[str, ToolProviderCredentials] = {}
credentials_need_to_validate: dict[str, ToolProviderCredentials] = {}
for credential_name in credentials_schema:
credentials_need_to_validate[credential_name] = credentials_schema[credential_name]
@@ -198,7 +198,7 @@ class ToolProviderController(BaseModel, ABC):
credentials[credential_name] = default_value
def validate_credentials(self, credentials: Dict[str, Any]) -> None:
def validate_credentials(self, credentials: dict[str, Any]) -> None:
"""
validate the credentials of the provider
@@ -212,7 +212,7 @@ class ToolProviderController(BaseModel, ABC):
self._validate_credentials(credentials)
@abstractmethod
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
"""
validate the credentials of the provider

12
api/core/tools/tool/api_tool.py
View File

@@ -1,6 +1,6 @@
import json
from json import dumps
from typing import Any, Dict, List, Union
from typing import Any, Union
import httpx
import requests
@@ -18,7 +18,7 @@ class ApiTool(Tool):
"""
Api tool
"""
def fork_tool_runtime(self, meta: Dict[str, Any]) -> 'Tool':
def fork_tool_runtime(self, meta: dict[str, Any]) -> 'Tool':
"""
fork a new tool with meta data
@@ -33,7 +33,7 @@ class ApiTool(Tool):
runtime=Tool.Runtime(**meta)
)
def validate_credentials(self, credentials: Dict[str, Any], parameters: Dict[str, Any], format_only: bool = False) -> str:
def validate_credentials(self, credentials: dict[str, Any], parameters: dict[str, Any], format_only: bool = False) -> str:
"""
validate the credentials for Api tool
"""
@@ -47,7 +47,7 @@ class ApiTool(Tool):
# validate response
return self.validate_and_parse_response(response)
def assembling_request(self, parameters: Dict[str, Any]) -> Dict[str, Any]:
def assembling_request(self, parameters: dict[str, Any]) -> dict[str, Any]:
headers = {}
credentials = self.runtime.credentials or {}
@@ -108,7 +108,7 @@ class ApiTool(Tool):
else:
raise ValueError(f'Invalid response type {type(response)}')
def do_http_request(self, url: str, method: str, headers: Dict[str, Any], parameters: Dict[str, Any]) -> httpx.Response:
def do_http_request(self, url: str, method: str, headers: dict[str, Any], parameters: dict[str, Any]) -> httpx.Response:
"""
do http request depending on api bundle
"""
@@ -225,7 +225,7 @@ class ApiTool(Tool):
return response
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> ToolInvokeMessage | List[ToolInvokeMessage]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage | list[ToolInvokeMessage]:
"""
invoke http request
"""

7
api/core/tools/tool/builtin_tool.py
View File

@@ -1,4 +1,3 @@
from typing import List
from core.model_runtime.entities.llm_entities import LLMResult
from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
@@ -22,7 +21,7 @@ class BuiltinTool(Tool):
"""
def invoke_model(
self, user_id: str, prompt_messages: List[PromptMessage], stop: List[str]
self, user_id: str, prompt_messages: list[PromptMessage], stop: list[str]
) -> LLMResult:
"""
invoke model
@@ -52,7 +51,7 @@ class BuiltinTool(Tool):
tenant_id=self.runtime.tenant_id,
)
def get_prompt_tokens(self, prompt_messages: List[PromptMessage]) -> int:
def get_prompt_tokens(self, prompt_messages: list[PromptMessage]) -> int:
"""
get prompt tokens
@@ -104,7 +103,7 @@ class BuiltinTool(Tool):
new_lines.append(line)
# merge lines into messages with max tokens
messages: List[str] = []
messages: list[str] = []
for i in new_lines:
if len(messages) == 0:
messages.append(i)

14
api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py
View File

@@ -1,5 +1,5 @@
import threading
from typing import List, Optional, Type
from typing import Optional
from flask import Flask, current_app
from langchain.tools import BaseTool
@@ -35,20 +35,20 @@ class DatasetMultiRetrieverToolInput(BaseModel):
class DatasetMultiRetrieverTool(BaseTool):
"""Tool for querying multi dataset."""
name: str = "dataset-"
args_schema: Type[BaseModel] = DatasetMultiRetrieverToolInput
args_schema: type[BaseModel] = DatasetMultiRetrieverToolInput
description: str = "dataset multi retriever and rerank. "
tenant_id: str
dataset_ids: List[str]
dataset_ids: list[str]
top_k: int = 2
score_threshold: Optional[float] = None
reranking_provider_name: str
reranking_model_name: str
return_resource: bool
retriever_from: str
hit_callbacks: List[DatasetIndexToolCallbackHandler] = []
hit_callbacks: list[DatasetIndexToolCallbackHandler] = []
@classmethod
def from_dataset(cls, dataset_ids: List[str], tenant_id: str, **kwargs):
def from_dataset(cls, dataset_ids: list[str], tenant_id: str, **kwargs):
return cls(
name=f'dataset-{tenant_id}',
tenant_id=tenant_id,
@@ -155,8 +155,8 @@ class DatasetMultiRetrieverTool(BaseTool):
async def _arun(self, tool_input: str) -> str:
raise NotImplementedError()
def _retriever(self, flask_app: Flask, dataset_id: str, query: str, all_documents: List,
hit_callbacks: List[DatasetIndexToolCallbackHandler]):
def _retriever(self, flask_app: Flask, dataset_id: str, query: str, all_documents: list,
hit_callbacks: list[DatasetIndexToolCallbackHandler]):
with flask_app.app_context():
dataset = db.session.query(Dataset).filter(
Dataset.tenant_id == self.tenant_id,

6
api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py
View File

@@ -1,5 +1,5 @@
import threading
from typing import List, Optional, Type
from typing import Optional
from flask import current_app
from langchain.tools import BaseTool
@@ -35,14 +35,14 @@ class DatasetRetrieverToolInput(BaseModel):
class DatasetRetrieverTool(BaseTool):
"""Tool for querying a Dataset."""
name: str = "dataset"
args_schema: Type[BaseModel] = DatasetRetrieverToolInput
args_schema: type[BaseModel] = DatasetRetrieverToolInput
description: str = "use this to retrieve a dataset. "
tenant_id: str
dataset_id: str
top_k: int = 2
score_threshold: Optional[float] = None
hit_callbacks: List[DatasetIndexToolCallbackHandler] = []
hit_callbacks: list[DatasetIndexToolCallbackHandler] = []
return_resource: bool
retriever_from: str

10
api/core/tools/tool/dataset_retriever_tool.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List
from typing import Any
from langchain.tools import BaseTool
@@ -20,7 +20,7 @@ class DatasetRetrieverTool(Tool):
return_resource: bool,
invoke_from: InvokeFrom,
hit_callback: DatasetIndexToolCallbackHandler
) -> List['DatasetRetrieverTool']:
) -> list['DatasetRetrieverTool']:
"""
get dataset tool
"""
@@ -65,7 +65,7 @@ class DatasetRetrieverTool(Tool):
return tools
def get_runtime_parameters(self) -> List[ToolParameter]:
def get_runtime_parameters(self) -> list[ToolParameter]:
return [
ToolParameter(name='query',
label=I18nObject(en_US='', zh_Hans=''),
@@ -77,7 +77,7 @@ class DatasetRetrieverTool(Tool):
default=''),
]
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> ToolInvokeMessage | List[ToolInvokeMessage]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage | list[ToolInvokeMessage]:
"""
invoke dataset retriever tool
"""
@@ -90,7 +90,7 @@ class DatasetRetrieverTool(Tool):
return self.create_text_message(text=result)
def validate_credentials(self, credentials: Dict[str, Any], parameters: Dict[str, Any]) -> None:
def validate_credentials(self, credentials: dict[str, Any], parameters: dict[str, Any]) -> None:
"""
validate the credentials for dataset retriever tool
"""

24
api/core/tools/tool/tool.py
View File

@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from pydantic import BaseModel
@@ -19,7 +19,7 @@ from core.tools.tool_file_manager import ToolFileManager
class Tool(BaseModel, ABC):
identity: ToolIdentity = None
parameters: Optional[List[ToolParameter]] = None
parameters: Optional[list[ToolParameter]] = None
description: ToolDescription = None
is_team_authorization: bool = False
agent_callback: Optional[DifyAgentCallbackHandler] = None
@@ -36,8 +36,8 @@ class Tool(BaseModel, ABC):
tenant_id: str = None
tool_id: str = None
credentials: Dict[str, Any] = None
runtime_parameters: Dict[str, Any] = None
credentials: dict[str, Any] = None
runtime_parameters: dict[str, Any] = None
runtime: Runtime = None
variables: ToolRuntimeVariablePool = None
@@ -53,7 +53,7 @@ class Tool(BaseModel, ABC):
class VARIABLE_KEY(Enum):
IMAGE = 'image'
def fork_tool_runtime(self, meta: Dict[str, Any], agent_callback: DifyAgentCallbackHandler = None) -> 'Tool':
def fork_tool_runtime(self, meta: dict[str, Any], agent_callback: DifyAgentCallbackHandler = None) -> 'Tool':
"""
fork a new tool with meta data
@@ -146,7 +146,7 @@ class Tool(BaseModel, ABC):
return file_binary[0]
def list_variables(self) -> List[ToolRuntimeVariable]:
def list_variables(self) -> list[ToolRuntimeVariable]:
"""
list all variables
@@ -157,7 +157,7 @@ class Tool(BaseModel, ABC):
return self.variables.pool
def list_default_image_variables(self) -> List[ToolRuntimeVariable]:
def list_default_image_variables(self) -> list[ToolRuntimeVariable]:
"""
list all image variables
@@ -174,7 +174,7 @@ class Tool(BaseModel, ABC):
return result
def invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> List[ToolInvokeMessage]:
def invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> list[ToolInvokeMessage]:
# update tool_parameters
if self.runtime.runtime_parameters:
tool_parameters.update(self.runtime.runtime_parameters)
@@ -209,7 +209,7 @@ class Tool(BaseModel, ABC):
return result
def _convert_tool_response_to_str(self, tool_response: List[ToolInvokeMessage]) -> str:
def _convert_tool_response_to_str(self, tool_response: list[ToolInvokeMessage]) -> str:
"""
Handle tool response
"""
@@ -233,10 +233,10 @@ class Tool(BaseModel, ABC):
return result
@abstractmethod
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
pass
def validate_credentials(self, credentials: Dict[str, Any], parameters: Dict[str, Any]) -> None:
def validate_credentials(self, credentials: dict[str, Any], parameters: dict[str, Any]) -> None:
"""
validate the credentials
@@ -245,7 +245,7 @@ class Tool(BaseModel, ABC):
"""
pass
def get_runtime_parameters(self) -> List[ToolParameter]:
def get_runtime_parameters(self) -> list[ToolParameter]:
"""
get the runtime parameters

9
api/core/tools/tool_file_manager.py
View File

@@ -4,8 +4,9 @@ import hmac
import logging
import os
import time
from collections.abc import Generator
from mimetypes import guess_extension, guess_type
from typing import Generator, Tuple, Union
from typing import Union
from uuid import uuid4
from flask import current_app
@@ -113,7 +114,7 @@ class ToolFileManager:
return tool_file
@staticmethod
def get_file_binary(id: str) -> Union[Tuple[bytes, str], None]:
def get_file_binary(id: str) -> Union[tuple[bytes, str], None]:
"""
get file binary
@@ -133,7 +134,7 @@ class ToolFileManager:
return blob, tool_file.mimetype
@staticmethod
def get_file_binary_by_message_file_id(id: str) -> Union[Tuple[bytes, str], None]:
def get_file_binary_by_message_file_id(id: str) -> Union[tuple[bytes, str], None]:
"""
get file binary
@@ -162,7 +163,7 @@ class ToolFileManager:
return blob, tool_file.mimetype
@staticmethod
def get_file_generator_by_message_file_id(id: str) -> Union[Tuple[Generator, str], None]:
def get_file_generator_by_message_file_id(id: str) -> Union[tuple[Generator, str], None]:
"""
get file binary

26
api/core/tools/tool_manager.py
View File

@@ -3,7 +3,7 @@ import json
import logging
import mimetypes
from os import listdir, path
from typing import Any, Dict, List, Tuple, Union
from typing import Any, Union
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.model_runtime.entities.message_entities import PromptMessage
@@ -35,10 +35,10 @@ class ToolManager:
provider: str,
tool_id: str,
tool_name: str,
tool_parameters: Dict[str, Any],
credentials: Dict[str, Any],
prompt_messages: List[PromptMessage],
) -> List[ToolInvokeMessage]:
tool_parameters: dict[str, Any],
credentials: dict[str, Any],
prompt_messages: list[PromptMessage],
) -> list[ToolInvokeMessage]:
"""
invoke the assistant
@@ -200,7 +200,7 @@ class ToolManager:
raise ToolProviderNotFoundError(f'provider type {provider_type} not found')
@staticmethod
def get_builtin_provider_icon(provider: str) -> Tuple[str, str]:
def get_builtin_provider_icon(provider: str) -> tuple[str, str]:
"""
get the absolute path of the icon of the builtin provider
@@ -223,14 +223,14 @@ class ToolManager:
return absolute_path, mime_type
@staticmethod
def list_builtin_providers() -> List[BuiltinToolProviderController]:
def list_builtin_providers() -> list[BuiltinToolProviderController]:
global _builtin_providers
# use cache first
if len(_builtin_providers) > 0:
return list(_builtin_providers.values())
builtin_providers: List[BuiltinToolProviderController] = []
builtin_providers: list[BuiltinToolProviderController] = []
for provider in listdir(path.join(path.dirname(path.realpath(__file__)), 'provider', 'builtin')):
if provider.startswith('__'):
continue
@@ -289,8 +289,8 @@ class ToolManager:
def user_list_providers(
user_id: str,
tenant_id: str,
) -> List[UserToolProvider]:
result_providers: Dict[str, UserToolProvider] = {}
) -> list[UserToolProvider]:
result_providers: dict[str, UserToolProvider] = {}
# get builtin providers
builtin_providers = ToolManager.list_builtin_providers()
# append builtin providers
@@ -325,7 +325,7 @@ class ToolManager:
result_providers[provider.identity.name].allow_delete = False
# get db builtin providers
db_builtin_providers: List[BuiltinToolProvider] = db.session.query(BuiltinToolProvider). \
db_builtin_providers: list[BuiltinToolProvider] = db.session.query(BuiltinToolProvider). \
filter(BuiltinToolProvider.tenant_id == tenant_id).all()
for db_builtin_provider in db_builtin_providers:
@@ -346,7 +346,7 @@ class ToolManager:
result_providers[provider_name].team_credentials = masked_credentials
# get db api providers
db_api_providers: List[ApiToolProvider] = db.session.query(ApiToolProvider). \
db_api_providers: list[ApiToolProvider] = db.session.query(ApiToolProvider). \
filter(ApiToolProvider.tenant_id == tenant_id).all()
for db_api_provider in db_api_providers:
@@ -394,7 +394,7 @@ class ToolManager:
return BuiltinToolProviderSort.sort(list(result_providers.values()))
@staticmethod
def get_api_provider_controller(tenant_id: str, provider_id: str) -> Tuple[ApiBasedToolProviderController, Dict[str, Any]]:
def get_api_provider_controller(tenant_id: str, provider_id: str) -> tuple[ApiBasedToolProviderController, dict[str, Any]]:
"""
get the api provider

10
api/core/tools/utils/configuration.py
View File

@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from pydantic import BaseModel
@@ -12,13 +12,13 @@ class ToolConfiguration(BaseModel):
tenant_id: str
provider_controller: ToolProviderController
def _deep_copy(self, credentials: Dict[str, str]) -> Dict[str, str]:
def _deep_copy(self, credentials: dict[str, str]) -> dict[str, str]:
"""
deep copy credentials
"""
return {key: value for key, value in credentials.items()}
def encrypt_tool_credentials(self, credentials: Dict[str, str]) -> Dict[str, str]:
def encrypt_tool_credentials(self, credentials: dict[str, str]) -> dict[str, str]:
"""
encrypt tool credentials with tenant id
@@ -36,7 +36,7 @@ class ToolConfiguration(BaseModel):
return credentials
def mask_tool_credentials(self, credentials: Dict[str, Any]) -> Dict[str, Any]:
def mask_tool_credentials(self, credentials: dict[str, Any]) -> dict[str, Any]:
"""
mask tool credentials
@@ -59,7 +59,7 @@ class ToolConfiguration(BaseModel):
return credentials
def decrypt_tool_credentials(self, credentials: Dict[str, str]) -> Dict[str, str]:
def decrypt_tool_credentials(self, credentials: dict[str, str]) -> dict[str, str]:
"""
decrypt tool credentials with tenant id

5
api/core/tools/utils/encoder.py
View File

@@ -1,11 +1,10 @@
from typing import List
from pydantic import BaseModel
def serialize_base_model_array(l: List[BaseModel]) -> str:
def serialize_base_model_array(l: list[BaseModel]) -> str:
class _BaseModel(BaseModel):
__root__: List[BaseModel]
__root__: list[BaseModel]
"""
{"__root__": [BaseModel, BaseModel, ...]}

15
api/core/tools/utils/parser.py
View File

@@ -1,6 +1,5 @@
from json import loads as json_loads
from typing import List, Tuple
from requests import get
from yaml import FullLoader, load
@@ -13,7 +12,7 @@ from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolPro
class ApiBasedToolSchemaParser:
@staticmethod
def parse_openapi_to_tool_bundle(openapi: dict, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_openapi_to_tool_bundle(openapi: dict, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
warning = warning if warning is not None else {}
extra_info = extra_info if extra_info is not None else {}
@@ -132,7 +131,7 @@ class ApiBasedToolSchemaParser:
return bundles
@staticmethod
def parse_openapi_yaml_to_tool_bundle(yaml: str, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_openapi_yaml_to_tool_bundle(yaml: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
"""
parse openapi yaml to tool bundle
@@ -148,7 +147,7 @@ class ApiBasedToolSchemaParser:
return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
@staticmethod
def parse_openapi_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_openapi_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
"""
parse openapi yaml to tool bundle
@@ -232,7 +231,7 @@ class ApiBasedToolSchemaParser:
return openapi
@staticmethod
def parse_swagger_yaml_to_tool_bundle(yaml: str, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_swagger_yaml_to_tool_bundle(yaml: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
"""
parse swagger yaml to tool bundle
@@ -248,7 +247,7 @@ class ApiBasedToolSchemaParser:
return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
@staticmethod
def parse_swagger_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_swagger_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
"""
parse swagger yaml to tool bundle
@@ -264,7 +263,7 @@ class ApiBasedToolSchemaParser:
return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
@staticmethod
def parse_openai_plugin_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> List[ApiBasedToolBundle]:
def parse_openai_plugin_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
"""
parse openapi plugin yaml to tool bundle
@@ -296,7 +295,7 @@ class ApiBasedToolSchemaParser:
return ApiBasedToolSchemaParser.parse_openapi_yaml_to_tool_bundle(response.text, extra_info=extra_info, warning=warning)
@staticmethod
def auto_parse_to_tool_bundle(content: str, extra_info: dict = None, warning: dict = None) -> Tuple[List[ApiBasedToolBundle], str]:
def auto_parse_to_tool_bundle(content: str, extra_info: dict = None, warning: dict = None) -> tuple[list[ApiBasedToolBundle], str]:
"""
auto parse to tool bundle

6
api/core/tools/utils/web_reader_tool.py
View File

@@ -7,7 +7,7 @@ import subprocess
import tempfile
import unicodedata
from contextlib import contextmanager
from typing import Any, Type
from typing import Any
import requests
from bs4 import BeautifulSoup, CData, Comment, NavigableString
@@ -56,7 +56,7 @@ class WebReaderTool(BaseTool):
"""Reader tool for getting website title and contents. Gives more control than SimpleReaderTool."""
name: str = "web_reader"
args_schema: Type[BaseModel] = WebReaderToolInput
args_schema: type[BaseModel] = WebReaderToolInput
description: str = "use this to read a website. " \
"If you can answer the question based on the information provided, " \
"there is no need to use."
@@ -211,7 +211,7 @@ def extract_using_readabilipy(html):
subprocess.check_call(["node", "ExtractArticle.js", "-i", html_path, "-o", article_json_path])
# Read output of call to Readability.parse() from JSON file and return as Python dictionary
with open(article_json_path, "r", encoding="utf-8") as json_file:
with open(article_json_path, encoding="utf-8") as json_file:
input_json = json.loads(json_file.read())
# Deleting files after processing