enhancement: introduce Ruff for Python linter for reordering and removing unused imports with automated pre-commit and sytle check (#2366)

This commit is contained in:
Bowen Liang
2024-02-06 13:21:13 +08:00
committed by GitHub
parent 42344795cd
commit 843280f82b
333 changed files with 2109 additions and 1050 deletions

View File

@@ -1,8 +1,9 @@
from typing import Any, Dict, List, Optional
from core.tools.entities.tool_entities import ToolParameter, ToolProviderType
from pydantic import BaseModel
from core.tools.entities.tool_entities import ToolParameter, ToolProviderType
class ApiBasedToolBundle(BaseModel):
"""

View File

@@ -1,9 +1,10 @@
from enum import Enum
from typing import Any, Dict, List, Optional, Union, cast
from core.tools.entities.common_entities import I18nObject
from pydantic import BaseModel, Field
from core.tools.entities.common_entities import I18nObject
class ToolProviderType(Enum):
"""

View File

@@ -1,10 +1,11 @@
from enum import Enum
from typing import Dict, List, Optional
from pydantic import BaseModel
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolProviderCredentials
from core.tools.tool.tool import ToolParameter
from pydantic import BaseModel
class UserToolProvider(BaseModel):

View File

@@ -11,8 +11,13 @@ from core.model_manager import ModelManager
from core.model_runtime.entities.llm_entities import LLMResult
from core.model_runtime.entities.message_entities import PromptMessage
from core.model_runtime.entities.model_entities import ModelType
from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError,
InvokeRateLimitError, InvokeServerUnavailableError)
from core.model_runtime.errors.invoke import (
InvokeAuthorizationError,
InvokeBadRequestError,
InvokeConnectionError,
InvokeRateLimitError,
InvokeServerUnavailableError,
)
from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel, ModelPropertyKey
from core.model_runtime.utils.encoders import jsonable_encoder
from core.tools.model.errors import InvokeModelError

View File

@@ -2,8 +2,12 @@ from typing import Any, Dict, List
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiBasedToolBundle
from core.tools.entities.tool_entities import (ApiProviderAuthType, ToolCredentialsOption, ToolProviderCredentials,
ToolProviderType)
from core.tools.entities.tool_entities import (
ApiProviderAuthType,
ToolCredentialsOption,
ToolProviderCredentials,
ToolProviderType,
)
from core.tools.provider.tool_provider import ToolProviderController
from core.tools.tool.api_tool import ApiTool
from core.tools.tool.tool import Tool

View File

@@ -1,9 +1,9 @@
from core.tools.entities.user_entities import UserToolProvider
from core.tools.entities.tool_entities import ToolProviderType
from typing import List
from yaml import load, FullLoader
import os.path
from typing import List
from yaml import FullLoader, load
from core.tools.entities.user_entities import UserToolProvider
position = {}

View File

@@ -1,10 +1,10 @@
from base64 import b64decode
from os.path import join
from typing import Any, Dict, List, Union
from openai import AzureOpenAI
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from openai import AzureOpenAI
class DallE3Tool(BuiltinTool):

View File

@@ -1,9 +1,9 @@
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
from typing import Any, Dict
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.bing.tools.bing_web_search import BingSearchTool
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
from typing import Any, Dict, List
class BingProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: Dict[str, Any]) -> None:

View File

@@ -1,10 +1,11 @@
from core.tools.tool.builtin_tool import BuiltinTool
from core.tools.entities.tool_entities import ToolInvokeMessage
from typing import Any, Dict, List, Union
from os import path
from requests import get
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
class BingSearchTool(BuiltinTool):
url = 'https://api.bing.microsoft.com/v7.0/search'

View File

@@ -1,4 +1,5 @@
import matplotlib.pyplot as plt
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.chart.tools.line import LinearChartTool
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController

View File

@@ -2,6 +2,7 @@ import io
from typing import Any, Dict, List, Union
import matplotlib.pyplot as plt
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool

View File

@@ -2,6 +2,7 @@ import io
from typing import Any, Dict, List, Union
import matplotlib.pyplot as plt
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool

View File

@@ -2,6 +2,7 @@ import io
from typing import Any, Dict, List, Union
import matplotlib.pyplot as plt
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool

View File

@@ -2,9 +2,10 @@ from base64 import b64decode
from os.path import join
from typing import Any, Dict, List, Union
from openai import OpenAI
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from openai import OpenAI
class DallE2Tool(BuiltinTool):

View File

@@ -2,9 +2,10 @@ from base64 import b64decode
from os.path import join
from typing import Any, Dict, List, Union
from openai import OpenAI
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from openai import OpenAI
class DallE3Tool(BuiltinTool):

View File

@@ -1,6 +1,7 @@
import urllib.parse
import requests
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController

View File

@@ -2,6 +2,7 @@ import json
from typing import Any, Dict, List, Union
import requests
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool

View File

@@ -1,4 +1,5 @@
import requests
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController

View File

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Union
from urllib.parse import quote
import requests
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool

View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List
from typing import Any, Dict
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.google.tools.google_search import GoogleSearchTool

View File

@@ -2,9 +2,10 @@ import os
import sys
from typing import Any, Dict, List, Union
from serpapi import GoogleSearch
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from serpapi import GoogleSearch
class HiddenPrints:

View File

@@ -1,11 +1,11 @@
import logging
from datetime import datetime, timezone
from typing import Any, Dict, List, Union
import numexpr as ne
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from pytz import timezone as pytz_timezone
import numexpr as ne
class EvaluateExpressionTool(BuiltinTool):
def _invoke(self,

View File

@@ -5,12 +5,13 @@ from copy import deepcopy
from os.path import join
from typing import Any, Dict, List, Union
from httpx import get, post
from PIL import Image
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.tool.builtin_tool import BuiltinTool
from httpx import get, post
from PIL import Image
DRAW_TEXT_OPTIONS = {
"prompt": "",

View File

@@ -1,9 +1,10 @@
from datetime import datetime, timezone
from typing import Any, Dict, List, Union
from pytz import timezone as pytz_timezone
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from pytz import timezone as pytz_timezone
class CurrentTimeTool(BuiltinTool):

View File

@@ -1,11 +1,12 @@
from base64 import b64decode
from typing import Any, Dict, List, Union
from httpx import post
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.vectorizer.tools.test_data import VECTORIZER_ICON_PNG
from core.tools.tool.builtin_tool import BuiltinTool
from httpx import post
class VectorizerTool(BuiltinTool):

View File

@@ -1,4 +1,4 @@
from typing import Any, Dict, List
from typing import Any, Dict
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.webscraper.tools.webscraper import WebscraperTool

View File

@@ -1,11 +1,12 @@
from typing import Any, Dict, List, Union
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from langchain import WikipediaAPIWrapper
from langchain.tools import WikipediaQueryRun
from pydantic import BaseModel, Field
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
class WikipediaInput(BaseModel):
query: str = Field(..., description="search query.")

View File

@@ -1,9 +1,10 @@
from typing import Any, Dict, List, Union
from httpx import get
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.errors import ToolInvokeError, ToolProviderCredentialValidationError
from core.tools.tool.builtin_tool import BuiltinTool
from httpx import get
class WolframAlphaTool(BuiltinTool):

View File

@@ -1,10 +1,8 @@
from typing import Any, Dict, List
from typing import Any, Dict
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin.wolframalpha.tools.wolframalpha import WolframAlphaTool
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
from core.tools.tool.tool import Tool
class GoogleProvider(BuiltinToolProviderController):

View File

@@ -2,11 +2,12 @@ from datetime import datetime
from typing import Any, Dict, List, Union
import pandas as pd
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from requests.exceptions import HTTPError, ReadTimeout
from yfinance import download
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
class YahooFinanceAnalyticsTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \

View File

@@ -1,9 +1,10 @@
from typing import Any, Dict, List, Union
import yfinance
from requests.exceptions import HTTPError, ReadTimeout
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from requests.exceptions import HTTPError, ReadTimeout
class YahooFinanceSearchTickerTool(BuiltinTool):

View File

@@ -1,9 +1,10 @@
from typing import Any, Dict, List, Union
from requests.exceptions import HTTPError, ReadTimeout
from yfinance import Ticker
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from requests.exceptions import HTTPError, ReadTimeout
from yfinance import Ticker
class YahooFinanceSearchTickerTool(BuiltinTool):

View File

@@ -1,9 +1,10 @@
from datetime import datetime
from typing import Any, Dict, List, Union
from googleapiclient.discovery import build
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
from googleapiclient.discovery import build
class YoutubeVideosAnalyticsTool(BuiltinTool):

View File

@@ -3,14 +3,19 @@ from abc import abstractmethod
from os import listdir, path
from typing import Any, Dict, List
from yaml import FullLoader, load
from core.tools.entities.tool_entities import ToolParameter, ToolProviderCredentials, ToolProviderType
from core.tools.entities.user_entities import UserToolProviderCredentials
from core.tools.errors import (ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError,
ToolProviderNotFoundError)
from core.tools.errors import (
ToolNotFoundError,
ToolParameterValidationError,
ToolProviderCredentialValidationError,
ToolProviderNotFoundError,
)
from core.tools.provider.tool_provider import ToolProviderController
from core.tools.tool.builtin_tool import BuiltinTool
from core.tools.tool.tool import Tool
from yaml import FullLoader, load
class BuiltinToolProviderController(ToolProviderController):

View File

@@ -1,12 +1,17 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
from core.tools.entities.tool_entities import (ToolParameter, ToolProviderCredentials, ToolProviderIdentity,
ToolProviderType)
from pydantic import BaseModel
from core.tools.entities.tool_entities import (
ToolParameter,
ToolProviderCredentials,
ToolProviderIdentity,
ToolProviderType,
)
from core.tools.entities.user_entities import UserToolProviderCredentials
from core.tools.errors import ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError
from core.tools.tool.tool import Tool
from pydantic import BaseModel
class ToolProviderController(BaseModel, ABC):

View File

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Union
import httpx
import requests
import core.helper.ssrf_proxy as ssrf_proxy
from core.tools.entities.tool_bundle import ApiBasedToolBundle
from core.tools.entities.tool_entities import ToolInvokeMessage

View File

@@ -1,4 +1,3 @@
from enum import Enum
from typing import List
from core.model_runtime.entities.llm_entities import LLMResult

View File

@@ -1,7 +1,10 @@
import json
import threading
from typing import List, Optional, Type
from flask import Flask, current_app
from langchain.tools import BaseTool
from pydantic import BaseModel, Field
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.embedding.cached_embedding import CacheEmbedding
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -10,10 +13,7 @@ from core.model_manager import ModelManager
from core.model_runtime.entities.model_entities import ModelType
from core.rerank.rerank import RerankRunner
from extensions.ext_database import db
from flask import Flask, current_app
from langchain.tools import BaseTool
from models.dataset import Dataset, Document, DocumentSegment
from pydantic import BaseModel, Field
from services.retrieval_service import RetrievalService
default_retrieval_model = {

View File

@@ -1,6 +1,10 @@
import threading
from typing import List, Optional, Type
from flask import current_app
from langchain.tools import BaseTool
from pydantic import BaseModel, Field
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.embedding.cached_embedding import CacheEmbedding
from core.index.keyword_table_index.keyword_table_index import KeywordTableConfig, KeywordTableIndex
@@ -9,10 +13,7 @@ from core.model_runtime.entities.model_entities import ModelType
from core.model_runtime.errors.invoke import InvokeAuthorizationError
from core.rerank.rerank import RerankRunner
from extensions.ext_database import db
from flask import current_app
from langchain.tools import BaseTool
from models.dataset import Dataset, Document, DocumentSegment
from pydantic import BaseModel, Field
from services.retrieval_service import RetrievalService
default_retrieval_model = {

View File

@@ -1,4 +1,6 @@
from typing import Any, Dict, List, Union
from typing import Any, Dict, List
from langchain.tools import BaseTool
from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
from core.entities.application_entities import DatasetRetrieveConfigEntity, InvokeFrom
@@ -6,7 +8,6 @@ from core.features.dataset_retrieval import DatasetRetrievalFeature
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter
from core.tools.tool.tool import Tool
from langchain.tools import BaseTool
class DatasetRetrieverTool(Tool):

View File

@@ -2,12 +2,20 @@ from abc import ABC, abstractmethod
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.tools.entities.tool_entities import (ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter,
ToolRuntimeImageVariable, ToolRuntimeVariable, ToolRuntimeVariablePool)
from core.tools.tool_file_manager import ToolFileManager
from pydantic import BaseModel
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.tools.entities.tool_entities import (
ToolDescription,
ToolIdentity,
ToolInvokeMessage,
ToolParameter,
ToolRuntimeImageVariable,
ToolRuntimeVariable,
ToolRuntimeVariablePool,
)
from core.tools.tool_file_manager import ToolFileManager
class Tool(BaseModel, ABC):
identity: ToolIdentity = None

View File

@@ -8,10 +8,11 @@ from mimetypes import guess_extension, guess_type
from typing import Generator, Tuple, Union
from uuid import uuid4
from extensions.ext_database import db
from extensions.ext_storage import storage
from flask import current_app
from httpx import get
from extensions.ext_database import db
from extensions.ext_storage import storage
from models.model import MessageFile
from models.tools import ToolFile

View File

@@ -1,10 +1,12 @@
from typing import Dict, Any
from typing import Any, Dict
from pydantic import BaseModel
from core.helper import encrypter
from core.helper.tool_provider_cache import ToolProviderCredentialsCache, ToolProviderCredentialsCacheType
from core.tools.entities.tool_entities import ToolProviderCredentials
from core.tools.provider.tool_provider import ToolProviderController
from core.helper import encrypter
from core.helper.tool_provider_cache import ToolProviderCredentialsCacheType, ToolProviderCredentialsCache
class ToolConfiguration(BaseModel):
tenant_id: str

View File

@@ -1,4 +1,3 @@
from enum import Enum
from typing import List
from pydantic import BaseModel

View File

@@ -1,15 +1,15 @@
from json import dumps as json_dumps
from json import loads as json_loads
from typing import List, Tuple
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiBasedToolBundle
from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolParameter, ToolParameterOption
from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolProviderNotFoundError
from requests import get
from yaml import FullLoader, load
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiBasedToolBundle
from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolParameter
from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolProviderNotFoundError
class ApiBasedToolSchemaParser:
@staticmethod

View File

@@ -11,10 +11,6 @@ from typing import Any, Type
import requests
from bs4 import BeautifulSoup, CData, Comment, NavigableString
from core.chain.llm_chain import LLMChain
from core.data_loader import file_extractor
from core.data_loader.file_extractor import FileExtractor
from core.entities.application_entities import ModelConfigEntity
from langchain.chains import RefineDocumentsChain
from langchain.chains.summarize import refine_prompts
from langchain.schema import Document
@@ -24,6 +20,11 @@ from newspaper import Article
from pydantic import BaseModel, Field
from regex import regex
from core.chain.llm_chain import LLMChain
from core.data_loader import file_extractor
from core.data_loader.file_extractor import FileExtractor
from core.entities.application_entities import ModelConfigEntity
FULL_TEMPLATE = """
TITLE: {title}
AUTHORS: {authors}