admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat/enhance the multi-modal support (#8818)

Browse Source
This commit is contained in:
-LAN-
2024-10-21 10:43:49 +08:00
committed by GitHub
parent 7a1d6fe509
commit e61752bd3a
267 changed files with 6263 additions and 3523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/services/agent_service.py
View File

@@ -68,7 +68,7 @@ class AgentService:
"iterations": len(agent_thoughts),
},
"iterations": [],
"files": message.files,
"files": message.message_files,
}
agent_config = AgentConfigManager.convert(app_model.app_model_config.to_dict())

20
api/services/app_dsl_service.py
View File

@@ -3,9 +3,9 @@ import logging
import httpx
import yaml # type: ignore
from core.app.segments import factory
from events.app_event import app_model_config_was_updated, app_was_created
from extensions.ext_database import db
from factories import variable_factory
from models.account import Account
from models.model import App, AppMode, AppModelConfig
from models.workflow import Workflow
@@ -254,14 +254,18 @@ class AppDslService:
# init draft workflow
environment_variables_list = workflow_data.get("environment_variables") or []
environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list]
environment_variables = [
variable_factory.build_variable_from_mapping(obj) for obj in environment_variables_list
]
conversation_variables_list = workflow_data.get("conversation_variables") or []
conversation_variables = [factory.build_variable_from_mapping(obj) for obj in conversation_variables_list]
conversation_variables = [
variable_factory.build_variable_from_mapping(obj) for obj in conversation_variables_list
]
workflow_service = WorkflowService()
draft_workflow = workflow_service.sync_draft_workflow(
app_model=app,
graph=workflow_data.get("graph", {}),
features=workflow_data.get("../core/app/features", {}),
features=workflow_data.get("features", {}),
unique_hash=None,
account=account,
environment_variables=environment_variables,
@@ -295,9 +299,13 @@ class AppDslService:
# sync draft workflow
environment_variables_list = workflow_data.get("environment_variables") or []
environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list]
environment_variables = [
variable_factory.build_variable_from_mapping(obj) for obj in environment_variables_list
]
conversation_variables_list = workflow_data.get("conversation_variables") or []
conversation_variables = [factory.build_variable_from_mapping(obj) for obj in conversation_variables_list]
conversation_variables = [
variable_factory.build_variable_from_mapping(obj) for obj in conversation_variables_list
]
draft_workflow = workflow_service.sync_draft_workflow(
app_model=app_model,
graph=workflow_data.get("graph", {}),

4
api/services/app_generate_service.py
View File

@@ -1,4 +1,4 @@
from collections.abc import Generator
from collections.abc import Generator, Mapping
from typing import Any, Union
from openai._exceptions import RateLimitError
@@ -23,7 +23,7 @@ class AppGenerateService:
cls,
app_model: App,
user: Union[Account, EndUser],
args: Any,
args: Mapping[str, Any],
invoke_from: InvokeFrom,
streaming: bool = True,
):

84
api/services/file_service.py
View File

@@ -9,72 +9,55 @@ from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import NotFound
from configs import dify_config
from core.file.upload_file_parser import UploadFileParser
from constants import (
AUDIO_EXTENSIONS,
DOCUMENT_EXTENSIONS,
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
from core.file import helpers as file_helpers
from core.rag.extractor.extract_processor import ExtractProcessor
from extensions.ext_database import db
from extensions.ext_storage import storage
from models.account import Account
from models.model import EndUser, UploadFile
from services.errors.file import FileTooLargeError, UnsupportedFileTypeError
IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
ALLOWED_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
UNSTRUCTURED_ALLOWED_EXTENSIONS = [
"txt",
"markdown",
"md",
"pdf",
"html",
"htm",
"xlsx",
"xls",
"docx",
"csv",
"eml",
"msg",
"pptx",
"ppt",
"xml",
"epub",
]
from services.errors.file import FileNotExistsError, FileTooLargeError, UnsupportedFileTypeError
PREVIEW_WORDS_LIMIT = 3000
class FileService:
@staticmethod
def upload_file(file: FileStorage, user: Union[Account, EndUser], only_image: bool = False) -> UploadFile:
def upload_file(file: FileStorage, user: Union[Account, EndUser]) -> UploadFile:
# get file name
filename = file.filename
extension = file.filename.split(".")[-1]
if not filename:
raise FileNotExistsError
extension = filename.split(".")[-1]
if len(filename) > 200:
filename = filename.split(".")[0][:200] + "." + extension
etl_type = dify_config.ETL_TYPE
allowed_extensions = (
UNSTRUCTURED_ALLOWED_EXTENSIONS + IMAGE_EXTENSIONS
if etl_type == "Unstructured"
else ALLOWED_EXTENSIONS + IMAGE_EXTENSIONS
)
if extension.lower() not in allowed_extensions or only_image and extension.lower() not in IMAGE_EXTENSIONS:
raise UnsupportedFileTypeError()
# read file content
file_content = file.read()
# get file size
file_size = len(file_content)
if extension.lower() in IMAGE_EXTENSIONS:
# select file size limit
if extension in IMAGE_EXTENSIONS:
file_size_limit = dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT * 1024 * 1024
elif extension in VIDEO_EXTENSIONS:
file_size_limit = dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT * 1024 * 1024
elif extension in AUDIO_EXTENSIONS:
file_size_limit = dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT * 1024 * 1024
else:
file_size_limit = dify_config.UPLOAD_FILE_SIZE_LIMIT * 1024 * 1024
# check if the file size is exceeded
if file_size > file_size_limit:
message = f"File size exceeded. {file_size} > {file_size_limit}"
raise FileTooLargeError(message)
# user uuid as file name
# generate file key
file_uuid = str(uuid.uuid4())
if isinstance(user, Account):
@@ -150,9 +133,7 @@ class FileService:
# extract text from file
extension = upload_file.extension
etl_type = dify_config.ETL_TYPE
allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == "Unstructured" else ALLOWED_EXTENSIONS
if extension.lower() not in allowed_extensions:
if extension.lower() not in DOCUMENT_EXTENSIONS:
raise UnsupportedFileTypeError()
text = ExtractProcessor.load_from_upload_file(upload_file, return_text=True)
@@ -161,8 +142,10 @@ class FileService:
return text
@staticmethod
def get_image_preview(file_id: str, timestamp: str, nonce: str, sign: str) -> tuple[Generator, str]:
result = UploadFileParser.verify_image_file_signature(file_id, timestamp, nonce, sign)
def get_image_preview(file_id: str, timestamp: str, nonce: str, sign: str):
result = file_helpers.verify_image_signature(
upload_file_id=file_id, timestamp=timestamp, nonce=nonce, sign=sign
)
if not result:
raise NotFound("File not found or signature is invalid")
@@ -180,6 +163,21 @@ class FileService:
return generator, upload_file.mime_type
@staticmethod
def get_signed_file_preview(file_id: str, timestamp: str, nonce: str, sign: str):
result = file_helpers.verify_file_signature(upload_file_id=file_id, timestamp=timestamp, nonce=nonce, sign=sign)
if not result:
raise NotFound("File not found or signature is invalid")
upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
if not upload_file:
raise NotFound("File not found or signature is invalid")
generator = storage.load(upload_file.key, stream=True)
return generator, upload_file.mime_type
@staticmethod
def get_public_image_preview(file_id: str) -> tuple[Generator, str]:
upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()

26
api/services/tools/workflow_tools_manage_service.py
View File

@@ -1,6 +1,7 @@
import json
from collections.abc import Mapping
from datetime import datetime
from typing import Optional
from typing import Any, Optional
from sqlalchemy import or_
@@ -21,9 +22,9 @@ class WorkflowToolManageService:
Service class for managing workflow tools.
"""
@classmethod
@staticmethod
def create_workflow_tool(
cls,
*,
user_id: str,
tenant_id: str,
workflow_app_id: str,
@@ -31,22 +32,10 @@ class WorkflowToolManageService:
label: str,
icon: dict,
description: str,
parameters: list[dict],
parameters: Mapping[str, Any],
privacy_policy: str = "",
labels: Optional[list[str]] = None,
) -> dict:
"""
Create a workflow tool.
:param user_id: the user id
:param tenant_id: the tenant id
:param name: the name
:param icon: the icon
:param description: the description
:param parameters: the parameters
:param privacy_policy: the privacy policy
:param labels: labels
:return: the created tool
"""
WorkflowToolConfigurationUtils.check_parameter_configurations(parameters)
# check if the name is unique
@@ -63,12 +52,11 @@ class WorkflowToolManageService:
if existing_workflow_tool_provider is not None:
raise ValueError(f"Tool with name {name} or app_id {workflow_app_id} already exists")
app: App = db.session.query(App).filter(App.id == workflow_app_id, App.tenant_id == tenant_id).first()
app = db.session.query(App).filter(App.id == workflow_app_id, App.tenant_id == tenant_id).first()
if app is None:
raise ValueError(f"App {workflow_app_id} not found")
workflow: Workflow = app.workflow
workflow = app.workflow
if workflow is None:
raise ValueError(f"Workflow not found for app {workflow_app_id}")

6
api/services/workflow/workflow_converter.py
View File

@@ -13,12 +13,12 @@ from core.app.app_config.entities import (
from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager
from core.app.apps.chat.app_config_manager import ChatAppConfigManager
from core.app.apps.completion.app_config_manager import CompletionAppConfigManager
from core.file.file_obj import FileExtraConfig
from core.file.models import FileExtraConfig
from core.helper import encrypter
from core.model_runtime.entities.llm_entities import LLMMode
from core.model_runtime.utils.encoders import jsonable_encoder
from core.prompt.simple_prompt_transform import SimplePromptTransform
from core.workflow.entities.node_entities import NodeType
from core.workflow.nodes import NodeType
from events.app_event import app_was_created
from extensions.ext_database import db
from models.account import Account
@@ -522,7 +522,7 @@ class WorkflowConverter:
"vision": {
"enabled": file_upload is not None,
"variable_selector": ["sys", "files"] if file_upload is not None else None,
"configs": {"detail": file_upload.image_config["detail"]}
"configs": {"detail": file_upload.image_config.detail}
if file_upload is not None and file_upload.image_config is not None
else None,
},

10
api/services/workflow_app_service.py
View File

@@ -4,9 +4,9 @@ from flask_sqlalchemy.pagination import Pagination
from sqlalchemy import and_, or_
from extensions.ext_database import db
from models import CreatedByRole
from models.model import App, EndUser
from models.workflow import WorkflowAppLog, WorkflowRun, WorkflowRunStatus
from models import App, EndUser, WorkflowAppLog, WorkflowRun
from models.enums import CreatedByRole
from models.workflow import WorkflowRunStatus
class WorkflowAppService:
@@ -21,7 +21,7 @@ class WorkflowAppService:
WorkflowAppLog.tenant_id == app_model.tenant_id, WorkflowAppLog.app_id == app_model.id
)
status = WorkflowRunStatus.value_of(args.get("status")) if args.get("status") else None
status = WorkflowRunStatus.value_of(args.get("status", "")) if args.get("status") else None
keyword = args["keyword"]
if keyword or status:
query = query.join(WorkflowRun, WorkflowRun.id == WorkflowAppLog.workflow_run_id)
@@ -42,7 +42,7 @@ class WorkflowAppService:
query = query.outerjoin(
EndUser,
and_(WorkflowRun.created_by == EndUser.id, WorkflowRun.created_by_role == CreatedByRole.END_USER.value),
and_(WorkflowRun.created_by == EndUser.id, WorkflowRun.created_by_role == CreatedByRole.END_USER),
).filter(or_(*keyword_conditions))
if status:

2
api/services/workflow_run_service.py
View File

@@ -1,11 +1,11 @@
from extensions.ext_database import db
from libs.infinite_scroll_pagination import InfiniteScrollPagination
from models.enums import WorkflowRunTriggeredFrom
from models.model import App
from models.workflow import (
WorkflowNodeExecution,
WorkflowNodeExecutionTriggeredFrom,
WorkflowRun,
WorkflowRunTriggeredFrom,
)

17
api/services/workflow_service.py
View File

@@ -6,19 +6,20 @@ from typing import Optional
from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
from core.app.segments import Variable
from core.model_runtime.utils.encoders import jsonable_encoder
from core.workflow.entities.node_entities import NodeRunResult, NodeType
from core.variables import Variable
from core.workflow.entities.node_entities import NodeRunResult
from core.workflow.errors import WorkflowNodeRunFailedError
from core.workflow.nodes import NodeType
from core.workflow.nodes.event import RunCompletedEvent
from core.workflow.nodes.node_mapping import node_classes
from core.workflow.nodes.node_mapping import node_type_classes_mapping
from core.workflow.workflow_entry import WorkflowEntry
from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated
from extensions.ext_database import db
from models.account import Account
from models.enums import CreatedByRole
from models.model import App, AppMode
from models.workflow import (
CreatedByRole,
Workflow,
WorkflowNodeExecution,
WorkflowNodeExecutionStatus,
@@ -175,7 +176,7 @@ class WorkflowService:
"""
# return default block config
default_block_configs = []
for node_type, node_class in node_classes.items():
for node_type, node_class in node_type_classes_mapping.items():
default_config = node_class.get_default_config()
if default_config:
default_block_configs.append(default_config)
@@ -189,10 +190,10 @@ class WorkflowService:
:param filters: filter by node config parameters.
:return:
"""
node_type_enum: NodeType = NodeType.value_of(node_type)
node_type_enum: NodeType = NodeType(node_type)
# return default block config
node_class = node_classes.get(node_type_enum)
node_class = node_type_classes_mapping.get(node_type_enum)
if not node_class:
return None
@@ -251,7 +252,7 @@ class WorkflowService:
workflow_node_execution.triggered_from = WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP.value
workflow_node_execution.index = 1
workflow_node_execution.node_id = node_id
workflow_node_execution.node_type = node_instance.node_type.value
workflow_node_execution.node_type = node_instance.node_type
workflow_node_execution.title = node_instance.node_data.title
workflow_node_execution.elapsed_time = time.perf_counter() - start_at
workflow_node_execution.created_by_role = CreatedByRole.ACCOUNT.value