Feat/assistant app (#2086)

Co-authored-by: chenhe <guchenhe@gmail.com>
Co-authored-by: Pascal M <11357019+perzeuss@users.noreply.github.com>

api/core/tools/provider/builtin/dalle/dalle.py

@@ -0,0 +1,23 @@
+from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
+from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool
+from core.tools.errors import ToolProviderCredentialValidationError
+
+from typing import Any, Dict
+
+class DALLEProvider(BuiltinToolProviderController):
+    def _validate_credentials(self, credentials: Dict[str, Any]) -> None:
+        try:
+            DallE2Tool().fork_tool_runtime(
+                meta={
+                    "credentials": credentials,
+                }
+            ).invoke(
+                user_id='',
+                tool_paramters={
+                    "prompt": "cute girl, blue eyes, white hair, anime style",
+                    "size": "small",
+                    "n": 1
+                },
+            )
+        except Exception as e:
+            raise ToolProviderCredentialValidationError(str(e))

api/core/tools/provider/builtin/dalle/dalle.yaml

@@ -0,0 +1,47 @@
+identity:
+  author: Dify
+  name: dalle
+  label:
+    en_US: DALL-E
+    zh_Hans: DALL-E 绘画
+  description:
+    en_US: DALL-E art
+    zh_Hans: DALL-E 绘画
+  icon: icon.png
+credentails_for_provider:
+  openai_api_key:
+    type: secret-input
+    required: true
+    label:
+      en_US: OpenAI API key
+      zh_Hans: OpenAI API key
+    help:
+      en_US: Please input your OpenAI API key
+      zh_Hans: 请输入你的 OpenAI API key
+    placeholder:
+      en_US: Please input your OpenAI API key
+      zh_Hans: 请输入你的 OpenAI API key
+  openai_organizaion_id:
+    type: text-input
+    required: false
+    label:
+      en_US: OpenAI organization ID
+      zh_Hans: OpenAI organization ID
+    help:
+      en_US: Please input your OpenAI organization ID
+      zh_Hans: 请输入你的 OpenAI organization ID
+    placeholder:
+      en_US: Please input your OpenAI organization ID
+      zh_Hans: 请输入你的 OpenAI organization ID
+  openai_base_url:
+    type: text-input
+    required: false
+    label:
+      en_US: OpenAI base URL
+      zh_Hans: OpenAI base URL
+    help:
+      en_US: Please input your OpenAI base URL
+      zh_Hans: 请输入你的 OpenAI base URL
+    placeholder:
+      en_US: Please input your OpenAI base URL
+      zh_Hans: 请输入你的 OpenAI base URL

api/core/tools/provider/builtin/dalle/tools/dalle2.py

@@ -0,0 +1,66 @@
+from typing import Any, Dict, List, Union
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+from base64 import b64decode
+from os.path import join
+
+from openai import OpenAI
+
+class DallE2Tool(BuiltinTool):
+    def _invoke(self, 
+                user_id: str, 
+               tool_paramters: Dict[str, Any], 
+        ) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
+        """
+            invoke tools
+        """
+        openai_organization = self.runtime.credentials.get('openai_organizaion_id', None)
+        if not openai_organization:
+            openai_organization = None
+        openai_base_url = self.runtime.credentials.get('openai_base_url', None)
+        if not openai_base_url:
+            openai_base_url = None
+        else:
+            openai_base_url = join(openai_base_url, 'v1')
+
+        client = OpenAI(
+            api_key=self.runtime.credentials['openai_api_key'],
+            base_url=openai_base_url,
+            organization=openai_organization
+        )
+
+        SIZE_MAPPING = {
+            'small': '256x256',
+            'medium': '512x512',
+            'large': '1024x1024',
+        }
+
+        # prompt
+        prompt = tool_paramters.get('prompt', '')
+        if not prompt:
+            return self.create_text_message('Please input prompt')
+        
+        # get size
+        size = SIZE_MAPPING[tool_paramters.get('size', 'large')]
+
+        # get n
+        n = tool_paramters.get('n', 1)
+
+        # call openapi dalle2
+        response = client.images.generate(
+            prompt=prompt,
+            model='dall-e-2',
+            size=size,
+            n=n,
+            response_format='b64_json'
+        )
+
+        result = []
+
+        for image in response.data:
+            result.append(self.create_blob_message(blob=b64decode(image.b64_json), 
+                                                   meta={ 'mime_type': 'image/png' },
+                                                    save_as=self.VARIABLE_KEY.IMAGE.value))
+
+        return result

api/core/tools/provider/builtin/dalle/tools/dalle2.yaml

@@ -0,0 +1,63 @@
+identity:
+  name: dalle2
+  author: Dify
+  label:
+    en_US: DALL-E 2
+    zh_Hans: DALL-E 2 绘画
+  description:
+    en_US: DALL-E 2 is a powerful drawing tool that can draw the image you want based on your prompt
+    zh_Hans: DALL-E 2 是一个强大的绘画工具，它可以根据您的提示词绘制出您想要的图像
+description:
+  human:
+    en_US: DALL-E is a text to image tool
+    zh_Hans: DALL-E 是一个文本到图像的工具
+  llm: DALL-E is a tool used to generate images from text
+parameters:
+  - name: prompt
+    type: string
+    required: true
+    label:
+      en_US: Prompt
+      zh_Hans: 提示词
+    human_description:
+      en_US: Image prompt, you can check the official documentation of DallE 2
+      zh_Hans: 图像提示词，您可以查看DallE 2 的官方文档
+    llm_description: Image prompt of DallE 2, you should describe the image you want to generate as a list of words as possible as detailed
+    form: llm
+  - name: size
+    type: select
+    required: true
+    human_description:
+      en_US: used for selecting the image size
+      zh_Hans: 用于选择图像大小
+    label:
+      en_US: Image size
+      zh_Hans: 图像大小
+    form: form
+    options:
+      - value: small
+        label:
+          en_US: Small(256x256)
+          zh_Hans: 小(256x256)
+      - value: medium
+        label:
+          en_US: Medium(512x512)
+          zh_Hans: 中(512x512)
+      - value: large
+        label:
+          en_US: Large(1024x1024)
+          zh_Hans: 大(1024x1024)
+    default: large
+  - name: n
+    type: number
+    required: true
+    human_description:
+      en_US: used for selecting the number of images
+      zh_Hans: 用于选择图像数量
+    label:
+      en_US: Number of images
+      zh_Hans: 图像数量
+    form: form
+    default: 1
+    min: 1
+    max: 10

api/core/tools/provider/builtin/dalle/tools/dalle3.py

@@ -0,0 +1,74 @@
+from typing import Any, Dict, List, Union
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+from base64 import b64decode
+from os.path import join
+
+from openai import OpenAI
+
+class DallE3Tool(BuiltinTool):
+    def _invoke(self, 
+                user_id: str, 
+               tool_paramters: Dict[str, Any], 
+        ) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
+        """
+            invoke tools
+        """
+        openai_organization = self.runtime.credentials.get('openai_organizaion_id', None)
+        if not openai_organization:
+            openai_organization = None
+        openai_base_url = self.runtime.credentials.get('openai_base_url', None)
+        if not openai_base_url:
+            openai_base_url = None
+        else:
+            openai_base_url = join(openai_base_url, 'v1')
+
+        client = OpenAI(
+            api_key=self.runtime.credentials['openai_api_key'],
+            base_url=openai_base_url,
+            organization=openai_organization
+        )
+
+        SIZE_MAPPING = {
+            'square': '1024x1024',
+            'vertical': '1024x1792',
+            'horizontal': '1792x1024',
+        }
+
+        # prompt
+        prompt = tool_paramters.get('prompt', '')
+        if not prompt:
+            return self.create_text_message('Please input prompt')
+        # get size
+        size = SIZE_MAPPING[tool_paramters.get('size', 'square')]
+        # get n
+        n = tool_paramters.get('n', 1)
+        # get quality
+        quality = tool_paramters.get('quality', 'standard')
+        if quality not in ['standard', 'hd']:
+            return self.create_text_message('Invalid quality')
+        # get style
+        style = tool_paramters.get('style', 'vivid')
+        if style not in ['natural', 'vivid']:
+            return self.create_text_message('Invalid style')
+
+        # call openapi dalle3
+        response = client.images.generate(
+            prompt=prompt,
+            model='dall-e-3',
+            size=size,
+            n=n,
+            style=style,
+            quality=quality,
+            response_format='b64_json'
+        )
+
+        result = []
+
+        for image in response.data:
+            result.append(self.create_blob_message(blob=b64decode(image.b64_json), 
+                                                   meta={ 'mime_type': 'image/png' },
+                                                    save_as=self.VARIABLE_KEY.IMAGE.value))
+
+        return result

api/core/tools/provider/builtin/dalle/tools/dalle3.yaml

@@ -0,0 +1,103 @@
+identity:
+  name: dalle3
+  author: Dify
+  label:
+    en_US: DALL-E 3
+    zh_Hans: DALL-E 3 绘画
+  description:
+    en_US: DALL-E 3 is a powerful drawing tool that can draw the image you want based on your prompt, compared to DallE 2, DallE 3 has stronger drawing ability, but it will consume more resources
+    zh_Hans: DALL-E 3 是一个强大的绘画工具，它可以根据您的提示词绘制出您想要的图像，相比于DallE 2， DallE 3拥有更强的绘画能力，但会消耗更多的资源
+description:
+  human:
+    en_US: DALL-E is a text to image tool
+    zh_Hans: DALL-E 是一个文本到图像的工具
+  llm: DALL-E is a tool used to generate images from text
+parameters:
+  - name: prompt
+    type: string
+    required: true
+    label:
+      en_US: Prompt
+      zh_Hans: 提示词
+    human_description:
+      en_US: Image prompt, you can check the official documentation of DallE 3
+      zh_Hans: 图像提示词，您可以查看DallE 3 的官方文档
+    llm_description: Image prompt of DallE 3, you should describe the image you want to generate as a list of words as possible as detailed
+    form: llm
+  - name: size
+    type: select
+    required: true
+    human_description:
+      en_US: selecting the image size
+      zh_Hans: 选择图像大小
+    label:
+      en_US: Image size
+      zh_Hans: 图像大小
+    form: form
+    options:
+      - value: square
+        label:
+          en_US: Squre(1024x1024)
+          zh_Hans: 方(1024x1024)
+      - value: vertical
+        label:
+          en_US: Vertical(1024x1792)
+          zh_Hans: 竖屏(1024x1792)
+      - value: horizontal
+        label:
+          en_US: Horizontal(1792x1024)
+          zh_Hans: 横屏(1792x1024)
+    default: square
+  - name: n
+    type: number
+    required: true
+    human_description:
+      en_US: selecting the number of images
+      zh_Hans: 选择图像数量
+    label:
+      en_US: Number of images
+      zh_Hans: 图像数量
+    form: form
+    min: 1
+    max: 1
+    default: 1
+  - name: quality
+    type: select
+    required: true
+    human_description:
+      en_US: selecting the image quality
+      zh_Hans: 选择图像质量
+    label:
+      en_US: Image quality
+      zh_Hans: 图像质量
+    form: form
+    options:
+      - value: standard
+        label:
+          en_US: Standard
+          zh_Hans: 标准
+      - value: hd
+        label:
+          en_US: HD
+          zh_Hans: 高清
+    default: standard
+  - name: style
+    type: select
+    required: true
+    human_description:
+      en_US: selecting the image style
+      zh_Hans: 选择图像风格
+    label:
+      en_US: Image style
+      zh_Hans: 图像风格
+    form: form
+    options:
+      - value: vivid
+        label:
+          en_US: Vivid
+          zh_Hans: 生动
+      - value: natural
+        label:
+          en_US: Natural
+          zh_Hans: 自然
+    default: vivid