Feat/support multimodal embedding (#29115)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/core/model_runtime/entities/text_embedding_entities.py

@@ -19,7 +19,7 @@ class EmbeddingUsage(ModelUsage):
     latency: float
 
 
-class TextEmbeddingResult(BaseModel):
+class EmbeddingResult(BaseModel):
     """
     Model class for text embedding result.
     """
@@ -27,3 +27,13 @@ class TextEmbeddingResult(BaseModel):
     model: str
     embeddings: list[list[float]]
     usage: EmbeddingUsage
+
+
+class FileEmbeddingResult(BaseModel):
+    """
+    Model class for file embedding result.
+    """
+
+    model: str
+    embeddings: list[list[float]]
+    usage: EmbeddingUsage

api/core/model_runtime/model_providers/__base/rerank_model.py

@@ -50,3 +50,43 @@ class RerankModel(AIModel):
             )
         except Exception as e:
             raise self._transform_invoke_error(e)
+
+    def invoke_multimodal_rerank(
+        self,
+        model: str,
+        credentials: dict,
+        query: dict,
+        docs: list[dict],
+        score_threshold: float | None = None,
+        top_n: int | None = None,
+        user: str | None = None,
+    ) -> RerankResult:
+        """
+        Invoke multimodal rerank model
+        :param model: model name
+        :param credentials: model credentials
+        :param query: search query
+        :param docs: docs for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n
+        :param user: unique user id
+        :return: rerank result
+        """
+        try:
+            from core.plugin.impl.model import PluginModelClient
+
+            plugin_model_manager = PluginModelClient()
+            return plugin_model_manager.invoke_multimodal_rerank(
+                tenant_id=self.tenant_id,
+                user_id=user or "unknown",
+                plugin_id=self.plugin_id,
+                provider=self.provider_name,
+                model=model,
+                credentials=credentials,
+                query=query,
+                docs=docs,
+                score_threshold=score_threshold,
+                top_n=top_n,
+            )
+        except Exception as e:
+            raise self._transform_invoke_error(e)

api/core/model_runtime/model_providers/__base/text_embedding_model.py

@@ -2,7 +2,7 @@ from pydantic import ConfigDict
 
 from core.entities.embedding_type import EmbeddingInputType
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
+from core.model_runtime.entities.text_embedding_entities import EmbeddingResult
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
 
@@ -20,16 +20,18 @@ class TextEmbeddingModel(AIModel):
         self,
         model: str,
         credentials: dict,
-        texts: list[str],
+        texts: list[str] | None = None,
+        multimodel_documents: list[dict] | None = None,
         user: str | None = None,
         input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT,
-    ) -> TextEmbeddingResult:
+    ) -> EmbeddingResult:
         """
         Invoke text embedding model
 
         :param model: model name
         :param credentials: model credentials
         :param texts: texts to embed
+        :param files: files to embed
         :param user: unique user id
         :param input_type: input type
         :return: embeddings result
@@ -38,16 +40,29 @@ class TextEmbeddingModel(AIModel):
 
         try:
             plugin_model_manager = PluginModelClient()
-            return plugin_model_manager.invoke_text_embedding(
-                tenant_id=self.tenant_id,
-                user_id=user or "unknown",
-                plugin_id=self.plugin_id,
-                provider=self.provider_name,
-                model=model,
-                credentials=credentials,
-                texts=texts,
-                input_type=input_type,
-            )
+            if texts:
+                return plugin_model_manager.invoke_text_embedding(
+                    tenant_id=self.tenant_id,
+                    user_id=user or "unknown",
+                    plugin_id=self.plugin_id,
+                    provider=self.provider_name,
+                    model=model,
+                    credentials=credentials,
+                    texts=texts,
+                    input_type=input_type,
+                )
+            if multimodel_documents:
+                return plugin_model_manager.invoke_multimodal_embedding(
+                    tenant_id=self.tenant_id,
+                    user_id=user or "unknown",
+                    plugin_id=self.plugin_id,
+                    provider=self.provider_name,
+                    model=model,
+                    credentials=credentials,
+                    documents=multimodel_documents,
+                    input_type=input_type,
+                )
+            raise ValueError("No texts or files provided")
         except Exception as e:
             raise self._transform_invoke_error(e)