Feat/add weaviate tokenization configurable (#28159)

Co-authored-by: lijiezhao <lijiezhao@perfect99.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

api/.env.example

@@ -176,6 +176,7 @@ WEAVIATE_ENDPOINT=http://localhost:8080
 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
 WEAVIATE_GRPC_ENABLED=false
 WEAVIATE_BATCH_SIZE=100
+WEAVIATE_TOKENIZATION=word
 
 # OceanBase Vector configuration
 OCEANBASE_VECTOR_HOST=127.0.0.1

api/configs/middleware/vdb/weaviate_config.py

@@ -31,3 +31,8 @@ class WeaviateConfig(BaseSettings):
         description="Number of objects to be processed in a single batch operation (default is 100)",
         default=100,
     )
+
+    WEAVIATE_TOKENIZATION: str | None = Field(
+        description="Tokenization for Weaviate (default is word)",
+        default="word",
+    )

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -167,13 +167,18 @@ class WeaviateVector(BaseVector):
 
             try:
                 if not self._client.collections.exists(self._collection_name):
+                    tokenization = (
+                        wc.Tokenization(dify_config.WEAVIATE_TOKENIZATION)
+                        if dify_config.WEAVIATE_TOKENIZATION
+                        else wc.Tokenization.WORD
+                    )
                     self._client.collections.create(
                         name=self._collection_name,
                         properties=[
                             wc.Property(
                                 name=Field.TEXT_KEY.value,
                                 data_type=wc.DataType.TEXT,
-                                tokenization=wc.Tokenization.WORD,
+                                tokenization=tokenization,
                             ),
                             wc.Property(name="document_id", data_type=wc.DataType.TEXT),
                             wc.Property(name="doc_id", data_type=wc.DataType.TEXT),

api/tests/integration_tests/.env.example

@@ -62,6 +62,7 @@ WEAVIATE_ENDPOINT=http://localhost:8080
 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
 WEAVIATE_GRPC_ENABLED=false
 WEAVIATE_BATCH_SIZE=100
+WEAVIATE_TOKENIZATION=word
 
 
 # Upload configuration