admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: ensure vector database cleanup on dataset deletion regardless of document presence (affects all 33 vector databases) (#23574)

Browse Source 
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
yunqiqiliang
2025-08-08 09:18:43 +08:00
committed by GitHub
parent 4b0480c8b3
commit 62772e8871
5 changed files with 97 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
api/tests/integration_tests/vdb/clickzetta/test_clickzetta.py
View File

@@ -39,10 +39,7 @@ class TestClickzettaVector(AbstractVectorTest):
)
with setup_mock_redis():
vector = ClickzettaVector(
collection_name="test_collection_" + str(os.getpid()),
config=config
)
vector = ClickzettaVector(collection_name="test_collection_" + str(os.getpid()), config=config)
yield vector
@@ -114,7 +111,7 @@ class TestClickzettaVector(AbstractVectorTest):
"category": "technical" if i % 2 == 0 else "general",
"document_id": f"doc_{i // 3}", # Group documents
"importance": i,
}
},
)
documents.append(doc)
# Create varied embeddings
@@ -124,22 +121,14 @@ class TestClickzettaVector(AbstractVectorTest):
# Test vector search with document filter
query_vector = [0.5, 1.0, 1.5, 2.0]
results = vector_store.search_by_vector(
query_vector,
top_k=5,
document_ids_filter=["doc_0", "doc_1"]
)
results = vector_store.search_by_vector(query_vector, top_k=5, document_ids_filter=["doc_0", "doc_1"])
assert len(results) > 0
# All results should belong to doc_0 or doc_1 groups
for result in results:
assert result.metadata["document_id"] in ["doc_0", "doc_1"]
# Test score threshold
results = vector_store.search_by_vector(
query_vector,
top_k=10,
score_threshold=0.5
)
results = vector_store.search_by_vector(query_vector, top_k=10, score_threshold=0.5)
# Check that all results have a score above threshold
for result in results:
assert result.metadata.get("score", 0) >= 0.5
@@ -154,7 +143,7 @@ class TestClickzettaVector(AbstractVectorTest):
for i in range(batch_size):
doc = Document(
page_content=f"Batch document {i}: This is a test document for batch processing.",
metadata={"doc_id": f"batch_doc_{i}", "batch": "test_batch"}
metadata={"doc_id": f"batch_doc_{i}", "batch": "test_batch"},
)
documents.append(doc)
embeddings.append([0.1 * (i % 10), 0.2 * (i % 10), 0.3 * (i % 10), 0.4 * (i % 10)])
@@ -179,7 +168,7 @@ class TestClickzettaVector(AbstractVectorTest):
# Test special characters in content
special_doc = Document(
page_content="Special chars: 'quotes', \"double\", \\backslash, \n newline",
metadata={"doc_id": "special_doc", "test": "edge_case"}
metadata={"doc_id": "special_doc", "test": "edge_case"},
)
embeddings = [[0.1, 0.2, 0.3, 0.4]]
@@ -199,20 +188,18 @@ class TestClickzettaVector(AbstractVectorTest):
# Prepare documents with various language content
documents = [
Document(
page_content="云器科技提供强大的Lakehouse解决方案",
metadata={"doc_id": "cn_doc_1", "lang": "chinese"}
page_content="云器科技提供强大的Lakehouse解决方案", metadata={"doc_id": "cn_doc_1", "lang": "chinese"}
),
Document(
page_content="Clickzetta provides powerful Lakehouse solutions",
metadata={"doc_id": "en_doc_1", "lang": "english"}
metadata={"doc_id": "en_doc_1", "lang": "english"},
),
Document(
page_content="Lakehouse是现代数据架构的重要组成部分",
metadata={"doc_id": "cn_doc_2", "lang": "chinese"}
page_content="Lakehouse是现代数据架构的重要组成部分", metadata={"doc_id": "cn_doc_2", "lang": "chinese"}
),
Document(
page_content="Modern data architecture includes Lakehouse technology",
metadata={"doc_id": "en_doc_2", "lang": "english"}
metadata={"doc_id": "en_doc_2", "lang": "english"},
),
]

22
api/tests/integration_tests/vdb/clickzetta/test_docker_integration.py
View File

@@ -2,6 +2,7 @@
"""
Test Clickzetta integration in Docker environment
"""
import os
import time
@@ -20,7 +21,7 @@ def test_clickzetta_connection():
service=os.getenv("CLICKZETTA_SERVICE", "api.clickzetta.com"),
workspace=os.getenv("CLICKZETTA_WORKSPACE", "test_workspace"),
vcluster=os.getenv("CLICKZETTA_VCLUSTER", "default"),
database=os.getenv("CLICKZETTA_SCHEMA", "dify")
database=os.getenv("CLICKZETTA_SCHEMA", "dify"),
)
with conn.cursor() as cursor:
@@ -36,7 +37,7 @@ def test_clickzetta_connection():
# Check if test collection exists
test_collection = "collection_test_dataset"
if test_collection in [t[1] for t in tables if t[0] == 'dify']:
if test_collection in [t[1] for t in tables if t[0] == "dify"]:
cursor.execute(f"DESCRIBE dify.{test_collection}")
columns = cursor.fetchall()
print(f"✓ Table structure for {test_collection}:")
@@ -55,6 +56,7 @@ def test_clickzetta_connection():
print(f"✗ Connection test failed: {e}")
return False
def test_dify_api():
"""Test Dify API with Clickzetta backend"""
print("\n=== Testing Dify API ===")
@@ -83,6 +85,7 @@ def test_dify_api():
print(f"✗ API test failed: {e}")
return False
def verify_table_structure():
"""Verify the table structure meets Dify requirements"""
print("\n=== Verifying Table Structure ===")
@@ -91,15 +94,10 @@ def verify_table_structure():
"id": "VARCHAR",
"page_content": "VARCHAR",
"metadata": "VARCHAR", # JSON stored as VARCHAR in Clickzetta
"vector": "ARRAY<FLOAT>"
"vector": "ARRAY<FLOAT>",
}
expected_metadata_fields = [
"doc_id",
"doc_hash",
"document_id",
"dataset_id"
]
expected_metadata_fields = ["doc_id", "doc_hash", "document_id", "dataset_id"]
print("✓ Expected table structure:")
for col, dtype in expected_columns.items():
@@ -117,6 +115,7 @@ def verify_table_structure():
return True
def main():
"""Run all tests"""
print("Starting Clickzetta integration tests for Dify Docker\n")
@@ -137,9 +136,9 @@ def main():
results.append((test_name, False))
# Summary
print("\n" + "="*50)
print("\n" + "=" * 50)
print("Test Summary:")
print("="*50)
print("=" * 50)
passed = sum(1 for _, success in results if success)
total = len(results)
@@ -161,5 +160,6 @@ def main():
print("\n⚠️ Some tests failed. Please check the errors above.")
return 1
if __name__ == "__main__":
exit(main())