Co-authored-by: -LAN- <laipz8200@outlook.com>
This commit is contained in:
@@ -5,20 +5,15 @@ from core.rag.datasource.vdb.milvus.milvus_vector import MilvusConfig
|
||||
|
||||
|
||||
def test_default_value():
|
||||
valid_config = {
|
||||
'host': 'localhost',
|
||||
'port': 19530,
|
||||
'user': 'root',
|
||||
'password': 'Milvus'
|
||||
}
|
||||
valid_config = {"host": "localhost", "port": 19530, "user": "root", "password": "Milvus"}
|
||||
|
||||
for key in valid_config:
|
||||
config = valid_config.copy()
|
||||
del config[key]
|
||||
with pytest.raises(ValidationError) as e:
|
||||
MilvusConfig(**config)
|
||||
assert e.value.errors()[0]['msg'] == f'Value error, config MILVUS_{key.upper()} is required'
|
||||
assert e.value.errors()[0]["msg"] == f"Value error, config MILVUS_{key.upper()} is required"
|
||||
|
||||
config = MilvusConfig(**valid_config)
|
||||
assert config.secure is False
|
||||
assert config.database == 'default'
|
||||
assert config.database == "default"
|
||||
|
||||
@@ -9,19 +9,17 @@ from tests.unit_tests.core.rag.extractor.test_notion_extractor import _mock_resp
|
||||
|
||||
def test_firecrawl_web_extractor_crawl_mode(mocker):
|
||||
url = "https://firecrawl.dev"
|
||||
api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-'
|
||||
base_url = 'https://api.firecrawl.dev'
|
||||
firecrawl_app = FirecrawlApp(api_key=api_key,
|
||||
base_url=base_url)
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY") or "fc-"
|
||||
base_url = "https://api.firecrawl.dev"
|
||||
firecrawl_app = FirecrawlApp(api_key=api_key, base_url=base_url)
|
||||
params = {
|
||||
'crawlerOptions': {
|
||||
"crawlerOptions": {
|
||||
"includes": [],
|
||||
"excludes": [],
|
||||
"generateImgAltText": True,
|
||||
"maxDepth": 1,
|
||||
"limit": 1,
|
||||
'returnOnlyUrls': False,
|
||||
|
||||
"returnOnlyUrls": False,
|
||||
}
|
||||
}
|
||||
mocked_firecrawl = {
|
||||
|
||||
@@ -8,11 +8,8 @@ page_id = "page1"
|
||||
|
||||
|
||||
extractor = notion_extractor.NotionExtractor(
|
||||
notion_workspace_id='x',
|
||||
notion_obj_id='x',
|
||||
notion_page_type='page',
|
||||
tenant_id='x',
|
||||
notion_access_token='x')
|
||||
notion_workspace_id="x", notion_obj_id="x", notion_page_type="page", tenant_id="x", notion_access_token="x"
|
||||
)
|
||||
|
||||
|
||||
def _generate_page(page_title: str):
|
||||
@@ -21,16 +18,10 @@ def _generate_page(page_title: str):
|
||||
"id": page_id,
|
||||
"properties": {
|
||||
"Page": {
|
||||
"type": "title",
|
||||
"title": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": {"content": page_title},
|
||||
"plain_text": page_title
|
||||
}
|
||||
]
|
||||
"type": "title",
|
||||
"title": [{"type": "text", "text": {"content": page_title}, "plain_text": page_title}],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -38,10 +29,7 @@ def _generate_block(block_id: str, block_type: str, block_text: str):
|
||||
return {
|
||||
"object": "block",
|
||||
"id": block_id,
|
||||
"parent": {
|
||||
"type": "page_id",
|
||||
"page_id": page_id
|
||||
},
|
||||
"parent": {"type": "page_id", "page_id": page_id},
|
||||
"type": block_type,
|
||||
"has_children": False,
|
||||
block_type: {
|
||||
@@ -49,10 +37,11 @@ def _generate_block(block_id: str, block_type: str, block_text: str):
|
||||
{
|
||||
"type": "text",
|
||||
"text": {"content": block_text},
|
||||
"plain_text": block_text,
|
||||
}]
|
||||
}
|
||||
}
|
||||
"plain_text": block_text,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _mock_response(data):
|
||||
@@ -63,7 +52,7 @@ def _mock_response(data):
|
||||
|
||||
|
||||
def _remove_multiple_new_lines(text):
|
||||
while '\n\n' in text:
|
||||
while "\n\n" in text:
|
||||
text = text.replace("\n\n", "\n")
|
||||
return text.strip()
|
||||
|
||||
@@ -71,21 +60,21 @@ def _remove_multiple_new_lines(text):
|
||||
def test_notion_page(mocker):
|
||||
texts = ["Head 1", "1.1", "paragraph 1", "1.1.1"]
|
||||
mocked_notion_page = {
|
||||
"object": "list",
|
||||
"results": [
|
||||
_generate_block("b1", "heading_1", texts[0]),
|
||||
_generate_block("b2", "heading_2", texts[1]),
|
||||
_generate_block("b3", "paragraph", texts[2]),
|
||||
_generate_block("b4", "heading_3", texts[3])
|
||||
],
|
||||
"next_cursor": None
|
||||
"object": "list",
|
||||
"results": [
|
||||
_generate_block("b1", "heading_1", texts[0]),
|
||||
_generate_block("b2", "heading_2", texts[1]),
|
||||
_generate_block("b3", "paragraph", texts[2]),
|
||||
_generate_block("b4", "heading_3", texts[3]),
|
||||
],
|
||||
"next_cursor": None,
|
||||
}
|
||||
mocker.patch("requests.request", return_value=_mock_response(mocked_notion_page))
|
||||
|
||||
page_docs = extractor._load_data_as_documents(page_id, "page")
|
||||
assert len(page_docs) == 1
|
||||
content = _remove_multiple_new_lines(page_docs[0].page_content)
|
||||
assert content == '# Head 1\n## 1.1\nparagraph 1\n### 1.1.1'
|
||||
assert content == "# Head 1\n## 1.1\nparagraph 1\n### 1.1.1"
|
||||
|
||||
|
||||
def test_notion_database(mocker):
|
||||
@@ -93,10 +82,10 @@ def test_notion_database(mocker):
|
||||
mocked_notion_database = {
|
||||
"object": "list",
|
||||
"results": [_generate_page(i) for i in page_title_list],
|
||||
"next_cursor": None
|
||||
"next_cursor": None,
|
||||
}
|
||||
mocker.patch("requests.post", return_value=_mock_response(mocked_notion_database))
|
||||
database_docs = extractor._load_data_as_documents(database_id, "database")
|
||||
assert len(database_docs) == 1
|
||||
content = _remove_multiple_new_lines(database_docs[0].page_content)
|
||||
assert content == '\n'.join([f'Page:{i}' for i in page_title_list])
|
||||
assert content == "\n".join([f"Page:{i}" for i in page_title_list])
|
||||
|
||||
Reference in New Issue
Block a user