admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix variable typo (#8084)

Browse Source
This commit is contained in:
Nam Vu
2024-09-08 12:14:11 +07:00
committed by GitHub
parent b1918dae5e
commit 2d7954c7da
215 changed files with 599 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/core/helper/position_helper.py
View File

@@ -79,7 +79,7 @@ def is_filtered(
name_func: Callable[[Any], str],
) -> bool:
"""
Chcek if the object should be filtered out.
Check if the object should be filtered out.
Overall logic: exclude > include > pin
:param include_set: the set of names to be included
:param exclude_set: the set of names to be excluded

2
api/core/indexing_runner.py
View File

@@ -494,7 +494,7 @@ class IndexingRunner:
hash = helper.generate_text_hash(document_node.page_content)
document_node.metadata['doc_id'] = doc_id
document_node.metadata['doc_hash'] = hash
# delete Spliter character
# delete Splitter character
page_content = document_node.page_content
if page_content.startswith(".") or page_content.startswith("。"):
page_content = page_content[1:]

2
api/core/llm_generator/prompts.py
View File

@@ -87,7 +87,7 @@ Here is a task description for which I would like you to create a high-quality p
{{TASK_DESCRIPTION}}
</task_description>
Based on task description, please create a well-structured prompt template that another AI could use to consistently complete the task. The prompt template should include:
- Do not inlcude <input> or <output> section and variables in the prompt, assume user will add them at their own will.
- Do not include <input> or <output> section and variables in the prompt, assume user will add them at their own will.
- Clear instructions for the AI that will be using this prompt, demarcated with <instructions> tags. The instructions should provide step-by-step directions on how to complete the task using the input variables. Also Specifies in the instructions that the output should not contain any xml tag.
- Relevant examples if needed to clarify the task further, demarcated with <example> tags. Do not include variables in the prompt. Give three pairs of input and output examples.
- Include other relevant sections demarcated with appropriate XML tags like <examples>, <instructions>.

4
api/core/model_runtime/docs/en_US/schema.md
View File

@@ -52,7 +52,7 @@
- `mode` (string) voice model.（available for model type `tts`）
- `name` (string) voice model display name.（available for model type `tts`）
- `language` (string) the voice model supports languages.（available for model type `tts`）
- `word_limit` (int) Single conversion word limit, paragraphwise by default（available for model type `tts`）
- `word_limit` (int) Single conversion word limit, paragraph-wise by default（available for model type `tts`）
- `audio_type` (string) Support audio file extension format, e.g.：mp3,wav（available for model type `tts`）
- `max_workers` (int) Number of concurrent workers supporting text and audio conversion（available for model type`tts`）
- `max_characters_per_chunk` (int) Maximum characters per chunk (available for model type `moderation`)
@@ -150,7 +150,7 @@
- `input` (float) Input price, i.e., Prompt price
- `output` (float) Output price, i.e., returned content price
- `unit` (float) Pricing unit, e.g., if the price is meausred in 1M tokens, the corresponding token amount for the unit price is `0.000001`.
- `unit` (float) Pricing unit, e.g., if the price is measured in 1M tokens, the corresponding token amount for the unit price is `0.000001`.
- `currency` (string) Currency unit
### ProviderCredentialSchema

2
api/core/model_runtime/model_providers/__base/tts_model.py
View File

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class TTSModel(AIModel):
"""
Model class for ttstext model.
Model class for TTS model.
"""
model_type: ModelType = ModelType.TTS

2
api/core/model_runtime/model_providers/azure_openai/llm/llm.py
View File

@@ -284,7 +284,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel):
try:
schema = json.loads(json_schema)
except:
raise ValueError(f"not currect json_schema format: {json_schema}")
raise ValueError(f"not correct json_schema format: {json_schema}")
model_parameters.pop("json_schema")
model_parameters["response_format"] = {"type": "json_schema", "json_schema": schema}
else:

2
api/core/model_runtime/model_providers/baichuan/llm/llm.py
View File

@@ -37,7 +37,7 @@ from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors impor
)
class BaichuanLarguageModel(LargeLanguageModel):
class BaichuanLanguageModel(LargeLanguageModel):
def _invoke(
self,

2
api/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py
View File

@@ -60,7 +60,7 @@ class BaichuanTextEmbeddingModel(TextEmbeddingModel):
token_usage = 0
for chunk in chunks:
# embeding chunk
# embedding chunk
chunk_embeddings, chunk_usage = self.embedding(
model=model,
api_key=api_key,

6
api/core/model_runtime/model_providers/bedrock/llm/llm.py
View File

@@ -793,11 +793,11 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
"""
Map model invoke error to unified error
The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller
The value is the md = genai.GenerativeModel(model)error type thrown by the model,
The key is the ermd = genai.GenerativeModel(model) error type thrown to the caller
The value is the md = genai.GenerativeModel(model) error type thrown by the model,
which needs to be converted into a unified error type for the caller.
:return: Invoke emd = genai.GenerativeModel(model)rror mapping
:return: Invoke emd = genai.GenerativeModel(model) error mapping
"""
return {
InvokeConnectionError: [],

6
api/core/model_runtime/model_providers/bedrock/text_embedding/text_embedding.py
View File

@@ -130,11 +130,11 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
"""
Map model invoke error to unified error
The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller
The value is the md = genai.GenerativeModel(model)error type thrown by the model,
The key is the ermd = genai.GenerativeModel(model) error type thrown to the caller
The value is the md = genai.GenerativeModel(model) error type thrown by the model,
which needs to be converted into a unified error type for the caller.
:return: Invoke emd = genai.GenerativeModel(model)rror mapping
:return: Invoke emd = genai.GenerativeModel(model) error mapping
"""
return {
InvokeConnectionError: [],

6
api/core/model_runtime/model_providers/google/llm/llm.py
View File

@@ -416,11 +416,11 @@ class GoogleLargeLanguageModel(LargeLanguageModel):
def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
"""
Map model invoke error to unified error
The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller
The value is the md = genai.GenerativeModel(model)error type thrown by the model,
The key is the ermd = genai.GenerativeModel(model) error type thrown to the caller
The value is the md = genai.GenerativeModel(model) error type thrown by the model,
which needs to be converted into a unified error type for the caller.
:return: Invoke emd = genai.GenerativeModel(model)rror mapping
:return: Invoke emd = genai.GenerativeModel(model) error mapping
"""
return {
InvokeConnectionError: [

2
api/core/model_runtime/model_providers/minimax/llm/llm.py
View File

@@ -86,7 +86,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel):
Calculate num tokens for minimax model
not like ChatGLM, Minimax has a special prompt structure, we could not find a proper way
to caculate the num tokens, so we use str() to convert the prompt to string
to calculate the num tokens, so we use str() to convert the prompt to string
Minimax does not provide their own tokenizer of adab5.5 and abab5 model
therefore, we use gpt2 tokenizer instead

1
api/core/model_runtime/model_providers/novita/llm/llm.py
View File

@@ -10,6 +10,7 @@ from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAI
class NovitaLargeLanguageModel(OAIAPICompatLargeLanguageModel):
def _update_endpoint_url(self, credentials: dict):
credentials['endpoint_url'] = "https://api.novita.ai/v3/openai"
credentials['extra_headers'] = { 'X-Novita-Source': 'dify.ai' }
return credentials

8
api/core/model_runtime/model_providers/oci/llm/llm.py
View File

@@ -243,7 +243,7 @@ class OCILargeLanguageModel(LargeLanguageModel):
request_args["compartmentId"] = compartment_id
request_args["servingMode"]["modelId"] = model
chathistory = []
chat_history = []
system_prompts = []
#if "meta.llama" in model:
# request_args["chatRequest"]["apiFormat"] = "GENERIC"
@@ -273,16 +273,16 @@ class OCILargeLanguageModel(LargeLanguageModel):
if isinstance(message.content, str):
text = message.content
if isinstance(message, UserPromptMessage):
chathistory.append({"role": "USER", "message": text})
chat_history.append({"role": "USER", "message": text})
else:
chathistory.append({"role": "CHATBOT", "message": text})
chat_history.append({"role": "CHATBOT", "message": text})
if isinstance(message, SystemPromptMessage):
if isinstance(message.content, str):
system_prompts.append(message.content)
args = {"apiFormat": "COHERE",
"preambleOverride": ' '.join(system_prompts),
"message": prompt_messages[-1].content,
"chatHistory": chathistory, }
"chatHistory": chat_history, }
request_args["chatRequest"].update(args)
elif model.startswith("meta"):
#print("run meta " * 10)

2
api/core/model_runtime/model_providers/openai/llm/llm.py
View File

@@ -552,7 +552,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel):
try:
schema = json.loads(json_schema)
except:
raise ValueError(f"not currect json_schema format: {json_schema}")
raise ValueError(f"not correct json_schema format: {json_schema}")
model_parameters.pop("json_schema")
model_parameters["response_format"] = {"type": "json_schema", "json_schema": schema}
else:

8
api/core/model_runtime/model_providers/tencent/speech2text/flash_recognizer.py
View File

@@ -67,7 +67,7 @@ class FlashRecognitionRequest:
class FlashRecognizer:
"""
reponse:
response:
request_id string
status Integer
message String
@@ -132,9 +132,9 @@ class FlashRecognizer:
signstr = self._format_sign_string(query)
signature = self._sign(signstr, secret_key)
header["Authorization"] = signature
requrl = "https://"
requrl += signstr[4::]
return requrl
req_url = "https://"
req_url += signstr[4::]
return req_url
def _create_query_arr(self, req):
return {

6
api/core/model_runtime/model_providers/vertex_ai/llm/llm.py
View File

@@ -695,11 +695,11 @@ class VertexAiLargeLanguageModel(LargeLanguageModel):
def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
"""
Map model invoke error to unified error
The key is the ermd = gml.GenerativeModel(model)ror type thrown to the caller
The value is the md = gml.GenerativeModel(model)error type thrown by the model,
The key is the ermd = gml.GenerativeModel(model) error type thrown to the caller
The value is the md = gml.GenerativeModel(model) error type thrown by the model,
which needs to be converted into a unified error type for the caller.
:return: Invoke emd = gml.GenerativeModel(model)rror mapping
:return: Invoke emd = gml.GenerativeModel(model) error mapping
"""
return {
InvokeConnectionError: [

8
api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py
View File

@@ -135,16 +135,16 @@ class HttpClient:
**kwargs,
)
def _object_to_formfata(self, key: str, value: Data | Mapping[object, object]) -> list[tuple[str, str]]:
def _object_to_formdata(self, key: str, value: Data | Mapping[object, object]) -> list[tuple[str, str]]:
items = []
if isinstance(value, Mapping):
for k, v in value.items():
items.extend(self._object_to_formfata(f"{key}[{k}]", v))
items.extend(self._object_to_formdata(f"{key}[{k}]", v))
return items
if isinstance(value, list | tuple):
for v in value:
items.extend(self._object_to_formfata(key + "[]", v))
items.extend(self._object_to_formdata(key + "[]", v))
return items
def _primitive_value_to_str(val) -> str:
@@ -165,7 +165,7 @@ class HttpClient:
def _make_multipartform(self, data: Mapping[object, object]) -> dict[str, object]:
items = flatten([self._object_to_formfata(k, v) for k, v in data.items()])
items = flatten([self._object_to_formdata(k, v) for k, v in data.items()])
serialized: dict[str, object] = {}
for key, value in items:

2
api/core/rag/cleaner/unstructured/unstructured_non_ascii_chars_cleaner.py
View File

@@ -8,5 +8,5 @@ class UnstructuredNonAsciiCharsCleaner(BaseCleaner):
"""clean document content."""
from unstructured.cleaners.core import clean_non_ascii_chars
# Returns "This text containsnon-ascii characters!"
# Returns "This text contains non-ascii characters!"
return clean_non_ascii_chars(content)

24
api/core/rag/datasource/retrieval_service.py
View File

@@ -7,7 +7,7 @@ from core.rag.data_post_processor.data_post_processor import DataPostProcessor
from core.rag.datasource.keyword.keyword_factory import Keyword
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.rerank.constants.rerank_mode import RerankMode
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from extensions.ext_database import db
from models.dataset import Dataset
@@ -26,7 +26,7 @@ default_retrieval_model = {
class RetrievalService:
@classmethod
def retrieve(cls, retrival_method: str, dataset_id: str, query: str,
def retrieve(cls, retrieval_method: str, dataset_id: str, query: str,
top_k: int, score_threshold: Optional[float] = .0,
reranking_model: Optional[dict] = None, reranking_mode: Optional[str] = 'reranking_model',
weights: Optional[dict] = None):
@@ -39,7 +39,7 @@ class RetrievalService:
threads = []
exceptions = []
# retrieval_model source with keyword
if retrival_method == 'keyword_search':
if retrieval_method == 'keyword_search':
keyword_thread = threading.Thread(target=RetrievalService.keyword_search, kwargs={
'flask_app': current_app._get_current_object(),
'dataset_id': dataset_id,
@@ -51,7 +51,7 @@ class RetrievalService:
threads.append(keyword_thread)
keyword_thread.start()
# retrieval_model source with semantic
if RetrievalMethod.is_support_semantic_search(retrival_method):
if RetrievalMethod.is_support_semantic_search(retrieval_method):
embedding_thread = threading.Thread(target=RetrievalService.embedding_search, kwargs={
'flask_app': current_app._get_current_object(),
'dataset_id': dataset_id,
@@ -60,19 +60,19 @@ class RetrievalService:
'score_threshold': score_threshold,
'reranking_model': reranking_model,
'all_documents': all_documents,
'retrival_method': retrival_method,
'retrieval_method': retrieval_method,
'exceptions': exceptions,
})
threads.append(embedding_thread)
embedding_thread.start()
# retrieval source with full text
if RetrievalMethod.is_support_fulltext_search(retrival_method):
if RetrievalMethod.is_support_fulltext_search(retrieval_method):
full_text_index_thread = threading.Thread(target=RetrievalService.full_text_index_search, kwargs={
'flask_app': current_app._get_current_object(),
'dataset_id': dataset_id,
'query': query,
'retrival_method': retrival_method,
'retrieval_method': retrieval_method,
'score_threshold': score_threshold,
'top_k': top_k,
'reranking_model': reranking_model,
@@ -89,7 +89,7 @@ class RetrievalService:
exception_message = ';\n'.join(exceptions)
raise Exception(exception_message)
if retrival_method == RetrievalMethod.HYBRID_SEARCH.value:
if retrieval_method == RetrievalMethod.HYBRID_SEARCH.value:
data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_mode,
reranking_model, weights, False)
all_documents = data_post_processor.invoke(
@@ -124,7 +124,7 @@ class RetrievalService:
@classmethod
def embedding_search(cls, flask_app: Flask, dataset_id: str, query: str,
top_k: int, score_threshold: Optional[float], reranking_model: Optional[dict],
all_documents: list, retrival_method: str, exceptions: list):
all_documents: list, retrieval_method: str, exceptions: list):
with flask_app.app_context():
try:
dataset = db.session.query(Dataset).filter(
@@ -146,7 +146,7 @@ class RetrievalService:
)
if documents:
if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value:
if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrieval_method == RetrievalMethod.SEMANTIC_SEARCH.value:
data_post_processor = DataPostProcessor(str(dataset.tenant_id),
RerankMode.RERANKING_MODEL.value,
reranking_model, None, False)
@@ -164,7 +164,7 @@ class RetrievalService:
@classmethod
def full_text_index_search(cls, flask_app: Flask, dataset_id: str, query: str,
top_k: int, score_threshold: Optional[float], reranking_model: Optional[dict],
all_documents: list, retrival_method: str, exceptions: list):
all_documents: list, retrieval_method: str, exceptions: list):
with flask_app.app_context():
try:
dataset = db.session.query(Dataset).filter(
@@ -180,7 +180,7 @@ class RetrievalService:
top_k=top_k
)
if documents:
if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value:
if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrieval_method == RetrievalMethod.FULL_TEXT_SEARCH.value:
data_post_processor = DataPostProcessor(str(dataset.tenant_id),
RerankMode.RERANKING_MODEL.value,
reranking_model, None, False)

0
api/core/rag/extractor/blod/blod.py → api/core/rag/extractor/blob/blob.py
View File

8
api/core/rag/extractor/notion_extractor.py
View File

@@ -275,10 +275,10 @@ class NotionExtractor(BaseExtractor):
data = res.json()
# get table headers text
table_header_cell_texts = []
tabel_header_cells = data["results"][0]['table_row']['cells']
for tabel_header_cell in tabel_header_cells:
if tabel_header_cell:
for table_header_cell_text in tabel_header_cell:
table_header_cells = data["results"][0]['table_row']['cells']
for table_header_cell in table_header_cells:
if table_header_cell:
for table_header_cell_text in table_header_cell:
text = table_header_cell_text["text"]["content"]
table_header_cell_texts.append(text)
else:

2
api/core/rag/extractor/pdf_extractor.py
View File

@@ -2,7 +2,7 @@
from collections.abc import Iterator
from typing import Optional
from core.rag.extractor.blod.blod import Blob
from core.rag.extractor.blob.blob import Blob
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
from extensions.ext_storage import storage

2
api/core/rag/index_processor/index_processor_base.py
View File

@@ -34,7 +34,7 @@ class BaseIndexProcessor(ABC):
raise NotImplementedError
@abstractmethod
def retrieve(self, retrival_method: str, query: str, dataset: Dataset, top_k: int,
def retrieve(self, retrieval_method: str, query: str, dataset: Dataset, top_k: int,
score_threshold: float, reranking_model: dict) -> list[Document]:
raise NotImplementedError

6
api/core/rag/index_processor/processor/paragraph_index_processor.py
View File

@@ -42,7 +42,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
hash = helper.generate_text_hash(document_node.page_content)
document_node.metadata['doc_id'] = doc_id
document_node.metadata['doc_hash'] = hash
# delete Spliter character
# delete Splitter character
page_content = document_node.page_content
if page_content.startswith(".") or page_content.startswith("。"):
page_content = page_content[1:].strip()
@@ -76,10 +76,10 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
else:
keyword.delete()
def retrieve(self, retrival_method: str, query: str, dataset: Dataset, top_k: int,
def retrieve(self, retrieval_method: str, query: str, dataset: Dataset, top_k: int,
score_threshold: float, reranking_model: dict) -> list[Document]:
# Set search parameters.
results = RetrievalService.retrieve(retrival_method=retrival_method, dataset_id=dataset.id, query=query,
results = RetrievalService.retrieve(retrieval_method=retrieval_method, dataset_id=dataset.id, query=query,
top_k=top_k, score_threshold=score_threshold,
reranking_model=reranking_model)
# Organize results.

6
api/core/rag/index_processor/processor/qa_index_processor.py
View File

@@ -50,7 +50,7 @@ class QAIndexProcessor(BaseIndexProcessor):
hash = helper.generate_text_hash(document_node.page_content)
document_node.metadata['doc_id'] = doc_id
document_node.metadata['doc_hash'] = hash
# delete Spliter character
# delete Splitter character
page_content = document_node.page_content
if page_content.startswith(".") or page_content.startswith("。"):
page_content = page_content[1:]
@@ -107,10 +107,10 @@ class QAIndexProcessor(BaseIndexProcessor):
else:
vector.delete()
def retrieve(self, retrival_method: str, query: str, dataset: Dataset, top_k: int,
def retrieve(self, retrieval_method: str, query: str, dataset: Dataset, top_k: int,
score_threshold: float, reranking_model: dict):
# Set search parameters.
results = RetrievalService.retrieve(retrival_method=retrival_method, dataset_id=dataset.id, query=query,
results = RetrievalService.retrieve(retrieval_method=retrieval_method, dataset_id=dataset.id, query=query,
top_k=top_k, score_threshold=score_threshold,
reranking_model=reranking_model)
# Organize results.

20
api/core/rag/retrieval/dataset_retrieval.py
View File

@@ -21,7 +21,7 @@ from core.rag.data_post_processor.data_post_processor import DataPostProcessor
from core.rag.datasource.keyword.jieba.jieba_keyword_table_handler import JiebaKeywordTableHandler
from core.rag.datasource.retrieval_service import RetrievalService
from core.rag.models.document import Document
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.rag.retrieval.router.multi_dataset_function_call_router import FunctionCallMultiDatasetRouter
from core.rag.retrieval.router.multi_dataset_react_route import ReactMultiDatasetRouter
from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
@@ -261,9 +261,9 @@ class DatasetRetrieval:
top_k = retrieval_model_config['top_k']
# get retrieval method
if dataset.indexing_technique == "economy":
retrival_method = 'keyword_search'
retrieval_method = 'keyword_search'
else:
retrival_method = retrieval_model_config['search_method']
retrieval_method = retrieval_model_config['search_method']
# get reranking model
reranking_model = retrieval_model_config['reranking_model'] \
if retrieval_model_config['reranking_enable'] else None
@@ -275,7 +275,7 @@ class DatasetRetrieval:
with measure_time() as timer:
results = RetrievalService.retrieve(
retrival_method=retrival_method, dataset_id=dataset.id,
retrieval_method=retrieval_method, dataset_id=dataset.id,
query=query,
top_k=top_k, score_threshold=score_threshold,
reranking_model=reranking_model,
@@ -285,7 +285,7 @@ class DatasetRetrieval:
self._on_query(query, [dataset_id], app_id, user_from, user_id)
if results:
self._on_retrival_end(results, message_id, timer)
self._on_retrieval_end(results, message_id, timer)
return results
return []
@@ -347,14 +347,14 @@ class DatasetRetrieval:
self._on_query(query, dataset_ids, app_id, user_from, user_id)
if all_documents:
self._on_retrival_end(all_documents, message_id, timer)
self._on_retrieval_end(all_documents, message_id, timer)
return all_documents
def _on_retrival_end(
def _on_retrieval_end(
self, documents: list[Document], message_id: Optional[str] = None, timer: Optional[dict] = None
) -> None:
"""Handle retrival end."""
"""Handle retrieval end."""
for document in documents:
query = db.session.query(DocumentSegment).filter(
DocumentSegment.index_node_id == document.metadata['doc_id']
@@ -419,7 +419,7 @@ class DatasetRetrieval:
if dataset.indexing_technique == "economy":
# use keyword table query
documents = RetrievalService.retrieve(retrival_method='keyword_search',
documents = RetrievalService.retrieve(retrieval_method='keyword_search',
dataset_id=dataset.id,
query=query,
top_k=top_k
@@ -429,7 +429,7 @@ class DatasetRetrieval:
else:
if top_k > 0:
# retrieval source
documents = RetrievalService.retrieve(retrival_method=retrieval_model['search_method'],
documents = RetrievalService.retrieve(retrieval_method=retrieval_model['search_method'],
dataset_id=dataset.id,
query=query,
top_k=top_k,

0
api/core/rag/retrieval/retrival_methods.py → api/core/rag/retrieval/retrieval_methods.py
View File

2
api/core/tools/provider/builtin/dalle/dalle.yaml
View File

@@ -29,7 +29,7 @@ credentials_for_provider:
en_US: Please input your OpenAI API key
zh_Hans: 请输入你的 OpenAI API key
pt_BR: Please input your OpenAI API key
openai_organizaion_id:
openai_organization_id:
type: text-input
required: false
label:

2
api/core/tools/provider/builtin/dalle/tools/dalle2.py
View File

@@ -16,7 +16,7 @@ class DallE2Tool(BuiltinTool):
"""
invoke tools
"""
openai_organization = self.runtime.credentials.get('openai_organizaion_id', None)
openai_organization = self.runtime.credentials.get('openai_organization_id', None)
if not openai_organization:
openai_organization = None
openai_base_url = self.runtime.credentials.get('openai_base_url', None)

2
api/core/tools/provider/builtin/dalle/tools/dalle3.py
View File

@@ -17,7 +17,7 @@ class DallE3Tool(BuiltinTool):
"""
invoke tools
"""
openai_organization = self.runtime.credentials.get('openai_organizaion_id', None)
openai_organization = self.runtime.credentials.get('openai_organization_id', None)
if not openai_organization:
openai_organization = None
openai_base_url = self.runtime.credentials.get('openai_base_url', None)

2
api/core/tools/provider/builtin/github/github.py
View File

@@ -4,7 +4,7 @@ from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
class GihubProvider(BuiltinToolProviderController):
class GithubProvider(BuiltinToolProviderController):
def _validate_credentials(self, credentials: dict) -> None:
try:
if 'access_tokens' not in credentials or not credentials.get('access_tokens'):

2
api/core/tools/provider/builtin/github/tools/github_repositories.py
View File

@@ -9,7 +9,7 @@ from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
class GihubRepositoriesTool(BuiltinTool):
class GithubRepositoriesTool(BuiltinTool):
def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
invoke tools

4
api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py
View File

@@ -60,7 +60,7 @@ class GitlabCommitsTool(BuiltinTool):
project_name = project['name']
print(f"Project: {project_name}")
# Get all of proejct commits
# Get all of project commits
commits_url = f"{domain}/api/v4/projects/{project_id}/repository/commits"
params = {
'since': start_time,
@@ -83,7 +83,7 @@ class GitlabCommitsTool(BuiltinTool):
diffs = diff_response.json()
for diff in diffs:
# Caculate code lines of changed
# Calculate code lines of changed
added_lines = diff['diff'].count('\n+')
removed_lines = diff['diff'].count('\n-')
total_changes = added_lines + removed_lines

2
api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml
View File

@@ -6,7 +6,7 @@ identity:
zh_Hans: GitLab 提交内容查询
description:
human:
en_US: A tool for query GitLab commits, Input should be a exists username or projec.
en_US: A tool for query GitLab commits, Input should be a exists username or project.
zh_Hans: 一个用于查询 GitLab 代码提交内容的工具，输入的内容应该是一个已存在的用户名或者项目名。
llm: A tool for query GitLab commits, Input should be a exists username or project.
parameters:

2
api/core/tools/provider/builtin/openweather/tools/weather.py
View File

@@ -29,7 +29,7 @@ class OpenweatherTool(BuiltinTool):
# request URL
url = "https://api.openweathermap.org/data/2.5/weather"
# request parmas
# request params
params = {
"q": city,
"appid": self.runtime.credentials.get("api_key"),

16
api/core/tools/provider/builtin/spark/tools/spark_img_generation.py
View File

@@ -35,20 +35,20 @@ def sha256base64(data):
return digest
def parse_url(requset_url):
stidx = requset_url.index("://")
host = requset_url[stidx + 3 :]
schema = requset_url[: stidx + 3]
def parse_url(request_url):
stidx = request_url.index("://")
host = request_url[stidx + 3 :]
schema = request_url[: stidx + 3]
edidx = host.index("/")
if edidx <= 0:
raise AssembleHeaderException("invalid request url:" + requset_url)
raise AssembleHeaderException("invalid request url:" + request_url)
path = host[edidx:]
host = host[:edidx]
u = Url(host, path, schema)
return u
def assemble_ws_auth_url(requset_url, method="GET", api_key="", api_secret=""):
u = parse_url(requset_url)
def assemble_ws_auth_url(request_url, method="GET", api_key="", api_secret=""):
u = parse_url(request_url)
host = u.host
path = u.path
now = datetime.now()
@@ -69,7 +69,7 @@ def assemble_ws_auth_url(requset_url, method="GET", api_key="", api_secret=""):
)
values = {"host": host, "date": date, "authorization": authorization}
return requset_url + "?" + urlencode(values)
return request_url + "?" + urlencode(values)
def get_body(appid, text):

2
api/core/tools/provider/builtin/spider/tools/scraper_crawler.py
View File

@@ -42,6 +42,6 @@ class ScrapeTool(BuiltinTool):
result += "URL: " + i.get('url', '') + "\n"
result += "CONTENT: " + i.get('content', '') + "\n\n"
except Exception as e:
return self.create_text_message("An error occured", str(e))
return self.create_text_message("An error occurred", str(e))
return self.create_text_message(result)

6
api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py
View File

@@ -8,7 +8,7 @@ from core.model_manager import ModelManager
from core.model_runtime.entities.model_entities import ModelType
from core.rag.datasource.retrieval_service import RetrievalService
from core.rag.rerank.rerank_model import RerankModelRunner
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.tools.tool.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
from extensions.ext_database import db
from models.dataset import Dataset, Document, DocumentSegment
@@ -163,7 +163,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
if dataset.indexing_technique == "economy":
# use keyword table query
documents = RetrievalService.retrieve(retrival_method='keyword_search',
documents = RetrievalService.retrieve(retrieval_method='keyword_search',
dataset_id=dataset.id,
query=query,
top_k=self.top_k
@@ -173,7 +173,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
else:
if self.top_k > 0:
# retrieval source
documents = RetrievalService.retrieve(retrival_method=retrieval_model['search_method'],
documents = RetrievalService.retrieve(retrieval_method=retrieval_model['search_method'],
dataset_id=dataset.id,
query=query,
top_k=self.top_k,

6
api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py
View File

@@ -2,7 +2,7 @@
from pydantic import BaseModel, Field
from core.rag.datasource.retrieval_service import RetrievalService
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.tools.tool.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
from extensions.ext_database import db
from models.dataset import Dataset, Document, DocumentSegment
@@ -63,7 +63,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model
if dataset.indexing_technique == "economy":
# use keyword table query
documents = RetrievalService.retrieve(retrival_method='keyword_search',
documents = RetrievalService.retrieve(retrieval_method='keyword_search',
dataset_id=dataset.id,
query=query,
top_k=self.top_k
@@ -72,7 +72,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
else:
if self.top_k > 0:
# retrieval source
documents = RetrievalService.retrieve(retrival_method=retrieval_model.get('search_method', 'semantic_search'),
documents = RetrievalService.retrieve(retrieval_method=retrieval_model.get('search_method', 'semantic_search'),
dataset_id=dataset.id,
query=query,
top_k=self.top_k,

16
api/core/tools/tool/dataset_retriever_tool.py
View File

@@ -17,7 +17,7 @@ from core.tools.tool.tool import Tool
class DatasetRetrieverTool(Tool):
retrival_tool: DatasetRetrieverBaseTool
retrieval_tool: DatasetRetrieverBaseTool
@staticmethod
def get_dataset_tools(tenant_id: str,
@@ -42,7 +42,7 @@ class DatasetRetrieverTool(Tool):
# Agent only support SINGLE mode
original_retriever_mode = retrieve_config.retrieve_strategy
retrieve_config.retrieve_strategy = DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE
retrival_tools = feature.to_dataset_retriever_tool(
retrieval_tools = feature.to_dataset_retriever_tool(
tenant_id=tenant_id,
dataset_ids=dataset_ids,
retrieve_config=retrieve_config,
@@ -53,17 +53,17 @@ class DatasetRetrieverTool(Tool):
# restore retrieve strategy
retrieve_config.retrieve_strategy = original_retriever_mode
# convert retrival tools to Tools
# convert retrieval tools to Tools
tools = []
for retrival_tool in retrival_tools:
for retrieval_tool in retrieval_tools:
tool = DatasetRetrieverTool(
retrival_tool=retrival_tool,
identity=ToolIdentity(provider='', author='', name=retrival_tool.name, label=I18nObject(en_US='', zh_Hans='')),
retrieval_tool=retrieval_tool,
identity=ToolIdentity(provider='', author='', name=retrieval_tool.name, label=I18nObject(en_US='', zh_Hans='')),
parameters=[],
is_team_authorization=True,
description=ToolDescription(
human=I18nObject(en_US='', zh_Hans=''),
llm=retrival_tool.description),
llm=retrieval_tool.description),
runtime=DatasetRetrieverTool.Runtime()
)
@@ -95,7 +95,7 @@ class DatasetRetrieverTool(Tool):
return self.create_text_message(text='please input query')
# invoke dataset retriever tool
result = self.retrival_tool._run(query=query)
result = self.retrieval_tool._run(query=query)
return self.create_text_message(text=result)

26
api/core/tools/utils/web_reader_tool.py
View File

@@ -189,8 +189,8 @@ def extract_text_blocks_as_plain_text(paragraph_html):
def plain_text_leaf_node(element):
# Extract all text, stripped of any child HTML elements and normalise it
plain_text = normalise_text(element.get_text())
# Extract all text, stripped of any child HTML elements and normalize it
plain_text = normalize_text(element.get_text())
if plain_text != "" and element.name == "li":
plain_text = "* {}, ".format(plain_text)
if plain_text == "":
@@ -231,8 +231,8 @@ def plain_element(element, content_digests, node_indexes):
# For leaf node elements, extract the text content, discarding any HTML tags
# 1. Get element contents as text
plain_text = element.get_text()
# 2. Normalise the extracted text string to a canonical representation
plain_text = normalise_text(plain_text)
# 2. Normalize the extracted text string to a canonical representation
plain_text = normalize_text(plain_text)
# 3. Update element content to be plain text
element.string = plain_text
elif is_text(element):
@@ -243,7 +243,7 @@ def plain_element(element, content_digests, node_indexes):
element = type(element)("")
else:
plain_text = element.string
plain_text = normalise_text(plain_text)
plain_text = normalize_text(plain_text)
element = type(element)(plain_text)
else:
# If not a leaf node or leaf type call recursively on child nodes, replacing
@@ -267,12 +267,12 @@ def add_node_indexes(element, node_index="0"):
return element
def normalise_text(text):
"""Normalise unicode and whitespace."""
# Normalise unicode first to try and standardise whitespace characters as much as possible before normalising them
def normalize_text(text):
"""Normalize unicode and whitespace."""
# Normalize unicode first to try and standardize whitespace characters as much as possible before normalizing them
text = strip_control_characters(text)
text = normalise_unicode(text)
text = normalise_whitespace(text)
text = normalize_unicode(text)
text = normalize_whitespace(text)
return text
@@ -291,14 +291,14 @@ def strip_control_characters(text):
return "".join(["" if (unicodedata.category(char) in control_chars) and (char not in retained_chars) else char for char in text])
def normalise_unicode(text):
"""Normalise unicode such that things that are visually equivalent map to the same unicode string where possible."""
def normalize_unicode(text):
"""Normalize unicode such that things that are visually equivalent map to the same unicode string where possible."""
normal_form = "NFKC"
text = unicodedata.normalize(normal_form, text)
return text
def normalise_whitespace(text):
def normalize_whitespace(text):
"""Replace runs of whitespace characters with a single space as this is what happens when HTML text is displayed."""
text = regex.sub(r"\s+", " ", text)
# Remove leading and trailing whitespace

2
api/core/workflow/entities/variable_pool.py
View File

@@ -31,7 +31,7 @@ class VariablePool:
# 'files': []
# }
# Varaible dictionary is a dictionary for looking up variables by their selector.
# Variable dictionary is a dictionary for looking up variables by their selector.
# The first element of the selector is the node id, it's the first-level key in the dictionary.
# Other elements of the selector are the keys in the second-level dictionary. To get the key, we hash the
# elements of the selector except the first one.

2
api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py
View File

@@ -11,7 +11,7 @@ from core.model_manager import ModelInstance, ModelManager
from core.model_runtime.entities.model_entities import ModelFeature, ModelType
from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.workflow.entities.base_node_data_entities import BaseNodeData
from core.workflow.entities.node_entities import NodeRunResult, NodeType
from core.workflow.entities.variable_pool import VariablePool