admin/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

chore: add ast-grep rule to convert Optional[T] to T | None (#25560)

Browse Source 
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
-LAN-
2025-09-15 13:06:33 +08:00
committed by GitHub
parent 2e44ebe98d
commit bab4975809
394 changed files with 2555 additions and 2792 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
api/core/rag/extractor/blob/blob.py
View File

@@ -12,7 +12,7 @@ import mimetypes
from collections.abc import Generator, Mapping
from io import BufferedReader, BytesIO
from pathlib import Path, PurePath
from typing import Any, Optional, Union
from typing import Any, Union
from pydantic import BaseModel, ConfigDict, model_validator
@@ -30,17 +30,17 @@ class Blob(BaseModel):
"""
data: Union[bytes, str, None] = None # Raw data
mimetype: Optional[str] = None # Not to be confused with a file extension
mimetype: str | None = None # Not to be confused with a file extension
encoding: str = "utf-8" # Use utf-8 as default encoding, if decoding to string
# Location where the original content was found
# Represent location on the local file system
# Useful for situations where downstream code assumes it must work with file paths
# rather than in-memory content.
path: Optional[PathLike] = None
path: PathLike | None = None
model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
@property
def source(self) -> Optional[str]:
def source(self) -> str | None:
"""The source location of the blob as string if known otherwise none."""
return str(self.path) if self.path else None
@@ -91,7 +91,7 @@ class Blob(BaseModel):
path: PathLike,
*,
encoding: str = "utf-8",
mime_type: Optional[str] = None,
mime_type: str | None = None,
guess_type: bool = True,
) -> Blob:
"""Load the blob from a path like object.
@@ -120,8 +120,8 @@ class Blob(BaseModel):
data: Union[str, bytes],
*,
encoding: str = "utf-8",
mime_type: Optional[str] = None,
path: Optional[str] = None,
mime_type: str | None = None,
path: str | None = None,
) -> Blob:
"""Initialize the blob from in-memory data.

7
api/core/rag/extractor/csv_extractor.py
View File

@@ -1,7 +1,6 @@
"""Abstract interface for document loader implementations."""
import csv
from typing import Optional
import pandas as pd
@@ -21,10 +20,10 @@ class CSVExtractor(BaseExtractor):
def __init__(
self,
file_path: str,
encoding: Optional[str] = None,
encoding: str | None = None,
autodetect_encoding: bool = False,
source_column: Optional[str] = None,
csv_args: Optional[dict] = None,
source_column: str | None = None,
csv_args: dict | None = None,
):
"""Initialize with file path."""
self._file_path = file_path

12
api/core/rag/extractor/entity/extract_setting.py
View File

@@ -1,5 +1,3 @@
from typing import Optional
from pydantic import BaseModel, ConfigDict
from models.dataset import Document
@@ -14,7 +12,7 @@ class NotionInfo(BaseModel):
notion_workspace_id: str
notion_obj_id: str
notion_page_type: str
document: Optional[Document] = None
document: Document | None = None
tenant_id: str
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -43,10 +41,10 @@ class ExtractSetting(BaseModel):
"""
datasource_type: str
upload_file: Optional[UploadFile] = None
notion_info: Optional[NotionInfo] = None
website_info: Optional[WebsiteInfo] = None
document_model: Optional[str] = None
upload_file: UploadFile | None = None
notion_info: NotionInfo | None = None
website_info: WebsiteInfo | None = None
document_model: str | None = None
model_config = ConfigDict(arbitrary_types_allowed=True)
def __init__(self, **data):

4
api/core/rag/extractor/excel_extractor.py
View File

@@ -1,7 +1,7 @@
"""Abstract interface for document loader implementations."""
import os
from typing import Optional, cast
from typing import cast
import pandas as pd
from openpyxl import load_workbook
@@ -18,7 +18,7 @@ class ExcelExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, encoding: Optional[str] = None, autodetect_encoding: bool = False):
def __init__(self, file_path: str, encoding: str | None = None, autodetect_encoding: bool = False):
"""Initialize with file path."""
self._file_path = file_path
self._encoding = encoding

6
api/core/rag/extractor/extract_processor.py
View File

@@ -1,7 +1,7 @@
import re
import tempfile
from pathlib import Path
from typing import Optional, Union
from typing import Union
from urllib.parse import unquote
from configs import dify_config
@@ -90,7 +90,7 @@ class ExtractProcessor:
@classmethod
def extract(
cls, extract_setting: ExtractSetting, is_automatic: bool = False, file_path: Optional[str] = None
cls, extract_setting: ExtractSetting, is_automatic: bool = False, file_path: str | None = None
) -> list[Document]:
if extract_setting.datasource_type == DatasourceType.FILE.value:
with tempfile.TemporaryDirectory() as temp_dir:
@@ -104,7 +104,7 @@ class ExtractProcessor:
input_file = Path(file_path)
file_extension = input_file.suffix.lower()
etl_type = dify_config.ETL_TYPE
extractor: Optional[BaseExtractor] = None
extractor: BaseExtractor | None = None
if etl_type == "Unstructured":
unstructured_api_url = dify_config.UNSTRUCTURED_API_URL or ""
unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or ""

6
api/core/rag/extractor/helpers.py
View File

@@ -1,17 +1,17 @@
"""Document loader helpers."""
import concurrent.futures
from typing import NamedTuple, Optional, cast
from typing import NamedTuple, cast
class FileEncoding(NamedTuple):
"""A file encoding as the NamedTuple."""
encoding: Optional[str]
encoding: str | None
"""The encoding of the file."""
confidence: float
"""The confidence of the encoding."""
language: Optional[str]
language: str | None
"""The language of the file."""

9
api/core/rag/extractor/markdown_extractor.py
View File

@@ -2,7 +2,6 @@
import re
from pathlib import Path
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.extractor.helpers import detect_file_encodings
@@ -22,7 +21,7 @@ class MarkdownExtractor(BaseExtractor):
file_path: str,
remove_hyperlinks: bool = False,
remove_images: bool = False,
encoding: Optional[str] = None,
encoding: str | None = None,
autodetect_encoding: bool = True,
):
"""Initialize with file path."""
@@ -45,13 +44,13 @@ class MarkdownExtractor(BaseExtractor):
return documents
def markdown_to_tups(self, markdown_text: str) -> list[tuple[Optional[str], str]]:
def markdown_to_tups(self, markdown_text: str) -> list[tuple[str | None, str]]:
"""Convert a markdown file to a dictionary.
The keys are the headers and the values are the text under each header.
"""
markdown_tups: list[tuple[Optional[str], str]] = []
markdown_tups: list[tuple[str | None, str]] = []
lines = markdown_text.split("\n")
current_header = None
@@ -94,7 +93,7 @@ class MarkdownExtractor(BaseExtractor):
content = re.sub(pattern, r"\1", content)
return content
def parse_tups(self, filepath: str) -> list[tuple[Optional[str], str]]:
def parse_tups(self, filepath: str) -> list[tuple[str | None, str]]:
"""Parse file into tuples."""
content = ""
try:

8
api/core/rag/extractor/notion_extractor.py
View File

@@ -1,7 +1,7 @@
import json
import logging
import operator
from typing import Any, Optional, cast
from typing import Any, cast
import requests
from sqlalchemy import select
@@ -36,8 +36,8 @@ class NotionExtractor(BaseExtractor):
notion_obj_id: str,
notion_page_type: str,
tenant_id: str,
document_model: Optional[DocumentModel] = None,
notion_access_token: Optional[str] = None,
document_model: DocumentModel | None = None,
notion_access_token: str | None = None,
):
self._notion_access_token = None
self._document_model = document_model
@@ -328,7 +328,7 @@ class NotionExtractor(BaseExtractor):
result_lines = "\n".join(result_lines_arr)
return result_lines
def update_last_edited_time(self, document_model: Optional[DocumentModel]):
def update_last_edited_time(self, document_model: DocumentModel | None):
if not document_model:
return

3
api/core/rag/extractor/pdf_extractor.py
View File

@@ -2,7 +2,6 @@
import contextlib
from collections.abc import Iterator
from typing import Optional
from core.rag.extractor.blob.blob import Blob
from core.rag.extractor.extractor_base import BaseExtractor
@@ -18,7 +17,7 @@ class PdfExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, file_cache_key: Optional[str] = None):
def __init__(self, file_path: str, file_cache_key: str | None = None):
"""Initialize with file path."""
self._file_path = file_path
self._file_cache_key = file_cache_key

3
api/core/rag/extractor/text_extractor.py
View File

@@ -1,7 +1,6 @@
"""Abstract interface for document loader implementations."""
from pathlib import Path
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.extractor.helpers import detect_file_encodings
@@ -16,7 +15,7 @@ class TextExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, encoding: Optional[str] = None, autodetect_encoding: bool = False):
def __init__(self, file_path: str, encoding: str | None = None, autodetect_encoding: bool = False):
"""Initialize with file path."""
self._file_path = file_path
self._encoding = encoding

3
api/core/rag/extractor/unstructured/unstructured_eml_extractor.py
View File

@@ -1,7 +1,6 @@
import base64
import contextlib
import logging
from typing import Optional
from bs4 import BeautifulSoup
@@ -17,7 +16,7 @@ class UnstructuredEmailExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

3
api/core/rag/extractor/unstructured/unstructured_epub_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
import pypandoc # type: ignore
@@ -20,7 +19,7 @@ class UnstructuredEpubExtractor(BaseExtractor):
def __init__(
self,
file_path: str,
api_url: Optional[str] = None,
api_url: str | None = None,
api_key: str = "",
):
"""Initialize with file path."""

3
api/core/rag/extractor/unstructured/unstructured_markdown_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -16,7 +15,7 @@ class UnstructuredMarkdownExtractor(BaseExtractor):
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

3
api/core/rag/extractor/unstructured/unstructured_msg_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -15,7 +14,7 @@ class UnstructuredMsgExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

3
api/core/rag/extractor/unstructured/unstructured_ppt_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -15,7 +14,7 @@ class UnstructuredPPTExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

3
api/core/rag/extractor/unstructured/unstructured_pptx_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -15,7 +14,7 @@ class UnstructuredPPTXExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

3
api/core/rag/extractor/unstructured/unstructured_xml_extractor.py
View File

@@ -1,5 +1,4 @@
import logging
from typing import Optional
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -15,7 +14,7 @@ class UnstructuredXmlExtractor(BaseExtractor):
file_path: Path to the file to load.
"""
def __init__(self, file_path: str, api_url: Optional[str] = None, api_key: str = ""):
def __init__(self, file_path: str, api_url: str | None = None, api_key: str = ""):
"""Initialize with file path."""
self._file_path = file_path
self._api_url = api_url

4
api/core/rag/extractor/watercrawl/provider.py
View File

@@ -1,6 +1,6 @@
from collections.abc import Generator
from datetime import datetime
from typing import Any, Optional
from typing import Any
from core.rag.extractor.watercrawl.client import WaterCrawlAPIClient
@@ -9,7 +9,7 @@ class WaterCrawlProvider:
def __init__(self, api_key, base_url: str | None = None):
self.client = WaterCrawlAPIClient(api_key, base_url)
def crawl_url(self, url, options: Optional[dict | Any] = None):
def crawl_url(self, url, options: dict | Any | None = None):
options = options or {}
spider_options = {
"max_depth": 1,