more httpx (#25651)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2025-09-23 00:07:09 +09:00
parent 0c4193bd91
commit 8940decd1b
21 changed files with 120 additions and 117 deletions
--- a/api/tests/unit_tests/controllers/console/auth/test_oauth.py
+++ b/api/tests/unit_tests/controllers/console/auth/test_oauth.py
@@ -201,9 +201,9 @@ class TestOAuthCallback:
        mock_db.session.rollback = MagicMock()

        # Import the real requests module to create a proper exception
-        import requests
+        import httpx

-        request_exception = requests.exceptions.RequestException("OAuth error")
+        request_exception = httpx.RequestError("OAuth error")
        request_exception.response = MagicMock()
        request_exception.response.text = str(exception)

--- a/api/tests/unit_tests/libs/test_oauth_clients.py
+++ b/api/tests/unit_tests/libs/test_oauth_clients.py
@@ -1,8 +1,8 @@
 import urllib.parse
 from unittest.mock import MagicMock, patch

+import httpx
 import pytest
-import requests

 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo

@@ -68,7 +68,7 @@ class TestGitHubOAuth(BaseOAuthTest):
            ({}, None, True),
        ],
    )
-    @patch("requests.post")
+    @patch("httpx.post")
    def test_should_retrieve_access_token(
        self, mock_post, oauth, mock_response, response_data, expected_token, should_raise
    ):
@@ -105,7 +105,7 @@ class TestGitHubOAuth(BaseOAuthTest):
            ),
        ],
    )
-    @patch("requests.get")
+    @patch("httpx.get")
    def test_should_retrieve_user_info_correctly(self, mock_get, oauth, user_data, email_data, expected_email):
        user_response = MagicMock()
        user_response.json.return_value = user_data
@@ -121,11 +121,11 @@ class TestGitHubOAuth(BaseOAuthTest):
        assert user_info.name == user_data["name"]
        assert user_info.email == expected_email

-    @patch("requests.get")
+    @patch("httpx.get")
    def test_should_handle_network_errors(self, mock_get, oauth):
-        mock_get.side_effect = requests.exceptions.RequestException("Network error")
+        mock_get.side_effect = httpx.RequestError("Network error")

-        with pytest.raises(requests.exceptions.RequestException):
+        with pytest.raises(httpx.RequestError):
            oauth.get_raw_user_info("test_token")


@@ -167,7 +167,7 @@ class TestGoogleOAuth(BaseOAuthTest):
            ({}, None, True),
        ],
    )
-    @patch("requests.post")
+    @patch("httpx.post")
    def test_should_retrieve_access_token(
        self, mock_post, oauth, oauth_config, mock_response, response_data, expected_token, should_raise
    ):
@@ -201,7 +201,7 @@ class TestGoogleOAuth(BaseOAuthTest):
            ({"sub": "123", "email": "test@example.com", "name": "Test User"}, ""),  # Always returns empty string
        ],
    )
-    @patch("requests.get")
+    @patch("httpx.get")
    def test_should_retrieve_user_info_correctly(self, mock_get, oauth, mock_response, user_data, expected_name):
        mock_response.json.return_value = user_data
        mock_get.return_value = mock_response
@@ -217,12 +217,12 @@ class TestGoogleOAuth(BaseOAuthTest):
    @pytest.mark.parametrize(
        "exception_type",
        [
-            requests.exceptions.HTTPError,
-            requests.exceptions.ConnectionError,
-            requests.exceptions.Timeout,
+            httpx.HTTPError,
+            httpx.ConnectError,
+            httpx.TimeoutException,
        ],
    )
-    @patch("requests.get")
+    @patch("httpx.get")
    def test_should_handle_http_errors(self, mock_get, oauth, exception_type):
        mock_response = MagicMock()
        mock_response.raise_for_status.side_effect = exception_type("Error")
--- a/api/tests/unit_tests/services/auth/test_auth_integration.py
+++ b/api/tests/unit_tests/services/auth/test_auth_integration.py
@@ -6,8 +6,8 @@ import json
 from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import Mock, patch

+import httpx
 import pytest
-import requests

 from services.auth.api_key_auth_factory import ApiKeyAuthFactory
 from services.auth.api_key_auth_service import ApiKeyAuthService
@@ -26,7 +26,7 @@ class TestAuthIntegration:
        self.watercrawl_credentials = {"auth_type": "x-api-key", "config": {"api_key": "wc_test_key_789"}}

    @patch("services.auth.api_key_auth_service.db.session")
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    @patch("services.auth.api_key_auth_service.encrypter.encrypt_token")
    def test_end_to_end_auth_flow(self, mock_encrypt, mock_http, mock_session):
        """Test complete authentication flow: request → validation → encryption → storage"""
@@ -47,7 +47,7 @@ class TestAuthIntegration:
        mock_session.add.assert_called_once()
        mock_session.commit.assert_called_once()

-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_cross_component_integration(self, mock_http):
        """Test factory → provider → HTTP call integration"""
        mock_http.return_value = self._create_success_response()
@@ -97,7 +97,7 @@ class TestAuthIntegration:
        assert "another_secret" not in factory_str

    @patch("services.auth.api_key_auth_service.db.session")
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    @patch("services.auth.api_key_auth_service.encrypter.encrypt_token")
    def test_concurrent_creation_safety(self, mock_encrypt, mock_http, mock_session):
        """Test concurrent authentication creation safety"""
@@ -142,31 +142,31 @@ class TestAuthIntegration:
        with pytest.raises((ValueError, KeyError, TypeError, AttributeError)):
            ApiKeyAuthFactory(AuthType.FIRECRAWL, invalid_input)

-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_http_error_handling(self, mock_http):
        """Test proper HTTP error handling"""
        mock_response = Mock()
        mock_response.status_code = 401
        mock_response.text = '{"error": "Unauthorized"}'
-        mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("Unauthorized")
+        mock_response.raise_for_status.side_effect = httpx.HTTPError("Unauthorized")
        mock_http.return_value = mock_response

        # PT012: Split into single statement for pytest.raises
        factory = ApiKeyAuthFactory(AuthType.FIRECRAWL, self.firecrawl_credentials)
-        with pytest.raises((requests.exceptions.HTTPError, Exception)):
+        with pytest.raises((httpx.HTTPError, Exception)):
            factory.validate_credentials()

    @patch("services.auth.api_key_auth_service.db.session")
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_network_failure_recovery(self, mock_http, mock_session):
        """Test system recovery from network failures"""
-        mock_http.side_effect = requests.exceptions.RequestException("Network timeout")
+        mock_http.side_effect = httpx.RequestError("Network timeout")
        mock_session.add = Mock()
        mock_session.commit = Mock()

        args = {"category": self.category, "provider": AuthType.FIRECRAWL, "credentials": self.firecrawl_credentials}

-        with pytest.raises(requests.exceptions.RequestException):
+        with pytest.raises(httpx.RequestError):
            ApiKeyAuthService.create_provider_auth(self.tenant_id_1, args)

        mock_session.commit.assert_not_called()
--- a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py
+++ b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py
@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock, patch

+import httpx
 import pytest
-import requests

 from services.auth.firecrawl.firecrawl import FirecrawlAuth

@@ -64,7 +64,7 @@ class TestFirecrawlAuth:
            FirecrawlAuth(credentials)
        assert str(exc_info.value) == expected_error

-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_validate_valid_credentials_successfully(self, mock_post, auth_instance):
        """Test successful credential validation"""
        mock_response = MagicMock()
@@ -95,7 +95,7 @@ class TestFirecrawlAuth:
            (500, "Internal server error"),
        ],
    )
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_handle_http_errors(self, mock_post, status_code, error_message, auth_instance):
        """Test handling of various HTTP error codes"""
        mock_response = MagicMock()
@@ -115,7 +115,7 @@ class TestFirecrawlAuth:
            (401, "Not JSON", True, "Expecting value"),  # JSON decode error
        ],
    )
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_handle_unexpected_errors(
        self, mock_post, status_code, response_text, has_json_error, expected_error_contains, auth_instance
    ):
@@ -134,13 +134,13 @@ class TestFirecrawlAuth:
    @pytest.mark.parametrize(
        ("exception_type", "exception_message"),
        [
-            (requests.ConnectionError, "Network error"),
-            (requests.Timeout, "Request timeout"),
-            (requests.ReadTimeout, "Read timeout"),
-            (requests.ConnectTimeout, "Connection timeout"),
+            (httpx.ConnectError, "Network error"),
+            (httpx.TimeoutException, "Request timeout"),
+            (httpx.ReadTimeout, "Read timeout"),
+            (httpx.ConnectTimeout, "Connection timeout"),
        ],
    )
-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_handle_network_errors(self, mock_post, exception_type, exception_message, auth_instance):
        """Test handling of various network-related errors including timeouts"""
        mock_post.side_effect = exception_type(exception_message)
@@ -162,7 +162,7 @@ class TestFirecrawlAuth:
            FirecrawlAuth({"auth_type": "basic", "config": {"api_key": "super_secret_key_12345"}})
        assert "super_secret_key_12345" not in str(exc_info.value)

-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_use_custom_base_url_in_validation(self, mock_post):
        """Test that custom base URL is used in validation"""
        mock_response = MagicMock()
@@ -179,12 +179,12 @@ class TestFirecrawlAuth:
        assert result is True
        assert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl"

-    @patch("services.auth.firecrawl.firecrawl.requests.post")
+    @patch("services.auth.firecrawl.firecrawl.httpx.post")
    def test_should_handle_timeout_with_retry_suggestion(self, mock_post, auth_instance):
        """Test that timeout errors are handled gracefully with appropriate error message"""
-        mock_post.side_effect = requests.Timeout("The request timed out after 30 seconds")
+        mock_post.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")

-        with pytest.raises(requests.Timeout) as exc_info:
+        with pytest.raises(httpx.TimeoutException) as exc_info:
            auth_instance.validate_credentials()

        # Verify the timeout exception is raised with original message
--- a/api/tests/unit_tests/services/auth/test_jina_auth.py
+++ b/api/tests/unit_tests/services/auth/test_jina_auth.py
@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock, patch

+import httpx
 import pytest
-import requests

 from services.auth.jina.jina import JinaAuth

@@ -35,7 +35,7 @@ class TestJinaAuth:
            JinaAuth(credentials)
        assert str(exc_info.value) == "No API key provided"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_validate_valid_credentials_successfully(self, mock_post):
        """Test successful credential validation"""
        mock_response = MagicMock()
@@ -53,7 +53,7 @@ class TestJinaAuth:
            json={"url": "https://example.com"},
        )

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_http_402_error(self, mock_post):
        """Test handling of 402 Payment Required error"""
        mock_response = MagicMock()
@@ -68,7 +68,7 @@ class TestJinaAuth:
            auth.validate_credentials()
        assert str(exc_info.value) == "Failed to authorize. Status code: 402. Error: Payment required"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_http_409_error(self, mock_post):
        """Test handling of 409 Conflict error"""
        mock_response = MagicMock()
@@ -83,7 +83,7 @@ class TestJinaAuth:
            auth.validate_credentials()
        assert str(exc_info.value) == "Failed to authorize. Status code: 409. Error: Conflict error"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_http_500_error(self, mock_post):
        """Test handling of 500 Internal Server Error"""
        mock_response = MagicMock()
@@ -98,7 +98,7 @@ class TestJinaAuth:
            auth.validate_credentials()
        assert str(exc_info.value) == "Failed to authorize. Status code: 500. Error: Internal server error"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_unexpected_error_with_text_response(self, mock_post):
        """Test handling of unexpected errors with text response"""
        mock_response = MagicMock()
@@ -114,7 +114,7 @@ class TestJinaAuth:
            auth.validate_credentials()
        assert str(exc_info.value) == "Failed to authorize. Status code: 403. Error: Forbidden"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_unexpected_error_without_text(self, mock_post):
        """Test handling of unexpected errors without text response"""
        mock_response = MagicMock()
@@ -130,15 +130,15 @@ class TestJinaAuth:
            auth.validate_credentials()
        assert str(exc_info.value) == "Unexpected error occurred while trying to authorize. Status code: 404"

-    @patch("services.auth.jina.jina.requests.post")
+    @patch("services.auth.jina.jina.httpx.post")
    def test_should_handle_network_errors(self, mock_post):
        """Test handling of network connection errors"""
-        mock_post.side_effect = requests.ConnectionError("Network error")
+        mock_post.side_effect = httpx.ConnectError("Network error")

        credentials = {"auth_type": "bearer", "config": {"api_key": "test_api_key_123"}}
        auth = JinaAuth(credentials)

-        with pytest.raises(requests.ConnectionError):
+        with pytest.raises(httpx.ConnectError):
            auth.validate_credentials()

    def test_should_not_expose_api_key_in_error_messages(self):
--- a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
+++ b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock, patch

+import httpx
 import pytest
-import requests

 from services.auth.watercrawl.watercrawl import WatercrawlAuth

@@ -64,7 +64,7 @@ class TestWatercrawlAuth:
            WatercrawlAuth(credentials)
        assert str(exc_info.value) == expected_error

-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_validate_valid_credentials_successfully(self, mock_get, auth_instance):
        """Test successful credential validation"""
        mock_response = MagicMock()
@@ -87,7 +87,7 @@ class TestWatercrawlAuth:
            (500, "Internal server error"),
        ],
    )
-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_handle_http_errors(self, mock_get, status_code, error_message, auth_instance):
        """Test handling of various HTTP error codes"""
        mock_response = MagicMock()
@@ -107,7 +107,7 @@ class TestWatercrawlAuth:
            (401, "Not JSON", True, "Expecting value"),  # JSON decode error
        ],
    )
-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_handle_unexpected_errors(
        self, mock_get, status_code, response_text, has_json_error, expected_error_contains, auth_instance
    ):
@@ -126,13 +126,13 @@ class TestWatercrawlAuth:
    @pytest.mark.parametrize(
        ("exception_type", "exception_message"),
        [
-            (requests.ConnectionError, "Network error"),
-            (requests.Timeout, "Request timeout"),
-            (requests.ReadTimeout, "Read timeout"),
-            (requests.ConnectTimeout, "Connection timeout"),
+            (httpx.ConnectError, "Network error"),
+            (httpx.TimeoutException, "Request timeout"),
+            (httpx.ReadTimeout, "Read timeout"),
+            (httpx.ConnectTimeout, "Connection timeout"),
        ],
    )
-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_handle_network_errors(self, mock_get, exception_type, exception_message, auth_instance):
        """Test handling of various network-related errors including timeouts"""
        mock_get.side_effect = exception_type(exception_message)
@@ -154,7 +154,7 @@ class TestWatercrawlAuth:
            WatercrawlAuth({"auth_type": "bearer", "config": {"api_key": "super_secret_key_12345"}})
        assert "super_secret_key_12345" not in str(exc_info.value)

-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_use_custom_base_url_in_validation(self, mock_get):
        """Test that custom base URL is used in validation"""
        mock_response = MagicMock()
@@ -179,7 +179,7 @@ class TestWatercrawlAuth:
            ("https://app.watercrawl.dev//", "https://app.watercrawl.dev/api/v1/core/crawl-requests/"),
        ],
    )
-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_use_urljoin_for_url_construction(self, mock_get, base_url, expected_url):
        """Test that urljoin is used correctly for URL construction with various base URLs"""
        mock_response = MagicMock()
@@ -193,12 +193,12 @@ class TestWatercrawlAuth:
        # Verify the correct URL was called
        assert mock_get.call_args[0][0] == expected_url

-    @patch("services.auth.watercrawl.watercrawl.requests.get")
+    @patch("services.auth.watercrawl.watercrawl.httpx.get")
    def test_should_handle_timeout_with_retry_suggestion(self, mock_get, auth_instance):
        """Test that timeout errors are handled gracefully with appropriate error message"""
-        mock_get.side_effect = requests.Timeout("The request timed out after 30 seconds")
+        mock_get.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")

-        with pytest.raises(requests.Timeout) as exc_info:
+        with pytest.raises(httpx.TimeoutException) as exc_info:
            auth_instance.validate_credentials()

        # Verify the timeout exception is raised with original message