fix(http): Improve exhausted 5xx retry errors

CHANGELOG.md

@@ -4,6 +4,10 @@ Newer updates can be found here: [GitHub Release Notes](https://github.com/airby
 
 # Changelog
 
+## 6.5.3
+
+HttpClient: Improve retry-exhausted server error messages for HTTP 5xx responses.
+
 ## 6.5.2
 
 bugfix: Ensure that streams with partition router are not executed concurrently

airbyte_cdk/sources/streams/http/http_client.py

@@ -64,6 +64,12 @@
 MessageRepresentationAirbyteTracedErrors = AirbyteTracedException
 
 BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
+SERVER_ERROR_MESSAGE_BY_STATUS_CODE = {
+    500: "API server returned HTTP 500.",
+    502: "API server returned HTTP 502.",
+    503: "API server returned HTTP 503.",
+    504: "API server returned HTTP 504.",
+}
 
 
 def monkey_patched_get_item(self, key):  # type: ignore # this interface is a copy/paste from the requests_cache lib
@@ -314,6 +320,25 @@ def _send_with_retry(
                     stream_descriptor=StreamDescriptor(name=self._name),
                 )
 
+            if isinstance(e.response, requests.Response):
+                server_error_message = SERVER_ERROR_MESSAGE_BY_STATUS_CODE.get(
+                    e.response.status_code
+                )
+                if server_error_message:
+                    internal_message = (
+                        "Exhausted available request attempts. "
+                        f"Request URL: {e.request.url}, "
+                        f"Response Code: {e.response.status_code}, "
+                        f"Response Text: {e.response.text}"
+                    )
+                    raise AirbyteTracedException(
+                        internal_message=internal_message,
+                        message=server_error_message,
+                        failure_type=e.failure_type or FailureType.transient_error,
+                        exception=e,
+                        stream_descriptor=StreamDescriptor(name=self._name),
+                    )
+
             raise AirbyteTracedException(
                 internal_message=f"Exhausted available request attempts. Exception: {e}",
                 message=f"Exhausted available request attempts. Please see logs for more details. Exception: {e}",

unit_tests/sources/streams/http/test_http_client.py

@@ -840,6 +840,31 @@ def backoff_time(self, response_or_exception, attempt_count):
     assert e.value.failure_type == expected_failure_type
 
 
+@pytest.mark.usefixtures("mock_sleep")
+def test_send_with_retry_raises_specific_message_for_exhausted_server_error(requests_mock):
+    http_client = HttpClient(
+        name="test",
+        logger=MagicMock(spec=logging.Logger),
+        error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=1),
+    )
+
+    requests_mock.register_uri(
+        "GET",
+        "https://airbyte.io/",
+        status_code=500,
+        text="Internal server error.",
+        headers={},
+    )
+
+    with pytest.raises(AirbyteTracedException) as e:
+        http_client.send_request(http_method="get", url="https://airbyte.io/", request_kwargs={})
+
+    assert e.value.message == "API server returned HTTP 500."
+    assert e.value.failure_type == FailureType.transient_error
+    assert e.value.internal_message is not None
+    assert "Response Code: 500" in e.value.internal_message
+
+
 class MockOAuthAuthenticator:
     def __init__(self):
         self.access_token = "old_token"