diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e73e411..e0bfc412c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Newer updates can be found here: [GitHub Release Notes](https://github.com/airby # Changelog +## 6.5.3 + +HttpClient: Improve retry-exhausted server error messages for HTTP 5xx responses. + ## 6.5.2 bugfix: Ensure that streams with partition router are not executed concurrently diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index c1d0eabd6..03b7bb5fd 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -64,6 +64,12 @@ MessageRepresentationAirbyteTracedErrors = AirbyteTracedException BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH") +SERVER_ERROR_MESSAGE_BY_STATUS_CODE = { + 500: "API server returned HTTP 500.", + 502: "API server returned HTTP 502.", + 503: "API server returned HTTP 503.", + 504: "API server returned HTTP 504.", +} def monkey_patched_get_item(self, key): # type: ignore # this interface is a copy/paste from the requests_cache lib @@ -314,6 +320,25 @@ def _send_with_retry( stream_descriptor=StreamDescriptor(name=self._name), ) + if isinstance(e.response, requests.Response): + server_error_message = SERVER_ERROR_MESSAGE_BY_STATUS_CODE.get( + e.response.status_code + ) + if server_error_message: + internal_message = ( + "Exhausted available request attempts. " + f"Request URL: {e.request.url}, " + f"Response Code: {e.response.status_code}, " + f"Response Text: {e.response.text}" + ) + raise AirbyteTracedException( + internal_message=internal_message, + message=server_error_message, + failure_type=e.failure_type or FailureType.transient_error, + exception=e, + stream_descriptor=StreamDescriptor(name=self._name), + ) + raise AirbyteTracedException( internal_message=f"Exhausted available request attempts. Exception: {e}", message=f"Exhausted available request attempts. Please see logs for more details. Exception: {e}", diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 48d396cb6..449721da6 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -840,6 +840,31 @@ def backoff_time(self, response_or_exception, attempt_count): assert e.value.failure_type == expected_failure_type +@pytest.mark.usefixtures("mock_sleep") +def test_send_with_retry_raises_specific_message_for_exhausted_server_error(requests_mock): + http_client = HttpClient( + name="test", + logger=MagicMock(spec=logging.Logger), + error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=1), + ) + + requests_mock.register_uri( + "GET", + "https://airbyte.io/", + status_code=500, + text="Internal server error.", + headers={}, + ) + + with pytest.raises(AirbyteTracedException) as e: + http_client.send_request(http_method="get", url="https://airbyte.io/", request_kwargs={}) + + assert e.value.message == "API server returned HTTP 500." + assert e.value.failure_type == FailureType.transient_error + assert e.value.internal_message is not None + assert "Response Code: 500" in e.value.internal_message + + class MockOAuthAuthenticator: def __init__(self): self.access_token = "old_token"