diff --git a/README.md b/README.md index 68918ab..3b4bb9f 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,11 @@ the pages are specified in the query parameters, or by modifying the url. Usage is simple in both cases; paginator decorators take a Callable with two required arguments: - `by_query_params` -> callable takes `response` and `previous_page_params`. -- `by_url` -> callable takes `respones` and `previous_page_url`. +- `by_url` -> callable takes `response` and `previous_page_url`. + +The `response` argument is a `Response` object. Use `response.get_json()` to read the decoded body, +and `response.get_original()` to reach the underlying `requests.Response` (for example to read +`headers` or `links`). The callable will need to return either the params in the case of `by_query_params`, or a new url in the case of `by_url`. @@ -151,13 +155,13 @@ from apiclient.paginators import paginated def next_page_by_params(response, previous_page_params): # Function reads the response data and returns the query param # that tells the next request to go to. - return {"next": response["pages"]["next"]} + return {"next": response.get_json()["pages"]["next"]} def next_page_by_url(response, previous_page_url): # Function reads the response and returns the url as string # where the next page of data lives. - return response["pages"]["next"]["url"] + return response.get_json()["pages"]["next"]["url"] class MyClient(APIClient): @@ -567,10 +571,11 @@ class Endpoint: todo = "todos/{id}" -def get_next_page(response): +def get_next_page(response, previous_page_params): + body = response.get_json() return { - "limit": response["limit"], - "offset": response["offset"] + response["limit"], + "limit": body["limit"], + "offset": body["offset"] + body["limit"], } diff --git a/apiclient/request_strategies.py b/apiclient/request_strategies.py index 98dfb15..c50d4ec 100644 --- a/apiclient/request_strategies.py +++ b/apiclient/request_strategies.py @@ -55,23 +55,33 @@ def set_session(self, session: requests.Session): def post(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs): """Send data and return response data from POST endpoint.""" - return self._make_request(self.get_session().post, endpoint, data=data, params=params, **kwargs) + return self._decode_response_data( + self._make_request(self.get_session().post, endpoint, data=data, params=params, **kwargs) + ) def get(self, endpoint: str, params: dict | None = None, **kwargs): """Return response data from GET endpoint.""" - return self._make_request(self.get_session().get, endpoint, params=params, **kwargs) + return self._decode_response_data( + self._make_request(self.get_session().get, endpoint, params=params, **kwargs) + ) def put(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs): """Send data to overwrite resource and return response data from PUT endpoint.""" - return self._make_request(self.get_session().put, endpoint, data=data, params=params, **kwargs) + return self._decode_response_data( + self._make_request(self.get_session().put, endpoint, data=data, params=params, **kwargs) + ) def patch(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs): """Send data to update resource and return response data from PATCH endpoint.""" - return self._make_request(self.get_session().patch, endpoint, data=data, params=params, **kwargs) + return self._decode_response_data( + self._make_request(self.get_session().patch, endpoint, data=data, params=params, **kwargs) + ) def delete(self, endpoint: str, params: dict | None = None, **kwargs): """Remove resource with DELETE endpoint.""" - return self._make_request(self.get_session().delete, endpoint, params=params, **kwargs) + return self._decode_response_data( + self._make_request(self.get_session().delete, endpoint, params=params, **kwargs) + ) def _make_request( self, @@ -82,9 +92,10 @@ def _make_request( data: JsonType | None = None, **kwargs, ) -> Response: - """Make the request with the given method. + """Make the request with the given method and return the checked response. - Delegates response parsing to the response handler. + Decoding is left to the caller so that paginators retain access to the + underlying response (headers, links) via `Response.get_original()`. """ try: response = RequestsResponse( @@ -102,7 +113,7 @@ def _make_request( raise UnexpectedError(f"Error when contacting '{endpoint}'") from error else: self._check_response(response) - return self._decode_response_data(response) + return response def _get_request_params(self, params: dict | None) -> dict: """Return dictionary with any additional authentication query parameters.""" @@ -157,9 +168,11 @@ def get(self, endpoint: str, params: dict | None = None, **kwargs): while run: this_page_params = deepcopy(params) - response = super().get(endpoint, params=this_page_params, **kwargs) + response = self._make_request( + self.get_session().get, endpoint, params=this_page_params, **kwargs + ) - pages.append(response) + pages.append(self._decode_response_data(response)) next_page_params = self.get_next_page_params(response, previous_page_params=this_page_params) if next_page_params: @@ -183,9 +196,9 @@ def __init__(self, next_page: Callable): def get(self, endpoint: str, params: dict | None = None, **kwargs): pages = [] while endpoint: - response = super().get(endpoint, params=params, **kwargs) + response = self._make_request(self.get_session().get, endpoint, params=params, **kwargs) - pages.append(response) + pages.append(self._decode_response_data(response)) next_page_url = self.get_next_page_url(response, previous_page_url=endpoint) endpoint = next_page_url diff --git a/tests/integration_tests/client.py b/tests/integration_tests/client.py index 28bfbe6..a7f1a9a 100644 --- a/tests/integration_tests/client.py +++ b/tests/integration_tests/client.py @@ -8,8 +8,9 @@ def by_query_params_callable(response, prev_params): - if "nextPage" in response and response["nextPage"]: - return {"page": response["nextPage"]} + body = response.get_json() + if "nextPage" in body and body["nextPage"]: + return {"page": body["nextPage"]} class InternalError(APIRequestError): diff --git a/tests/test_paginators.py b/tests/test_paginators.py index ed6c1b9..f300785 100644 --- a/tests/test_paginators.py +++ b/tests/test_paginators.py @@ -10,13 +10,15 @@ def next_page_param(response, previous_page_params): - if response["next"]: - return {"page": response["next"]} + body = response.get_json() + if body["next"]: + return {"page": body["next"]} def next_page_url(response, previous_page_url): - if response["next"]: - return response["next"] + body = response.get_json() + if body["next"]: + return body["next"] class QueryPaginatedClient(APIClient): diff --git a/tests/test_request_strategies.py b/tests/test_request_strategies.py index 1deedbf..2a91acf 100644 --- a/tests/test_request_strategies.py +++ b/tests/test_request_strategies.py @@ -178,7 +178,8 @@ def test_query_param_paginated_strategy_delegates_to_callable(initial_params, mo ) def next_page_callback(response, previous_params): - return {"nextPage": response["nextPage"]} if response["nextPage"] else None + body = response.get_json() + return {"nextPage": body["nextPage"]} if body["nextPage"] else None strategy = request_strategy_factory(QueryParamPaginatedRequestStrategy, next_page=next_page_callback) @@ -211,7 +212,7 @@ def test_url_paginated_strategy_delegates_to_callable(mock_requests): ) def next_page_callback(response, previous_params): - return response["nextPage"] + return response.get_json()["nextPage"] strategy = request_strategy_factory(UrlPaginatedRequestStrategy, next_page=next_page_callback) @@ -231,6 +232,32 @@ def next_page_callback(response, previous_params): assert history[1].url == "mock://testserver.com/2" +def test_url_paginated_strategy_callback_can_read_response_headers(mock_requests): + # Given pages are linked via the Link header rather than the body (issue #86) + mock_requests.get( + "mock://testserver.com", + json={"data": ["element1", "element2"]}, + status_code=200, + headers={"Link": '; rel="next"'}, + ) + mock_requests.get("mock://testserver.com/2", json={"data": ["element3", "element4"]}, status_code=200) + + def next_page_callback(response, previous_url): + next_link = response.get_original().links.get("next") + return next_link["url"] if next_link else None + + strategy = request_strategy_factory(UrlPaginatedRequestStrategy, next_page=next_page_callback) + + # When we request the page, the callback follows the Link header to the next page + response = strategy.get("mock://testserver.com") + + assert list(response) == [ + {"data": ["element1", "element2"]}, + {"data": ["element3", "element4"]}, + ] + assert mock_requests.call_count == 2 + + def test_mock_strategy(): mock_strategy = Mock(spec=BaseRequestStrategy) client = APIClient(request_strategy=mock_strategy)