fix: paginator callbacks receive the Response object (#86)

README.md

@@ -136,7 +136,11 @@ the pages are specified in the query parameters, or by modifying the url.
 
 Usage is simple in both cases; paginator decorators take a Callable with two required arguments:
 - `by_query_params` -> callable takes `response` and `previous_page_params`.
-- `by_url` -> callable takes `respones` and `previous_page_url`.
+- `by_url` -> callable takes `response` and `previous_page_url`.
+
+The `response` argument is a `Response` object. Use `response.get_json()` to read the decoded body,
+and `response.get_original()` to reach the underlying `requests.Response` (for example to read
+`headers` or `links`).
 
 The callable will need to return either the params in the case of `by_query_params`, or a new url in the
 case of `by_url`.
@@ -151,13 +155,13 @@ from apiclient.paginators import paginated
 def next_page_by_params(response, previous_page_params):
     # Function reads the response data and returns the query param
     # that tells the next request to go to.
-    return {"next": response["pages"]["next"]}
+    return {"next": response.get_json()["pages"]["next"]}
 
 
 def next_page_by_url(response, previous_page_url):
     # Function reads the response and returns the url as string
     # where the next page of data lives.
-    return response["pages"]["next"]["url"]
+    return response.get_json()["pages"]["next"]["url"]
 
 
 class MyClient(APIClient):
@@ -567,10 +571,11 @@ class Endpoint:
     todo = "todos/{id}"
 
 
-def get_next_page(response):
+def get_next_page(response, previous_page_params):
+    body = response.get_json()
     return {
-        "limit": response["limit"],
-        "offset": response["offset"] + response["limit"],
+        "limit": body["limit"],
+        "offset": body["offset"] + body["limit"],
     }
 
 

apiclient/request_strategies.py

@@ -55,23 +55,33 @@ def set_session(self, session: requests.Session):
 
     def post(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs):
         """Send data and return response data from POST endpoint."""
-        return self._make_request(self.get_session().post, endpoint, data=data, params=params, **kwargs)
+        return self._decode_response_data(
+            self._make_request(self.get_session().post, endpoint, data=data, params=params, **kwargs)
+        )
 
     def get(self, endpoint: str, params: dict | None = None, **kwargs):
         """Return response data from GET endpoint."""
-        return self._make_request(self.get_session().get, endpoint, params=params, **kwargs)
+        return self._decode_response_data(
+            self._make_request(self.get_session().get, endpoint, params=params, **kwargs)
+        )
 
     def put(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs):
         """Send data to overwrite resource and return response data from PUT endpoint."""
-        return self._make_request(self.get_session().put, endpoint, data=data, params=params, **kwargs)
+        return self._decode_response_data(
+            self._make_request(self.get_session().put, endpoint, data=data, params=params, **kwargs)
+        )
 
     def patch(self, endpoint: str, data: JsonType, params: dict | None = None, **kwargs):
         """Send data to update resource and return response data from PATCH endpoint."""
-        return self._make_request(self.get_session().patch, endpoint, data=data, params=params, **kwargs)
+        return self._decode_response_data(
+            self._make_request(self.get_session().patch, endpoint, data=data, params=params, **kwargs)
+        )
 
     def delete(self, endpoint: str, params: dict | None = None, **kwargs):
         """Remove resource with DELETE endpoint."""
-        return self._make_request(self.get_session().delete, endpoint, params=params, **kwargs)
+        return self._decode_response_data(
+            self._make_request(self.get_session().delete, endpoint, params=params, **kwargs)
+        )
 
     def _make_request(
         self,
@@ -82,9 +92,10 @@ def _make_request(
         data: JsonType | None = None,
         **kwargs,
     ) -> Response:
-        """Make the request with the given method.
+        """Make the request with the given method and return the checked response.
 
-        Delegates response parsing to the response handler.
+        Decoding is left to the caller so that paginators retain access to the
+        underlying response (headers, links) via `Response.get_original()`.
         """
         try:
             response = RequestsResponse(
@@ -102,7 +113,7 @@ def _make_request(
             raise UnexpectedError(f"Error when contacting '{endpoint}'") from error
         else:
             self._check_response(response)
-        return self._decode_response_data(response)
+        return response
 
     def _get_request_params(self, params: dict | None) -> dict:
         """Return dictionary with any additional authentication query parameters."""
@@ -157,9 +168,11 @@ def get(self, endpoint: str, params: dict | None = None, **kwargs):
         while run:
             this_page_params = deepcopy(params)
 
-            response = super().get(endpoint, params=this_page_params, **kwargs)
+            response = self._make_request(
+                self.get_session().get, endpoint, params=this_page_params, **kwargs
+            )
 
-            pages.append(response)
+            pages.append(self._decode_response_data(response))
             next_page_params = self.get_next_page_params(response, previous_page_params=this_page_params)
 
             if next_page_params:
@@ -183,9 +196,9 @@ def __init__(self, next_page: Callable):
     def get(self, endpoint: str, params: dict | None = None, **kwargs):
         pages = []
         while endpoint:
-            response = super().get(endpoint, params=params, **kwargs)
+            response = self._make_request(self.get_session().get, endpoint, params=params, **kwargs)
 
-            pages.append(response)
+            pages.append(self._decode_response_data(response))
 
             next_page_url = self.get_next_page_url(response, previous_page_url=endpoint)
             endpoint = next_page_url

tests/integration_tests/client.py

@@ -8,8 +8,9 @@
 
 
 def by_query_params_callable(response, prev_params):
-    if "nextPage" in response and response["nextPage"]:
-        return {"page": response["nextPage"]}
+    body = response.get_json()
+    if "nextPage" in body and body["nextPage"]:
+        return {"page": body["nextPage"]}
 
 
 class InternalError(APIRequestError):

tests/test_paginators.py

@@ -10,13 +10,15 @@
 
 
 def next_page_param(response, previous_page_params):
-    if response["next"]:
-        return {"page": response["next"]}
+    body = response.get_json()
+    if body["next"]:
+        return {"page": body["next"]}
 
 
 def next_page_url(response, previous_page_url):
-    if response["next"]:
-        return response["next"]
+    body = response.get_json()
+    if body["next"]:
+        return body["next"]
 
 
 class QueryPaginatedClient(APIClient):

tests/test_request_strategies.py

@@ -178,7 +178,8 @@ def test_query_param_paginated_strategy_delegates_to_callable(initial_params, mo
     )
 
     def next_page_callback(response, previous_params):
-        return {"nextPage": response["nextPage"]} if response["nextPage"] else None
+        body = response.get_json()
+        return {"nextPage": body["nextPage"]} if body["nextPage"] else None
 
     strategy = request_strategy_factory(QueryParamPaginatedRequestStrategy, next_page=next_page_callback)
 
@@ -211,7 +212,7 @@ def test_url_paginated_strategy_delegates_to_callable(mock_requests):
     )
 
     def next_page_callback(response, previous_params):
-        return response["nextPage"]
+        return response.get_json()["nextPage"]
 
     strategy = request_strategy_factory(UrlPaginatedRequestStrategy, next_page=next_page_callback)
 
@@ -231,6 +232,32 @@ def next_page_callback(response, previous_params):
     assert history[1].url == "mock://testserver.com/2"
 
 
+def test_url_paginated_strategy_callback_can_read_response_headers(mock_requests):
+    # Given pages are linked via the Link header rather than the body (issue #86)
+    mock_requests.get(
+        "mock://testserver.com",
+        json={"data": ["element1", "element2"]},
+        status_code=200,
+        headers={"Link": '<mock://testserver.com/2>; rel="next"'},
+    )
+    mock_requests.get("mock://testserver.com/2", json={"data": ["element3", "element4"]}, status_code=200)
+
+    def next_page_callback(response, previous_url):
+        next_link = response.get_original().links.get("next")
+        return next_link["url"] if next_link else None
+
+    strategy = request_strategy_factory(UrlPaginatedRequestStrategy, next_page=next_page_callback)
+
+    # When we request the page, the callback follows the Link header to the next page
+    response = strategy.get("mock://testserver.com")
+
+    assert list(response) == [
+        {"data": ["element1", "element2"]},
+        {"data": ["element3", "element4"]},
+    ]
+    assert mock_requests.call_count == 2
+
+
 def test_mock_strategy():
     mock_strategy = Mock(spec=BaseRequestStrategy)
     client = APIClient(request_strategy=mock_strategy)