feat: Add pagination iterable helper (#125)

thousandeyes-sdk-core/README.md

@@ -1,3 +1,29 @@
 # thousandeyes-sdk-core
 
 This package provides core functionality for interacting with the ThousandEyes API and should be installed before using any of the published SDKs.
+
+`PaginationIterable` is unbounded, so wrap it with `itertools.islice` to cap the number of items and avoid making unintended, potentially expensive API calls. 
+Pick a slice size that matches your UI or batch size so you only fetch what you plan to process:
+
+```python
+from thousandeyes_sdk.core import Configuration, ApiClient, PaginationIterable
+from thousandeyes_sdk.dashboards import DashboardsApi
+from itertools import islice
+
+configuration = Configuration(
+    host = "https://api.thousandeyes.com/v7",
+    access_token = "an_access_token",
+)
+
+
+def get_dashboard_widget_data():
+    with ApiClient(configuration) as client:
+        dashboards_api = DashboardsApi(client)
+        for item in list(islice(PaginationIterable(
+            dashboards_api.get_dashboard_widget_data,
+            lambda response: response.data.tests,
+            dashboard_id="a_dashboard_id",
+            widget_id="a_widget_id",
+        ), 20)):
+            print(item.test_id)
+```

thousandeyes-sdk-core/src/thousandeyes_sdk/core/__init__.py

@@ -18,5 +18,6 @@ from . import exceptions
 from .api_client import ApiClient
 from .api_response import ApiResponse
 from .configuration import Configuration
+from .pagination_iterable import PaginationIterable
 
 import os.path

thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterable.py

@@ -0,0 +1,106 @@
+# Copyright 2024 Cisco Systems, Inc. and its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator
+from typing import Any, Mapping, Optional, TypeVar, Generic
+from urllib.parse import parse_qs, urlparse
+
+from typing_extensions import ParamSpec
+
+P = ParamSpec("P")
+R = TypeVar("R")
+I = TypeVar("I")
+
+class PaginationIterable(Generic[P, R, I]):
+    """Iterate over cursor-paginated responses.
+
+    Calls ``method`` repeatedly, passing a cursor parameter between calls,
+    and yields items obtained from ``items_getter``.
+    The next cursor is derived from ``response.data.links`` or ``response.data._links``
+    (or mapping equivalents), supporting these link formats:
+
+    - a direct ``href`` string
+    - a mapping with a ``href`` key
+    - an object with a ``href`` attribute
+
+    Iteration stops when no next cursor is found or the cursor repeats.
+    """
+
+    def __init__(
+        self,
+        method: Callable[P, R],
+        items_getter: Callable[[R], Iterable[I]],
+        *,
+        cursor_param: str = "cursor",
+        **params: P.kwargs,
+    ) -> None:
+        self._method = method
+        self._items_getter = items_getter
+        self._cursor_param = cursor_param
+        self._params: dict[str, Any] = dict(params)
+
+    def __iter__(self) -> Iterator[I]:
+        params = dict(self._params)
+        last_cursor = params.get(self._cursor_param)
+
+        while True:
+            response = self._method(**params)
+            items = self._items_getter(response)
+            for item in items if items else []:
+                yield item
+
+            next_cursor = self._next_cursor_from_response(response)
+            if not next_cursor or next_cursor == last_cursor:
+                break
+
+            params[self._cursor_param] = next_cursor
+            last_cursor = next_cursor
+
+    def _next_cursor_from_response(self, response: Any) -> Optional[str]:
+        links = getattr(response, "links", None)
+
+        if links is None:
+            links = getattr(response, "_links", None)
+
+        if links is None:
+            return None
+
+        next_link = getattr(links, "next", None)
+        if next_link is None and isinstance(links, Mapping):
+            next_link = links.get("next")
+
+        if next_link is None:
+            return None
+
+        if isinstance(next_link, str):
+            href = next_link
+        elif isinstance(next_link, Mapping):
+            href = next_link.get("href")
+        else:
+            href = getattr(next_link, "href", None)
+
+        if not href:
+            return None
+
+        parsed = urlparse(href)
+        query_params = parse_qs(parsed.query)
+        cursor_values = query_params.get(self._cursor_param)
+
+        if cursor_values:
+            return cursor_values[0]
+        return None

thousandeyes-sdk-core/test/test_pagination_iterable.py

@@ -0,0 +1,74 @@
+from types import SimpleNamespace
+
+from thousandeyes_sdk.core.pagination_iterable import PaginationIterable
+
+
+def test_iterable_uses_cursor_from_next_href():
+    calls = []
+
+    def method(**params):
+        calls.append(params.copy())
+        if params.get("cursor") is None:
+            links = SimpleNamespace(next="https://example.com/items?cursor=abc")
+            items = ["first", "second"]
+        else:
+            links = SimpleNamespace(next=None)
+            items = ["third"]
+        return SimpleNamespace(links=links, items=items)
+
+    responses = list(PaginationIterable(method, lambda response: response.items))
+
+    assert responses == ["first", "second", "third"]
+    assert calls == [{}, {"cursor": "abc"}]
+
+
+def test_iterable_reads_cursor_from_links_mapping():
+    calls = []
+
+    def method(**params):
+        calls.append(params.copy())
+        if params.get("pageCursor") is None:
+            links = {"next": {"href": "https://example.com?foo=1&pageCursor=xyz"}}
+            items = ["alpha"]
+        else:
+            links = {"next": None}
+            items = ["beta"]
+        return SimpleNamespace(links=links, items=items)
+
+    responses = list(PaginationIterable(method, lambda response: response.items, cursor_param="pageCursor"))
+
+    assert responses == ["alpha", "beta"]
+    assert calls == [{}, {"pageCursor": "xyz"}]
+
+
+def test_iterable_stops_when_no_cursor_param_present():
+    calls = []
+
+    def method(**params):
+        calls.append(params.copy())
+        if params.get("cursor") is None:
+            links = {"next": "/next/page"}
+            items = ["one"]
+        else:
+            links = {"next": None}
+            items = ["two"]
+        return SimpleNamespace(links=links, items=items)
+
+    responses = list(PaginationIterable(method, lambda response: response.items))
+
+    assert responses == ["one"]
+    assert calls == [{}]
+
+
+def test_iterable_stops_on_repeated_cursor():
+    calls = []
+
+    def method(**params):
+        calls.append(params.copy())
+        links = {"next": "https://example.com?cursor=same"}
+        return SimpleNamespace(links=links, items=["only"])
+
+    responses = list(PaginationIterable(method, lambda response: response.items, cursor="same"))
+
+    assert responses == ["only"]
+    assert calls == [{"cursor": "same"}]