From abea9a244afa082e78e5ca6e46ad1f8112e91651 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodrigues Date: Tue, 20 Jan 2026 14:27:14 +0000 Subject: [PATCH] CP-2451 yield items instead of pages --- thousandeyes-sdk-core/README.md | 15 +++++----- .../core/pagination_iterator.py | 17 +++++++---- .../test/test_pagination_iterator.py | 29 ++++++++++++------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/thousandeyes-sdk-core/README.md b/thousandeyes-sdk-core/README.md index 66f1ddd4..2f3c285b 100644 --- a/thousandeyes-sdk-core/README.md +++ b/thousandeyes-sdk-core/README.md @@ -6,13 +6,14 @@ Usage example for iterating paginated responses: ```python from thousandeyes_sdk.core import PaginatorIterator -from thousandeyes_sdk.usage.api.usage_api import UsageApi +from thousandeyes_sdk.dashboards.api.dashboards_api import DashboardsApi -usage_api = UsageApi() -for page in PaginatorIterator( - usage_api.get_enterprise_agents_units_usage, - start_date="2024-01-01T00:00:00Z", - end_date="2024-01-31T23:59:59Z", +dashboards_api = DashboardsApi() +for widget_data in PaginatorIterator( + dashboards_api.get_dashboard_widget_data, + lambda response: response.data.tests if response.data else [], + dashboard_id="dashboard-id", + widget_id="widget-id", ): - print(page) + print(widget_data) ``` diff --git a/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py index 57ca5613..11b90a9f 100644 --- a/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py +++ b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py @@ -16,19 +16,21 @@ from __future__ import annotations -from typing import Any, Callable, Iterator, Mapping, Optional, TypeVar, Generic +from collections.abc import Callable, Iterable, Iterator +from typing import Any, Mapping, Optional, TypeVar, Generic from urllib.parse import parse_qs, urlparse from typing_extensions import ParamSpec P = ParamSpec("P") R = TypeVar("R") +I = TypeVar("I") - -class PaginatorIterator(Generic[P, R]): +class PaginatorIterator(Generic[P, R, I]): """Iterate over cursor-paginated responses. - Calls ``method`` repeatedly, passing a cursor parameter between calls. + Calls ``method`` repeatedly, passing a cursor parameter between calls, + and yields items obtained from ``items_getter``. The next cursor is derived from ``response.data.links`` or ``response.data._links`` (or mapping equivalents), supporting these link formats: @@ -42,21 +44,24 @@ class PaginatorIterator(Generic[P, R]): def __init__( self, method: Callable[P, R], + items_getter: Callable[[R], Iterable[I]], *, cursor_param: str = "cursor", **params: P.kwargs, ) -> None: self._method = method + self._items_getter = items_getter self._cursor_param = cursor_param self._params: dict[str, Any] = dict(params) - def __iter__(self) -> Iterator[R]: + def __iter__(self) -> Iterator[I]: params = dict(self._params) last_cursor = params.get(self._cursor_param) while True: response = self._method(**params) - yield response + for item in self._items_getter(response): + yield item next_cursor = self._next_cursor_from_response(response) if not next_cursor or next_cursor == last_cursor: diff --git a/thousandeyes-sdk-core/test/test_pagination_iterator.py b/thousandeyes-sdk-core/test/test_pagination_iterator.py index bb2da6cf..f80a6bec 100644 --- a/thousandeyes-sdk-core/test/test_pagination_iterator.py +++ b/thousandeyes-sdk-core/test/test_pagination_iterator.py @@ -1,6 +1,6 @@ from types import SimpleNamespace -from thousandeyes_sdk.core.iterable import PaginatorIterator +from thousandeyes_sdk.core.pagination_iterator import PaginatorIterator def test_iterator_uses_cursor_from_next_href(): @@ -10,14 +10,16 @@ def test_iterator_uses_cursor_from_next_href(): calls.append(params.copy()) if params.get("cursor") is None: links = SimpleNamespace(next="https://example.com/items?cursor=abc") + items = ["first", "second"] else: links = SimpleNamespace(next=None) + items = ["third"] data = SimpleNamespace(links=links) - return SimpleNamespace(data=data) + return SimpleNamespace(data=data, items=items) - responses = list(PaginatorIterator(method)) + responses = list(PaginatorIterator(method, lambda response: response.items)) - assert len(responses) == 2 + assert responses == ["first", "second", "third"] assert calls == [{}, {"cursor": "abc"}] @@ -28,12 +30,15 @@ def test_iterator_reads_cursor_from_links_mapping(): calls.append(params.copy()) if params.get("pageCursor") is None: data = {"_links": {"next": {"href": "https://example.com?foo=1&pageCursor=xyz"}}} + items = ["alpha"] else: data = {"_links": {"next": None}} - return SimpleNamespace(data=data) + items = ["beta"] + return SimpleNamespace(data=data, items=items) - list(PaginatorIterator(method, cursor_param="pageCursor")) + responses = list(PaginatorIterator(method, lambda response: response.items, cursor_param="pageCursor")) + assert responses == ["alpha", "beta"] assert calls == [{}, {"pageCursor": "xyz"}] @@ -44,12 +49,15 @@ def test_iterator_stops_when_no_cursor_param_present(): calls.append(params.copy()) if params.get("cursor") is None: data = {"links": {"next": "/next/page"}} + items = ["one"] else: data = {"links": {"next": None}} - return SimpleNamespace(data=data) + items = ["two"] + return SimpleNamespace(data=data, items=items) - list(PaginatorIterator(method)) + responses = list(PaginatorIterator(method, lambda response: response.items)) + assert responses == ["one"] assert calls == [{}] @@ -59,8 +67,9 @@ def test_iterator_stops_on_repeated_cursor(): def method(**params): calls.append(params.copy()) data = {"links": {"next": "https://example.com?cursor=same"}} - return SimpleNamespace(data=data) + return SimpleNamespace(data=data, items=["only"]) - list(PaginatorIterator(method, cursor="same")) + responses = list(PaginatorIterator(method, lambda response: response.items, cursor="same")) + assert responses == ["only"] assert calls == [{"cursor": "same"}]