diff --git a/thousandeyes-sdk-core/README.md b/thousandeyes-sdk-core/README.md index 0999e656..267e4af2 100644 --- a/thousandeyes-sdk-core/README.md +++ b/thousandeyes-sdk-core/README.md @@ -1,3 +1,29 @@ # thousandeyes-sdk-core This package provides core functionality for interacting with the ThousandEyes API and should be installed before using any of the published SDKs. + +`PaginatorIterator` is unbounded, so wrap it with `itertools.islice` to cap the number of items and avoid making unintended, potentially expensive API calls. +Pick a slice size that matches your UI or batch size so you only fetch what you plan to process: + +```python +from thousandeyes_sdk.core import Configuration, ApiClient, PaginatorIterator +from thousandeyes_sdk.dashboards import DashboardsApi +from itertools import islice + +configuration = Configuration( + host = "https://api.thousandeyes.com/v7", + access_token = "an_access_token", +) + + +def get_dashboard_widget_data(): + with ApiClient(configuration) as client: + dashboards_api = DashboardsApi(client) + for item in list(islice(PaginatorIterator( + dashboards_api.get_dashboard_widget_data, + lambda response: response.data.tests, + dashboard_id="a_dashboard_id", + widget_id="a_widget_id", + ), 20)): + print(item.test_id) +``` diff --git a/thousandeyes-sdk-core/src/thousandeyes_sdk/core/__init__.py b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/__init__.py index 9bba574f..8c6173c7 100644 --- a/thousandeyes-sdk-core/src/thousandeyes_sdk/core/__init__.py +++ b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/__init__.py @@ -18,5 +18,6 @@ from . import exceptions from .api_client import ApiClient from .api_response import ApiResponse from .configuration import Configuration +from .pagination_iterator import PaginatorIterator import os.path diff --git a/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py new file mode 100644 index 00000000..cffe6fbd --- /dev/null +++ b/thousandeyes-sdk-core/src/thousandeyes_sdk/core/pagination_iterator.py @@ -0,0 +1,106 @@ +# Copyright 2024 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator +from typing import Any, Mapping, Optional, TypeVar, Generic +from urllib.parse import parse_qs, urlparse + +from typing_extensions import ParamSpec + +P = ParamSpec("P") +R = TypeVar("R") +I = TypeVar("I") + +class PaginatorIterator(Generic[P, R, I]): + """Iterate over cursor-paginated responses. + + Calls ``method`` repeatedly, passing a cursor parameter between calls, + and yields items obtained from ``items_getter``. + The next cursor is derived from ``response.data.links`` or ``response.data._links`` + (or mapping equivalents), supporting these link formats: + + - a direct ``href`` string + - a mapping with a ``href`` key + - an object with a ``href`` attribute + + Iteration stops when no next cursor is found or the cursor repeats. + """ + + def __init__( + self, + method: Callable[P, R], + items_getter: Callable[[R], Iterable[I]], + *, + cursor_param: str = "cursor", + **params: P.kwargs, + ) -> None: + self._method = method + self._items_getter = items_getter + self._cursor_param = cursor_param + self._params: dict[str, Any] = dict(params) + + def __iter__(self) -> Iterator[I]: + params = dict(self._params) + last_cursor = params.get(self._cursor_param) + + while True: + response = self._method(**params) + items = self._items_getter(response) + for item in items if items else []: + yield item + + next_cursor = self._next_cursor_from_response(response) + if not next_cursor or next_cursor == last_cursor: + break + + params[self._cursor_param] = next_cursor + last_cursor = next_cursor + + def _next_cursor_from_response(self, response: Any) -> Optional[str]: + links = getattr(response, "links", None) + + if links is None: + links = getattr(response, "_links", None) + + if links is None: + return None + + next_link = getattr(links, "next", None) + if next_link is None and isinstance(links, Mapping): + next_link = links.get("next") + + if next_link is None: + return None + + if isinstance(next_link, str): + href = next_link + elif isinstance(next_link, Mapping): + href = next_link.get("href") + else: + href = getattr(next_link, "href", None) + + if not href: + return None + + parsed = urlparse(href) + query_params = parse_qs(parsed.query) + cursor_values = query_params.get(self._cursor_param) + + if cursor_values: + return cursor_values[0] + return None diff --git a/thousandeyes-sdk-core/test/test_pagination_iterator.py b/thousandeyes-sdk-core/test/test_pagination_iterator.py new file mode 100644 index 00000000..e4717e55 --- /dev/null +++ b/thousandeyes-sdk-core/test/test_pagination_iterator.py @@ -0,0 +1,74 @@ +from types import SimpleNamespace + +from thousandeyes_sdk.core.pagination_iterator import PaginatorIterator + + +def test_iterator_uses_cursor_from_next_href(): + calls = [] + + def method(**params): + calls.append(params.copy()) + if params.get("cursor") is None: + links = SimpleNamespace(next="https://example.com/items?cursor=abc") + items = ["first", "second"] + else: + links = SimpleNamespace(next=None) + items = ["third"] + return SimpleNamespace(links=links, items=items) + + responses = list(PaginatorIterator(method, lambda response: response.items)) + + assert responses == ["first", "second", "third"] + assert calls == [{}, {"cursor": "abc"}] + + +def test_iterator_reads_cursor_from_links_mapping(): + calls = [] + + def method(**params): + calls.append(params.copy()) + if params.get("pageCursor") is None: + links = {"next": {"href": "https://example.com?foo=1&pageCursor=xyz"}} + items = ["alpha"] + else: + links = {"next": None} + items = ["beta"] + return SimpleNamespace(links=links, items=items) + + responses = list(PaginatorIterator(method, lambda response: response.items, cursor_param="pageCursor")) + + assert responses == ["alpha", "beta"] + assert calls == [{}, {"pageCursor": "xyz"}] + + +def test_iterator_stops_when_no_cursor_param_present(): + calls = [] + + def method(**params): + calls.append(params.copy()) + if params.get("cursor") is None: + links = {"next": "/next/page"} + items = ["one"] + else: + links = {"next": None} + items = ["two"] + return SimpleNamespace(links=links, items=items) + + responses = list(PaginatorIterator(method, lambda response: response.items)) + + assert responses == ["one"] + assert calls == [{}] + + +def test_iterator_stops_on_repeated_cursor(): + calls = [] + + def method(**params): + calls.append(params.copy()) + links = {"next": "https://example.com?cursor=same"} + return SimpleNamespace(links=links, items=["only"]) + + responses = list(PaginatorIterator(method, lambda response: response.items, cursor="same")) + + assert responses == ["only"] + assert calls == [{"cursor": "same"}]