diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index c2010a70..ca7c29ad 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -163,6 +163,41 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +Execute FetchXML queries with automatic paging cookie handling. FetchXML must contain ``. Returns an iterable of pages. + +```python +# Basic FetchXML query +fetchxml = """ + + + + + +""" +for page in client.query.fetchxml(fetchxml): + for record in page: + print(record["name"]) + +# FetchXML with filter and paging +fetchxml = """ + + + + + + + + + + +""" +for page in client.query.fetchxml(fetchxml, page_size=50): + for record in page: + print(record["fullname"]) +``` + ### Table Management #### Create Custom Tables diff --git a/README.md b/README.md index a83dcf9b..fbc3f77e 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - [Bulk operations](#bulk-operations) - [Upsert operations](#upsert-operations) - [Query data](#query-data) + - [FetchXML queries](#fetchxml-queries) - [Table management](#table-management) - [Relationship management](#relationship-management) - [File operations](#file-operations) @@ -36,7 +37,8 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - **🔄 CRUD Operations**: Create, read, update, and delete records with support for bulk operations and automatic retry - **⚡ True Bulk Operations**: Automatically uses Dataverse's native `CreateMultiple`, `UpdateMultiple`, `UpsertMultiple`, and `BulkDelete` Web API operations for maximum performance and transactional integrity -- **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter +- **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter +- **📋 FetchXML Queries**: Execute FetchXML queries with automatic paging cookie handling via `client.query.fetchxml()` - **🏗️ Table Management**: Create, inspect, and delete custom tables and columns programmatically - **🔗 Relationship Management**: Create one-to-many and many-to-many relationships between tables with full metadata control - **📎 File Operations**: Upload files to Dataverse file columns with automatic chunking for large files @@ -268,6 +270,48 @@ for page in client.records.get( > - **`expand`**: Navigation property names are case-sensitive and must match the exact server names > - **`select`** and **`orderby`**: Case-insensitive; automatically converted to lowercase +### FetchXML queries + +```python +# Basic FetchXML query +fetchxml = """ + + + + + +""" +for page in client.query.fetchxml(fetchxml): + for record in page: + print(record["name"]) + +# FetchXML with filter and paging +fetchxml = """ + + + + + + + + + + +""" +for page in client.query.fetchxml(fetchxml, page_size=50): + for record in page: + print(record["fullname"]) +``` + +> **FetchXML limitations:** +> - FetchXML queries are sent as URL-encoded GET parameters. Very large FetchXML strings may exceed the 32KB URL limit. +> - Do not use the `top` attribute with paging (`count`/`page`) — they are incompatible. +> - Aggregate queries (`aggregate='true'`) return a single result set and do not support paging. Aggregates are limited to 50,000 records. +> - Maximum of 15 `link-entity` (join) elements per query. +> - Maximum of 500 total `condition` and `link-entity` elements combined. +> - For consistent paging results, include an `` element with a unique column (e.g., primary key). +> - Queries with `distinct='true'` require at least one `` element. + ### Table management ```python diff --git a/src/PowerPlatform/Dataverse/core/_error_codes.py b/src/PowerPlatform/Dataverse/core/_error_codes.py index 12703198..27e77f07 100644 --- a/src/PowerPlatform/Dataverse/core/_error_codes.py +++ b/src/PowerPlatform/Dataverse/core/_error_codes.py @@ -41,6 +41,11 @@ # Validation subcodes VALIDATION_SQL_NOT_STRING = "validation_sql_not_string" VALIDATION_SQL_EMPTY = "validation_sql_empty" +VALIDATION_FETCHXML_NOT_STRING = "validation_fetchxml_not_string" +VALIDATION_FETCHXML_EMPTY = "validation_fetchxml_empty" +VALIDATION_FETCHXML_MALFORMED = "validation_fetchxml_malformed" +VALIDATION_FETCHXML_TOO_LONG = "validation_fetchxml_too_long" +VALIDATION_FETCHXML_INVALID_PAGE_SIZE = "validation_fetchxml_invalid_page_size" VALIDATION_ENUM_NO_MEMBERS = "validation_enum_no_members" VALIDATION_ENUM_NON_INT_VALUE = "validation_enum_non_int_value" VALIDATION_UNSUPPORTED_COLUMN_TYPE = "validation_unsupported_column_type" diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index eb341f22..8794a329 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -13,10 +13,12 @@ import re import json import uuid +import xml.etree.ElementTree as ET from datetime import datetime, timezone import importlib.resources as ir from contextlib import contextmanager from contextvars import ContextVar +from urllib.parse import quote as _url_encode, unquote as _url_decode from ..core._http import _HttpClient from ._upload import _FileUploadMixin @@ -27,6 +29,11 @@ _is_transient_status, VALIDATION_SQL_NOT_STRING, VALIDATION_SQL_EMPTY, + VALIDATION_FETCHXML_NOT_STRING, + VALIDATION_FETCHXML_EMPTY, + VALIDATION_FETCHXML_MALFORMED, + VALIDATION_FETCHXML_TOO_LONG, + VALIDATION_FETCHXML_INVALID_PAGE_SIZE, METADATA_ENTITYSET_NOT_FOUND, METADATA_ENTITYSET_NAME_MISSING, METADATA_TABLE_NOT_FOUND, @@ -41,6 +48,8 @@ _GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") _CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) _DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) +_MAX_FETCHXML_LENGTH = 16_000 # ~16 KB raw; caps input size to mitigate XML entity expansion attacks; after URL-encoding stays under 32 KB GET URL limit +_MAX_FETCHXML_PAGES = 10_000 @dataclass @@ -852,6 +861,198 @@ def _extract_logical_table(sql: str) -> str: raise ValueError("Unable to determine table logical name from SQL (expected 'FROM ').") return m.group(1).lower() + # ---------------------- FetchXML ------------------------- + @staticmethod + def _extract_entity_from_fetchxml_element(root: ET.Element) -> str: + """Extract entity logical name from in a parsed FetchXML root element.""" + local_tag = root.tag.split("}")[-1] if "}" in root.tag else root.tag + if local_tag != "fetch": + raise ValidationError( + "Invalid FetchXML: root element must be ", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + entity_elem = None + for child in root: + child_local = child.tag.split("}")[-1] if "}" in child.tag else child.tag + if child_local == "entity": + entity_elem = child + break + if entity_elem is None: + raise ValidationError( + "Invalid FetchXML: missing element", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + name = entity_elem.get("name") + if not name or not str(name).strip(): + raise ValidationError( + "Invalid FetchXML: must have a name attribute", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + return str(name).strip().lower() + + @staticmethod + def _extract_entity_from_fetchxml(fetchxml: str) -> str: + """Extract entity logical name from in FetchXML.""" + try: + root = ET.fromstring(fetchxml.strip()) + except ET.ParseError as e: + raise ValidationError( + f"Invalid FetchXML: malformed XML ({e})", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) from e + return _ODataClient._extract_entity_from_fetchxml_element(root) + + def _query_fetchxml( + self, + fetchxml: str, + *, + page_size: Optional[int] = None, + ) -> Iterable[List[Dict[str, Any]]]: + """Execute a FetchXML query with automatic paging cookie handling.""" + if not isinstance(fetchxml, str): + raise ValidationError( + "fetchxml must be a string", + subcode=VALIDATION_FETCHXML_NOT_STRING, + ) + if not fetchxml.strip(): + raise ValidationError( + "fetchxml must be a non-empty string", + subcode=VALIDATION_FETCHXML_EMPTY, + ) + + if page_size is not None: + if isinstance(page_size, bool) or not isinstance(page_size, int): + raise ValidationError( + "page_size must be an integer", + subcode=VALIDATION_FETCHXML_INVALID_PAGE_SIZE, + ) + ps = page_size + if ps <= 0: + raise ValidationError( + "page_size must be a positive integer", + subcode=VALIDATION_FETCHXML_INVALID_PAGE_SIZE, + ) + + if len(fetchxml) > _MAX_FETCHXML_LENGTH: + raise ValidationError( + f"FetchXML string exceeds maximum supported length ({_MAX_FETCHXML_LENGTH} characters)", + subcode=VALIDATION_FETCHXML_TOO_LONG, + ) + + try: + root = ET.fromstring(fetchxml.strip()) + except ET.ParseError as e: + raise ValidationError( + f"Invalid FetchXML: malformed XML ({e})", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) from e + + entity_name = self._extract_entity_from_fetchxml_element(root) + entity_set = self._entity_set_from_schema_name(entity_name) + + has_top = root.get("top") is not None + is_aggregate = (root.get("aggregate") or "").lower() == "true" + + if has_top and page_size is not None: + raise ValidationError( + "Cannot use page_size with FetchXML that has a 'top' attribute. " + "The 'top' and paging (count/page) attributes are incompatible.", + subcode=VALIDATION_FETCHXML_INVALID_PAGE_SIZE, + ) + + if is_aggregate and page_size is not None: + raise ValidationError( + "Cannot use page_size with aggregate FetchXML queries. " + "Aggregate queries return a single result set and do not support paging.", + subcode=VALIDATION_FETCHXML_INVALID_PAGE_SIZE, + ) + + if not is_aggregate and not has_top: + is_distinct = (root.get("distinct") or "").lower() == "true" + if is_distinct: + has_order = any( + (child.tag.split("}")[-1] if "}" in child.tag else child.tag) == "order" + for entity_elem in root + if (entity_elem.tag.split("}")[-1] if "}" in entity_elem.tag else entity_elem.tag) == "entity" + for child in entity_elem + ) + if not has_order: + raise ValidationError( + "FetchXML with distinct='true' requires at least one element for consistent paging results", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + if page_size is not None and root.get("count") is None: + root.set("count", str(page_size)) + if root.get("page") is None: + root.set("page", "1") + + headers = { + "Prefer": 'odata.include-annotations="Microsoft.Dynamics.CRM.fetchxmlpagingcookie,Microsoft.Dynamics.CRM.morerecords"' + } + + def _do_request(url: str) -> Dict[str, Any]: + r = self._request("get", url, headers=headers) + try: + return r.json() + except ValueError: + return {} + + page_count = 0 + while True: + page_count += 1 + if page_count > _MAX_FETCHXML_PAGES: + raise ValidationError( + f"FetchXML paging exceeded maximum page limit ({_MAX_FETCHXML_PAGES}). " + "This may indicate a query returning too many results or a paging loop.", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + fetchxml_str = ET.tostring(root, encoding="unicode") + encoded = _url_encode(fetchxml_str, safe="") + url = f"{self.api}/{entity_set}?fetchXml={encoded}" + data = _do_request(url) + + items = data.get("value") if isinstance(data, dict) else None + if isinstance(items, list) and items: + yield [x for x in items if isinstance(x, dict)] + + if has_top or is_aggregate: + break + + more_records = False + paging_cookie_raw = None + + if isinstance(data, dict): + more_records_raw = data.get("@Microsoft.Dynamics.CRM.morerecords") + more_records = more_records_raw is True or ( + isinstance(more_records_raw, str) and more_records_raw.lower() == "true" + ) + paging_cookie_raw = data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie") + + if not more_records: + break + + if paging_cookie_raw: + try: + cookie_elem = ET.fromstring(paging_cookie_raw) + except ET.ParseError as e: + raise ValidationError( + f"Failed to parse FetchXML paging cookie from server response: {e}", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) from e + + pagingcookie_value = cookie_elem.get("pagingcookie") + if not pagingcookie_value: + raise ValidationError( + "Paging cookie returned by server is missing 'pagingcookie' attribute", + subcode=VALIDATION_FETCHXML_MALFORMED, + ) + + decoded = _url_decode(_url_decode(pagingcookie_value)) + root.set("paging-cookie", decoded) + + current_page = int(root.get("page", "1")) + root.set("page", str(current_page + 1)) + # ---------------------- Entity set resolution ----------------------- def _entity_set_from_schema_name(self, table_schema_name: str) -> str: """Resolve entity set name (plural) from a schema name (singular) name using metadata. diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 9193fe9f..eb56fee5 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import Any, Dict, Iterable, List, Optional, TYPE_CHECKING if TYPE_CHECKING: from ..client import DataverseClient @@ -73,3 +73,70 @@ def sql(self, sql: str) -> List[Dict[str, Any]]: """ with self._client._scoped_odata() as od: return od._query_sql(sql) + + # -------------------------------------------------------------------- fetchxml + + def fetchxml( + self, + fetchxml: str, + *, + page_size: Optional[int] = None, + ) -> Iterable[List[Dict[str, Any]]]: + """Execute a FetchXML query with automatic paging cookie handling. + + Executes a FetchXML query against Dataverse via the Web API. The FetchXML + string must contain a root ```` element with a child ```` + element. Results are yielded as pages (lists of record dicts). + + :param fetchxml: Raw FetchXML string (e.g. ``...``). + :type fetchxml: :class:`str` + :param page_size: Optional page size override. Sets the ``count`` attribute on + ```` if not already present. + :type page_size: :class:`int` | ``None`` + + .. note:: + If the FetchXML already contains a ``count`` attribute, the ``page_size`` parameter + is ignored. The ``count`` in FetchXML takes precedence. + + :return: Generator yielding pages, where each page is a list of record dicts. + :rtype: :class:`~typing.Iterable` of :class:`list` of :class:`dict` + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``fetchxml`` is not a string, is empty, or has invalid structure. + + Example: + Basic FetchXML query:: + + fetchxml = ''' + + + + + + ''' + for page in client.query.fetchxml(fetchxml): + for record in page: + print(record["name"]) + + Paginated query with page size:: + + fetchxml = ''' + + + + + + + + + ''' + for page in client.query.fetchxml(fetchxml, page_size=50): + for record in page: + print(record["fullname"]) + """ + + def _paged() -> Iterable[List[Dict[str, Any]]]: + with self._client._scoped_odata() as od: + yield from od._query_fetchxml(fetchxml, page_size=page_size) + + return _paged() diff --git a/tests/unit/data/test_fetchxml.py b/tests/unit/data/test_fetchxml.py new file mode 100644 index 00000000..97f84228 --- /dev/null +++ b/tests/unit/data/test_fetchxml.py @@ -0,0 +1,525 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest +from unittest.mock import MagicMock, patch + +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError, ValidationError +from PowerPlatform.Dataverse.core._error_codes import ( + METADATA_ENTITYSET_NOT_FOUND, + VALIDATION_FETCHXML_NOT_STRING, + VALIDATION_FETCHXML_EMPTY, + VALIDATION_FETCHXML_MALFORMED, + VALIDATION_FETCHXML_TOO_LONG, + VALIDATION_FETCHXML_INVALID_PAGE_SIZE, +) +from PowerPlatform.Dataverse.data._odata import _ODataClient + + +def _make_odata_client() -> _ODataClient: + """Return an _ODataClient with HTTP calls mocked out.""" + mock_auth = MagicMock() + mock_auth._acquire_token.return_value = MagicMock(access_token="token") + client = _ODataClient(mock_auth, "https://example.crm.dynamics.com") + client._request = MagicMock() + return client + + +class TestExtractEntityFromFetchxml(unittest.TestCase): + """Unit tests for _ODataClient._extract_entity_from_fetchxml static helper.""" + + def test_extract_entity_from_fetchxml(self): + """Extract entity name from valid FetchXML.""" + fetchxml = "" + result = _ODataClient._extract_entity_from_fetchxml(fetchxml) + self.assertEqual(result, "account") + + def test_extract_entity_from_fetchxml_preserves_lowercase(self): + """Entity name is lowercased.""" + fetchxml = "" + result = _ODataClient._extract_entity_from_fetchxml(fetchxml) + self.assertEqual(result, "account") + + def test_extract_entity_from_fetchxml_missing_entity(self): + """Raise ValidationError on missing entity element.""" + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + _ODataClient._extract_entity_from_fetchxml(fetchxml) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + + def test_extract_entity_from_fetchxml_malformed_xml(self): + """Raise ValidationError on invalid XML.""" + fetchxml = ".""" + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + _ODataClient._extract_entity_from_fetchxml(fetchxml) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + self.assertIn("root element must be ", str(ctx.exception)) + + +class TestQueryFetchxml(unittest.TestCase): + """Unit tests for _ODataClient._query_fetchxml.""" + + def setUp(self): + self.od = _make_odata_client() + + def _setup_entity_set(self): + """Patch _entity_set_from_schema_name to return accounts without HTTP.""" + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + + def test_query_fetchxml_validation_not_string(self): + """Raise ValidationError for non-string input.""" + self._setup_entity_set() + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(123)) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_NOT_STRING) + + def test_query_fetchxml_validation_empty(self): + """Raise ValidationError for empty string.""" + self._setup_entity_set() + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml("")) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_EMPTY) + + def test_query_fetchxml_validation_whitespace_only(self): + """Raise ValidationError for whitespace-only string.""" + self._setup_entity_set() + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(" \n\t ")) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_EMPTY) + + def test_query_fetchxml_wrong_entity_case_raises_http_error(self): + """Wrong entity case (PascalCase) in FetchXML causes API 400; HttpError propagates.""" + self._setup_entity_set() + self.od._request.side_effect = HttpError( + "The entity with a name = 'Contact' was not found", + status_code=400, + ) + fetchxml = "" + with self.assertRaises(HttpError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertEqual(ctx.exception.status_code, 400) + + def test_query_fetchxml_wrong_entity_name_raises_metadata_error(self): + """Non-existent entity name raises MetadataError from entity set resolution.""" + self.od._entity_set_from_schema_name = MagicMock( + side_effect=MetadataError( + "Unable to resolve entity set for table schema name 'NonexistentEntity123'.", + subcode=METADATA_ENTITYSET_NOT_FOUND, + ) + ) + fetchxml = "" + with self.assertRaises(MetadataError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertEqual(ctx.exception.subcode, METADATA_ENTITYSET_NOT_FOUND) + self.assertIn("NonexistentEntity123", str(ctx.exception)) + + def test_query_fetchxml_wrong_attribute_name_raises_http_error(self): + """Invalid attribute name in FetchXML causes API 400; HttpError propagates.""" + self._setup_entity_set() + self.od._request.side_effect = HttpError( + "Invalid attribute 'InvalidColumnName123'", + status_code=400, + ) + fetchxml = "" + with self.assertRaises(HttpError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertEqual(ctx.exception.status_code, 400) + + def test_query_fetchxml_single_page(self): + """Mock HTTP response with no more records, verify single page yielded.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = { + "value": [{"accountid": "1", "name": "Contoso"}, {"accountid": "2", "name": "Fabrikam"}], + } + self.od._request.return_value = mock_response + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 1) + self.assertEqual(len(pages[0]), 2) + self.assertEqual(pages[0][0]["name"], "Contoso") + self.assertEqual(pages[0][1]["name"], "Fabrikam") + self.od._request.assert_called_once() + + def test_query_fetchxml_empty_result(self): + """Empty value list yields no pages (matches _get_multiple pattern).""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = {"value": []} + self.od._request.return_value = mock_response + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 0) + + def test_query_fetchxml_multi_page(self): + """Mock HTTP responses with paging cookie, verify multiple pages yielded.""" + self._setup_entity_set() + # First page: 2 records, more records exist + # Second page: 1 record, no more records + from urllib.parse import quote + + cookie_inner = quote(quote("")) + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": f'', + } + resp2 = MagicMock() + resp2.json.return_value = { + "value": [{"accountid": "3", "name": "C"}], + } + self.od._request.side_effect = [resp1, resp2] + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0], [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}]) + self.assertEqual(pages[1], [{"accountid": "3", "name": "C"}]) + self.assertEqual(self.od._request.call_count, 2) + + def test_query_fetchxml_paging_cookie_decode(self): + """Verify double URL-decode of paging cookie and injection into fetch element.""" + self._setup_entity_set() + from urllib.parse import quote, unquote + + # Cookie value is double-encoded + inner = "" + encoded_once = quote(inner) + encoded_twice = quote(encoded_once) + + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": f'', + } + resp2 = MagicMock() + resp2.json.return_value = {"value": []} + self.od._request.side_effect = [resp1, resp2] + + fetchxml = "" + list(self.od._query_fetchxml(fetchxml)) + + # Second request URL should contain the decoded paging cookie + second_call_url = self.od._request.call_args_list[1][0][1] + self.assertIn("fetchXml=", second_call_url) + # The decoded inner cookie should appear in the URL (after double decode) + self.assertIn("pagenumber", second_call_url) + + def test_query_fetchxml_page_size_injected(self): + """Verify count attribute set on fetch when page_size provided.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = {"value": []} + self.od._request.return_value = mock_response + + fetchxml = "" + list(self.od._query_fetchxml(fetchxml, page_size=50)) + + # The request URL should contain count=50 in the fetchXml + call_url = self.od._request.call_args[0][1] + self.assertIn("fetchXml=", call_url) + self.assertIn("count", call_url) + self.assertIn("50", call_url) + + def test_query_fetchxml_existing_count_not_overridden(self): + """page_size does not override existing count attribute in FetchXML.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = {"value": []} + self.od._request.return_value = mock_response + + fetchxml = "" + list(self.od._query_fetchxml(fetchxml, page_size=50)) + + call_url = self.od._request.call_args[0][1] + self.assertIn("count", call_url) + self.assertIn("10", call_url) + self.assertNotIn("50", call_url) + + def test_query_fetchxml_existing_page_preserved(self): + """Pre-set page attribute is preserved and not overridden to '1'.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = {"value": [{"accountid": "1"}]} + self.od._request.return_value = mock_response + + fetchxml = "" + list(self.od._query_fetchxml(fetchxml)) + + call_url = self.od._request.call_args[0][1] + self.assertIn("page", call_url) + self.assertIn("3", call_url) + + def test_query_fetchxml_morerecords_string_true(self): + """When morerecords is string 'true' (not boolean), paging continues.""" + self._setup_entity_set() + from urllib.parse import quote + + cookie_inner = quote(quote("")) + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1", "name": "A"}], + "@Microsoft.Dynamics.CRM.morerecords": "true", # string, not boolean + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": f'', + } + resp2 = MagicMock() + resp2.json.return_value = {"value": [{"accountid": "2", "name": "B"}]} + self.od._request.side_effect = [resp1, resp2] + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0], [{"accountid": "1", "name": "A"}]) + self.assertEqual(pages[1], [{"accountid": "2", "name": "B"}]) + self.assertEqual(self.od._request.call_count, 2) + + def test_query_fetchxml_simple_paging_fallback(self): + """When morerecords=True but no paging cookie, fall back to simple page increment.""" + self._setup_entity_set() + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1", "name": "A"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + } + resp2 = MagicMock() + resp2.json.return_value = { + "value": [{"accountid": "2", "name": "B"}], + } + self.od._request.side_effect = [resp1, resp2] + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0], [{"accountid": "1", "name": "A"}]) + self.assertEqual(pages[1], [{"accountid": "2", "name": "B"}]) + self.assertEqual(self.od._request.call_count, 2) + + second_call_url = self.od._request.call_args_list[1][0][1] + self.assertIn("page", second_call_url) + self.assertIn("2", second_call_url) + + def test_query_fetchxml_top_with_page_size_raises(self): + """Raise ValidationError when FetchXML has top and page_size is provided.""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=10)) + self.assertIn("top", str(ctx.exception).lower()) + self.assertIn("page_size", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_negative_page_size_raises(self): + """Raise ValidationError when page_size is negative.""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=-5)) + self.assertIn("page_size", str(ctx.exception).lower()) + self.assertIn("positive", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_zero_page_size_raises(self): + """Raise ValidationError when page_size is zero.""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=0)) + self.assertIn("page_size", str(ctx.exception).lower()) + self.assertIn("positive", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_boolean_true_page_size_raises(self): + """Raise ValidationError for boolean page_size (bool is int subclass).""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=True)) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_boolean_false_page_size_raises(self): + """Raise ValidationError for False page_size (bool is int subclass).""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=False)) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_non_numeric_page_size_raises(self): + """Raise ValidationError for non-numeric page_size.""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size="abc")) + self.assertIn("page_size", str(ctx.exception).lower()) + self.assertIn("integer", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_fractional_page_size_raises(self): + """Raise ValidationError for fractional page_size (e.g. from config/CLI parsing).""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=1.9)) + self.assertIn("page_size", str(ctx.exception).lower()) + self.assertIn("integer", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_top_no_paging(self): + """FetchXML with top returns single page, no page/count attributes injected.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = { + "value": [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}], + } + self.od._request.return_value = mock_response + + fetchxml = "" + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 1) + self.assertEqual(len(pages[0]), 2) + self.od._request.assert_called_once() + + call_url = self.od._request.call_args[0][1] + self.assertNotIn("page=", call_url) + self.assertNotIn("count=", call_url) + + def test_query_fetchxml_malformed_paging_cookie_raises(self): + """Raise ValidationError when paging cookie XML is malformed.""" + self._setup_entity_set() + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertIn("paging cookie", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + + def test_query_fetchxml_missing_pagingcookie_attr_raises(self): + """Raise ValidationError when paging cookie element lacks pagingcookie attribute.""" + self._setup_entity_set() + resp1 = MagicMock() + resp1.json.return_value = { + "value": [{"accountid": "1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": "", + } + self.od._request.return_value = resp1 + + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertIn("pagingcookie", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + + def test_query_fetchxml_aggregate_no_paging(self): + """Aggregate FetchXML returns single page, no page/count attributes injected.""" + self._setup_entity_set() + mock_response = MagicMock() + mock_response.json.return_value = { + "value": [{"accountid": "1", "aggregate_column": 42}], + } + self.od._request.return_value = mock_response + + fetchxml = ( + "" + "" + "" + "" + ) + pages = list(self.od._query_fetchxml(fetchxml)) + + self.assertEqual(len(pages), 1) + self.assertEqual(len(pages[0]), 1) + self.od._request.assert_called_once() + + call_url = self.od._request.call_args[0][1] + self.assertNotIn("page=", call_url) + self.assertNotIn("count=", call_url) + + def test_query_fetchxml_excessive_length_raises(self): + """Raise ValidationError when FetchXML exceeds maximum length.""" + self._setup_entity_set() + from PowerPlatform.Dataverse.data._odata import _MAX_FETCHXML_LENGTH + + fetchxml = "" + long_fetchxml = fetchxml + "x" * (_MAX_FETCHXML_LENGTH - len(fetchxml) + 1) + + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(long_fetchxml)) + self.assertIn("exceeds maximum", str(ctx.exception).lower()) + self.assertIn(str(_MAX_FETCHXML_LENGTH), str(ctx.exception)) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_TOO_LONG) + + def test_query_fetchxml_aggregate_with_page_size_raises(self): + """Raise ValidationError when page_size is used with aggregate FetchXML.""" + self._setup_entity_set() + fetchxml = ( + "" + "" + "" + "" + ) + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml, page_size=50)) + self.assertIn("aggregate", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_INVALID_PAGE_SIZE) + + def test_query_fetchxml_distinct_without_order_raises(self): + """Raise ValidationError for distinct query without order element.""" + self._setup_entity_set() + fetchxml = "" + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertIn("distinct", str(ctx.exception).lower()) + self.assertIn("order", str(ctx.exception).lower()) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + + def test_query_fetchxml_max_pages_exceeded_raises(self): + """Raise ValidationError when paging loop exceeds maximum page limit.""" + self._setup_entity_set() + from urllib.parse import quote + + cookie_inner = quote(quote("")) + resp_with_more = MagicMock() + resp_with_more.json.return_value = { + "value": [{"accountid": "1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": f'', + } + self.od._request.return_value = resp_with_more + + fetchxml = "" + with patch("PowerPlatform.Dataverse.data._odata._MAX_FETCHXML_PAGES", 3): + with self.assertRaises(ValidationError) as ctx: + list(self.od._query_fetchxml(fetchxml)) + self.assertIn("maximum page limit", str(ctx.exception).lower()) + self.assertIn("3", str(ctx.exception)) + self.assertEqual(ctx.exception.subcode, VALIDATION_FETCHXML_MALFORMED) + self.assertEqual(self.od._request.call_count, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_query_operations.py b/tests/unit/test_query_operations.py index a4cbd158..939da6b3 100644 --- a/tests/unit/test_query_operations.py +++ b/tests/unit/test_query_operations.py @@ -51,6 +51,49 @@ def test_sql_empty_result(self): self.assertIsInstance(result, list) self.assertEqual(result, []) + # -------------------------------------------------------------- fetchxml + + def test_fetchxml_basic(self): + """fetchxml() should call _query_fetchxml and return results.""" + expected_pages = [ + [{"accountid": "1", "name": "Contoso"}, {"accountid": "2", "name": "Fabrikam"}], + ] + self.client._odata._query_fetchxml.return_value = iter(expected_pages) + + fetchxml = "" + result = list(self.client.query.fetchxml(fetchxml)) + + self.client._odata._query_fetchxml.assert_called_once_with(fetchxml, page_size=None) + self.assertEqual(result, expected_pages) + + def test_fetchxml_with_page_size(self): + """fetchxml() should pass page_size through to _query_fetchxml.""" + self.client._odata._query_fetchxml.return_value = iter([]) + + fetchxml = "" + list(self.client.query.fetchxml(fetchxml, page_size=50)) + + self.client._odata._query_fetchxml.assert_called_once_with(fetchxml, page_size=50) + + def test_fetchxml_empty_result(self): + """fetchxml() should return empty generator when no results.""" + self.client._odata._query_fetchxml.return_value = iter([]) + + fetchxml = "" + result = list(self.client.query.fetchxml(fetchxml)) + + self.assertEqual(result, []) + + def test_fetchxml_returns_iterable(self): + """fetchxml() should return an iterable (generator).""" + self.client._odata._query_fetchxml.return_value = iter([[{"name": "A"}]]) + + fetchxml = "" + result = self.client.query.fetchxml(fetchxml) + + self.assertIsNotNone(iter(result)) + self.assertEqual(list(result), [[{"name": "A"}]]) + if __name__ == "__main__": unittest.main()