From 63fdead109771976e79b576f2fc2574db419668a Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Mon, 23 Feb 2026 18:30:14 +0100 Subject: [PATCH 1/8] separate graphql queries from source code - graphql queries now in dedicated files - added common FileTemplate (built from string.Template) to factorize code --- cosmo/clients/netbox_v4.py | 351 +----------------- .../clients/queries/connected_devices.graphql | 33 ++ cosmo/clients/queries/device.graphql | 188 ++++++++++ cosmo/clients/queries/l2vpn.graphql | 64 ++++ cosmo/clients/queries/loopback.graphql | 39 ++ cosmo/common.py | 10 + 6 files changed, 345 insertions(+), 340 deletions(-) create mode 100644 cosmo/clients/queries/connected_devices.graphql create mode 100644 cosmo/clients/queries/device.graphql create mode 100644 cosmo/clients/queries/l2vpn.graphql create mode 100644 cosmo/clients/queries/loopback.graphql diff --git a/cosmo/clients/netbox_v4.py b/cosmo/clients/netbox_v4.py index 45eaa6f..09a82b2 100644 --- a/cosmo/clients/netbox_v4.py +++ b/cosmo/clients/netbox_v4.py @@ -2,10 +2,12 @@ from abc import ABC, abstractmethod from builtins import map from multiprocessing import Manager -from string import Template +from os import PathLike +from pathlib import Path from cosmo.clients import get_client_mp_context from cosmo.clients.netbox_client import NetboxAPIClient +from cosmo.common import FileTemplate class ParallelQuery(ABC): @@ -16,6 +18,10 @@ def __init__(self, client: NetboxAPIClient, **kwargs): self.data_promise = None self.kwargs = kwargs + @staticmethod + def file_template(relpath: str | PathLike): + return FileTemplate(Path(__file__).parent.joinpath(Path(relpath))) + def fetch_data(self, pool): return pool.apply_async(self._fetch_data, args=(self.kwargs, pool)) @@ -43,43 +49,7 @@ def _fetch_data(self, kwargs, pool): if self.netbox_43_query_syntax else 'tag: "bgp_cpe"' ) - query_template = Template( - """ - query { - interface_list(filters: { $tag_filter }) { - __typename - id, - parent { - __typename - id, - connected_endpoints { - ... on InterfaceType { - __typename - name - device { - name - __typename - primary_ip4 { - __typename - address - } - interfaces { - id - name - __typename - ip_addresses { - __typename - address - } - } - } - } - } - } - } - } - """ - ) + query_template = self.file_template("queries/connected_devices.graphql") return self.client.query( query_template.substitute(tag_filter=tag_filter), "connected_devices_query" @@ -123,49 +93,7 @@ def _fetch_data(self, kwargs, pool): # Note: This does not use the device list, because we can have other participating devices # which are not in the same repository and thus are not appearing in device list. - query_template = Template( - """ - query{ - interface_list(filters: { - name: {starts_with: "lo"} - }) { - __typename - name, - child_interfaces { - __typename - name, - vrf { - __typename - id - name - description - rd - export_targets { - __typename - name - } - import_targets { - __typename - name - } - }, - ip_addresses { - __typename - address, - family { - __typename - value, - } - } - } - device{ - __typename - name, - } - } - } - """ - ) + query_template = self.file_template("queries/loopback.graphql") return self.client.query(query_template.substitute(), "loopback_query")["data"] @@ -205,74 +133,7 @@ def _merge_into(self, data: dict, query_data): class L2VPNDataQuery(ParallelQuery): def _fetch_data(self, kwargs, pool): - query_template = Template( - """ - query { - l2vpn_list (filters: {name: {starts_with: "WAN: "}}) { - __typename - id - name - type - identifier - terminations { - __typename - id - assigned_object { - __typename - ... on VLANType { - __typename - id - name - interfaces_as_tagged { - id - name - __typename - device { - __typename - id - name - } - } - interfaces_as_untagged { - id - name - __typename - device { - __typename - id - name - } - } - } - ... on InterfaceType { - __typename - id - name - custom_fields - untagged_vlan { - __typename - id - name - vid - } - tagged_vlans { - __typename - id - name - vid - } - device { - __typename - id - name - } - } - } - } - } - } - """ - ) + query_template = self.file_template("queries/l2vpn.graphql") return self.client.query(query_template.substitute(), "l2vpn_query")["data"] @@ -428,197 +289,7 @@ def __init__(self, *args, multiple_mac_addresses=False, **kwargs): def _fetch_data(self, kwargs, pool): device = kwargs.get("device") - query_template = Template( - """ - query { - device_list(filters: { - name: { i_exact: $device }, - }) { - __typename - id - name - custom_fields - - device_type { - __typename - slug - } - platform { - __typename - manufacturer { - __typename - slug - } - slug - } - primary_ip4 { - __typename - address - } - - interfaces { - __typename - id - name - enabled - type - mode - mtu - description - connected_endpoints { - ... on ProviderNetworkType { - __typename - display - } - ... on CircuitTerminationType { - __typename - display - } - ... on VirtualCircuitTerminationType { - __typename - display - } - ... on InterfaceType { - __typename - name - device { - __typename - name - } - } - ... on FrontPortType { - __typename - name - device { - __typename - name - } - } - ... on RearPortType { - __typename - name - device { - __typename - name - } - } - ... on ConsolePortType { - __typename - name - device { - __typename - name - } - } - ... on ConsoleServerPortType { - __typename - name - device { - __typename - name - } - } - } - link_peers { - ... on CircuitTerminationType { - __typename - display - } - ... on FrontPortType { - __typename - name - device { - __typename - name - } - } - ... on RearPortType { - __typename - name - device { - __typename - name - } - } - ... on ConsolePortType { - __typename - name - device { - __typename - name - } - } - ... on ConsoleServerPortType { - __typename - name - device { - __typename - name - } - } - ... on InterfaceType { - __typename - name - device { - __typename - name - } - } - } - vrf { - __typename - id - name - description - rd - export_targets { - __typename - name - } - import_targets { - __typename - name - } - } - lag { - __typename - id - name - } - ip_addresses { - __typename - address - role - } - untagged_vlan { - __typename - id - name - vid - } - tagged_vlans { - __typename - id - name - vid - } - tags { - __typename - id - name - slug - } - parent { - __typename - id - mtu - name - } - custom_fields - } - } - }""" - ) + query_template = self.file_template("queries/device.graphql") query = query_template.substitute( device=json.dumps(device), diff --git a/cosmo/clients/queries/connected_devices.graphql b/cosmo/clients/queries/connected_devices.graphql new file mode 100644 index 0000000..05f9af2 --- /dev/null +++ b/cosmo/clients/queries/connected_devices.graphql @@ -0,0 +1,33 @@ +query { + interface_list(filters: { $tag_filter }) { + __typename + id, + parent { + __typename + id, + connected_endpoints { + ... on InterfaceType { + __typename + name + device { + name + __typename + primary_ip4 { + __typename + address + } + interfaces { + id + name + __typename + ip_addresses { + __typename + address + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/cosmo/clients/queries/device.graphql b/cosmo/clients/queries/device.graphql new file mode 100644 index 0000000..40de4f7 --- /dev/null +++ b/cosmo/clients/queries/device.graphql @@ -0,0 +1,188 @@ +query { + device_list(filters: { + name: { i_exact: $device }, + }) { + __typename + id + name + custom_fields + + device_type { + __typename + slug + } + platform { + __typename + manufacturer { + __typename + slug + } + slug + } + primary_ip4 { + __typename + address + } + + interfaces { + __typename + id + name + enabled + type + mode + mtu + description + connected_endpoints { + ... on ProviderNetworkType { + __typename + display + } + ... on CircuitTerminationType { + __typename + display + } + ... on VirtualCircuitTerminationType { + __typename + display + } + ... on InterfaceType { + __typename + name + device { + __typename + name + } + } + ... on FrontPortType { + __typename + name + device { + __typename + name + } + } + ... on RearPortType { + __typename + name + device { + __typename + name + } + } + ... on ConsolePortType { + __typename + name + device { + __typename + name + } + } + ... on ConsoleServerPortType { + __typename + name + device { + __typename + name + } + } + } + link_peers { + ... on CircuitTerminationType { + __typename + display + } + ... on FrontPortType { + __typename + name + device { + __typename + name + } + } + ... on RearPortType { + __typename + name + device { + __typename + name + } + } + ... on ConsolePortType { + __typename + name + device { + __typename + name + } + } + ... on ConsoleServerPortType { + __typename + name + device { + __typename + name + } + } + ... on InterfaceType { + __typename + name + device { + __typename + name + } + } + } + vrf { + __typename + id + name + description + rd + export_targets { + __typename + name + } + import_targets { + __typename + name + } + } + lag { + __typename + id + name + } + ip_addresses { + __typename + address + role + } + untagged_vlan { + __typename + id + name + vid + } + tagged_vlans { + __typename + id + name + vid + } + tags { + __typename + id + name + slug + } + parent { + __typename + id + mtu + name + } + custom_fields + } + } +} \ No newline at end of file diff --git a/cosmo/clients/queries/l2vpn.graphql b/cosmo/clients/queries/l2vpn.graphql new file mode 100644 index 0000000..1b1fbb4 --- /dev/null +++ b/cosmo/clients/queries/l2vpn.graphql @@ -0,0 +1,64 @@ +query { + l2vpn_list (filters: {name: {starts_with: "WAN: "}}) { + __typename + id + name + type + identifier + terminations { + __typename + id + assigned_object { + __typename + ... on VLANType { + __typename + id + name + interfaces_as_tagged { + id + name + __typename + device { + __typename + id + name + } + } + interfaces_as_untagged { + id + name + __typename + device { + __typename + id + name + } + } + } + ... on InterfaceType { + __typename + id + name + custom_fields + untagged_vlan { + __typename + id + name + vid + } + tagged_vlans { + __typename + id + name + vid + } + device { + __typename + id + name + } + } + } + } + } +} \ No newline at end of file diff --git a/cosmo/clients/queries/loopback.graphql b/cosmo/clients/queries/loopback.graphql new file mode 100644 index 0000000..b65f4f8 --- /dev/null +++ b/cosmo/clients/queries/loopback.graphql @@ -0,0 +1,39 @@ +query{ + interface_list(filters: { + name: {starts_with: "lo"} + }) { + __typename + name, + child_interfaces { + __typename + name, + vrf { + __typename + id + name + description + rd + export_targets { + __typename + name + } + import_targets { + __typename + name + } + }, + ip_addresses { + __typename + address, + family { + __typename + value, + } + } + } + device{ + __typename + name, + } + } +} \ No newline at end of file diff --git a/cosmo/common.py b/cosmo/common.py index c5a164a..77ca4b5 100644 --- a/cosmo/common.py +++ b/cosmo/common.py @@ -1,3 +1,6 @@ +from os import PathLike +from pathlib import Path +from string import Template from abc import ABC, abstractmethod from typing import Optional, Union, Protocol, TypeVar, Sequence @@ -70,3 +73,10 @@ def without_keys(d, keys) -> dict: if type(keys) != list: keys = [keys] return {k: v for k, v in d.items() if k not in keys} + + +class FileTemplate(Template): + def __init__(self, template_file_path: str | bytes | PathLike): + with open(template_file_path, "r") as template_file: + template = template_file.read() + super().__init__(template) From 59159f088295bbeff3d41e3876de2defc7ee52a6 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 24 Feb 2026 16:19:49 +0100 Subject: [PATCH 2/8] add @without_feature decorator for tests also fix broken introspection for pytest --- cosmo/features.py | 16 ++++++++++++++-- cosmo/tests/test_features.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/cosmo/features.py b/cosmo/features.py index 235a233..1094bb5 100644 --- a/cosmo/features.py +++ b/cosmo/features.py @@ -1,5 +1,6 @@ # implementation guide # https://martinfowler.com/articles/feature-toggles.html +import functools from argparse import Action, ArgumentParser from typing import Never, Self, Optional, TextIO, Sequence, Any, Callable @@ -83,11 +84,14 @@ def __str__(self): return ", ".join(features_desc) -def with_feature(instance: FeatureToggle, feature_name: str): +def _feature_toggler_decorator_gen( + instance: FeatureToggle, feature_name: str, target_state: bool +): def decorator_with_feature(func: Callable): + @functools.wraps(func) def exe_with_feature(*args, **kwargs): previous_state = instance.featureIsEnabled(feature_name) - instance.setFeature(feature_name, True) + instance.setFeature(feature_name, target_state) func(*args, **kwargs) instance.setFeature(feature_name, previous_state) @@ -96,6 +100,14 @@ def exe_with_feature(*args, **kwargs): return decorator_with_feature +def with_feature(instance: FeatureToggle, feature_name: str): + return _feature_toggler_decorator_gen(instance, feature_name, True) + + +def without_feature(instance: FeatureToggle, feature_name: str): + return _feature_toggler_decorator_gen(instance, feature_name, False) + + features = FeatureToggle( { "interface-auto-descriptions": True, diff --git a/cosmo/tests/test_features.py b/cosmo/tests/test_features.py index 642fc82..6dc7016 100644 --- a/cosmo/tests/test_features.py +++ b/cosmo/tests/test_features.py @@ -7,6 +7,7 @@ NonExistingFeatureToggleException, FeatureToggle, with_feature, + without_feature, ) @@ -72,6 +73,17 @@ def execute_with_decorator(): assert not ft.featureIsEnabled("feature_a") +def test_without_feature_decorator(): + ft = FeatureToggle({"feature_a": True}) + + @without_feature(ft, "feature_a") + def execute_with_decorator(): + assert not ft.featureIsEnabled("feature_a") + + execute_with_decorator() + assert ft.featureIsEnabled("feature_a") + + def test_argparse_integration(): ft = FeatureToggle({"feature_a": False, "feature_b": False, "feature_c": True}) From f8e956c7eac66318446fceb1ce4b23d46ea98d72 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 25 Feb 2026 13:33:05 +0100 Subject: [PATCH 3/8] differenciated queries for autodesc + test cases (smoke tests for now) --- cosmo/clients/netbox_v4.py | 13 ++- cosmo/clients/queries/device.graphql | 91 +------------------ .../queries/device_autodesc_query.graphql | 90 ++++++++++++++++++ .../queries/device_no_autodesc_query.graphql | 0 cosmo/tests/integration_test.py | 33 ++++++- cosmo/tests/test_netboxclient.py | 2 +- cosmo/tests/utils.py | 57 +++++++----- 7 files changed, 166 insertions(+), 120 deletions(-) create mode 100644 cosmo/clients/queries/device_autodesc_query.graphql create mode 100644 cosmo/clients/queries/device_no_autodesc_query.graphql diff --git a/cosmo/clients/netbox_v4.py b/cosmo/clients/netbox_v4.py index 09a82b2..60b7077 100644 --- a/cosmo/clients/netbox_v4.py +++ b/cosmo/clients/netbox_v4.py @@ -8,6 +8,7 @@ from cosmo.clients import get_client_mp_context from cosmo.clients.netbox_client import NetboxAPIClient from cosmo.common import FileTemplate +from cosmo.features import features class ParallelQuery(ABC): @@ -289,10 +290,19 @@ def __init__(self, *args, multiple_mac_addresses=False, **kwargs): def _fetch_data(self, kwargs, pool): device = kwargs.get("device") - query_template = self.file_template("queries/device.graphql") + if features.featureIsEnabled("interface-auto-descriptions"): + autodesc_query_extension_template = self.file_template( + "queries/device_autodesc_query.graphql" + ) + else: + autodesc_query_extension_template = self.file_template( + "queries/device_no_autodesc_query.graphql" + ) + query_template = self.file_template("queries/device.graphql") query = query_template.substitute( device=json.dumps(device), + autodesc_query_extension=autodesc_query_extension_template.substitute(), ) query_result = self.client.query(query, f"device_query_{device}") @@ -343,6 +353,7 @@ def get_data(self, device_config): ( TobagoLineMembersDataQuery(client, device=d) if self.feature_flags["tobago"] + and features.featureIsEnabled("interface-auto-descriptions") else TobagoLineMemberDataDummyQuery(client, device=d) ), ] diff --git a/cosmo/clients/queries/device.graphql b/cosmo/clients/queries/device.graphql index 40de4f7..d41d861 100644 --- a/cosmo/clients/queries/device.graphql +++ b/cosmo/clients/queries/device.graphql @@ -34,96 +34,6 @@ query { mtu description connected_endpoints { - ... on ProviderNetworkType { - __typename - display - } - ... on CircuitTerminationType { - __typename - display - } - ... on VirtualCircuitTerminationType { - __typename - display - } - ... on InterfaceType { - __typename - name - device { - __typename - name - } - } - ... on FrontPortType { - __typename - name - device { - __typename - name - } - } - ... on RearPortType { - __typename - name - device { - __typename - name - } - } - ... on ConsolePortType { - __typename - name - device { - __typename - name - } - } - ... on ConsoleServerPortType { - __typename - name - device { - __typename - name - } - } - } - link_peers { - ... on CircuitTerminationType { - __typename - display - } - ... on FrontPortType { - __typename - name - device { - __typename - name - } - } - ... on RearPortType { - __typename - name - device { - __typename - name - } - } - ... on ConsolePortType { - __typename - name - device { - __typename - name - } - } - ... on ConsoleServerPortType { - __typename - name - device { - __typename - name - } - } ... on InterfaceType { __typename name @@ -132,6 +42,7 @@ query { name } } + $autodesc_query_extension } vrf { __typename diff --git a/cosmo/clients/queries/device_autodesc_query.graphql b/cosmo/clients/queries/device_autodesc_query.graphql new file mode 100644 index 0000000..7b5371d --- /dev/null +++ b/cosmo/clients/queries/device_autodesc_query.graphql @@ -0,0 +1,90 @@ + ... on ProviderNetworkType { + __typename + display + } + ... on CircuitTerminationType { + __typename + display + } + ... on VirtualCircuitTerminationType { + __typename + display + } + ... on FrontPortType { + __typename + name + device { + __typename + name + } + } + ... on RearPortType { + __typename + name + device { + __typename + name + } + } + ... on ConsolePortType { + __typename + name + device { + __typename + name + } + } + ... on ConsoleServerPortType { + __typename + name + device { + __typename + name + } + } + } + link_peers { + ... on CircuitTerminationType { + __typename + display + } + ... on FrontPortType { + __typename + name + device { + __typename + name + } + } + ... on RearPortType { + __typename + name + device { + __typename + name + } + } + ... on ConsolePortType { + __typename + name + device { + __typename + name + } + } + ... on ConsoleServerPortType { + __typename + name + device { + __typename + name + } + } + ... on InterfaceType { + __typename + name + device { + __typename + name + } + } diff --git a/cosmo/clients/queries/device_no_autodesc_query.graphql b/cosmo/clients/queries/device_no_autodesc_query.graphql new file mode 100644 index 0000000..e69de29 diff --git a/cosmo/tests/integration_test.py b/cosmo/tests/integration_test.py index d726924..5f1681c 100644 --- a/cosmo/tests/integration_test.py +++ b/cosmo/tests/integration_test.py @@ -8,6 +8,7 @@ import cosmo.tests.utils as utils from cosmo.__main__ import main as cosmoMain +from cosmo.features import with_feature, features, without_feature def test_missing_config(mocker): @@ -34,7 +35,7 @@ def test_limit_argument_with_commas(mocker): utils.CommonSetup( mocker, args=[utils.CommonSetup.PROGNAME, "--limit", "router1,router2"] ) - utils.RequestResponseMock.patchNetboxClient(mocker) + utils.RequestResponseMock().patchNetboxClient(mocker) assert cosmoMain() == 0 @@ -43,7 +44,7 @@ def test_limit_arguments_with_repeat(mocker): mocker, args=[utils.CommonSetup.PROGNAME, "--limit", "router1", "--limit", "router2"], ) - utils.RequestResponseMock.patchNetboxClient(mocker) + utils.RequestResponseMock().patchNetboxClient(mocker) assert cosmoMain() == 0 @@ -51,7 +52,7 @@ def test_device_generation_ansible(mocker): testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_ansible.yml") with open(f"cosmo/tests/test_case_l3vpn.yml") as f: test_data = yaml.safe_load(f) - utils.RequestResponseMock.patchNetboxClient(mocker, **test_data) + utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) assert cosmoMain() == 0 testEnv.stop() assert os.path.isfile("host_vars/test0001/generated-cosmo.yml") @@ -61,7 +62,7 @@ def test_device_generation_nix(mocker): testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_nix.yml") with open(f"cosmo/tests/test_case_l3vpn.yml") as f: test_data = yaml.safe_load(f) - utils.RequestResponseMock.patchNetboxClient(mocker, **test_data) + utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) assert cosmoMain() == 0 testEnv.stop() assert os.path.isfile("machines/test0001/generated-cosmo.json") @@ -71,7 +72,7 @@ def test_device_processing_error(mocker, capsys): testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_nix.yml") with open(f"cosmo/tests/test_case_vendor_unknown.yaml") as f: test_data = yaml.safe_load(f) - utils.RequestResponseMock.patchNetboxClient(mocker, **test_data) + utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) with pytest.raises( Exception, match="Cannot find suitable manufacturer for device .*" ): @@ -79,6 +80,28 @@ def test_device_processing_error(mocker, capsys): testEnv.stop() +@with_feature(features, "interface-auto-descriptions") +def test_autodesc_enabled(mocker): + testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_ansible.yml") + requestsMock = utils.RequestResponseMock() + with open("cosmo/tests/test_case_auto_descriptions.yaml") as f: + test_data = yaml.safe_load(f) + utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) + assert cosmoMain() == 0 + testEnv.stop() + + +@without_feature(features, "interface-auto-descriptions") +def test_autodesc_disabled(mocker): + testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_ansible.yml") + requestsMock = utils.RequestResponseMock() + with open(f"cosmo/tests/test_case_auto_descriptions.yaml") as f: + test_data = yaml.safe_load(f) + utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) + assert cosmoMain() == 0 + testEnv.stop() + + def test_invalid_config_file(mocker, capsys): testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.wrong.yml") with pytest.raises(jsonschema.exceptions.ValidationError): diff --git a/cosmo/tests/test_netboxclient.py b/cosmo/tests/test_netboxclient.py index 50368ff..c632985 100644 --- a/cosmo/tests/test_netboxclient.py +++ b/cosmo/tests/test_netboxclient.py @@ -22,7 +22,7 @@ def test_case_get_data(mocker): "l2vpn_list": [], "loopbacks": {}, } - [getMock, postMock] = utils.RequestResponseMock.patchNetboxClient(mocker) + [getMock, postMock] = utils.RequestResponseMock().patchNetboxClient(mocker) nc = NetboxClient(TEST_URL, TEST_TOKEN) assert nc.base_version == Version("4.1.2") diff --git a/cosmo/tests/utils.py b/cosmo/tests/utils.py index 89d5ddc..537f6d8 100644 --- a/cosmo/tests/utils.py +++ b/cosmo/tests/utils.py @@ -58,28 +58,46 @@ def stop(self): self.mocker.stop(patch) +class ResponseMock: + def __init__(self, status_code, obj): + self.status_code = status_code + self.text = json.dumps(obj) + self.obj = obj + + def json(self): + return self.obj + + class RequestResponseMock: + def __init__(self): + self.get_responses = dict() + self.post_responses = dict() - @staticmethod - def patchNetboxClient(mocker, **patchKwArgs): + def patchNetboxClient(self, mocker, **patchKwArgs): def patchGetFunc(url, **kwargs): if "/api/status" in url: - return ResponseMock( + r = ResponseMock( 200, { "netbox-version": "4.1.2+wobcom_0.4.2", - "plugins": [], + "plugins": [ + "netbox_plugin_tobago", + "netbox_plugin_ip_pools", + "netbox_plugin_routing", + ], }, ) - - return ResponseMock( - 200, - { - "next": None, - "results": [], - }, - ) + else: + r = ResponseMock( + 200, + { + "next": None, + "results": [], + }, + ) + self.get_responses[url] = r + return r def patchPostFunc(url, json, **kwargs): q = json.get("query") @@ -102,23 +120,16 @@ def patchPostFunc(url, json, **kwargs): elif rl in q: retVal[rl] = patchKwArgs.get(rl, []) - return ResponseMock(200, {"data": retVal}) + r = ResponseMock(200, {"data": retVal}) + # TODO: find out why value set is being trashed outside context ??? + self.post_responses[q] = r + return r getMock = mocker.patch("requests.get", side_effect=patchGetFunc) postMock = mocker.patch("requests.post", side_effect=patchPostFunc) return [getMock, postMock] -class ResponseMock: - def __init__(self, status_code, obj): - self.status_code = status_code - self.text = json.dumps(obj) - self.obj = obj - - def json(self): - return self.obj - - # it has to be stateful - so I'm making an object class PatchIoFilePath: def __init__(self, mocker, patchInNamespace, requestedPath, pathToRealData): From efd21c5daf18973753648ee488a097600114e392 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 25 Feb 2026 16:01:27 +0100 Subject: [PATCH 4/8] add tests for disabled autodesc + explicitely enable autodesc for autodesc tests --- .../tests/test_case_legacy_descriptions.yaml | 396 ++++++++++++++++++ .../test_case_switch_legacy_description.yaml | 193 +++++++++ cosmo/tests/test_serializer.py | 74 +++- 3 files changed, 662 insertions(+), 1 deletion(-) create mode 100644 cosmo/tests/test_case_legacy_descriptions.yaml create mode 100644 cosmo/tests/test_case_switch_legacy_description.yaml diff --git a/cosmo/tests/test_case_legacy_descriptions.yaml b/cosmo/tests/test_case_legacy_descriptions.yaml new file mode 100644 index 0000000..5d2b7d8 --- /dev/null +++ b/cosmo/tests/test_case_legacy_descriptions.yaml @@ -0,0 +1,396 @@ +device_list: +- device_type: + __typename: DeviceTypeType + slug: mx204 + __typename: DeviceType + id: '1747' + interfaces: + - custom_fields: + inner_tag: null + outer_tag: null + description: "test description" + __typename: InterfaceType + enabled: true + id: '30747' + ip_addresses: [] + connected_endpoints: + - name: "combo1" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik01" + lag: null + mac_address: 94:BF:41:41:41:F3 + mode: null + mtu: null + name: et-0/0/0 + tagged_vlans: [] + tags: [] + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + - custom_fields: + inner_tag: null + outer_tag: null + description: "do not overwrite me!" + __typename: InterfaceType + enabled: true + id: '30748' + ip_addresses: [] + connected_endpoints: + - name: "combo2" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik01" + lag: null + mac_address: 94:BF:41:41:41:F3 + mode: null + mtu: null + name: et-0/0/1 + tagged_vlans: [] + tags: [] + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + - custom_fields: + inner_tag: null + outer_tag: null + description: "test description et-0/0/1.2" + __typename: InterfaceType + enabled: true + id: '30749' + ip_addresses: [ ] + connected_endpoints: null + attached_tobago_line: + __typename: CosmoTobagoLine + component_type: CABLE + element: + description: '' + display: cable 000128934 + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + element_id: 93827 + element_type: dcim.cable + id: 5110 + index: 1 + termination_a: + _occupied: true + cable: + description: '' + display: cable 000128934n + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + description: '' + device: + description: '' + display: TEST0001 + id: 1747 + name: TEST0001 + url: https://netbox.example.com/api/dcim/devices/1747/ + display: et-0/0/1.2 + id: 30749 + name: et-0/0/1.2 + url: https://netbox.example.com/api/dcim/interfaces/30749/ + termination_a_id: 92387 + termination_a_type: dcim.interface + termination_b: + _occupied: true + cable: + description: '' + display: 031240 GF:5781 / 031241 GF:5782 + id: 39645 + label: 031240 GF:5781 / 031241 GF:5782 + url: https://netbox.example.com/api/dcim/cables/39645/ + circuit: + cid: FS 3298327 1-1-2 + description: '' + display: FS 3298327 1-1-2 + id: 138 + provider: + description: '' + display: Circuit Provider A + id: 12 + name: Circuit Provider A + slug: circuit-provider-a + url: https://netbox.example.com/api/circuits/providers/12/ + url: https://netbox.example.com/api/circuits/circuits/138/ + description: '' + display: 'FS 3298327 1-1-2' + id: 454 + term_side: A + url: https://netbox.example.com/api/circuits/circuit-terminations/454/ + termination_b_id: 454 + termination_b_type: circuits.circuittermination + version: + created: '2025-09-03T11:34:19.643456+01:00' + custom_fields: { } + id: 2617 + last_updated: '2025-09-03T11:34:40.819703+01:00' + tenant: + id: 43876 + url: https://netbox.example.com/api/plugins/tobago/tenants/43876/ + display: Contoso Ltd. + name: Contoso Ltd. + slug: contoso-ltd + line: + display: cl390287 + id: 9834 + name: '390287' + name_long: cl390287 + url: https://netbox.example.com/api/plugins/tobago/lines/9834/ + service: + id: 4893794 + url: https://netbox.example.com/api/plugins/tobago/services/4893794/ + display: "#9823 (cl390287)" + business_service: null + status: current + lag: null + mac_address: 94:BF:41:41:41:F3 + mode: ACCESS + mtu: null + name: et-0/0/1.2 + tagged_vlans: [ ] + tags: + - __typename: TagType + name: edge:peering-ixp + slug: edge_peering-ixp + type: virtual + untagged_vlan: + __typename: VLANType + id: '8237462' + name: TEST-VLAN-2 + vid: 2 + vrf: null + - custom_fields: + inner_tag: null + outer_tag: null + description: "test description et-0/0/1.3" + __typename: InterfaceType + enabled: true + id: '30750' + ip_addresses: [ ] + connected_endpoints: [ ] + attached_tobago_line: + __typename: CosmoTobagoLine + component_type: CABLE + element: + description: '' + display: cable 000128934 + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + element_id: 93827 + element_type: dcim.cable + id: 5110 + index: 1 + termination_a: + _occupied: true + cable: + description: '' + display: cable 000128934 + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + description: '' + device: + description: '' + display: TEST0001 + id: 1747 + name: TEST0001 + url: https://netbox.example.com/api/dcim/devices/1747/ + display: et-0/0/1.3 + id: 30750 + name: et-0/0/1.3 + url: https://netbox.example.com/api/dcim/interfaces/30750/ + termination_a_id: 92387 + termination_a_type: dcim.interface + termination_b: + _occupied: true + id: 198208 + url: https://netbox.example.com/api/dcim/interfaces/192878/ + display: xe-0/1/0 + device: + id: 39827 + url: https://netbox.example.com/api/dcim/device/39827/ + display: TEST0002 + name: TEST0002 + description: null + name: xe-0/1/0 + description: "customer A" + termination_b_id: 454 + termination_b_type: dcim.interface + version: + created: '2025-09-03T11:34:19.643456+01:00' + custom_fields: { } + id: 2617 + last_updated: '2025-09-03T11:34:40.819703+01:00' + tenant: + id: 43876 + url: https://netbox.example.com/api/plugins/tobago/tenants/43876/ + display: Contoso Ltd. + name: Contoso Ltd. + slug: contoso-ltd + line: + display: cl390287 + id: 9834 + name: '390287' + name_long: cl390287 + url: https://netbox.example.com/api/plugins/tobago/lines/9834/ + service: + id: 4893794 + url: https://netbox.example.com/api/plugins/tobago/services/4893794/ + display: "#9823 (cl390287)" + business_service: null + status: current + lag: null + mac_address: 94:BF:41:41:41:F3 + mode: ACCESS + mtu: null + name: et-0/0/1.3 + tagged_vlans: [ ] + tags: + - __typename: TagType + name: edge:customer + slug: edge_customer + type: virtual + untagged_vlan: + __typename: VLANType + id: '8237463' + name: TEST-VLAN-3 + vid: 3 + vrf: null + - custom_fields: + inner_tag: null + outer_tag: null + description: "test description et-0/0/1.4" + __typename: InterfaceType + enabled: true + id: '30754' + ip_addresses: [ ] + connected_endpoints: null + attached_tobago_line: + __typename: CosmoTobagoLine + component_type: CABLE + element: + description: '' + display: cable 000128935 + id: 39646 + label: cable 000128935 + url: https://netbox.example.com/api/dcim/cables/39646/ + element_id: 93828 + element_type: dcim.cable + id: 5111 + index: 1 + termination_a: + _occupied: true + cable: + description: '' + display: cable 000128935 + id: 39646 + label: cable 000128935 + url: https://netbox.example.com/api/dcim/cables/39646/ + description: '' + device: + description: '' + display: TEST0001 + id: 1747 + name: TEST0001 + url: https://netbox.example.com/api/dcim/devices/1747/ + display: et-0/0/1.4 + id: 30750 + name: et-0/0/1.4 + url: https://netbox.example.com/api/dcim/interfaces/30754/ + termination_a_id: 92388 + termination_a_type: dcim.interface + termination_b: + _occupied: true + id: 198209 + url: https://netbox.example.com/api/dcim/interfaces/192879/ + display: xe-0/1/4 + device: + id: 39827 + url: https://netbox.example.com/api/dcim/device/39827/ + display: TEST0002 + name: TEST0002 + description: null + name: xe-0/1/4 + description: "customer A" + termination_b_id: 454 + termination_b_type: dcim.interface + version: + created: '2025-09-03T11:34:19.643456+01:00' + custom_fields: { } + id: 2617 + last_updated: '2025-09-03T11:34:40.819703+01:00' + line: + display: cl390287 + id: 9834 + name: '390287' + name_long: cl390287 + url: https://netbox.example.com/api/plugins/tobago/lines/9834/ + service: null + status: current + lag: null + mac_address: 94:BF:41:41:41:F4 + mode: ACCESS + mtu: null + name: et-0/0/1.4 + tagged_vlans: [ ] + tags: + - __typename: TagType + name: edge:customer + slug: edge_customer + type: virtual + untagged_vlan: + __typename: VLANType + id: '8237463' + name: TEST-VLAN-4 + vid: 4 + vrf: null + - custom_fields: + inner_tag: null + outer_tag: null + description: "test description et-0/0/5" + __typename: InterfaceType + enabled: true + id: '30759' + ip_addresses: [ ] + link_peers: + - name: "port_45" + __typename: InterfaceType + device: + __typename: DeviceType + name: "Panel48673" + connected_endpoints: [] + attached_tobago_line: null + lag: null + mac_address: 94:BF:41:41:41:F3 + mode: null + mtu: null + name: et-0/0/5 + tagged_vlans: [ ] + tags: + - __typename: TagType + name: access:q_pppoe + slug: access_q_pppoe + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + name: TEST0001 + platform: + __typename: PlatformType + manufacturer: + __typename: ManufacturerType + slug: juniper + slug: junos-21-4r3-s5-4 + primary_ip4: + __typename: IPAddressType + address: 45.139.136.10/32 + staticroute_set: [] +l2vpn_list: [] +loopbacks: + TEST0001: + ipv4: 45.139.136.10/32 diff --git a/cosmo/tests/test_case_switch_legacy_description.yaml b/cosmo/tests/test_case_switch_legacy_description.yaml new file mode 100644 index 0000000..3a68d9e --- /dev/null +++ b/cosmo/tests/test_case_switch_legacy_description.yaml @@ -0,0 +1,193 @@ +device_list: +- device_type: + __typename: DeviceTypeType + slug: sn3420 + __typename: DeviceType + id: '1718' + interfaces: + - __typename: InterfaceType + custom_fields: + inner_tag: null + outer_tag: null + description: 'test description swp52' + enabled: true + id: '30370' + ip_addresses: [] + connected_endpoints: + - name: "combo1" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik01" + mac_address: null + mode: null + mtu: null + name: swp52 + tagged_vlans: [] + tags: [] + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + - __typename: InterfaceType + custom_fields: + inner_tag: null + outer_tag: null + description: 'do not overwrite me!' + enabled: true + id: '30378' + ip_addresses: [] + connected_endpoints: + - name: "combo2" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik01" + mac_address: null + mode: null + mtu: null + name: swp53 + tagged_vlans: [] + tags: [] + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + - __typename: InterfaceType + custom_fields: + inner_tag: null + outer_tag: null + description: "test description swp54" + enabled: true + id: '30379' + ip_addresses: [] + connected_endpoints: + - name: "combo3" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik02" + attached_tobago_line: + __typename: CosmoTobagoLine + component_type: CABLE + element: + description: '' + display: cable 000128934 + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + element_id: 93827 + element_type: dcim.cable + id: 5110 + index: 1 + termination_a: + _occupied: true + cable: + description: '' + display: cable 000128934 + id: 39645 + label: cable 000128934 + url: https://netbox.example.com/api/dcim/cables/39645/ + description: '' + device: + description: '' + display: TEST0001 + id: 1718 + name: TEST0001 + url: https://netbox.example.com/api/dcim/devices/1718/ + display: swp54 + id: 30379 + name: swp54 + url: https://netbox.example.com/api/dcim/interfaces/30379/ + termination_a_id: 92387 + termination_a_type: dcim.interface + termination_b: + _occupied: true + id: 198208 + url: https://netbox.example.com/api/dcim/interfaces/192878/ + display: DC10 duplex front 10b + device: + id: 39948 + url: https://netbox.example.com/api/dcim/device/39948/ + display: Panel C + name: Panel C + description: null + name: DC10 duplex front 10b + description: "" + termination_b_id: 454 + termination_b_type: dcim.frontport + version: + created: '2025-09-03T11:34:19.643456+01:00' + custom_fields: { } + id: 2617 + last_updated: '2025-09-03T11:34:40.819703+01:00' + tenant: + id: 43876 + url: https://netbox.example.com/api/plugins/tobago/tenants/43876/ + display: Contoso Ltd. + name: Contoso Ltd. + slug: contoso-ltd + line: + display: cl390287 + id: 9834 + name: '390287' + name_long: cl390287 + url: https://netbox.example.com/api/plugins/tobago/lines/9834/ + service: + id: 4893794 + url: https://netbox.example.com/api/plugins/tobago/services/4893794/ + display: "#9823 (cl390287)" + business_service: null + status: current + mac_address: null + mode: null + mtu: null + name: swp54 + tagged_vlans: [] + tags: + - __typename: TagType + name: "edge:customer" + slug: "edge_customer" + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + - __typename: InterfaceType + custom_fields: + inner_tag: null + outer_tag: null + description: "test description swp55" + enabled: true + id: '30380' + ip_addresses: [] + attached_tobago_line: null + link_peers: + - name: "port6" + __typename: InterfaceType + device: + __typename: DeviceType + name: "Panel98473" + connected_endpoints: + - name: "combo1" + __typename: InterfaceType + device: + __typename: DeviceType + name: "mikrotik09" + mac_address: null + mode: null + mtu: null + name: swp55 + tagged_vlans: [] + tags: [] + type: A_100GBASE_X_QSFP28 + untagged_vlan: null + vrf: null + name: TEST0001 + platform: + __typename: PlatformType + manufacturer: + __typename: ManufacturerType + slug: cumulus-networks + slug: cumulus-linux-4-3 + primary_ip4: + __typename: IPAddressType + address: 10.120.142.11/24 + staticroute_set: [] +l2vpn_list: [] diff --git a/cosmo/tests/test_serializer.py b/cosmo/tests/test_serializer.py index fac61f1..9aa1071 100644 --- a/cosmo/tests/test_serializer.py +++ b/cosmo/tests/test_serializer.py @@ -5,7 +5,7 @@ import copy from cosmo.common import DeviceSerializationError -from cosmo.features import with_feature, features +from cosmo.features import with_feature, features, without_feature from cosmo.manufacturers import ManufacturerFactoryFromDevice from cosmo.netbox_types import DeviceType @@ -245,6 +245,7 @@ def test_router_logical_interface(capsys): ) +@with_feature(features, "interface-auto-descriptions") def test_router_interface_auto_description(): [sd] = get_router_sd_from_path("./test_case_auto_descriptions.yaml") @@ -292,6 +293,40 @@ def test_router_interface_auto_description(): } == json.loads(sd["interfaces"]["et-0/0/5"]["description"]) +@without_feature(features, "interface-auto-descriptions") +def test_router_interface_legacy_description(): + [sd] = get_router_sd_from_path("./test_case_legacy_descriptions.yaml") + + assert "et-0/0/0" in sd["interfaces"] + assert "et-0/0/1" in sd["interfaces"] + assert 2 in sd["interfaces"]["et-0/0/1"]["units"] + assert 3 in sd["interfaces"]["et-0/0/1"]["units"] + assert "et-0/0/5" in sd["interfaces"] + + assert sd["interfaces"]["et-0/0/0"]["description"] == "test description" + + # normal description mode should add "Peering" in front when autodesc is disabled + assert ( + sd["interfaces"]["et-0/0/1"]["units"][2]["description"] + == "Peering: test description et-0/0/1.2" + ) + + # customer tag + assert ( + sd["interfaces"]["et-0/0/1"]["units"][3]["description"] + == "Customer: test description et-0/0/1.3" + ) + + # customer tag + assert ( + sd["interfaces"]["et-0/0/1"]["units"][4]["description"] + == "Customer: test description et-0/0/1.4" + ) + + # whole interface + assert sd["interfaces"]["et-0/0/5"]["description"] == "test description et-0/0/5" + + def test_router_lag(): [sd] = get_router_sd_from_path("./test_case_lag.yaml") @@ -805,6 +840,7 @@ def test_switch_lldp(): assert True == sd["cumulus__device_interfaces"]["swp52"]["lldp"] +@with_feature(features, "interface-auto-descriptions") def test_switch_auto_description(): [sd] = get_switch_sd_from_path("./test_case_switch_auto_description.yaml") @@ -838,6 +874,42 @@ def test_switch_auto_description(): } == json.loads(sd["cumulus__device_interfaces"]["swp55"]["description"]) +@without_feature(features, "interface-auto-descriptions") +def test_switch_legacy_description(): + [sd] = get_switch_sd_from_path("./test_case_switch_legacy_description.yaml") + + assert "swp52" in sd["cumulus__device_interfaces"] + assert "swp53" in sd["cumulus__device_interfaces"] + assert "swp54" in sd["cumulus__device_interfaces"] + assert "swp55" in sd["cumulus__device_interfaces"] + + assert "description" in sd["cumulus__device_interfaces"]["swp52"] + assert ( + "test description swp52" + == sd["cumulus__device_interfaces"]["swp52"]["description"] + ) + + assert "description" in sd["cumulus__device_interfaces"]["swp53"] + assert ( + "do not overwrite me!" + == sd["cumulus__device_interfaces"]["swp53"]["description"] + ) + + # no Customer: prefix when customer tag exists with switch legacy description + assert "description" in sd["cumulus__device_interfaces"]["swp54"] + assert ( + sd["cumulus__device_interfaces"]["swp54"]["description"] + == "test description swp54" + ) + + # same as above + assert "description" in sd["cumulus__device_interfaces"]["swp55"] + assert ( + sd["cumulus__device_interfaces"]["swp55"]["description"] + == "test description swp55" + ) + + def test_switch_vlans(): [sd] = get_switch_sd_from_path("./test_case_switch_vlan.yaml") From 213946d54ec9326d7d0ff36fe8aa6dd54945493d Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Mon, 2 Mar 2026 17:36:03 +0100 Subject: [PATCH 5/8] add sharedmock as a dependency (needed for multiprocess testing) --- poetry.lock | 16 +++++++++++++++- pyproject.toml | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index b4ddcfe..3311ece 100644 --- a/poetry.lock +++ b/poetry.lock @@ -835,6 +835,20 @@ files = [ {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] +[[package]] +name = "sharedmock" +version = "0.1.0" +description = "A multiprocessing-friendly Python mock object" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "sharedmock-0.1.0.tar.gz", hash = "sha256:807104fca48b7bc4cc4260cbf235e276948181ead15fc65353895d3a2a0becc0"}, +] + +[package.extras] +test = ["coveralls (==1.1)", "prospector (==0.12.4)", "pylint (==1.6.5)", "pytest (==3.0.5)", "pytest-cov (==2.4.0)"] + [[package]] name = "termcolor" version = "3.1.0" @@ -969,4 +983,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "e5ade6f47a41718a4e4c82306000c5e1287ee2e4fe1307ca8d9c61a20410c1fa" +content-hash = "f5e17ef11f29444b0e3cf0fbee459e838a2873305328f7409b3d59c7545dac2c" diff --git a/pyproject.toml b/pyproject.toml index 98ca477..084ea0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ mypy = "^1.15.0" types-pyyaml = "^6.0.12.20241230" types-requests = "^2.32.0.20250301" black = "^25.1.0" +sharedmock = "^0.1.0" [build-system] requires = ["poetry-core>=1.2.0"] From 7c54de3c8613e743f596318c15128f12d84147e5 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Mon, 2 Mar 2026 17:37:12 +0100 Subject: [PATCH 6/8] fix broken multiprocess tests for autodesc feature checks --- cosmo/tests/integration_test.py | 53 ++++++++++++++++++++++++++++++--- cosmo/tests/utils.py | 12 ++++---- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/cosmo/tests/integration_test.py b/cosmo/tests/integration_test.py index 5f1681c..4c431e7 100644 --- a/cosmo/tests/integration_test.py +++ b/cosmo/tests/integration_test.py @@ -1,5 +1,7 @@ import json import re +from sharedmock.mock import SharedMock # type: ignore +from unittest.mock import call, ANY import jsonschema.exceptions import yaml @@ -8,6 +10,7 @@ import cosmo.tests.utils as utils from cosmo.__main__ import main as cosmoMain +from cosmo.common import FileTemplate from cosmo.features import with_feature, features, without_feature @@ -82,23 +85,65 @@ def test_device_processing_error(mocker, capsys): @with_feature(features, "interface-auto-descriptions") def test_autodesc_enabled(mocker): + device_query_template = FileTemplate("cosmo/clients/queries/device.graphql") testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_ansible.yml") - requestsMock = utils.RequestResponseMock() + rrm = utils.RequestResponseMock() + get_mock = mocker.patch.object(rrm, "get_callback", new=SharedMock()) + post_mock = mocker.patch.object(rrm, "post_callback", new=SharedMock()) + with open("cosmo/tests/test_case_auto_descriptions.yaml") as f: test_data = yaml.safe_load(f) - utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) + rrm.patchNetboxClient(mocker, **test_data) + assert cosmoMain() == 0 + assert get_mock.call_count # must be at least called once + assert post_mock.call_count # same as above + assert call("https://netbox.example.com/api/status/", ANY) in get_mock.mock_calls + assert ( + call( + device_query_template.substitute( + device='"TEST0001"', + autodesc_query_extension=FileTemplate( + "cosmo/clients/queries/device_autodesc_query.graphql" + ).substitute(), + ), + ANY, + ) + in post_mock.mock_calls + ) + testEnv.stop() @without_feature(features, "interface-auto-descriptions") def test_autodesc_disabled(mocker): + device_query_template = FileTemplate("cosmo/clients/queries/device.graphql") testEnv = utils.CommonSetup(mocker, cfgFile="cosmo/tests/cosmo.devgen_ansible.yml") - requestsMock = utils.RequestResponseMock() + rrm = utils.RequestResponseMock() + get_mock = mocker.patch.object(rrm, "get_callback", new=SharedMock()) + post_mock = mocker.patch.object(rrm, "post_callback", new=SharedMock()) + with open(f"cosmo/tests/test_case_auto_descriptions.yaml") as f: test_data = yaml.safe_load(f) - utils.RequestResponseMock().patchNetboxClient(mocker, **test_data) + rrm.patchNetboxClient(mocker, **test_data) + assert cosmoMain() == 0 + assert get_mock.call_count # must be at least called once + assert post_mock.call_count # same as above + assert call("https://netbox.example.com/api/status/", ANY) in get_mock.mock_calls + assert ( + call( + device_query_template.substitute( + device='"TEST0001"', + autodesc_query_extension=FileTemplate( + "cosmo/clients/queries/device_no_autodesc_query.graphql" + ).substitute(), + ), + ANY, + ) + in post_mock.mock_calls + ) + testEnv.stop() diff --git a/cosmo/tests/utils.py b/cosmo/tests/utils.py index 537f6d8..555f812 100644 --- a/cosmo/tests/utils.py +++ b/cosmo/tests/utils.py @@ -69,9 +69,11 @@ def json(self): class RequestResponseMock: - def __init__(self): - self.get_responses = dict() - self.post_responses = dict() + def get_callback(self, k: str, v: ResponseMock): + pass + + def post_callback(self, k: str, v: ResponseMock): + pass def patchNetboxClient(self, mocker, **patchKwArgs): @@ -96,7 +98,7 @@ def patchGetFunc(url, **kwargs): "results": [], }, ) - self.get_responses[url] = r + self.get_callback(url, r) return r def patchPostFunc(url, json, **kwargs): @@ -122,7 +124,7 @@ def patchPostFunc(url, json, **kwargs): r = ResponseMock(200, {"data": retVal}) # TODO: find out why value set is being trashed outside context ??? - self.post_responses[q] = r + self.post_callback(q, r) return r getMock = mocker.patch("requests.get", side_effect=patchGetFunc) From dc403fd35f1a5a39f2c9c2ccc032e600a777ab56 Mon Sep 17 00:00:00 2001 From: Yureka Date: Wed, 4 Mar 2026 14:49:53 +0100 Subject: [PATCH 7/8] Nix: package sharedmock dependency --- package.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/package.nix b/package.nix index c1ccf56..73e7b66 100644 --- a/package.nix +++ b/package.nix @@ -11,6 +11,7 @@ pytestCheckHook, pytest-mock, coverage, + callPackage, }: buildPythonApplication rec { @@ -32,6 +33,21 @@ buildPythonApplication rec { termcolor multimethod jsonschema + (callPackage ( + { buildPythonPackage, fetchPypi, setuptools }: + buildPythonPackage rec { + pname = "sharedmock"; + version = "0.1.0"; + + format = "pyproject"; + build-system = [ setuptools ]; + + src = fetchPypi { + inherit pname version; + hash = "sha256-gHEE/KSLe8TMQmDL8jXidpSBgerRX8ZTU4ldOioL7MA="; + }; + } + ) {}) ]; nativeCheckInputs = [ From 1f057168a2c7feeb8b95ed292026dd97127a8c10 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 4 Mar 2026 17:05:45 +0100 Subject: [PATCH 8/8] clip max number of inflight requests --- cosmo/clients/netbox_v4.py | 9 +++++++-- cosmo/common.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cosmo/clients/netbox_v4.py b/cosmo/clients/netbox_v4.py index 60b7077..007ffe9 100644 --- a/cosmo/clients/netbox_v4.py +++ b/cosmo/clients/netbox_v4.py @@ -7,7 +7,7 @@ from cosmo.clients import get_client_mp_context from cosmo.clients.netbox_client import NetboxAPIClient -from cosmo.common import FileTemplate +from cosmo.common import FileTemplate, clip from cosmo.features import features @@ -318,6 +318,8 @@ def _merge_into(self, data: dict, query_data): class NetboxV4Strategy: + MAGIC_MIN_INFLIGHT = 1 + MAGIC_MAX_INFLIGHT = 15 def __init__( self, url, token, multiple_mac_addresses, netbox_43_query_syntax, feature_flags @@ -328,6 +330,9 @@ def __init__( self.netbox_43_query_syntax = netbox_43_query_syntax self.feature_flags = feature_flags + def worker_amount(self, n_queries: int): + return clip(n_queries, self.MAGIC_MIN_INFLIGHT, self.MAGIC_MAX_INFLIGHT) + def get_data(self, device_config): device_list = device_config["router"] + device_config["switch"] @@ -387,7 +392,7 @@ def get_data(self, device_config): # we are going to send. # Note: This will most likely screw the measured times, because Netbox cannot process too many requests at once # and will stall them eventually. So, if you are measuring times, reduce this to a reasonable amounts of 8 or something. - worker_amount = len(queries) + worker_amount = self.worker_amount(len(queries)) with manager.Pool(worker_amount) as pool: data_promises = list(map(lambda x: x.fetch_data(pool), queries)) diff --git a/cosmo/common.py b/cosmo/common.py index 77ca4b5..0757b27 100644 --- a/cosmo/common.py +++ b/cosmo/common.py @@ -61,6 +61,10 @@ def head(l): return None if not l else l[0] +def clip(x: int, lower: int, upper: int): + return min(max(x, lower), upper) + + def deepsort(e): if isinstance(e, list): return sorted(deepsort(v) for v in e)