From 76df57457fa664f0ed1ab1a1778ee707524747a5 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 27 Feb 2026 18:20:17 -0700 Subject: [PATCH 01/72] test: use rmadison form devscripts to determine hello pkg version Avoid maintenance per release of static map. --- .github/workflows/11-dispatch-common.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/11-dispatch-common.yml b/.github/workflows/11-dispatch-common.yml index 66643329e97..9a463e4d872 100644 --- a/.github/workflows/11-dispatch-common.yml +++ b/.github/workflows/11-dispatch-common.yml @@ -77,7 +77,7 @@ jobs: - name: Install Dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update - sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data + sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data devscripts - name: Run integration Tests run: | tox -e integration-tests -- --junitxml="${{ github.workspace }}/reports/junit-report-${{ inputs.platform }}-${{ inputs.release }}.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} From a2209a9e1d3ea990a4699f16dfeb64c12d1e5a2f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 27 Feb 2026 18:40:34 -0700 Subject: [PATCH 02/72] test: apt tests need to wait_for_cloud_init on potentially slow VMs --- tests/integration_tests/modules/test_apt_functionality.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 78daf954ebd..a2b7172e145 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -22,6 +22,7 @@ from tests.integration_tests.util import ( get_feature_flag_value, verify_clean_boot, + wait_for_cloud_init, ) logger = logging.getLogger(__name__) @@ -597,6 +598,7 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): user_data=INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES, launch_kwargs={"image_id": snapshot_id}, ) as minimal_client: + wait_for_cloud_init(minimal_client) log = minimal_client.read_from_file("/var/log/cloud-init.log") assert re.search(RE_GPG_SW_PROPERTIES_INSTALLED, log) gpg_installed = re.search( From 4db78c97a9ad67eb6cc32e6656b82571b01be60a Mon Sep 17 00:00:00 2001 From: Chris Patterson Date: Tue, 10 Mar 2026 21:15:39 -0400 Subject: [PATCH 03/72] feat(azure): introduce experimental skip_ready_report for Azure (#6771) There are some scenarios where we wish to skip cloud-init's health ready reporting for Azure when it completes init-local phase. Prefix it with experimental_ so it's clear it's not supported. We can rename and document it in the future if we want to keep it. There are also some edge cases that aren't fully supported as-is as this process skips fetching ssh keys from wireserver in the cases that require it (e.g. CRLF or x509 certs). To test, configure this flag in VM/image: ```yaml datasource: Azure: experimental_skip_ready_report: true ``` Verify behavior in logs: ```bash $ grep ready_report /var/log/cloud-init.log 2026-02-27 17:57:53,058 - DataSourceAzure.py[DEBUG]: Skipping final health report as experimental_skip_ready_report is enabled. ``` Signed-off-by: Chris Patterson --- cloudinit/sources/DataSourceAzure.py | 34 +++-- tests/unittests/sources/test_azure.py | 207 ++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 13 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 261993fe1ce..299f2c781c5 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -292,6 +292,7 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, "apply_network_config": True, # Use IMDS published network configuration "apply_network_config_for_secondary_ips": True, # Configure secondary ips + "experimental_skip_ready_report": False, # Skip final ready report } BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG = { @@ -838,21 +839,28 @@ def crawl_metadata(self): crawled_data["metadata"]["instance-id"] = self._iid() if self._negotiated is False and self._is_ephemeral_networking_up(): - # Report ready and fetch public-keys from Wireserver, if required. - pubkey_info = self._determine_wireserver_pubkey_info( - cfg=cfg, imds_md=imds_md - ) - try: - ssh_keys = self._report_ready(pubkey_info=pubkey_info) - except Exception: - # Failed to report ready, but continue with best effort. - pass + if self.ds_cfg.get("experimental_skip_ready_report", False): + LOG.debug( + "Skipping final health report as " + "experimental_skip_ready_report is enabled." + ) else: - LOG.debug("negotiating returned %s", ssh_keys) - if ssh_keys: - crawled_data["metadata"]["public-keys"] = ssh_keys + # Report ready and fetch public-keys from Wireserver, + # if required. + pubkey_info = self._determine_wireserver_pubkey_info( + cfg=cfg, imds_md=imds_md + ) + try: + ssh_keys = self._report_ready(pubkey_info=pubkey_info) + except Exception: + # Failed to report ready, but continue with best effort. + pass + else: + LOG.debug("negotiating returned %s", ssh_keys) + if ssh_keys: + crawled_data["metadata"]["public-keys"] = ssh_keys - self._cleanup_markers() + self._cleanup_markers() return crawled_data diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 67373e1ef1e..06bbff5d871 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -5212,6 +5212,213 @@ def test_os_disk_pps(self, mock_sleep, subp_side_effect): assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + @pytest.mark.parametrize("pps_type", ["None", "Running", "Savable"]) + def test_skip_ready_report(self, pps_type): + """Verify ready report is skipped when configured to.""" + self.azure_ds.ds_cfg["experimental_skip_ready_report"] = True + + is_pps = pps_type in ("Running", "Savable") + + imds_md_source = copy.deepcopy(self.imds_md) + imds_md_source["extended"]["compute"]["ppsType"] = pps_type + + nl_sock = mock.MagicMock() + self.mock_netlink.create_bound_netlink_socket.return_value = nl_sock + if pps_type == "Savable": + self.mock_netlink.wait_for_nic_detach_event.return_value = "eth9" + self.mock_netlink.wait_for_nic_attach_event.return_value = ( + "ethAttached1" + ) + + if is_pps: + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(imds_md_source).encode()), + mock.MagicMock( + contents=construct_ovf_env( + provision_guest_proxy_agent=False + ).encode() + ), + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + else: + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() + + assert self.mock_subp_subp.mock_calls == [] + + # Verify IMDS calls. + if is_pps: + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + infinite=True, + log_req_resp=True, + timeout=30, + ), + mock.call( + "http://169.254.169.254/metadata/reprovisiondata?" + "api-version=2019-06-01", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + log_req_resp=False, + infinite=True, + timeout=30, + ), + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + exception_cb=mock.ANY, + headers_cb=imds.headers_cb, + infinite=True, + log_req_resp=True, + timeout=30, + ), + ] + else: + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + timeout=30, + headers_cb=imds.headers_cb, + exception_cb=mock.ANY, + infinite=True, + log_req_resp=True, + ), + ] + + # Verify DHCP setup. + if pps_type == "Running": + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [ + mock.call(timeout_minutes=20), + mock.call(timeout_minutes=5), + ] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + ] + ) + elif pps_type == "Savable": + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [ + mock.call(timeout_minutes=20), + mock.call( + iface="ethAttached1", + timeout_minutes=20, + report_failure_if_not_primary=False, + ), + ] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + mock.call( + self.azure_ds.distro, + "ethAttached1", + dsaz.dhcp_log_cb, + ), + ] + ) + else: + assert ( + self.mock_wrapping_setup_ephemeral_networking.mock_calls + == [mock.call(timeout_minutes=20)] + ) + assert ( + self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls + == [ + mock.call(self.azure_ds.distro, None, dsaz.dhcp_log_cb), + ] + ) + + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" + assert self.azure_ds._is_ephemeral_networking_up() is False + + # Verify DMI usage. + assert self.mock_dmi_read_dmi_data.mock_calls == [ + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), + ] + assert ( + self.azure_ds.metadata["instance-id"] + == "50109936-ef07-47fe-ac82-890c853f60d5" + ) + + # Verify IMDS metadata. + assert self.azure_ds.metadata["imds"] == self.imds_md + + # PPS types still report ready once (source), no-PPS skips entirely. + if is_pps: + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ + mock.call( + endpoint="10.11.12.13", + distro=self.azure_ds.distro, + iso_dev="/dev/sr0", + pubkey_info=None, + ), + ] + else: + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [] + + # Verify netlink operations. + if pps_type == "Running": + assert self.mock_netlink.mock_calls == [ + mock.call.create_bound_netlink_socket(), + mock.call.wait_for_media_disconnect_connect( + mock.ANY, "ethBoot0" + ), + mock.call.create_bound_netlink_socket().close(), + ] + elif pps_type == "Savable": + assert self.mock_netlink.mock_calls == [ + mock.call.create_bound_netlink_socket(), + mock.call.wait_for_nic_detach_event(nl_sock), + mock.call.wait_for_nic_attach_event(nl_sock, ["ethAttached1"]), + mock.call.create_bound_netlink_socket().close(), + ] + else: + assert self.mock_netlink.mock_calls == [] + + # Verify reported_ready marker cleaned up. + if is_pps: + assert self.wrapped_util_write_file.mock_calls[0] == mock.call( + self.patched_reported_ready_marker_path.as_posix(), + mock.ANY, + ) + else: + assert self.wrapped_util_write_file.mock_calls == [] + assert self.patched_reported_ready_marker_path.exists() is False + + # Verify KVP reports. + assert not self.mock_kvp_report_via_kvp.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls + expected_kvp_count = 1 if is_pps else 0 + assert ( + len(self.mock_kvp_report_success_to_host.mock_calls) + == expected_kvp_count + ) + assert ( + len(self.mock_report_dmesg_to_kvp.mock_calls) == expected_kvp_count + ) + def test_imds_failure_results_in_provisioning_failure(self): self.mock_readurl.side_effect = url_helper.UrlError( requests.ConnectionError( From 7132625c6f9294ea73eedcfdab0c4e2dae6a4d0d Mon Sep 17 00:00:00 2001 From: Alejandro Perez <69607775+aale24@users.noreply.github.com> Date: Tue, 10 Mar 2026 18:17:32 -0700 Subject: [PATCH 04/72] feat(util): fail early when hostname is not resolvable in is_resolvable (#6772) Building on the previous optimization in #2040, add an additional optimization to fail early when the hostname is not resolvable at all. --- cloudinit/util.py | 14 ++++--- tests/unittests/config/test_apt_source_v3.py | 35 +++++++++++++---- tests/unittests/test_util.py | 40 ++++++++++++++++++++ 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index cac5926f10f..fa258456b81 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1300,6 +1300,10 @@ def is_resolvable(url) -> bool: with suppress(ValueError): if net.is_ip_address(parsed_url.netloc.strip("[]")): return True + try: + hostname_result = socket.getaddrinfo(name, None) + except (socket.gaierror, socket.error): + return False if _DNS_REDIRECT_IP is None: badips = set() @@ -1324,13 +1328,11 @@ def is_resolvable(url) -> bool: if badresults: LOG.debug("detected dns redirection: %s", badresults) - try: - result = socket.getaddrinfo(name, None) - # check first result's sockaddr field - addr = result[0][4][0] - return addr not in _DNS_REDIRECT_IP - except (socket.gaierror, socket.error): + # check first result's sockaddr field + addr = hostname_result[0][4][0] + if addr in _DNS_REDIRECT_IP: return False + return True def get_hostname(): diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index 2e766d77ed5..b7c76ded350 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -927,20 +927,32 @@ def test_apt_v3_url_resolvable(self): # former tests can leave this set (or not if the test is ran directly) # do a hard reset to ensure a stable result util._DNS_REDIRECT_IP = None + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) bad = [(None, None, None, "badname", ["10.3.2.1"])] good = [(None, None, None, "goodname", ["10.2.3.4"])] with mock.patch.object( - socket, "getaddrinfo", side_effect=[bad, bad, bad, good, good] + socket, "getaddrinfo", side_effect=[good, bad, bad, bad] ) as mocksock: ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") - ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") - mocksock.assert_any_call( - "does-not-exist.example.com.", None, 0, 0, 1, 2 - ) - mocksock.assert_any_call("example.invalid.", None, 0, 0, 1, 2) + for badname in badnames: + mocksock.assert_any_call(badname, None, 0, 0, 1, 2) mocksock.assert_any_call("us.archive.ubuntu.com", None) - assert ret is True + + # IP addresses skip DNS checks entirely + with mock.patch.object(socket, "getaddrinfo") as mocksock: + ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") + mocksock.assert_not_called() + # Verify badnames were NOT checked for IP addresses + for badname in badnames: + assert ( + mock.call(badname, None, 0, 0, 1, 2) + not in mocksock.call_args_list + ) assert ret2 is True # side effect need only bad ret after initial call @@ -952,6 +964,15 @@ def test_apt_v3_url_resolvable(self): mocksock.assert_has_calls(calls) assert ret3 is False + # Test unresolvable hostname + with mock.patch.object( + socket, "getaddrinfo", side_effect=[bad] + ) as mocksock: + ret4 = util.is_resolvable_url("http://instance.:3336") + calls = [call("instance.", None)] + mocksock.assert_has_calls(calls) + assert ret4 is False + def test_apt_v3_disable_suites(self): """test_disable_suites - disable_suites with many configurations""" release = "xenial" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 90206489987..789897504ab 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -3173,6 +3173,12 @@ def test_from_str(self, str_ver, cls_ver): @pytest.mark.allow_dns_lookup class TestResolvable: + @pytest.fixture(autouse=True) + def reset_dns_redirect_ip(self): + util._DNS_REDIRECT_IP = None + yield # Test runs here + util._DNS_REDIRECT_IP = None + @mock.patch.object(util, "_DNS_REDIRECT_IP", return_value=True) @mock.patch.object(util.socket, "getaddrinfo") def test_ips_need_not_be_resolved(self, m_getaddr, m_dns): @@ -3185,6 +3191,40 @@ def test_ips_need_not_be_resolved(self, m_getaddr, m_dns): assert util.is_resolvable("http://[fd00:ec2::254]/") is True assert not m_getaddr.called + @mock.patch.object(util.net, "is_ip_address") + @mock.patch.object(util.socket, "getaddrinfo") + def test_hostnames_require_dns_resolution(self, m_getaddr, m_is_ip): + """Hostnames should go through DNS resolution.""" + m_is_ip.return_value = False + + def mock_getaddrinfo(host, port, *args, **kwargs): + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) + if host in badnames: + return [(None, None, None, "badname", ("192.0.2.1", 0))] + return [(None, None, None, "example.com", ("10.2.3.4", 0))] + + m_getaddr.side_effect = mock_getaddrinfo + + assert util.is_resolvable("http://example.com/") is True + assert m_getaddr.called + + assert m_getaddr.call_args_list[0] == mock.call("example.com", None) + + badnames = ( + "does-not-exist.example.com.", + "example.invalid.", + "__cloud_init_expected_not_found__", + ) + called_hosts = [call[0][0] for call in m_getaddr.call_args_list[1:]] + for badname in badnames: + assert ( + badname in called_hosts + ), f"Expected badname {badname} to be checked" + class TestMaybeB64Decode: """Test the maybe_b64decode helper function.""" From 5c685d62932bd4759750295b50860a946dfc65b4 Mon Sep 17 00:00:00 2001 From: Carlos Nihelton Date: Mon, 16 Mar 2026 10:42:44 -0300 Subject: [PATCH 05/72] WSL: Subprocess cmd.exe with /U to output UTF-16LE (#6717) As we manipulate paths acquired by subprocessing cmd.exe inside WSL, by using it in UTF-16 mode we ensure a predictable output when the strings are not ASCII-compatible, such as reading the user profile when it contains special characters. Fixes GH-6716 --- cloudinit/sources/DataSourceWSL.py | 12 +++++++++--- tests/unittests/sources/test_wsl.py | 13 +++++++++++++ tools/ds-identify | 10 +++++++--- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceWSL.py b/cloudinit/sources/DataSourceWSL.py index 222ec2a0f63..607f8686368 100644 --- a/cloudinit/sources/DataSourceWSL.py +++ b/cloudinit/sources/DataSourceWSL.py @@ -90,6 +90,7 @@ def find_home() -> PurePath: raises: IOError when no mountpoint with cmd.exe is found ProcessExecutionError when either cmd.exe is unable to retrieve the user's home directory + UnicodeDecodeError when cmd.exe /U outputs invalid UTF16LE """ cmd = cmd_executable() @@ -97,8 +98,13 @@ def find_home() -> PurePath: # But we know that `/init` is the interpreter, so we can run it directly. # See /proc/sys/fs/binfmt_misc/WSLInterop[-late] # inside any WSL instance for more details. - home, _ = subp.subp(["/init", cmd.as_posix(), "/C", "echo %USERPROFILE%"]) - home = home.rstrip() + # Invoking with "/U" makes it output UTF-16LE, which is more predictable + # than ANSI Code Pages for anything above the ASCII range. + home, _ = subp.subp( + ["/init", cmd.as_posix(), "/U", "/C", "echo.%USERPROFILE%"], + decode=False, + ) + home = home.decode("utf-16-le").rstrip() if not home: raise subp.ProcessExecutionError( "No output from cmd.exe to show the user profile dir." @@ -443,7 +449,7 @@ def _get_data(self) -> bool: try: user_home = find_home() - except IOError as e: + except (IOError, ValueError) as e: LOG.debug("Unable to detect WSL datasource: %s", e) return False diff --git a/tests/unittests/sources/test_wsl.py b/tests/unittests/sources/test_wsl.py index a5300ff8af0..75c7fb5b524 100644 --- a/tests/unittests/sources/test_wsl.py +++ b/tests/unittests/sources/test_wsl.py @@ -144,6 +144,19 @@ def test_cmd_exe_no_win_mounts(self, m_mounts, m_os_access): with pytest.raises(IOError): wsl.cmd_executable() + @mock.patch("cloudinit.sources.DataSourceWSL.cmd_executable") + @mock.patch("cloudinit.util.subp.subp") + def test_find_home_raises(self, m_subp, m_cmd): + # The value really doesn't matter. + m_cmd.return_value = PurePath("/mnt/c/cmd.exe") + m_subp.return_value = util.subp.SubpResult( + "I am UTF-8 🦄 !".encode("utf-8"), "\r\n".encode("utf-8") + ) + # Checking for ValueError instead of UnicodeDecodeError because + # that's what we catch at the call sites. + with pytest.raises(ValueError): + wsl.find_home() + @pytest.mark.parametrize( "linux_distro_value,files", ( diff --git a/tools/ds-identify b/tools/ds-identify index 02c70a62c2a..d0117e83c47 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -1708,7 +1708,9 @@ WSL_path() { WSL_run_cmd() { local val="" exepath="$1" shift - _RET=$(/init "$exepath" /c "$@" 2>/dev/null) + # Using the '/u' flag to enforce Unicode (UTF-16 LE), thus we need to decode it afterwards. + # It's more reliable than the default ANSI Code Pages for anything above the ASCII range. + _RET=$(/init "$exepath" /u /c "$@" 2>/dev/null | iconv --from-code UTF-16LE --to-code UTF-8) } WSL_profile_dir() { @@ -1719,10 +1721,12 @@ WSL_profile_dir() { for m in $@; do cmdexe="$m/Windows/System32/cmd.exe" if command -v "$cmdexe" > /dev/null 2>&1; then - # Here WSL's proprietary `/init` is used to start the Windows cmd.exe + # Here WSL's `/init` is used to start the Windows cmd.exe # to output the Windows user profile directory path, which is # held by the environment variable %USERPROFILE%. - WSL_run_cmd "$cmdexe" "echo %USERPROFILE%" + # See https://wsl.dev/technical-documentation/interop/ for more information on how /init + # is used to launch Windows binaries. + WSL_run_cmd "$cmdexe" "echo.%USERPROFILE%" profiledir="${_RET%%[[:cntrl:]]}" if [ -n "$profiledir" ]; then # wslpath is a program supplied by WSL itself that translates Windows and Linux paths, From 724e085ad096335eea079980bbaf189af1157c39 Mon Sep 17 00:00:00 2001 From: Cade Jacobson <91996442+cadejacobson@users.noreply.github.com> Date: Mon, 16 Mar 2026 07:03:45 -0700 Subject: [PATCH 06/72] fix(azure): catch import error as reportable (#6714) Adds the ReportableErrorImportError class to catch a known issue with unhandled exceptions when importing crypt and passlib, but both are missing. Fixes GH-6770 --- cloudinit/sources/DataSourceAzure.py | 61 +++++---- cloudinit/sources/azure/errors.py | 7 + tests/unittests/sources/azure/test_errors.py | 9 ++ tests/unittests/sources/test_azure.py | 130 +++++++++++++++++-- 4 files changed, 166 insertions(+), 41 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 299f2c781c5..535268d9f01 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -5,7 +5,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 -import functools import logging import os import os.path @@ -50,31 +49,6 @@ ) from cloudinit.url_helper import UrlError -try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - import crypt # pylint: disable=W4901 - - blowfish_hash: Any = functools.partial( - crypt.crypt, salt=f"$6${util.rand_str(strlen=16)}" - ) -except (ImportError, AttributeError): - try: - import passlib.hash - - blowfish_hash = passlib.hash.sha512_crypt.hash - except ImportError: - - def blowfish_hash(_): - """Raise when called so that importing this module doesn't throw - ImportError when ds_detect() returns false. In this case, crypt - and passlib are not needed. - """ - raise ImportError( - "crypt and passlib not found, missing dependency" - ) - - LOG = logging.getLogger(__name__) DS_NAME = "Azure" @@ -166,6 +140,35 @@ def find_dev_from_busdev(camcontrol_out: str, busdev: str) -> Optional[str]: return None +def hash_password(password: str) -> str: + """Hash a password using SHA-512 crypt. + + Try to use crypt, falling back to passlib. + + If neither are available, raise ReportableErrorImportError. + + :param password: plaintext password to hash. + :return: The hashed password string. + :raises ReportableErrorImportError: If crypt and passlib are unavailable. + """ + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + import crypt # pylint: disable=W4901 + + salt = crypt.mksalt(crypt.METHOD_SHA512) + return crypt.crypt(password, salt) + except (ImportError, AttributeError): + pass + + try: + import passlib.hash + + return passlib.hash.sha512_crypt.hash(password) + except ImportError as error: + raise errors.ReportableErrorImportError(error=error) from error + + def normalize_mac_address(mac: str) -> str: """Normalize mac address with colons and lower-case.""" if len(mac) == 12: @@ -1992,7 +1995,7 @@ def read_azure_ovf(contents): if ovf_env.password: defuser["lock_passwd"] = False if DEF_PASSWD_REDACTION != ovf_env.password: - defuser["hashed_passwd"] = encrypt_pass(ovf_env.password) + defuser["hashed_passwd"] = hash_password(ovf_env.password) if defuser: cfg["system_info"] = {"default_user": defuser} @@ -2017,10 +2020,6 @@ def read_azure_ovf(contents): return (md, ud, cfg) -def encrypt_pass(password): - return blowfish_hash(password) - - def find_primary_nic(): candidate_nics = net.find_candidate_nics() if candidate_nics: diff --git a/cloudinit/sources/azure/errors.py b/cloudinit/sources/azure/errors.py index ae676a08d29..8ef9515304b 100644 --- a/cloudinit/sources/azure/errors.py +++ b/cloudinit/sources/azure/errors.py @@ -168,6 +168,13 @@ def __init__(self, *, exception: ValueError) -> None: self.supporting_data["exception"] = repr(exception) +class ReportableErrorImportError(ReportableError): + def __init__(self, *, error: ImportError) -> None: + super().__init__(f"error importing {error.name} library") + + self.supporting_data["error"] = repr(error) + + class ReportableErrorOsDiskPpsFailure(ReportableError): def __init__(self) -> None: super().__init__("error waiting for host shutdown") diff --git a/tests/unittests/sources/azure/test_errors.py b/tests/unittests/sources/azure/test_errors.py index d9dcf13d555..7b405368d39 100644 --- a/tests/unittests/sources/azure/test_errors.py +++ b/tests/unittests/sources/azure/test_errors.py @@ -211,6 +211,15 @@ def test_imds_metadata_parsing_exception(): assert error.supporting_data["exception"] == repr(exception) +def test_import_error(): + exception = ImportError("No module named 'foobar'", name="foobar") + + error = errors.ReportableErrorImportError(error=exception) + + assert error.reason == "error importing foobar library" + assert error.supporting_data["error"] == repr(exception) + + def test_ovf_parsing_exception(): error = None try: diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 06bbff5d871..e032c25f279 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -1,16 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. # pylint: disable=attribute-defined-outside-init +import builtins import copy import datetime import json import logging import os import stat +import sys import xml.etree.ElementTree as ET from pathlib import Path -import passlib.hash +try: + import passlib.hash +except ImportError: + passlib = None # type: ignore import pytest import requests @@ -1732,12 +1737,13 @@ def test_username_used(self, get_ds): assert "ssh_pwauth" not in dsrc.cfg + @pytest.mark.skipif(passlib is None, reason="passlib not installed") def test_password_given(self, get_ds, mocker): # The crypt module has platform-specific behavior and the purpose of # this test isn't to verify the differences between crypt and passlib, # so hardcode passlib usage as crypt is deprecated. mocker.patch.object( - dsaz, "blowfish_hash", passlib.hash.sha512_crypt.hash + dsaz, "hash_password", passlib.hash.sha512_crypt.hash ) data = { "ovfcontent": construct_ovf_env( @@ -2434,6 +2440,19 @@ def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self, tmp_path): == cm.value.reason ) + def test_import_error_from_failed_import(self): + """Attempt to import a module that is not present""" + try: + import nonexistent_module_that_will_never_exist # type: ignore[import-not-found] # noqa: F401 # isort:skip + except ImportError as error: + reportable_error = errors.ReportableErrorImportError(error=error) + + assert ( + reportable_error.reason == "error importing " + "nonexistent_module_that_will_never_exist library" + ) + assert reportable_error.supporting_data["error"] == repr(error) + class TestReadAzureOvf: def test_invalid_xml_raises_non_azure_ds(self): @@ -5837,14 +5856,6 @@ def test_missing_secondary( assert azure_ds.validate_imds_network_metadata(imds_md) is False -class TestDependencyFallback: - def test_dependency_fallback(self): - """Ensure that crypt/passlib import failover gets exercised on all - Python versions - """ - assert dsaz.encrypt_pass("`") - - class TestQueryVmId: @mock.patch.object( identity, "query_system_uuid", side_effect=["test-system-uuid"] @@ -5907,3 +5918,102 @@ def test_query_vm_id_vm_id_conversion_failure( mock_query_system_uuid.assert_called_once() mock_convert_uuid.assert_called_once_with("test-system-uuid") + + +class TestHashPassword: + """Tests for the hash_password function.""" + + def test_dependency_fallback(self): + """Ensure that crypt/passlib import failover gets exercised on all + Python versions + """ + result = dsaz.hash_password("`") + assert result + assert result.startswith("$6$") + + def test_crypt_working(self): + """Test that hash_password uses crypt when available.""" + mock_crypt = mock.MagicMock() + mock_crypt.METHOD_SHA512 = "sha512" + mock_crypt.mksalt.return_value = "$6$saltvalue" + mock_crypt.crypt.return_value = "$6$saltvalue$hashedpassword" + + with mock.patch.dict("sys.modules", {"crypt": mock_crypt}): + result = dsaz.hash_password("testpassword") + + mock_crypt.mksalt.assert_called_once_with("sha512") + mock_crypt.crypt.assert_called_once_with( + "testpassword", "$6$saltvalue" + ) + assert result == "$6$saltvalue$hashedpassword" + + def test_crypt_not_installed_passlib_fallback(self): + """Test that hash_password falls back to passlib when missing crypt.""" + real_import = builtins.__import__ + passlib_available = True + try: + import passlib.hash as _passlib_hash + except ImportError: + passlib_available = False + + if passlib_available: + # passlib is installed; block crypt and let passlib work normally + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + result = dsaz.hash_password("testpassword") + + # Verify we got a valid SHA-512 hash from passlib + assert result.startswith("$6$") + assert _passlib_hash.sha512_crypt.verify("testpassword", result) + else: + # passlib is not installed; mock it to return a known hash + mock_passlib_hash = mock.MagicMock() + mock_passlib_hash.sha512_crypt.hash.return_value = ( + "$6$mocksalt$mockedhash" + ) + + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + if name == "passlib.hash": + mod = mock.MagicMock() + mod.hash = mock_passlib_hash + sys.modules["passlib"] = mod + sys.modules["passlib.hash"] = mock_passlib_hash + return mod + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + result = dsaz.hash_password("testpassword") + + assert result == "$6$mocksalt$mockedhash" + mock_passlib_hash.sha512_crypt.hash.assert_called_once_with( + "testpassword" + ) + + def test_crypt_and_passlib_unavailable_raises_error(self): + """Test that hash_password raises ReportableErrorImportError.""" + real_import = builtins.__import__ + + def mock_import(name, *args, **kwargs): + if name == "crypt": + raise ImportError("No module named 'crypt'") + if name == "passlib.hash": + raise ImportError("No module named 'passlib'", name="passlib") + return real_import(name, *args, **kwargs) + + with mock.patch.object( + builtins, "__import__", side_effect=mock_import + ): + with pytest.raises(errors.ReportableErrorImportError) as exc_info: + dsaz.hash_password("testpassword") + + assert "passlib" in exc_info.value.reason From 0052a66dfa463186dc286d75079d451030468e0f Mon Sep 17 00:00:00 2001 From: Hyacinthe Cartiaux Date: Mon, 16 Mar 2026 15:13:14 +0100 Subject: [PATCH 07/72] fix(bsd): correct the _ROOT_TMPDIR path for *BSD system (#6794) fix(bsd): correct the _ROOT_TMPDIR path for *BSD system (solves #5789) Fixes GH-5789 --- cloudinit/temp_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index faa4aaa287a..f6b52e35d98 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -20,7 +20,10 @@ def get_tmp_ancestor(odir=None, needs_exe: bool = False): if needs_exe: return _EXE_ROOT_TMPDIR if os.getuid() == 0: - return _ROOT_TMPDIR + if util.is_BSD(): + return "/var/" + _ROOT_TMPDIR + else: + return _ROOT_TMPDIR return os.environ.get("TMPDIR", "/tmp") From e2ba66e5e83208c6f0ff0231126b8522712adb94 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 24 Mar 2026 17:41:32 -0600 Subject: [PATCH 08/72] ci: add cloud-init integration test logs upon failure (#6803) Set LOCAL_LOG_PATH to ./cloudinit_logs for artifact harvest on failure. --- .github/workflows/24-pr-integration.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/24-pr-integration.yml b/.github/workflows/24-pr-integration.yml index f25849eed3f..fda996dd63e 100644 --- a/.github/workflows/24-pr-integration.yml +++ b/.github/workflows/24-pr-integration.yml @@ -67,4 +67,11 @@ jobs: echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - name: Run integration Tests run: | - CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} tox -e integration-tests-ci -- --color=yes tests/integration_tests/ + CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} LOCAL_LOG_PATH=./cloudinit_logs tox -e integration-tests-ci -- --color=yes tests/integration_tests/ + - name: Upload cloudinit logs on failure + if: failure() && hashFiles('./cloudinit_logs/**') != '' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: 'cloudinit-logs' + path: './cloudinit_logs' + retention-days: 3 From 01446415d0d5a4ef3015b3f16c540b014feee34a Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 24 Mar 2026 21:24:20 -0600 Subject: [PATCH 09/72] chore(net): log when klibc is parsed (#6799) Absence of this log indicates that the klibc code is not used. --- cloudinit/net/cmdline.py | 9 +++++++-- cloudinit/sources/DataSourceOracle.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index c2c1d5af45b..784f7baad09 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -63,9 +63,14 @@ def __init__(self, _files=None, _mac_addrs=None, _cmdline=None): self._mac_addrs[k] = mac_addr def is_applicable(self) -> bool: - """ - Return whether this system has klibc initramfs network config or not + """Return whether this system has klibc initramfs network config.""" + + if is_applicable := self._is_applicable(): + LOG.debug("Using initramfs network config from klibc") + return is_applicable + def _is_applicable(self) -> bool: + """ Will return True if: (a) klibc files exist in /run, AND (b) either: diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py index e74e1cbad15..4a846a897b8 100644 --- a/cloudinit/sources/DataSourceOracle.py +++ b/cloudinit/sources/DataSourceOracle.py @@ -63,8 +63,8 @@ class KlibcOracleNetworkConfigSource(cmdline.KlibcNetworkConfigSource): `/run/initramfs/open-iscsi.interface` does not exist. """ - def is_applicable(self) -> bool: - """Override is_applicable""" + def _is_applicable(self) -> bool: + """Override _is_applicable""" return bool(self._files) From a5f6267630cea999355d7310db653cd2f1d317fe Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 24 Mar 2026 22:10:17 -0600 Subject: [PATCH 10/72] chore: auto-format shell (#6780) Make `tox` -e do_format` and `tox -e check_format` enforce formatting of json and shell scripts. Remove unmaintained shell scripts. Update GH Actions. --- .github/workflows/21-pr-check-format.yml | 21 +- Makefile | 33 +- tools/Z99-cloud-locale-test.sh | 11 +- tools/Z99-cloudinit-warnings.sh | 4 +- tools/benchmark.sh | 8 - tools/build-on-freebsd | 5 +- tools/build-on-netbsd | 5 +- tools/build-on-openbsd | 13 +- tools/check_json_format.sh | 11 - tools/cloud-init-hotplugd | 10 +- tools/cloud-init-per | 49 +- tools/ds-identify | 469 +++++++++------ tools/hook-hotplug | 2 +- tools/make-tarball | 36 +- tools/motd-hook | 23 - tools/run-container | 266 ++++++--- tools/run-lint | 22 - tools/tox-venv | 183 ------ tools/uncloud-init | 194 ++++--- tools/write-ssh-key-fingerprints | 2 - tools/xkvm | 707 ----------------------- tox.ini | 17 + 22 files changed, 679 insertions(+), 1412 deletions(-) delete mode 100755 tools/benchmark.sh delete mode 100755 tools/check_json_format.sh delete mode 100755 tools/motd-hook delete mode 100755 tools/run-lint delete mode 100755 tools/tox-venv delete mode 100755 tools/xkvm diff --git a/.github/workflows/21-pr-check-format.yml b/.github/workflows/21-pr-check-format.yml index 9fb9063e2cb..afe4f25208b 100644 --- a/.github/workflows/21-pr-check-format.yml +++ b/.github/workflows/21-pr-check-format.yml @@ -13,11 +13,11 @@ defaults: shell: sh -ex {0} jobs: - check_format: + check_linters: strategy: fail-fast: false matrix: - env: [ruff, mypy, pylint, black, isort] + env: [check_format] name: Check ${{ matrix.env }} runs-on: ubuntu-latest env: @@ -30,6 +30,7 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox + sudo snap install shfmt - name: Print version run: python3 --version @@ -39,20 +40,12 @@ jobs: # matrix env: not to be confused w/environment variables or testenv TOXENV: ${{ matrix.env }} run: tox - schema-format: - strategy: - fail-fast: false - name: Check json format - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Test format + - name: Formatting check failed + if: failure() run: | - tools/check_json_format.sh cloudinit/config/schemas/schema-cloud-config-v1.json - tools/check_json_format.sh cloudinit/config/schemas/schema-network-config-v1.json - tools/check_json_format.sh cloudinit/config/schemas/versions.schema.cloud-config.json + echo "[31mResolve formatting errors with `tox -e do_format` (requires shfmt to format shell).\e[0m" + echo "[31mFor mypy and pylint failures see the warnings above.\e[0m" doc: strategy: diff --git a/Makefile b/Makefile index a425420c9d8..7e2ef07211e 100644 --- a/Makefile +++ b/Makefile @@ -9,40 +9,18 @@ distro ?= redhat GENERATOR_F=./systemd/cloud-init-generator DS_IDENTIFY=./tools/ds-identify -BENCHMARK=./tools/benchmark.sh all: check check: test -style-check: lint - -lint: - @$(CWD)/tools/run-lint - unittest: clean_pyc $(PYTHON) -m pytest -v tests/unittests cloudinit render-template: $(PYTHON) ./tools/render-template --variant=$(VARIANT) $(FILE) $(subst .tmpl,,$(FILE)) -# from systemd-generator(7) regarding generators: -# "We do recommend C code however, since generators are executed -# synchronously and hence delay the entire boot if they are slow." -# -# Our generator is a shell script. Make it easy to measure the -# generator. This should be monitored for performance regressions -benchmark-generator: FILE=$(GENERATOR_F).tmpl -benchmark-generator: VARIANT="benchmark" -benchmark-generator: export ITER=$(NUM_ITER) -benchmark-generator: render-template - $(BENCHMARK) $(GENERATOR_F) - -benchmark-ds-identify: export ITER=$(NUM_ITER) -benchmark-ds-identify: - $(BENCHMARK) $(DS_IDENTIFY) - ci-deps-ubuntu: @$(PYTHON) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro @@ -103,13 +81,6 @@ deb-src: doc: tox -e doc -fmt: - tox -e do_format && tox -e check_format - -fmt-tip: - tox -e do_format_tip && tox -e check_format_tip - - -.PHONY: all check test lint clean rpm srpm deb deb-src clean_pyc -.PHONY: unittest style-check render-template benchmark-generator +.PHONY: all check test clean rpm srpm deb deb-src clean_pyc +.PHONY: unittest render-template .PHONY: clean_pytest clean_packaging clean_release doc diff --git a/tools/Z99-cloud-locale-test.sh b/tools/Z99-cloud-locale-test.sh index e051fa7369a..9ba7716532f 100644 --- a/tools/Z99-cloud-locale-test.sh +++ b/tools/Z99-cloud-locale-test.sh @@ -6,12 +6,12 @@ # (c) 2012, Canonical Group, Ltd. # # This file is part of cloud-init. See LICENSE file for license information. - + # Purpose: Detect invalid locale settings and inform the user # of how to fix them. locale_warn() { - command -v local >/dev/null && local _local="local" || + command -v local > /dev/null && local _local="local" || typeset _local="typeset" $_local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv="" @@ -27,13 +27,14 @@ locale_warn() { # locale: Cannot set LC_SOMETHING to default locale while read -r w1 w2 w3 w4 remain; do case "$w1" in - locale:) bad_names="${bad_names} ${w4}";; + locale:) bad_names="${bad_names} ${w4}" ;; *) key=${w1%%=*} val=${w1#*=} val=${val#\"} val=${val%\"} - vars="${vars} $key=$val";; + vars="${vars} $key=$val" + ;; esac done for bad in $bad_names; do @@ -93,7 +94,7 @@ locale_warn() { printf "_____________________________________________________________________\n\n" # only show the message once - : > ~/.cloud-locale-test.skip 2>/dev/null || : + : > ~/.cloud-locale-test.skip 2> /dev/null || : } [ -f ~/.cloud-locale-test.skip -o -f /var/lib/cloud/instance/locale-check.skip ] || diff --git a/tools/Z99-cloudinit-warnings.sh b/tools/Z99-cloudinit-warnings.sh index 560902350ca..297fbee8380 100644 --- a/tools/Z99-cloudinit-warnings.sh +++ b/tools/Z99-cloudinit-warnings.sh @@ -4,7 +4,7 @@ # Purpose: show user warnings on login. cloud_init_warnings() { - command -v local >/dev/null && local _local="local" || + command -v local > /dev/null && local _local="local" || typeset _local="typeset" $_local warning="" idir="/var/lib/cloud/instance" n=0 $_local warndir="$idir/warnings" @@ -16,7 +16,7 @@ cloud_init_warnings() { for warning in "$warndir"/*; do [ -f "$warning" ] || continue cat "$warning" - n=$((n+1)) + n=$((n + 1)) done [ $n -eq 0 ] && return 0 echo "" diff --git a/tools/benchmark.sh b/tools/benchmark.sh deleted file mode 100755 index c382f374472..00000000000 --- a/tools/benchmark.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/bash - -BIN="$@" -chmod +x "$BIN" - -time for _ in $(seq 1 $ITER); do - "$BIN"; -done diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index d7b3e354062..c5cc6e23662 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -8,7 +8,10 @@ set -eux -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON="${PYTHON:-python3}" if [ ! $(which ${PYTHON}) ]; then diff --git a/tools/build-on-netbsd b/tools/build-on-netbsd index b743d591b6e..25db86f66ec 100755 --- a/tools/build-on-netbsd +++ b/tools/build-on-netbsd @@ -1,6 +1,9 @@ #!/bin/sh -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON="${PYTHON:-python3}" if [ ! $(which ${PYTHON}) ]; then diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd index 94c66dd1744..47c6677e8a2 100755 --- a/tools/build-on-openbsd +++ b/tools/build-on-openbsd @@ -1,9 +1,12 @@ #!/bin/sh -fail() { echo "FAILED:" "$@" 1>&2; exit 1; } +fail() { + echo "FAILED:" "$@" 1>&2 + exit 1 +} PYTHON=${PYTHON:-python3} -if ! command -v ${PYTHON} >/dev/null 2>&1; then +if ! command -v ${PYTHON} > /dev/null 2>&1; then echo "Please install python first." exit 1 fi @@ -27,10 +30,10 @@ pkgs=" wget " -[ -f $depschecked ] || echo "Installing the following packages: $pkgs"; output=$(pkg_add -zI $pkgs 2>&1) - +[ -f $depschecked ] || echo "Installing the following packages: $pkgs" +output=$(pkg_add -zI $pkgs 2>&1) -if echo "$output" | grep -q -e "Can't find" -e "Ambiguous"; then +if echo "$output" | grep -q -e "Can't find" -e "Ambiguous"; then echo "Failed to find or install one or more packages" echo "Failed Package(s):" echo "$output" diff --git a/tools/check_json_format.sh b/tools/check_json_format.sh deleted file mode 100755 index 62f7d6cd74e..00000000000 --- a/tools/check_json_format.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# -# Run python's json.tool and check for changes -# -# requires python 3.9 for --indent -# -file=$1 -before=$(cat "$file") && - python3 -m json.tool --indent 2 "$file" "$file" && - after=$(cat "$file") && - test "$before" = "$after" diff --git a/tools/cloud-init-hotplugd b/tools/cloud-init-hotplugd index eb811d69b10..4df5dd55693 100755 --- a/tools/cloud-init-hotplugd +++ b/tools/cloud-init-hotplugd @@ -14,9 +14,9 @@ PIPE="/run/cloud-init/share/hook-hotplug-cmd" [ -p $PIPE ] || mkfifo -m700 $PIPE while true; do - # shellcheck disable=SC2162 - if read args < $PIPE; then - # shellcheck disable=SC2086 - /usr/bin/cloud-init devel hotplug-hook $args - fi + # shellcheck disable=SC2162 + if read args < $PIPE; then + # shellcheck disable=SC2086 + /usr/bin/cloud-init devel hotplug-hook $args + fi done diff --git a/tools/cloud-init-per b/tools/cloud-init-per index fcd1ea796df..07adf40890f 100755 --- a/tools/cloud-init-per +++ b/tools/cloud-init-per @@ -5,7 +5,7 @@ DATA_PRE="/var/lib/cloud/sem/bootper" INST_PRE="/var/lib/cloud/instance/sem/bootper" Usage() { - cat <&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } +fail() { + [ $# -eq 0 ] || error "$@" + exit 1 +} # support the old 'cloud-init-run-module freq name "execute" cmd arg1' # if < 3 arguments, it will fail below on usage. if [ "${0##*/}" = "cloud-init-run-module" ]; then - if [ $# -le 2 -o "$3" = "execute" ]; then - error "Warning: ${0##*/} is deprecated. Please use cloud-init-per." - freq=$1; name=$2; - [ $# -le 2 ] || shift 3; - set -- "$freq" "$name" "$@" - else - fail "legacy cloud-init-run-module only supported with module 'execute'" - fi + if [ $# -le 2 -o "$3" = "execute" ]; then + error "Warning: ${0##*/} is deprecated. Please use cloud-init-per." + freq=$1 + name=$2 + [ $# -le 2 ] || shift 3 + set -- "$freq" "$name" "$@" + else + fail "legacy cloud-init-run-module only supported with module 'execute'" + fi fi -[ "$1" = "-h" -o "$1" = "--help" ] && { Usage ; exit 0; } -[ $# -ge 3 ] || { Usage 1>&2; exit 1; } +[ "$1" = "-h" -o "$1" = "--help" ] && { + Usage + exit 0 +} +[ $# -ge 3 ] || { + Usage 1>&2 + exit 1 +} freq=$1 name=$(echo $2 | sed 's/-/_/g') -shift 2; +shift 2 [ "${name#*/}" = "${name}" ] || fail "name cannot contain a /" [ "$(id -u)" = "0" ] || fail "must be root" case "$freq" in - once|always) sem="${DATA_PRE}.$name.$freq";; - instance) sem="${INST_PRE}.$name.$freq";; - *) Usage 1>&2; fail "invalid frequency: $freq";; + once | always) sem="${DATA_PRE}.$name.$freq" ;; + instance) sem="${INST_PRE}.$name.$freq" ;; + *) + Usage 1>&2 + fail "invalid frequency: $freq" + ;; esac [ -d "${sem%/*}" ] || mkdir -p "${sem%/*}" || - fail "failed to make directory for ${sem}" + fail "failed to make directory for ${sem}" # Rename legacy sem files with dashes in their names. Do not overwrite existing # sem files to prevent clobbering those which may have been created from calls @@ -63,5 +76,5 @@ sem_legacy=$(echo $sem | sed 's/_/-/g') "$@" ret=$? printf "%s\t%s\n" "$ret" "$(date +%s)" > "$sem" || - fail "failed to write to $sem" + fail "failed to write to $sem" exit $ret diff --git a/tools/ds-identify b/tools/ds-identify index d0117e83c47..82f6b036bab 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -143,7 +143,7 @@ DI_EC2_STRICT_ID_DEFAULT="true" _IS_IBM_CLOUD="" error() { - set -- "ERROR:" "$@"; + set -- "ERROR:" "$@" debug 0 "$@" stderr "$@" } @@ -166,7 +166,7 @@ debug() { if [ "$_DI_LOGGED" != "$DI_LOG" ]; then # first time here, open file descriptor for append case "$DI_LOG" in - stderr) :;; + stderr) : ;; ?*/*) if [ ! -d "${DI_LOG%/*}" ]; then mkdir -p "${DI_LOG%/*}" || { @@ -174,12 +174,13 @@ debug() { DI_LOG="stderr" } fi + ;; esac if [ "$DI_LOG" = "stderr" ]; then exec 3>&2 else - ( exec 3>>"$DI_LOG" ) && exec 3>>"$DI_LOG" || { - stderr "ERROR: failed writing to $DI_LOG. logging to stderr."; + (exec 3>> "$DI_LOG") && exec 3>> "$DI_LOG" || { + stderr "ERROR: failed writing to $DI_LOG. logging to stderr." exec 3>&2 DI_LOG="stderr" } @@ -191,71 +192,77 @@ debug() { get_kenv_field() { local sys_field="$1" kenv_field="" val="" - command -v kenv >/dev/null 2>&1 || { + command -v kenv > /dev/null 2>&1 || { warn "No kenv program. Cannot read $sys_field." return 1 } case "$sys_field" in - board_asset_tag) kenv_field="smbios.planar.tag";; - board_vendor) kenv_field='smbios.planar.maker';; - board_name) kenv_field='smbios.planar.product';; - board_serial) kenv_field='smbios.planar.serial';; - board_version) kenv_field='smbios.planar.version';; - bios_date) kenv_field='smbios.bios.reldate';; - bios_vendor) kenv_field='smbios.bios.vendor';; - bios_version) kenv_field='smbios.bios.version';; - chassis_asset_tag) kenv_field='smbios.chassis.tag';; - chassis_vendor) kenv_field='smbios.chassis.maker';; - chassis_serial) kenv_field='smbios.chassis.serial';; - chassis_version) kenv_field='smbios.chassis.version';; - sys_vendor) kenv_field='smbios.system.maker';; - product_name) kenv_field='smbios.system.product';; - product_serial) kenv_field='smbios.system.serial';; - product_uuid) kenv_field='smbios.system.uuid';; - *) error "Unknown field $sys_field. Cannot call kenv." - return 1;; + board_asset_tag) kenv_field="smbios.planar.tag" ;; + board_vendor) kenv_field='smbios.planar.maker' ;; + board_name) kenv_field='smbios.planar.product' ;; + board_serial) kenv_field='smbios.planar.serial' ;; + board_version) kenv_field='smbios.planar.version' ;; + bios_date) kenv_field='smbios.bios.reldate' ;; + bios_vendor) kenv_field='smbios.bios.vendor' ;; + bios_version) kenv_field='smbios.bios.version' ;; + chassis_asset_tag) kenv_field='smbios.chassis.tag' ;; + chassis_vendor) kenv_field='smbios.chassis.maker' ;; + chassis_serial) kenv_field='smbios.chassis.serial' ;; + chassis_version) kenv_field='smbios.chassis.version' ;; + sys_vendor) kenv_field='smbios.system.maker' ;; + product_name) kenv_field='smbios.system.product' ;; + product_serial) kenv_field='smbios.system.serial' ;; + product_uuid) kenv_field='smbios.system.uuid' ;; + *) + error "Unknown field $sys_field. Cannot call kenv." + return 1 + ;; esac - val=$(kenv -q "$kenv_field" 2>/dev/null) || return 1 + val=$(kenv -q "$kenv_field" 2> /dev/null) || return 1 _RET="$val" } get_sysctl_field() { local sys_field="$1" sysctl_field="" val="" - command -v sysctl >/dev/null 2>&1 || { + command -v sysctl > /dev/null 2>&1 || { warn "No sysctl program. Cannot read $sys_field." return 1 } case "$sys_field" in - chassis_vendor) sysctl_field='hw.vendor';; - chassis_serial) sysctl_field='hw.type';; - chassis_version) sysctl_field='hw.uuid';; - sys_vendor) sysctl_field='hw.vendor';; - product_name) sysctl_field='hw.product';; - product_serial) sysctl_field='hw.uuid';; - product_uuid) sysctl_field='hw.uuid';; - *) error "Unknown field $sys_field. Cannot call sysctl." - return 1;; + chassis_vendor) sysctl_field='hw.vendor' ;; + chassis_serial) sysctl_field='hw.type' ;; + chassis_version) sysctl_field='hw.uuid' ;; + sys_vendor) sysctl_field='hw.vendor' ;; + product_name) sysctl_field='hw.product' ;; + product_serial) sysctl_field='hw.uuid' ;; + product_uuid) sysctl_field='hw.uuid' ;; + *) + error "Unknown field $sys_field. Cannot call sysctl." + return 1 + ;; esac - val=$(sysctl -nq "$sysctl_field" 2>/dev/null) || return 1 + val=$(sysctl -nq "$sysctl_field" 2> /dev/null) || return 1 _RET="$val" } dmi_decode() { local sys_field="$1" dmi_field="" val="" - command -v dmidecode >/dev/null 2>&1 || { + command -v dmidecode > /dev/null 2>&1 || { warn "No dmidecode program. Cannot read $sys_field." return 1 } case "$sys_field" in - sys_vendor) dmi_field="system-manufacturer";; - product_name) dmi_field="system-product-name";; - product_uuid) dmi_field="system-uuid";; - product_serial) dmi_field="system-serial-number";; - chassis_asset_tag) dmi_field="chassis-asset-tag";; - *) error "Unknown field $sys_field. Cannot call dmidecode." - return 1;; + sys_vendor) dmi_field="system-manufacturer" ;; + product_name) dmi_field="system-product-name" ;; + product_uuid) dmi_field="system-uuid" ;; + product_serial) dmi_field="system-serial-number" ;; + chassis_asset_tag) dmi_field="chassis-asset-tag" ;; + *) + error "Unknown field $sys_field. Cannot call dmidecode." + return 1 + ;; esac - val=$(dmidecode --quiet "--string=$dmi_field" 2>/dev/null) || return 1 + val=$(dmidecode --quiet "--string=$dmi_field" 2> /dev/null) || return 1 _RET="$val" } @@ -295,7 +302,7 @@ ensure_sane_path() { local t for t in /sbin /usr/sbin /bin /usr/bin; do case ":$PATH:" in - *:$t:*|*:$t/:*) continue;; + *:$t:* | *:$t/:*) continue ;; esac PATH="${PATH:+${PATH}:}$t" done @@ -341,20 +348,28 @@ read_fs_info_linux() { # empty lines in "$@" below. # shellcheck disable=2086 - { IFS="$CR"; set -- $DI_BLKID_EXPORT_OUT; IFS="$oifs"; } + { + IFS="$CR" + set -- $DI_BLKID_EXPORT_OUT + IFS="$oifs" + } for line in "$@"; do case "${line}" in DEVNAME=*) [ -n "$dev" -a "$ftype" = "iso9660" ] && isodevs="${isodevs},${dev}=$label" - ftype=""; dev=""; label=""; - dev=${line#DEVNAME=};; - LABEL=*|LABEL_FATBOOT=*) - label="${line#*=}"; - labels="${labels}${label}${delim}";; - TYPE=*) ftype=${line#TYPE=};; - UUID=*) uuids="${uuids}${line#UUID=}$delim";; + ftype="" + dev="" + label="" + dev=${line#DEVNAME=} + ;; + LABEL=* | LABEL_FATBOOT=*) + label="${line#*=}" + labels="${labels}${label}${delim}" + ;; + TYPE=*) ftype=${line#TYPE=} ;; + UUID=*) uuids="${uuids}${line#UUID=}$delim" ;; esac done [ -n "$dev" -a "$ftype" = "iso9660" ] && @@ -395,7 +410,11 @@ read_fs_info_freebsd() { # iso9660/cidata N/A vtbd2 # shellcheck disable=2086 - { IFS="$CR"; set -- $DI_GEOM_LABEL_STATUS_OUT; IFS="$oifs"; } + { + IFS="$CR" + set -- $DI_GEOM_LABEL_STATUS_OUT + IFS="$oifs" + } for line in "$@"; do # shellcheck disable=2086 @@ -451,16 +470,16 @@ detect_virt() { fi debug 2 "detected $virt via ds-identify" fi - elif command -v virt-what >/dev/null 2>&1; then + elif command -v virt-what > /dev/null 2>&1; then # Map virt-what's names to those systemd-detect-virt that # don't match up. out=$(virt-what 2>&1 | head -n 1) && { case "$out" in - ibm_systemz-zvm) virt="zvm" ;; - hyperv) virt="microsoft" ;; - virtualbox) virt="oracle" ;; - xen-domU) virt="xen" ;; - *) virt="$out" + ibm_systemz-zvm) virt="zvm" ;; + hyperv) virt="microsoft" ;; + virtualbox) virt="oracle" ;; + xen-domU) virt="xen" ;; + *) virt="$out" ;; esac } elif [ "$DI_UNAME_KERNEL_NAME" = "FreeBSD" -o "$DI_UNAME_KERNEL_NAME" = "Dragonfly" ]; then @@ -480,15 +499,15 @@ detect_virt() { # parallels | parallels # bhyve | bhyve # vm-other | generic - out=$(sysctl -qn kern.vm_guest 2>/dev/null) && { + out=$(sysctl -qn kern.vm_guest 2> /dev/null) && { case "$out" in hv) virt="microsoft" ;; vbox) virt="oracle" ;; - generic) virt="vm-other";; - *) virt="$out" + generic) virt="vm-other" ;; + *) virt="$out" ;; esac } - out=$(sysctl -qn security.jail.jailed 2>/dev/null) && { + out=$(sysctl -qn security.jail.jailed 2> /dev/null) && { if [ "$out" = "1" ]; then virt="jail" fi @@ -505,8 +524,8 @@ read_virt() { is_container() { case "${DI_VIRT}" in - container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt|jail) return 0;; - *) return 1;; + container-other | lxc | lxc-libvirt | systemd-nspawn | docker | rkt | jail) return 0 ;; + *) return 1 ;; esac } @@ -524,7 +543,7 @@ read_kernel_cmdline() { cmdline=$x fi elif [ -f "$fpath" ]; then - read cmdline <"$fpath" + read cmdline < "$fpath" else cmdline="${UNAVAILABLE}:no-cmdline" fi @@ -609,7 +628,11 @@ parse_yaml_array() { val=${val#"["} val=${val%"]"} # shellcheck disable=2086 - { IFS=","; set -- $val; IFS="$oifs"; } + { + IFS="," + set -- $val + IFS="$oifs" + } for tok in "$@"; do trim "$tok" unquote "$_RET" @@ -655,9 +678,13 @@ read_pid1_product_name() { local oifs="$IFS" out="" tok="" key="" val="" product_name="${UNAVAILABLE}" cached "${DI_PID_1_PRODUCT_NAME}" && return [ -r "${PATH_PROC_1_ENVIRON}" ] || return - out=$(tr '\0' '\n' <"${PATH_PROC_1_ENVIRON}") + out=$(tr '\0' '\n' < "${PATH_PROC_1_ENVIRON}") # shellcheck disable=2086 - { IFS="$CR"; set -- $out; IFS="$oifs"; } + { + IFS="$CR" + set -- $out + IFS="$oifs" + } for tok in "$@"; do key=${tok%%=*} [ "$key" != "$tok" ] || continue @@ -671,7 +698,7 @@ dmi_chassis_asset_tag_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_CHASSIS_ASSET_TAG}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -680,7 +707,7 @@ dmi_product_name_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_PRODUCT_NAME}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -689,7 +716,7 @@ dmi_product_serial_matches() { is_container && return 1 # shellcheck disable=2254 case "${DI_DMI_PRODUCT_SERIAL}" in - $1) return 0;; + $1) return 0 ;; esac return 1 } @@ -701,7 +728,7 @@ dmi_sys_vendor_is() { has_fs_with_uuid() { case ",${DI_FS_UUIDS}," in - *,$1,*) return 0;; + *,$1,*) return 0 ;; esac return 1 } @@ -712,7 +739,7 @@ has_fs_with_label() { local label="" for label in "$@"; do case ",${DI_FS_LABELS}," in - *,$label,*) return 0;; + *,$label,*) return 0 ;; esac done return 1 @@ -768,16 +795,25 @@ probe_floppy() { local fpath=/dev/floppy [ -b "$fpath" ] || - { STATE_FLOPPY_PROBED=1; return 1; } + { + STATE_FLOPPY_PROBED=1 + return 1 + } # Use "-b" option as Busybox modprobe doesn't support long-option - modprobe -b floppy >/dev/null 2>&1 || - { STATE_FLOPPY_PROBED=1; return 1; } + modprobe -b floppy > /dev/null 2>&1 || + { + STATE_FLOPPY_PROBED=1 + return 1 + } # Some Linux distros/non-Linux OSes may not have udev if command -v udevadm; then udevadm settle "--exit-if-exists=$fpath" || - { STATE_FLOPPY_PROBED=1; return 1; } + { + STATE_FLOPPY_PROBED=1 + return 1 + } fi [ -b "$fpath" ] @@ -827,7 +863,11 @@ check_config() { files="$*" fi # shellcheck disable=2086 - { set +f; set -- $files; set -f; } + { + set +f + set -- $files + set -f + } if [ "$1" = "$files" -a ! -f "$1" ]; then return 1 fi @@ -846,11 +886,11 @@ check_config() { # key: [ some_value ] # key : [ "some value" ] # key\t:\t[\tsome_value\t]\t - # + # # The syntax warned about is not valid posix shell, and we are not # attempting to access an index of arrays. Silence it. # shellcheck disable=1087 - out=$(grep "$key[\"\']*[[:space:]]*:" "$@" 2>/dev/null) + out=$(grep "$key[\"\']*[[:space:]]*:" "$@" 2> /dev/null) IFS=${CR} for line in $out; do # drop '# comment' @@ -869,8 +909,8 @@ check_config() { continue fi fi - ret=${line#*: }; - found=$((found+1)) + ret=${line#*: } + found=$((found + 1)) done IFS="$oifs" if [ $found -ne 0 ]; then @@ -959,15 +999,17 @@ dscheck_LXD() { return ${DS_NOT_FOUND} fi # Temporarily enable globbing to walk virtio_ports_path. - set +f; set -- "${virtio_ports_path}/"*; set -f; + set +f + set -- "${virtio_ports_path}/"* + set -f for port_dir in "$@"; do local name_file="${port_dir}/name" port_name [ ! -f "${name_file}" ] && continue - read port_name < "${name_file}" || \ - warn "unable to read file: $name_file" + read port_name < "${name_file}" || + warn "unable to read file: $name_file" # Check for both current and legacy LXD serial names - if [ "${port_name}" = "com.canonical.lxd" ] || \ - [ "${port_name}" = "org.linuxcontainers.lxd" ]; then + if [ "${port_name}" = "com.canonical.lxd" ] || + [ "${port_name}" = "org.linuxcontainers.lxd" ]; then return ${DS_FOUND} fi done @@ -977,7 +1019,7 @@ dscheck_LXD() { dscheck_NoCloud() { local fslabel="cidata CIDATA" d="" case " ${DI_DMI_PRODUCT_SERIAL} " in - *\ ds=nocloud*) return ${DS_FOUND};; + *\ ds=nocloud*) return ${DS_FOUND} ;; esac for d in nocloud nocloud-net; do @@ -1012,7 +1054,9 @@ check_configdrive_v2() { local d="" local vlc_config_drive_path="${PATH_VAR_LIB_CLOUD}/seed/config_drive" for d in /config-drive $vlc_config_drive_path; do - set +f; set -- "$d/openstack/"2???-??-??/meta_data.json; set -f; + set +f + set -- "$d/openstack/"2???-??-??/meta_data.json + set -f [ -f "$1" ] && return ${DS_FOUND} done # at least one cloud (softlayer) seeds config drive with only 'latest'. @@ -1081,13 +1125,15 @@ vmware_guest_customization() { local ppath="plugins/vmsvc/libdeployPkgPlugin.so" for pkg in vmware-tools open-vm-tools; do if [ -f "$pre/$pkg/$ppath" -o -f "${pre}64/$pkg/$ppath" ]; then - found="$pkg"; break; + found="$pkg" + break fi # search in multiarch dir - if [ -f "$pre/$x86/$pkg/$ppath" ] || \ - [ -f "$pre/$aarch/$pkg/$ppath" ] || \ - [ -f "$pre/$i386/$pkg/$ppath" ]; then - found="$pkg"; break; + if [ -f "$pre/$x86/$pkg/$ppath" ] || + [ -f "$pre/$aarch/$pkg/$ppath" ] || + [ -f "$pre/$i386/$pkg/$ppath" ]; then + found="$pkg" + break fi done [ -n "$found" ] || return 1 @@ -1098,8 +1144,8 @@ vmware_guest_customization() { if check_config "$key" && get_value "$key" "$_RET"; then debug 2 "${_RET_fname} set $key to $_RET" case "$_RET" in - 0|false|False) return 0;; - *) return 1;; + 0 | false | False) return 0 ;; + *) return 1 ;; esac fi @@ -1107,11 +1153,11 @@ vmware_guest_customization() { } vmware_has_rpctool() { - command -v vmware-rpctool >/dev/null 2>&1 + command -v vmware-rpctool > /dev/null 2>&1 } vmware_rpctool_guestinfo() { - vmware-rpctool "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]" + vmware-rpctool "info-get guestinfo.${1}" 2> /dev/null | grep "[[:alnum:]]" } vmware_rpctool_guestinfo_err() { @@ -1119,11 +1165,11 @@ vmware_rpctool_guestinfo_err() { } vmware_has_vmtoolsd() { - command -v vmtoolsd >/dev/null 2>&1 + command -v vmtoolsd > /dev/null 2>&1 } vmware_vmtoolsd_guestinfo() { - vmtoolsd --cmd "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]" + vmtoolsd --cmd "info-get guestinfo.${1}" 2> /dev/null | grep "[[:alnum:]]" } vmware_vmtoolsd_guestinfo_err() { @@ -1167,9 +1213,11 @@ ovf_vmware_transport_guestinfo() { return 1 fi case "$out" in - "/dev/null 2>&1; then + if { vmware_guestinfo_metadata || + vmware_guestinfo_userdata || + vmware_guestinfo_vendordata; } > /dev/null 2>&1; then return "${DS_FOUND}" fi @@ -1710,7 +1783,7 @@ WSL_run_cmd() { shift # Using the '/u' flag to enforce Unicode (UTF-16 LE), thus we need to decode it afterwards. # It's more reliable than the default ANSI Code Pages for anything above the ASCII range. - _RET=$(/init "$exepath" /u /c "$@" 2>/dev/null | iconv --from-code UTF-16LE --to-code UTF-8) + _RET=$(/init "$exepath" /u /c "$@" 2> /dev/null | iconv --from-code UTF-16LE --to-code UTF-8) } WSL_profile_dir() { @@ -1724,8 +1797,8 @@ WSL_profile_dir() { # Here WSL's `/init` is used to start the Windows cmd.exe # to output the Windows user profile directory path, which is # held by the environment variable %USERPROFILE%. - # See https://wsl.dev/technical-documentation/interop/ for more information on how /init - # is used to launch Windows binaries. + # See https://wsl.dev/technical-documentation/interop/ for more information on how /init + # is used to launch Windows binaries. WSL_run_cmd "$cmdexe" "echo.%USERPROFILE%" profiledir="${_RET%%[[:cntrl:]]}" if [ -n "$profiledir" ]; then @@ -1867,7 +1940,7 @@ write_result() { pre=" " fi for line in "$@"; do - echo "${pre}$line"; + echo "${pre}$line" done } > "$runcfg" ret=$? @@ -1909,8 +1982,8 @@ found() { fi # if None is not already in the list, then add it last. case " $list " in - *\ None,\ *|*\ None\ ) :;; - *) list=${list:+${list}, None};; + *\ None,\ * | *\ None\ ) : ;; + *) list=${list:+${list}, None} ;; esac write_result "datasource_list: [ $list ]" "$@" return @@ -1919,8 +1992,14 @@ found() { trim() { # trim all whitespace from the string, assign output to _RET local tmp="" cur="$*" - until tmp="${cur#[[:space:]]}"; [ "$tmp" = "$cur" ]; do cur="$tmp"; done - until tmp="${cur%[[:space:]]}"; [ "$tmp" = "$cur" ]; do cur="$tmp"; done + until + tmp="${cur#[[:space:]]}" + [ "$tmp" = "$cur" ] + do cur="$tmp"; done + until + tmp="${cur%[[:space:]]}" + [ "$tmp" = "$cur" ] + do cur="$tmp"; done _RET="$tmp" } @@ -1929,8 +2008,10 @@ unquote() { local quote='"' tick="'" local val="$1" case "$val" in - ${quote}*${quote}|${tick}*${tick}) - val=${val#?}; val=${val%?};; + ${quote}*${quote} | ${tick}*${tick}) + val=${val#?} + val=${val%?} + ;; esac _RET="$val" } @@ -1964,8 +2045,8 @@ _read_config() { fi case "$key" in - datasource) _rc_dsname="$val";; - policy) _rc_policy="$val";; + datasource) _rc_dsname="$val" ;; + policy) _rc_policy="$val" ;; esac done if [ "$keyname" = "_unset" ]; then @@ -2003,10 +2084,10 @@ parse_policy() { local def="" case "$DI_UNAME_MACHINE" in # these have dmi data - i?86|x86_64) def=${DI_DEFAULT_POLICY};; + i?86 | x86_64) def=${DI_DEFAULT_POLICY} ;; # aarch64 has dmi, but not currently used (LP: #1663304) - aarch64) def=${DI_DEFAULT_POLICY_NO_DMI};; - *) def=${DI_DEFAULT_POLICY_NO_DMI};; + aarch64) def=${DI_DEFAULT_POLICY_NO_DMI} ;; + *) def=${DI_DEFAULT_POLICY_NO_DMI} ;; esac local policy="$1" local _def_mode="" _def_report="" _def_found="" _def_maybe="" @@ -2019,23 +2100,30 @@ parse_policy() { local mode="" report="" found="" maybe="" notfound="" local oifs="$IFS" tok="" val="" # shellcheck disable=2086 - { IFS=","; set -- $policy; IFS="$oifs"; } + { + IFS="," + set -- $policy + IFS="$oifs" + } for tok in "$@"; do val=${tok#*=} case "$tok" in - "${DI_ENABLED}"|"${DI_DISABLED}"|search|report) mode=$tok;; - found=all|found=first) found=$val;; - maybe=all|maybe=none) maybe=$val;; - notfound="${DI_ENABLED}"|notfound="${DI_DISABLED}") notfound=$val;; + "${DI_ENABLED}" | "${DI_DISABLED}" | search | report) mode=$tok ;; + found=all | found=first) found=$val ;; + maybe=all | maybe=none) maybe=$val ;; + notfound="${DI_ENABLED}" | notfound="${DI_DISABLED}") notfound=$val ;; found=*) - parse_warn found "$val" "${_def_found}" - found=${_def_found};; + parse_warn found "$val" "${_def_found}" + found=${_def_found} + ;; maybe=*) - parse_warn maybe "$val" "${_def_maybe}" - maybe=${_def_maybe};; + parse_warn maybe "$val" "${_def_maybe}" + maybe=${_def_maybe} + ;; notfound=*) - parse_warn notfound "$val" "${_def_notfound}" - notfound=${_def_notfound};; + parse_warn notfound "$val" "${_def_notfound}" + notfound=${_def_notfound} + ;; esac done report=${report:-${_def_report:-false}} @@ -2064,17 +2152,17 @@ read_config() { # discard anything after the first delimiter val=${val%%;*} case "$key" in - ds) _rc_dsname="$val";; - ci.ds) _rc_dsname="$val";; - ci.datasource) _rc_dsname="$val";; - ci.di.policy) _rc_policy="$val";; + ds) _rc_dsname="$val" ;; + ci.ds) _rc_dsname="$val" ;; + ci.datasource) _rc_dsname="$val" ;; + ci.di.policy) _rc_policy="$val" ;; esac done local _rc_mode _rc_report _rc_found _rc_maybe _rc_notfound parse_policy "${_rc_policy}" debug 1 "policy loaded: mode=${_rc_mode} report=${_rc_report}" \ - "found=${_rc_found} maybe=${_rc_maybe} notfound=${_rc_notfound}" + "found=${_rc_found} maybe=${_rc_maybe} notfound=${_rc_notfound}" DI_MODE=${_rc_mode} DI_ON_FOUND=${_rc_found} DI_ON_MAYBE=${_rc_maybe} @@ -2084,7 +2172,6 @@ read_config() { return $ret } - manual_clean_and_existing() { [ -f "${PATH_VAR_LIB_CLOUD}/instance/manual-clean" ] } @@ -2115,7 +2202,7 @@ set_run_path() { # testing only - NOT use for production code, it is NOT supported get_environment() { if [ -f "$PATH_DI_ENV" ]; then - debug 0 "WARN: loading environment file [${PATH_DI_ENV}]"; + debug 0 "WARN: loading environment file [${PATH_DI_ENV}]" # shellcheck source=/dev/null . "$PATH_DI_ENV" fi @@ -2146,8 +2233,9 @@ _main() { ;; "${DI_ENABLED}") debug 1 "mode=$DI_ENABLED. returning $ret_en" - return $ret_en;; - search|report) :;; + return $ret_en + ;; + search | report) : ;; esac if [ -n "${DI_DSNAME}" ]; then @@ -2165,7 +2253,7 @@ _main() { # shellcheck disable=2086 set -- $DI_DSLIST # if there is only a single entry in $DI_DSLIST - if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ] ; then + if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ]; then debug 1 "single entry in datasource_list ($DI_DSLIST) use that." if [ $# -eq 1 ]; then write_result "datasource_list: [ $1 ]" @@ -2180,7 +2268,7 @@ _main() { for ds in ${DI_DSLIST}; do dscheck_fn="dscheck_${ds}" debug 2 "Checking for datasource '$ds' via '$dscheck_fn'" - if ! type "$dscheck_fn" >/dev/null 2>&1; then + if ! type "$dscheck_fn" > /dev/null 2>&1; then warn "No check method '$dscheck_fn' for datasource '$ds'" continue fi @@ -2189,14 +2277,16 @@ _main() { ret="$?" case "$ret" in "${DS_FOUND}") - debug 1 "check for '$ds' returned found"; + debug 1 "check for '$ds' returned found" exfound_cfg="${exfound_cfg:+${exfound_cfg}${CR}}${_RET_excfg}" - found="${found} $ds";; + found="${found} $ds" + ;; "${DS_MAYBE}") - debug 1 "check for '$ds' returned maybe"; + debug 1 "check for '$ds' returned maybe" exmaybe_cfg="${exmaybe_cfg:+${exmaybe_cfg}${CR}}${_RET_excfg}" - maybe="${maybe} $ds";; - *) debug 2 "check for '$ds' returned not-found[$ret]";; + maybe="${maybe} $ds" + ;; + *) debug 2 "check for '$ds' returned not-found[$ret]" ;; esac done @@ -2233,17 +2323,21 @@ _main() { case "$DI_MODE:$DI_ON_NOTFOUND" in report:"${DI_DISABLED}") msg="$basemsg Would disable cloud-init [$ret_dis]" - ret=$ret_en;; + ret=$ret_en + ;; report:"${DI_ENABLED}") msg="$basemsg Would enable cloud-init [$ret_en]" - ret=$ret_en;; + ret=$ret_en + ;; search:"${DI_DISABLED}") msg="$basemsg Disabled cloud-init [$ret_dis]" - ret=$ret_dis;; + ret=$ret_dis + ;; search:"${DI_ENABLED}") msg="$basemsg Enabled cloud-init [$ret_en]" - ret=$ret_en;; - *) error "Unexpected result";; + ret=$ret_en + ;; + *) error "Unexpected result" ;; esac debug 1 "$msg" return "$ret" @@ -2262,7 +2356,7 @@ main() { if read ret < "$PATH_RUN_DI_RESULT"; then if [ "$ret" = "0" ] || [ "$ret" = "1" ] || [ "$ret" = "2" ]; then debug 2 "used cached result $ret. pass --force to re-run." - return "$ret"; + return "$ret" fi debug 1 "previous run returned unexpected '$ret'. Re-running." else @@ -2284,11 +2378,12 @@ noop() { get_environment case "${DI_MAIN}" in # builtin DI_MAIN implementations - main|print_info|noop) "${DI_MAIN}" "$@";; + main | print_info | noop) "${DI_MAIN}" "$@" ;; # side-load an alternate implementation # testing only - NOT use for production code, it is NOT supported *) - debug 0 "WARN: side-loading alternate implementation: [${DI_MAIN}]"; - exec "${DI_MAIN}" "$@";; + debug 0 "WARN: side-loading alternate implementation: [${DI_MAIN}]" + exec "${DI_MAIN}" "$@" + ;; esac diff --git a/tools/hook-hotplug b/tools/hook-hotplug index f142d4b9548..485cff19ec5 100755 --- a/tools/hook-hotplug +++ b/tools/hook-hotplug @@ -25,7 +25,7 @@ if ! should_run; then fi # open cloud-init's hotplug-hook fifo rw -exec 3<>$fifo +exec 3<> $fifo env_params=" --subsystem=${SUBSYSTEM} handle --devpath=${DEVPATH} --udevaction=${ACTION}" # write params to cloud-init's hotplug-hook fifo echo "${env_params}" >&3 diff --git a/tools/make-tarball b/tools/make-tarball index 48442d0b080..11dddc26f52 100755 --- a/tools/make-tarball +++ b/tools/make-tarball @@ -8,7 +8,7 @@ cleanup() { trap cleanup EXIT Usage() { - cat <&2; exit 1; } + eval set -- "${getopt_out}" || { + Usage 1>&2 + exit 1 +} long_opt="" orig_opt="" version="" while [ $# -ne 0 ]; do - cur=$1; next=$2 + cur=$1 + next=$2 case "$cur" in - -h|--help) Usage; exit 0;; - -o|--output) output=$next; shift;; - --version) version=$next; shift;; - --long) long_opt="--long";; - --orig-tarball) orig_opt=".orig";; - --) shift; break;; + -h | --help) + Usage + exit 0 + ;; + -o | --output) + output=$next + shift + ;; + --version) + version=$next + shift + ;; + --long) long_opt="--long" ;; + --orig-tarball) orig_opt=".orig" ;; + --) + shift + break + ;; esac - shift; + shift done rev=${1:-HEAD} diff --git a/tools/motd-hook b/tools/motd-hook deleted file mode 100755 index 73d9792c1ab..00000000000 --- a/tools/motd-hook +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# Copyright (C) 2010 Canonical Ltd. -# -# Authors: Scott Moser -# -# This file is part of cloud-init. See LICENSE file for license information. - -# 92-ec2-upgrade-available - update-motd script - -# Determining if updates are available is possibly slow. -# a cronjob runs occasioinally and updates a file with information -# on the latest available release (if newer than current) - -BUILD_FILE=/var/lib/cloud/data/available.build - -[ -s "${BUILD_FILE}" ] || exit 0 - -read suite build_name name serial other < "${BUILD_FILE}" - -cat <&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } -errorrc() { local r=$?; error "$@" "ret=$r"; return $r; } +fail() { + [ $# -eq 0 ] || error "$@" + exit 1 +} +errorrc() { + local r=$? + error "$@" "ret=$r" + return $r +} Usage() { - cat <&2; [ $# -eq 0 ] || error "$@"; return 1; } +bad_Usage() { + Usage 1>&2 + [ $# -eq 0 ] || error "$@" + return 1 +} cleanup() { if [ -n "$CONTAINER" ]; then if [ "$KEEP" = "true" ]; then @@ -60,12 +71,12 @@ cleanup() { } debug() { - local level=${1}; shift; + local level=${1} + shift [ "${level}" -gt "${VERBOSITY}" ] && return error "${@}" } - inside_as() { # inside_as(container_name, user, cmd[, args]) # executes cmd with args inside container as user in users home dir. @@ -80,7 +91,7 @@ inside_as() { stuffed=${stuffed# -- } b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0) inside "$name" su "$user" -c \ - 'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"'; + 'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"' } inside_as_cd() { @@ -95,21 +106,22 @@ inside() { $LXC exec "$name" -- "$@" } -inject_cloud_init(){ +inject_cloud_init() { # take current cloud-init git dir and put it inside $name at # ~$user/cloud-init. local name="$1" user="$2" dirty="$3" local dname="cloud-init" gitdir="" commitish="" gitdir=$(git rev-parse --git-dir) || { - errorrc "Failed to get git dir in $PWD"; + errorrc "Failed to get git dir in $PWD" return } local t=${gitdir%/*} case "$t" in - */worktrees) + */worktrees) if [ -f "${t%worktrees}/config" ]; then gitdir="${t%worktrees}" fi + ;; esac # attempt to get branch name. @@ -182,7 +194,10 @@ get_os_info_in() { # prep the container (install very basic dependencies) [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] && return 0 data=$(run_self_inside "$name" os_info) || - { errorrc "Failed to get os-info in container $name"; return; } + { + errorrc "Failed to get os-info in container $name" + return + } eval "$data" && [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] || return debug 1 "determined $name is $OS_NAME/$OS_VERSION" } @@ -204,33 +219,42 @@ get_os_info() { local pname="" pname=$(sh -c '. /etc/os-release; echo $PRETTY_NAME') case "$pname" in - *buster*) OS_VERSION=10;; - *sid*) OS_VERSION="sid";; + *buster*) OS_VERSION=10 ;; + *sid*) OS_VERSION="sid" ;; esac fi elif [ -f /etc/centos-release ]; then local line="" read line < /etc/centos-release case "$line" in - CentOS\ *\ 6.*) OS_VERSION="6"; OS_NAME="centos";; + CentOS\ *\ 6.*) + OS_VERSION="6" + OS_NAME="centos" + ;; esac fi [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] || - { error "Unable to determine OS_NAME/OS_VERSION"; return 1; } + { + error "Unable to determine OS_NAME/OS_VERSION" + return 1 + } } yum_install() { local n=0 max=10 ret bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" - while n=$((n+1)); do - error ":: running $bcmd $* [$n/$max]" - $bcmd "$@" - ret=$? - [ $ret -eq 0 ] && break - [ $n -ge $max ] && { error "gave up on $bcmd"; exit $ret; } - nap=$((n*5)) - error ":: failed [$ret] ($n/$max). sleeping $nap." - sleep $nap + while n=$((n + 1)); do + error ":: running $bcmd $* [$n/$max]" + $bcmd "$@" + ret=$? + [ $ret -eq 0 ] && break + [ $n -ge $max ] && { + error "gave up on $bcmd" + exit $ret + } + nap=$((n * 5)) + error ":: failed [$ret] ($n/$max). sleeping $nap." + sleep $nap done error ":: running yum install --cacheonly --assumeyes $*" yum install --cacheonly --assumeyes "$@" @@ -251,11 +275,13 @@ apt_install() { install_packages() { get_os_info || return case "$OS_NAME" in - centos|rocky*|fedora) yum_install "$@";; - opensuse*) zypper_install "$@";; - debian|ubuntu) apt_install "$@" -y;; - *) error "Do not know how to install packages on ${OS_NAME}"; - return 1;; + centos | rocky* | fedora) yum_install "$@" ;; + opensuse*) zypper_install "$@" ;; + debian | ubuntu) apt_install "$@" -y ;; + *) + error "Do not know how to install packages on ${OS_NAME}" + return 1 + ;; esac } @@ -270,7 +296,8 @@ prep() { local py3pkg="python3" case "$OS_NAME" in opensuse) - py3pkg="python3-base";; + py3pkg="python3-base" + ;; esac pairs="$pairs python3:$py3pkg" @@ -278,7 +305,7 @@ prep() { for pair in $pairs; do pkg=${pair#*:} cmd=${pair%%:*} - command -v "$cmd" >/dev/null 2>&1 || needed="${needed} $pkg" + command -v "$cmd" > /dev/null 2>&1 || needed="${needed} $pkg" done needed=${needed# } if [ -z "$needed" ]; then @@ -302,15 +329,16 @@ is_done_cloudinit() { is_done_systemd() { local s="" num="$1" - s=$(systemctl is-system-running 2>&1); + s=$(systemctl is-system-running 2>&1) _RET="$? $s" case "$s" in - initializing|starting) return 1;; + initializing | starting) return 1 ;; *[Ff]ailed*connect*bus*) # warn if not the first run. [ "$num" -lt 5 ] || - error "Failed to connect to systemd bus [${_RET%% *}]"; - return 1;; + error "Failed to connect to systemd bus [${_RET%% *}]" + return 1 + ;; esac return 0 } @@ -323,20 +351,20 @@ is_done_other() { wait_inside() { local name="$1" max="${2:-${WAIT_MAX}}" debug=${3:-0} - local i=0 check="is_done_other"; + local i=0 check="is_done_other" if [ -e /run/systemd ]; then check=is_done_systemd elif [ -x /usr/bin/cloud-init ]; then check=is_done_cloudinit fi [ "$debug" != "0" ] && debug 1 "check=$check" - while ! $check $i && i=$((i+1)); do + while ! $check $i && i=$((i + 1)); do [ "$i" -ge "$max" ] && exit 1 [ "$debug" = "0" ] || echo -n . sleep 1 done if [ "$debug" != "0" ]; then - read up _ /dev/null && system_up=true && break + inside "$name" true 2> /dev/null && system_up=true && break done - [ $system_up == true ] || { errorrc "exec command inside $name failed."; return; } + [ $system_up == true ] || { + errorrc "exec command inside $name failed." + return + } get_os_info_in "$name" [ "$OS_NAME" = "debian" ] && wtime=300 && debug 1 "on debian we wait for ${wtime}s" debug 1 "waiting for boot of $name" run_self_inside "$name" wait_inside "$name" "$wtime" "$VERBOSITY" || - { errorrc "wait inside $name failed."; return; } + { + errorrc "wait inside $name failed." + return + } if [ -n "${http_proxy-}" ]; then if [ "$OS_NAME" = "centos" -o "$OS_NAME" = "fedora" ]; then @@ -364,21 +398,21 @@ wait_for_boot() { inside "$name" sh -c "sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" - if [ "$OS_NAME" = "centos" ]; then - CENTOS_REPO="/etc/yum.repos.d/centos.repo" - inside "$name" sh -c "grep -q baseurl $CENTOS_REPO" - if [ $? -eq 1 ]; then - # CentOS 9 does not provide baseurl definitions - inside "$name" sh -c "sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO}" - inside "$name" sh -c "sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO}" - inside "$name" sh -c "sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO}" + if [ "$OS_NAME" = "centos" ]; then + CENTOS_REPO="/etc/yum.repos.d/centos.repo" + inside "$name" sh -c "grep -q baseurl $CENTOS_REPO" + if [ $? -eq 1 ]; then + # CentOS 9 does not provide baseurl definitions + inside "$name" sh -c "sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO}" CENTOS_EXTRAS_REPO="/etc/yum.repos.d/centos-addons.repo" - inside "$name" sh -c "sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO}" - inside "$name" sh -c "dnf install -y 'dnf-command(config-manager)'" - inside "$name" sh -c "dnf config-manager --set-enabled crb" - inside "$name" sh -c "dnf config-manager --set-disabled epel-cisco-openh264" || true + inside "$name" sh -c "sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO}" + inside "$name" sh -c "dnf install -y 'dnf-command(config-manager)'" + inside "$name" sh -c "dnf config-manager --set-enabled crb" + inside "$name" sh -c "dnf config-manager --set-disabled epel-cisco-openh264" || true fi - fi + fi else debug 1 "do not know how to configure proxy on $OS_NAME" fi @@ -391,7 +425,7 @@ start_instance() { launch_flags=() [ "$use_vm" == true ] && launch_flags+=(--vm) $LXC launch "$src" "$name" "${launch_flags[@]}" || { - errorrc "Failed to start container '$name' from '$src'"; + errorrc "Failed to start container '$name' from '$src'" return } CONTAINER=$name @@ -407,13 +441,13 @@ run_self_inside() { # run_self_inside(container, args) local name="$1" shift - inside "$name" bash -s "$@" <"$0" + inside "$name" bash -s "$@" < "$0" } run_self_inside_as_cd() { local name="$1" user="$2" dir="$3" shift 3 - inside_as_cd "$name" "$user" "$dir" bash -s "$@" <"$0" + inside_as_cd "$name" "$user" "$dir" bash -s "$@" < "$0" } main() { @@ -423,7 +457,10 @@ main() { getopt_out=$(getopt --name "${0##*/}" \ --options "${short_opts}" --long "${long_opts}" -- "$@") && eval set -- "${getopt_out}" || - { bad_Usage; return; } + { + bad_Usage + return + } local cur="" next="" local package=false srcpackage=false unittest="" name="" @@ -431,31 +468,50 @@ main() { local use_vm=false while [ $# -ne 0 ]; do - cur="${1:-}"; next="${2:-}"; + cur="${1:-}" + next="${2:-}" case "$cur" in - -a|--artifacts) artifact_d="$next";; - --dirty) dirty=true;; - -h|--help) Usage ; exit 0;; - -k|--keep) KEEP=true;; - -n|--name) name="$next"; shift;; - -p|--package) package=true;; - -s|--source-package) srcpackage=true;; - -u|--unittest) unittest=1;; - -v|--verbose) VERBOSITY=$((VERBOSITY+1));; - --vm) use_vm=true;; - --wait-max) WAIT_MAX="$next"; shift;; - --commitish) COMMITISH="$next"; shift;; - --) shift; break;; + -a | --artifacts) artifact_d="$next" ;; + --dirty) dirty=true ;; + -h | --help) + Usage + exit 0 + ;; + -k | --keep) KEEP=true ;; + -n | --name) + name="$next" + shift + ;; + -p | --package) package=true ;; + -s | --source-package) srcpackage=true ;; + -u | --unittest) unittest=1 ;; + -v | --verbose) VERBOSITY=$((VERBOSITY + 1)) ;; + --vm) use_vm=true ;; + --wait-max) + WAIT_MAX="$next" + shift + ;; + --commitish) + COMMITISH="$next" + shift + ;; + --) + shift + break + ;; esac - shift; + shift done COMMITISH=${COMMITISH:-HEAD} - [ $# -eq 1 ] || { bad_Usage "Expected 1 arg, got $# ($*)"; return; } + [ $# -eq 1 ] || { + bad_Usage "Expected 1 arg, got $# ($*)" + return + } local img_ref_in="$1" case "${img_ref_in}" in - *:*) img_ref="${img_ref_in}";; - *) img_ref="images:${img_ref_in}";; + *:*) img_ref="${img_ref_in}" ;; + *) img_ref="images:${img_ref_in}" ;; esac # program starts here @@ -476,18 +532,30 @@ main() { trap cleanup EXIT start_instance "$img_ref" "$name" "$use_vm" || - { errorrc "Failed to start container for $img_ref"; return; } + { + errorrc "Failed to start container for $img_ref" + return + } get_os_info_in "$name" || - { errorrc "failed to get os_info in $name"; return; } + { + errorrc "failed to get os_info in $name" + return + } # prep the container (install very basic dependencies) run_self_inside "$name" prep || - { errorrc "Failed to prep container $name"; return; } + { + errorrc "Failed to prep container $name" + return + } # add the user inside "$name" useradd "$user" --create-home "--home-dir=$home" || - { errorrc "Failed to add user '$user' in '$name'"; return 1; } + { + errorrc "Failed to add user '$user' in '$name'" + return 1 + } debug 1 "inserting cloud-init" inject_cloud_init "$name" "$user" "$dirty" || { @@ -501,36 +569,38 @@ main() { return } - local errors=( ) + local errors=() inside_as_cd "$name" "$user" "$cdir" git status || { errorrc "git checkout failed." - errors[${#errors[@]}]="git checkout"; + errors[${#errors[@]}]="git checkout" } if [ -n "$unittest" ]; then debug 1 "running unit tests." run_self_inside_as_cd "$name" "$user" "$cdir" pytest \ tests/unittests cloudinit/ || { - errorrc "pytest failed."; - errors[${#errors[@]}]="pytest" - } + errorrc "pytest failed." + errors[${#errors[@]}]="pytest" + } fi local build_pkg="" build_srcpkg="" pkg_ext="" distflag="" case "$OS_NAME" in - centos|rocky*|fedora) distflag="--distro=redhat";; - opensuse*) distflag="--distro=suse";; + centos | rocky* | fedora) distflag="--distro=redhat" ;; + opensuse*) distflag="--distro=suse" ;; esac case "$OS_NAME" in - debian|ubuntu) - build_pkg="./packages/bddeb -d" + debian | ubuntu) + build_pkg="./packages/bddeb -d" build_srcpkg="./packages/bddeb -S -d" - pkg_ext=".deb";; - centos|opensuse*|rocky*|fedora) + pkg_ext=".deb" + ;; + centos | opensuse* | rocky* | fedora) build_pkg="./packages/brpm $distflag" build_srcpkg="./packages/brpm $distflag --srpm" - pkg_ext=".rpm";; + pkg_ext=".rpm" + ;; esac if [ "$srcpackage" = "true" ]; then [ -n "$build_srcpkg" ] || { @@ -540,7 +610,7 @@ main() { debug 1 "building source package with $build_srcpkg." # shellcheck disable=SC2086 inside_as_cd "$name" "$user" "$cdir" python3 $build_srcpkg || { - errorrc "failed: $build_srcpkg"; + errorrc "failed: $build_srcpkg" errors[${#errors[@]}]="source package" } fi @@ -553,7 +623,7 @@ main() { debug 1 "building binary package with $build_pkg." # shellcheck disable=SC2086 inside_as_cd "$name" "$user" "$cdir" python3 $build_pkg || { - errorrc "failed: $build_pkg"; + errorrc "failed: $build_pkg" errors[${#errors[@]}]="binary package" } fi @@ -588,6 +658,10 @@ main() { } case "${1:-}" in - prep|os_info|wait_inside|pytest) _n=$1; shift; "$_n" "$@";; - *) main "$@";; + prep | os_info | wait_inside | pytest) + _n=$1 + shift + "$_n" "$@" + ;; + *) main "$@" ;; esac diff --git a/tools/run-lint b/tools/run-lint deleted file mode 100755 index 2bd0ab17ac9..00000000000 --- a/tools/run-lint +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This file runs flake8 for compatibility's sake. As soon as we move off python 3.6, this should be changed to use ruff. - -CR=" -" -pycheck_dirs=( "cloudinit/" "tests/" "tools/" "setup.py" ) - -set -f -if [ $# -eq 0 ]; then - files=( "${pycheck_dirs[@]}" ) -else - files=( "$@" ) -fi - -if [ -z "$PYTHON" ]; then - PYTHON="python3" -fi -cmd=( "$PYTHON" -m "flake8" "${files[@]}" ) - -echo "Running: " "${cmd[@]}" 1>&2 -exec "${cmd[@]}" diff --git a/tools/tox-venv b/tools/tox-venv deleted file mode 100755 index c22f6faca4d..00000000000 --- a/tools/tox-venv +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/sh -# https://gist.github.com/smoser/2d4100a6a5d230ca937f - -CR=' -' -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } -get_env_dirs() { - # read 'tox --showconfig'. return list of - # envname:dir - local key="" equal="" val="" curenv="" out="" - while read key equal val; do - case "$key" in - "[testenv:"*) - curenv=${key#*:}; - curenv=${curenv%%"]"*}; - continue;; - esac - if [ "${key#*=}" != "$key" ]; then - # older tox shows key=value or key= value - # newer tox shows: key = value - key=${key%%=*} - val=${equal} - fi - [ "$key" = "envdir" ] || [ "$key" = "env_dir" ] || continue - out="${out:+${out}${CR}}${curenv}:$val" - done - echo "$out" -} - -load_config() { - local tox_ini="$1" out="" envs="" - if [ "$tox_ini" = "${CACHED_ENVS_INI}" ]; then - _RET="$CACHED_ENVS" - return - fi - out=$(tox -c "$tox_ini" --showconfig) || return 1 - envs=$(echo "$out" | get_env_dirs) || return 1 - CACHED_ENVS="$envs" - CACHED_ENVS_INI="$tox_ini" - _RET="$envs" -} - -list_environments() { - local tox_ini="$1" prefix=" " out="" envs="" oifs="$IFS" - load_config "$tox_ini" || return 1 - envs="${_RET}" - IFS="$CR" - for d in ${envs}; do - env=${d%%:*} - dir=${d#*:} - [ -f "$dir/bin/activate" ] && s="*" || s="" - echo "${prefix}$env$s"; - done - IFS="$oifs" -} - -get_command() { - local tox_ini="$1" env="$2" out="" - shift 2 - out=$( - sed -e ':x; /\\$/ { N; s/\\\n[ ]*//; tx };' "${tox_ini}" | - gawk ' - $1 ~ /^\[testenv.*\]/ { - name=$1; - sub("\\[", "", name); sub(".*:", "", name); - sub("].*", "", name); - curenv=name; }; - $1 == "basepython" && (name == "testenv" || name == n) { python=$3 } - $1 == "commands" && (name == "testenv" || name == n) { - sub("commands = ", ""); cmd = $0; }; - END { - sub("{envpython}", python, cmd); - sub("{toxinidir}", toxinidir, cmd); - if (inargs == "") replacement = "\\1" - else replacement = inargs - cmd = gensub(/{posargs:?([^}]*)}/, replacement, "global", cmd) - print(cmd); - }' n="$env" toxinidir="$(dirname $tox_ini)" inargs="$*") - if [ -z "$out" ]; then - error "Failed to find command for $env in $tox_ini" - return 1 - fi - echo "$out" -} - -get_env_dir() { - local tox_ini="$1" env="$2" oifs="$IFS" t="" d="" envs="" - if [ "${TOX_VENV_SHORTCUT:-1}" != "0" ]; then - local stox_d="${tox_ini%/*}/.tox/${env}" - if [ -e "${stox_d}/bin/activate" ]; then - _RET="${stox_d}" - return - fi - fi - load_config "$tox_ini" && envs="$_RET" || return 1 - IFS="$CR" - for t in $envs; do - [ "$env" = "${t%%:*}" ] && d="${t#*:}" && break - done - IFS=${oifs} - [ -n "$d" ] || return 1 - _RET="$d" -} - -Usage() { - local tox_ini="$1" - cat <&2; exit 1; } -[ "$1" = "-h" -o "$1" = "--help" ] && { Usage "$tox_ini"; exit 0; } - -[ -f "$tox_ini" ] || fail "$tox_ini: did not find tox.ini" - -if [ "$1" = "-l" -o "$1" = "--list" ]; then - list_environments "$tox_ini" - exit -fi - -nocreate="false" -if [ "$1" = "--no-create" ]; then - nocreate="true" - shift -fi - -env="$1" -shift -[ "$1" = "--" ] && shift -get_env_dir "$tox_ini" "$env" && activate="$_RET/bin/activate" || activate="" - -if [ -z "$activate" -o ! -f "$activate" ]; then - if $nocreate; then - fail "tox env '$env' did not exist, and no-create specified" - elif [ -n "$activate" ]; then - error "attempting to create $env:" - error " tox -c $tox_ini --recreate --notest -e $env" - tox -c "$tox_ini" --recreate --notest -e "$env" || - fail "failed creation of env $env" - else - error "$env: not a valid tox environment?" - error "found tox_ini=$tox_ini" - error "try one of:" - list_environments "$tox_ini" 1>&2 - fail - fi -fi -. "$activate" - -[ $# -eq 0 ] && set -- cmd -if [ "$1" = "cmd" -o "$1" = "-" ]; then - shift - out=$(get_command "$tox_ini" "$env" "$@") || exit - eval set -- "$out" -fi -echo "inside tox:$env running: $*" 1>&2 -debian_chroot="tox:$env" exec "$@" diff --git a/tools/uncloud-init b/tools/uncloud-init index 7900ab73229..68dfa3a6b14 100755 --- a/tools/uncloud-init +++ b/tools/uncloud-init @@ -14,122 +14,156 @@ MARK=/var/lib/cloud/sem/uncloud-init.once ROOT_RW="" doexec() { - if [ -n "$ROOT_RW" ]; then - mkdir -p "${MARK%/*}"; - date > "${MARK}"; - fi - cleanup; - log "invoking /sbin/init $*" - exec /sbin/init "$@"; + if [ -n "$ROOT_RW" ]; then + mkdir -p "${MARK%/*}" + date > "${MARK}" + fi + cleanup + log "invoking /sbin/init $*" + exec /sbin/init "$@" } log() { echo "::${0##*/}:" "$@"; } cleanup() { - [ -z "${UMOUNT}" ] || { umount "${UMOUNT}" && unset UMOUNT; } - [ -z "${RMDIR}" ] || { rm -Rf "${RMDIR}" && unset RMDIR; } - [ -z "${ROOT_RW}" ] || { mount -o remount,ro / ; unset ROOT_RW; } + [ -z "${UMOUNT}" ] || { umount "${UMOUNT}" && unset UMOUNT; } + [ -z "${RMDIR}" ] || { rm -Rf "${RMDIR}" && unset RMDIR; } + [ -z "${ROOT_RW}" ] || { + mount -o remount,ro / + unset ROOT_RW + } } updateFrom() { - local dev=$1 fmt=$2 - local mp=""; - - [ "${fmt}" = "tar" -o "${fmt}" = "mnt" ] || - { log FAIL "unknown format ${fmt}"; return 1; } - - log INFO "updating from ${dev} format ${fmt}" - [ ! -e "${dev}" -a -e "/dev/${dev}" ] && dev="/dev/${dev}" - [ -e "${dev}" ] || { echo "no file $dev"; return 2; } - - mp=$(mktemp -d "${TEMPDIR:-/tmp}/update.XXXXXX") && - RMDIR="${mp}" || - { log FAIL "failed to mktemp"; return 1; } - - if [ "$fmt" = "tar" ]; then - dd "if=${dev}" | ( tar -C "${mp}" -xf - ) || - { log FAIL "failed to extract ${dev}"; return 1; } - elif [ "$fmt" = "mnt" ]; then - mount -o ro "${dev}" "${mp}" && UMOUNT=${mp} || - { log FAIL "failed mount ${mp}"; return 1; } - else - log FAIL "unknown format ${fmt}"; return 1; - fi - - if [ -d "${mp}/updates" ]; then - rsync -av "${mp}/updates/" "/" || - { log FAIL "failed rsync updates/ /"; return 1; } - fi - if [ -f "${mp}/updates.tar" ]; then - tar -C / -xvf "${mp}/updates.tar" || - { log FAIL "failed tar -C / -xvf ${mp}/updates.tar"; return 1; } - fi - script="${mp}/updates.script" - if [ -f "${script}" -a -x "${script}" ]; then - MP_DIR=${mp} "${mp}/updates.script" || - { log FAIL "failed to run updates.script"; return 1; } - fi + local dev=$1 fmt=$2 + local mp="" + + [ "${fmt}" = "tar" -o "${fmt}" = "mnt" ] || + { + log FAIL "unknown format ${fmt}" + return 1 + } + + log INFO "updating from ${dev} format ${fmt}" + [ ! -e "${dev}" -a -e "/dev/${dev}" ] && dev="/dev/${dev}" + [ -e "${dev}" ] || { + echo "no file $dev" + return 2 + } + + mp=$(mktemp -d "${TEMPDIR:-/tmp}/update.XXXXXX") && + RMDIR="${mp}" || + { + log FAIL "failed to mktemp" + return 1 + } + + if [ "$fmt" = "tar" ]; then + dd "if=${dev}" | (tar -C "${mp}" -xf -) || + { + log FAIL "failed to extract ${dev}" + return 1 + } + elif [ "$fmt" = "mnt" ]; then + mount -o ro "${dev}" "${mp}" && UMOUNT=${mp} || + { + log FAIL "failed mount ${mp}" + return 1 + } + else + log FAIL "unknown format ${fmt}" + return 1 + fi + + if [ -d "${mp}/updates" ]; then + rsync -av "${mp}/updates/" "/" || + { + log FAIL "failed rsync updates/ /" + return 1 + } + fi + if [ -f "${mp}/updates.tar" ]; then + tar -C / -xvf "${mp}/updates.tar" || + { + log FAIL "failed tar -C / -xvf ${mp}/updates.tar" + return 1 + } + fi + script="${mp}/updates.script" + if [ -f "${script}" -a -x "${script}" ]; then + MP_DIR=${mp} "${mp}/updates.script" || + { + log FAIL "failed to run updates.script" + return 1 + } + fi } -fail() { { [ $# -eq 0 ] && log "FAILING" ; } || log "$@"; exit 1; } +fail() { + { [ $# -eq 0 ] && log "FAILING"; } || log "$@" + exit 1 +} -[ -s "$MARK" ] && { log "already updated" ; doexec "$@"; } +[ -s "$MARK" ] && { + log "already updated" + doexec "$@" +} mount -o remount,rw / || fail "failed to mount rw" ROOT_RW=1 if [ ! -e /proc/cmdline ]; then - mount -t proc /proc /proc - read cmdline < /proc/cmdline - umount /proc + mount -t proc /proc /proc + read cmdline < /proc/cmdline + umount /proc else - read cmdline < /proc/cmdline + read cmdline < /proc/cmdline fi ubuntu_pass="" for x in ${cmdline}; do - case "$x" in - ${KEY}=*) - val=${x#${KEY}=} - dev=${val%:*} - [ "${dev}" = "${val}" ] && fmt="" || fmt=${val#${dev}:} - log "update from ${dev},${fmt}" - updateFrom "${dev}" "${fmt}" || fail "update failed" - log "end update ${dev},${fmt}" - ;; - ubuntu-pass=*|ubuntu_pass=*) ubuntu_pass=${x#*=};; - helpmount) helpmount=1;; - root=*) rootspec=${x#root=};; - esac + case "$x" in + ${KEY}=*) + val=${x#${KEY}=} + dev=${val%:*} + [ "${dev}" = "${val}" ] && fmt="" || fmt=${val#${dev}:} + log "update from ${dev},${fmt}" + updateFrom "${dev}" "${fmt}" || fail "update failed" + log "end update ${dev},${fmt}" + ;; + ubuntu-pass=* | ubuntu_pass=*) ubuntu_pass=${x#*=} ;; + helpmount) helpmount=1 ;; + root=*) rootspec=${x#root=} ;; + esac done if [ "${ubuntu_pass}" = "R" -o "${ubuntu_pass}" = "random" ]; then - ubuntu_pass=$(python -c 'import string, random; + ubuntu_pass=$(python -c 'import string, random; random.seed(); print "".join(random.sample(string.letters+string.digits, 8))') - log "setting ubuntu pass = ${ubuntu_pass}" - printf "\n===\nubuntu_pass = %s\n===\n" "${ubuntu_pass}" >/dev/ttyS0 + log "setting ubuntu pass = ${ubuntu_pass}" + printf "\n===\nubuntu_pass = %s\n===\n" "${ubuntu_pass}" > /dev/ttyS0 fi [ -z "${ubuntu_pass}" ] || - printf "ubuntu:%s\n" "${ubuntu_pass}" > /root/ubuntu-user-pass + printf "ubuntu:%s\n" "${ubuntu_pass}" > /root/ubuntu-user-pass if [ -e /root/ubuntu-user-pass ]; then - log "changing ubuntu user's password!" - chpasswd < /root/ubuntu-user-pass || - log "FAIL: failed changing pass" + log "changing ubuntu user's password!" + chpasswd < /root/ubuntu-user-pass || + log "FAIL: failed changing pass" fi cp /etc/init/tty2.conf /etc/init/ttyS0.conf && - sed -i s,tty2,ttyS0,g /etc/init/ttyS0.conf 2>/dev/null && - log "enabled console on ttyS0" + sed -i s,tty2,ttyS0,g /etc/init/ttyS0.conf 2> /dev/null && + log "enabled console on ttyS0" pa=PasswordAuthentication -sed -i "s,${pa} no,${pa} yes," /etc/ssh/sshd_config 2>/dev/null && - log "enabled passwd auth in ssh" || - log "failed to enable passwd ssh" +sed -i "s,${pa} no,${pa} yes," /etc/ssh/sshd_config 2> /dev/null && + log "enabled passwd auth in ssh" || + log "failed to enable passwd ssh" grep -q vga16fb /etc/modprobe.d/blacklist.conf || { - echo "blacklist vga16fb" >> /etc/modprobe.d/blacklist.conf && - log "blacklisted vga16fb" + echo "blacklist vga16fb" >> /etc/modprobe.d/blacklist.conf && + log "blacklisted vga16fb" } #lstr="${rootspec}" diff --git a/tools/write-ssh-key-fingerprints b/tools/write-ssh-key-fingerprints index 9409257dba0..55711d1da24 100755 --- a/tools/write-ssh-key-fingerprints +++ b/tools/write-ssh-key-fingerprints @@ -1,7 +1,6 @@ #!/bin/sh # This file is part of cloud-init. See LICENSE file for license information. - do_syslog() { log_message=$1 @@ -17,7 +16,6 @@ do_syslog() { logger $logger_opts "$log_message" } - # Redirect stderr to stdout exec 2>&1 diff --git a/tools/xkvm b/tools/xkvm deleted file mode 100755 index b030dc43f05..00000000000 --- a/tools/xkvm +++ /dev/null @@ -1,707 +0,0 @@ -#!/bin/bash -# This file is part of cloud-init. -# See LICENSE file for copyright and license info. - -set -f - -VERBOSITY=0 -KVM_PID="" -DRY_RUN=false -TEMP_D="" -DEF_BRIDGE="virbr0" -TAPDEVS=( ) -# OVS_CLEANUP gets populated with bridge:devname pairs used with ovs -OVS_CLEANUP=( ) -MAC_PREFIX="52:54:00:12:34" -# allow this to be set externally. -_QEMU_SUPPORTS_FILE_LOCKING="${_QEMU_SUPPORTS_FILE_LOCKING}" -KVM="kvm" -declare -A KVM_DEVOPTS - -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } - -bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } -randmac() { - # return random mac addr within final 3 tokens - local random="" - random=$(printf "%02x:%02x:%02x" \ - "$((${RANDOM}%256))" "$((${RANDOM}%256))" "$((${RANDOM}%256))") - padmac "$random" -} - -cleanup() { - [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" - [ -z "${KVM_PID}" ] || kill "$KVM_PID" - if [ ${#TAPDEVS[@]} -ne 0 ]; then - local name item - for item in "${TAPDEVS[@]}"; do - [ "${item}" = "skip" ] && continue - debug 1 "removing" "$item" - name="${item%:*}" - if $DRY_RUN; then - error ip tuntap del mode tap "$name" - else - ip tuntap del mode tap "$name" - fi - [ $? -eq 0 ] || error "failed removal of $name" - done - if [ ${#OVS_CLEANUP[@]} -ne 0 ]; then - # with linux bridges, there seems to be no harm in just deleting - # the device (not detaching from the bridge). However, with - # ovs, you have to remove them from the bridge, or later it - # will refuse to add the same name. - error "cleaning up ovs ports: ${OVS_CLEANUP[@]}" - if ${DRY_RUN}; then - error sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" - else - sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" - fi - fi - fi -} - -debug() { - local level=${1}; shift; - [ "${level}" -gt "${VERBOSITY}" ] && return - error "${@}" -} - -Usage() { - cat <&1) && - out=$(echo "$out" | sed -e "s,[^.]*[.],," -e 's,=.*,,') && - KVM_DEVOPTS[$model]="$out" || - { error "bad device model $model?"; exit 1; } - fi - opts=( ${KVM_DEVOPTS[$model]} ) - for opt in "${opts[@]}"; do - [ "$input" = "$opt" ] && return 0 - done - return 1 -} - -qemu_supports_file_locking() { - # hackily check if qemu has file.locking in -drive params (LP: #1716028) - if [ -z "$_QEMU_SUPPORTS_FILE_LOCKING" ]; then - # The only way we could find to check presence of file.locking is - # qmp (query-qmp-schema). Simply checking if the virtio-blk driver - # supports 'share-rw' is expected to be equivalent and simpler. - isdevopt virtio-blk share-rw && - _QEMU_SUPPORTS_FILE_LOCKING=true || - _QEMU_SUPPORTS_FILE_LOCKING=false - debug 1 "qemu supports file locking = ${_QEMU_SUPPORTS_FILE_LOCKING}" - fi - [ "$_QEMU_SUPPORTS_FILE_LOCKING" = "true" ] - return -} - -padmac() { - # return a full mac, given a subset. - # assume whatever is input is the last portion to be - # returned, and fill it out with entries from MAC_PREFIX - local mac="$1" num="$2" prefix="${3:-$MAC_PREFIX}" itoks="" ptoks="" - # if input is empty set to :$num - [ -n "$mac" ] || mac=$(printf "%02x" "$num") || return - itoks=( ${mac//:/ } ) - ptoks=( ${prefix//:/ } ) - rtoks=( ) - for r in ${ptoks[@]:0:6-${#itoks[@]}} ${itoks[@]}; do - rtoks[${#rtoks[@]}]="0x$r" - done - _RET=$(printf "%02x:%02x:%02x:%02x:%02x:%02x" "${rtoks[@]}") -} - -make_nics_Usage() { - cat <: for each tap created - # type is one of "ovs" or "brctl" - local short_opts="v" - local long_opts="--verbose" - local getopt_out="" - getopt_out=$(getopt --name "${0##*/} make-nics" \ - --options "${short_opts}" --long "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || { make_nics_Usage 1>&2; return 1; } - - local cur="" next="" - while [ $# -ne 0 ]; do - cur=${1}; next=${2}; - case "$cur" in - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - --) shift; break;; - esac - shift; - done - - [ $# -ne 0 ] || { - make_nics_Usage 1>&2; error "must give bridge"; - return 1; - } - - local owner="" ovsbrs="" tap="" tapnum="0" brtype="" bridge="" - [ "$(id -u)" = "0" ] || { error "must be root for make-nics"; return 1; } - owner="${SUDO_USER:-root}" - ovsbrs="" - if command -v ovs-vsctl >/dev/null 2>&1; then - out=$(ovs-vsctl list-br) - out=$(echo "$out" | sed "s/\n/,/") - ovsbrs=",$out," - fi - for bridge in "$@"; do - [ "$bridge" = "user" ] && echo skip && continue - [ "${ovsbrs#*,${bridge},}" != "$ovsbrs" ] && - btype="ovs" || btype="brctl" - tapnum=0; - while [ -e /sys/class/net/tapvm$tapnum ]; do tapnum=$(($tapnum+1)); done - tap="tapvm$tapnum" - debug 1 "creating $tap:$btype on $bridge" 1>&2 - ip tuntap add mode tap user "$owner" "$tap" || - { error "failed to create tap '$tap' for '$owner'"; return 1; } - ip link set "$tap" up 1>&2 || { - error "failed to bring up $tap"; - ip tuntap del mode tap "$tap"; - return 1; - } - if [ "$btype" = "ovs" ]; then - ovs-vsctl add-port "$bridge" "$tap" 1>&2 || { - error "failed: ovs-vsctl add-port $bridge $tap"; - ovs-vsctl del-port "$bridge" "$tap" - return 1; - } - else - ip link set "$tap" master "$bridge" 1>&2 || { - error "failed to add tap '$tap' to '$bridge'" - ip tuntap del mode tap "$tap"; - return 1 - } - fi - echo "$tap:$btype" - done -} - -ovs_cleanup() { - [ "$(id -u)" = "0" ] || - { error "must be root for ovs-cleanup"; return 1; } - local item="" errors=0 - # TODO: if get owner (SUDO_USERNAME) and if that isn't - # the owner, then do not delete. - for item in "$@"; do - name=${item#*:} - bridge=${item%:*} - ovs-vsctl del-port "$bridge" "$name" || errors=$((errors+1)) - done - return $errors -} - -quote_cmd() { - local quote='"' x="" vline="" - for x in "$@"; do - if [ "${x#* }" != "${x}" ]; then - if [ "${x#*$quote}" = "${x}" ]; then - x="\"$x\"" - else - x="'$x'" - fi - fi - vline="${vline} $x" - done - echo "$vline" -} - -get_bios_opts() { - # get_bios_opts(bios, uefi, nvram) - # bios is a explicit bios to boot. - # uefi is boolean indicating uefi - # nvram is optional and indicates that ovmf vars should be copied - # to that file if it does not exist. if it exists, use it. - local bios="$1" uefi="${2:-false}" nvram="$3" - local ovmf_dir="/usr/share/OVMF" - local bios_opts="" pflash_common="if=pflash,format=raw" - unset _RET - _RET=( ) - if [ -n "$bios" ]; then - _RET=( -drive "${pflash_common},file=$bios" ) - return 0 - elif ! $uefi; then - return 0 - fi - - # ovmf in older releases (14.04) shipped only a single file - # /usr/share/ovmf/OVMF.fd - # newer ovmf ships split files - # /usr/share/OVMF/OVMF_CODE.fd - # /usr/share/OVMF/OVMF_VARS.fd - # with single file, pass only one file and read-write - # with split, pass code as readonly and vars as read-write - local joined="/usr/share/ovmf/OVMF.fd" - local code="/usr/share/OVMF/OVMF_CODE.fd" - local vars="/usr/share/OVMF/OVMF_VARS.fd" - local split="" nvram_src="" - if [ -e "$code" -o -e "$vars" ]; then - split=true - nvram_src="$vars" - elif [ -e "$joined" ]; then - split=false - nvram_src="$joined" - elif [ -n "$nvram" -a -e "$nvram" ]; then - error "WARN: nvram given, but did not find expected ovmf files." - error " assuming this is code and vars (OVMF.fd)" - split=false - else - error "uefi support requires ovmf bios: apt-get install -qy ovmf" - return 1 - fi - - if [ -n "$nvram" ]; then - if [ ! -f "$nvram" ]; then - cp "$nvram_src" "$nvram" || - { error "failed copy $nvram_src to $nvram"; return 1; } - debug 1 "copied $nvram_src to $nvram" - fi - else - debug 1 "uefi without --uefi-nvram storage." \ - "nvram settings likely will not persist." - nvram="${nvram_src}" - fi - - if [ ! -w "$nvram" ]; then - debug 1 "nvram file ${nvram} is readonly" - nvram_ro="readonly" - fi - - if $split; then - # to ensure bootability firmware must be first, then variables - _RET=( -drive "${pflash_common},file=$code,readonly" ) - fi - _RET=( "${_RET[@]}" - -drive "${pflash_common},file=$nvram${nvram_ro:+,${nvram_ro}}" ) -} - -main() { - local short_opts="hd:n:v" - local long_opts="bios:,help,dowait,disk:,dry-run,kvm:,no-dowait,netdev:,uefi,uefi-nvram:,verbose" - local getopt_out="" - getopt_out=$(getopt --name "${0##*/}" \ - --options "${short_opts}" --long "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || { bad_Usage; return 1; } - - local bridge="$DEF_BRIDGE" oifs="$IFS" - local netdevs="" need_tap="" ret="" p="" i="" pt="" cur="" conn="" - local kvm="" kvmcmd="" archopts="" - local def_disk_driver=${DEF_DISK_DRIVER:-"virtio-blk"} - local def_netmodel=${DEF_NETMODEL:-"virtio-net-pci"} - local bios="" uefi=false uefi_nvram="" - - archopts=( ) - kvmcmd=( ) - netdevs=( ) - addargs=( ) - diskdevs=( ) - diskargs=( ) - - # dowait: run qemu-system with a '&' and then 'wait' on the pid. - # the reason to do this or not do this has to do with interactivity - # if detached with &, then user input will not go to xkvm. - # if *not* detached, then signal handling is blocked until - # the foreground subprocess returns. which means we can't handle - # a sigterm and kill the qemu-system process. - # We default to dowait=false if input and output are a terminal - local dowait="" - [ -t 0 -a -t 1 ] && dowait=false || dowait=true - while [ $# -ne 0 ]; do - cur=${1}; next=${2}; - case "$cur" in - -h|--help) Usage; exit 0;; - -d|--disk) - diskdevs[${#diskdevs[@]}]="$next"; shift;; - --dry-run) DRY_RUN=true;; - --kvm) kvm="$next"; shift;; - -n|--netdev) - netdevs[${#netdevs[@]}]=$next; shift;; - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - --dowait) dowait=true;; - --no-dowait) dowait=false;; - --bios) bios="$next"; shift;; - --uefi) uefi=true;; - --uefi-nvram) uefi=true; uefi_nvram="$next"; shift;; - --) shift; break;; - esac - shift; - done - - [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" ) - pt=( "$@" ) - - local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" virtio_rng_device="virtio-rng-pci" - [ -n "$kvm" ] && kvm_pkg="none" - case $(uname -m) in - i?86) - [ -n "$kvm" ] || - { kvm="qemu-system-i386"; kvm_pkg="qemu-system-x86"; } - ;; - x86_64) - [ -n "$kvm" ] || - { kvm="qemu-system-x86_64"; kvm_pkg="qemu-system-x86"; } - ;; - s390x) - [ -n "$kvm" ] || - { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; } - def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"} - # disable virtio-scsi-bus - virtio_scsi_bus="virtio-scsi-ccw" - virtio_blk_bus="virtio-blk-ccw" - virtio_rng_device="virtio-rng-ccw" - ;; - ppc64*) - [ -n "$kvm" ] || - { kvm="qemu-system-ppc64"; kvm_pkg="qemu-system-ppc"; } - def_netmodel="virtio-net-pci" - # virtio seems functional on in 14.10, but might want scsi here - #def_diskif="scsi" - archopts=( "${archopts[@]}" -machine pseries,usb=off ) - archopts=( "${archopts[@]}" -device spapr-vscsi ) - ;; - *) kvm=qemu-system-$(uname -m);; - esac - KVM="$kvm" - kvmcmd=( $kvm -enable-kvm ) - - local bios_opts="" - if [ -n "$bios" ] && $uefi; then - error "--uefi (or --uefi-nvram) is incompatible with --bios" - return 1 - fi - get_bios_opts "$bios" "$uefi" "$uefi_nvram" || - { error "failed to get bios opts"; return 1; } - bios_opts=( "${_RET[@]}" ) - - local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts="" - local busorindex="" driveopts="" cur="" val="" file="" wwn="" - for((i=0;i<${#diskdevs[@]};i++)); do - cur=${diskdevs[$i]} - IFS=","; set -- $cur; IFS="$oifs" - driver="" - id=$(printf "disk%02d" "$i") - file="" - fmt="" - bus="" - unit="" - index="" - serial="" - wwn="" - for tok in "$@"; do - [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok" - val=${tok#*=} - case "$tok" in - driver=*) driver=$val;; - if=virtio) driver=virtio-blk;; - if=scsi) driver=scsi-hd;; - if=pflash) driver=;; - if=sd|if=mtd|floppy) fail "do not know what to do with $tok on $cur";; - id=*) id=$val;; - file=*) file=$val;; - fmt=*|format=*) fmt=$val;; - serial=*) serial=$val;; - wwn=*) wwn=$val;; - bus=*) bus=$val;; - unit=*) unit=$val;; - index=*) index=$val;; - esac - done - [ -z "$file" ] && fail "did not read a file from $cur" - if [ -f "$file" -a -z "$fmt" ]; then - out=$(LANG=C qemu-img info "$file") && - fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') || - { error "failed to determine format of $file"; return 1; } - elif [ -z "$fmt" ]; then - fmt=raw - fi - if [ -z "$driver" ]; then - driver="$def_disk_driver" - fi - if [ -z "$serial" ]; then - # use filename as serial if not provided a wwn - if [ -n "$wwn" ]; then - serial="$wwn" - else - serial="${file##*/}" - fi - fi - - # make sure we add either bus= or index= - if [ -n "$bus" -o "$unit" ] && [ -n "$index" ]; then - fail "bus and index cant be specified together: $cur" - elif [ -z "$bus" -a -z "$unit" -a -z "$index" ]; then - index=$i - elif [ -n "$bus" -a -z "$unit" ]; then - unit=$i - fi - - busorindex="${bus:+bus=$bus,unit=$unit}${index:+index=${index}}" - diskopts="file=${file},id=$id,if=none,format=$fmt,$busorindex" - devopts="$driver,drive=$id${serial:+,serial=${serial}}" - for tok in "$@"; do - case "$tok" in - id=*|if=*|driver=*|$file|file=*) continue;; - fmt=*|format=*) continue;; - serial=*|bus=*|unit=*|index=*) continue;; - file.locking=*) - qemu_supports_file_locking || { - debug 2 "qemu has no file locking." \ - "Dropping '$tok' from: $cur" - continue - };; - esac - isdevopt "$driver" "$tok" && devopts="${devopts},$tok" || - diskopts="${diskopts},${tok}" - done - case $driver in - virtio-blk-ccw) - # disable scsi when using virtio-blk-ccw - devopts="${devopts},scsi=off";; - esac - diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" ) - done - - local mnics_vflag="" - for((i=0;i<${VERBOSITY}-1;i++)); do mnics_vflag="${mnics_vflag}v"; done - [ -n "$mnics_vflag" ] && mnics_vflag="-${mnics_vflag}" - - # now go through and split out options - # -device virtio-net-pci,netdev=virtnet0,mac=52:54:31:15:63:02 - # -netdev type=tap,id=virtnet0,vhost=on,script=/etc/kvm/kvm-ifup.br0,downscript=no - local netopts="" devopts="" id="" need_taps=0 model="" - local device_args netdev_args - device_args=( ) - netdev_args=( ) - connections=( ) - for((i=0;i<${#netdevs[@]};i++)); do - id=$(printf "net%02d" "$i") - netopts=""; - devopts="" - # mac=auto is 'unspecified' (let qemu assign one) - mac="auto" - #vhost="off" - - IFS=","; set -- ${netdevs[$i]}; IFS="$oifs" - bridge=$1; shift; - if [ "$bridge" = "user" ]; then - netopts="type=user" - ntype="user" - connections[$i]="user" - else - need_taps=1 - ntype="tap" - netopts="type=tap" - connections[$i]="$bridge" - fi - netopts="${netopts},id=$id" - [ "$ntype" = "tap" ] && netopts="${netopts},script=no,downscript=no" - - model="${def_netmodel}" - for tok in "$@"; do - [ "${tok#model=}" = "${tok}" ] && continue - case "${tok#model=}" in - virtio) model=virtio-net-pci;; - *) model=${tok#model=};; - esac - done - - for tok in "$@"; do - case "$tok" in - mac=*) mac="${tok#mac=}"; continue;; - macaddr=*) mac=${tok#macaddr=}; continue;; - model=*) continue;; - esac - - isdevopt "$model" "$tok" && devopts="${devopts},$tok" || - netopts="${netopts},${tok}" - done - devopts=${devopts#,} - netopts=${netopts#,} - - if [ "$mac" != "auto" ]; then - [ "$mac" = "random" ] && randmac && mac="$_RET" - padmac "$mac" "$i" - devopts="${devopts:+${devopts},}mac=$_RET" - fi - devopts="$model,netdev=$id${devopts:+,${devopts}}" - #netopts="${netopts},vhost=${vhost}" - - device_args[$i]="$devopts" - netdev_args[$i]="$netopts" - done - - trap cleanup EXIT - - reqs=( "$kvm" ) - pkgs=( "$kvm_pkg" ) - for((i=0;i<${#reqs[@]};i++)); do - req=${reqs[$i]} - pkg=${pkgs[$i]} - [ "$pkg" = "none" ] && continue - command -v "$req" >/dev/null || { - missing="${missing:+${missing} }${req}" - missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg" - } - done - if [ -n "$missing" ]; then - local reply cmd="" - cmd=( sudo apt-get --quiet install ${missing_pkgs} ) - error "missing prereqs: $missing"; - error "install them now with the following?: ${cmd[*]}" - read reply && [ "$reply" = "y" -o "$reply" = "Y" ] || - { error "run: apt-get install ${missing_pkgs}"; return 1; } - "${cmd[@]}" || { error "failed to install packages"; return 1; } - fi - - if [ $need_taps -ne 0 ]; then - local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg="" - for i in "${connections[@]}"; do - [ "$i" = "user" -o -e "/sys/class/net/$i" ] || - missing="${missing} $i" - done - [ -z "$missing" ] || { - error "cannot create connection on: ${missing# }." - error "bridges do not exist."; - return 1; - } - error "creating tap devices: ${connections[*]}" - if $DRY_RUN; then - error "sudo $0 tap-control make-nics" \ - $mnics_vflag "${connections[@]}" - taps="" - for((i=0;i<${#connections[@]};i++)); do - if [ "${connections[$i]}" = "user" ]; then - taps="${taps} skip" - else - taps="${taps} dryruntap$i:brctl" - fi - done - else - taps=$(sudo "$0" tap-control make-nics \ - ${mnics_vflag} "${connections[@]}") || - { error "$failed to make-nics ${connections[*]}"; return 1; } - fi - TAPDEVS=( ${taps} ) - for((i=0;i<${#TAPDEVS[@]};i++)); do - cur=${TAPDEVS[$i]} - [ "${cur#*:}" = "ovs" ] || continue - conn=${connections[$i]} - OVS_CLEANUP[${#OVS_CLEANUP[@]}]="${conn}:${cur%:*}" - done - - debug 2 "tapdevs='${TAPDEVS[@]}'" - [ ${#OVS_CLEANUP[@]} -eq 0 ] || error "OVS_CLEANUP='${OVS_CLEANUP[*]}'" - - for((i=0;i<${#TAPDEVS[@]};i++)); do - cur=${TAPDEVS[$i]} - [ "$cur" = "skip" ] && continue - netdev_args[$i]="${netdev_args[$i]},ifname=${cur%:*}"; - done - fi - - netargs=() - for((i=0;i<${#device_args[@]};i++)); do - netargs=( "${netargs[@]}" -device "${device_args[$i]}" - -netdev "${netdev_args[$i]}") - done - - local bus_devices - if [ -n "${virtio_scsi_bus}" ]; then - bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" ) - fi - local rng_devices - rng_devices=( -object "rng-random,filename=/dev/urandom,id=objrng0" - -device "$virtio_rng_device,rng=objrng0,id=rng0" ) - cmd=( "${kvmcmd[@]}" "${archopts[@]}" - "${bios_opts[@]}" - "${bus_devices[@]}" - "${rng_devices[@]}" - "${netargs[@]}" - "${diskargs[@]}" "${pt[@]}" ) - local pcmd=$(quote_cmd "${cmd[@]}") - error "$pcmd" - ${DRY_RUN} && return 0 - - if $dowait; then - "${cmd[@]}" & - KVM_PID=$! - debug 1 "kvm pid=$KVM_PID. my pid=$$" - wait - ret=$? - KVM_PID="" - else - "${cmd[@]}" - ret=$? - fi - return $ret -} - - -if [ "$1" = "tap-control" ]; then - shift - mode=$1 - shift || fail "must give mode to tap-control" - case "$mode" in - make-nics) make_nics "$@";; - ovs-cleanup) ovs_cleanup "$@";; - *) fail "tap mode must be either make-nics or ovs-cleanup";; - esac -else - main "$@" -fi diff --git a/tox.ini b/tox.ini index d4c750463bb..58e1207caf8 100644 --- a/tox.ini +++ b/tox.ini @@ -94,27 +94,40 @@ deps = -r{toxinidir}/integration-requirements.txt {[testenv]deps} {[pinned_versions]deps} +allowlist_externals = + shfmt + sh commands = {envpython} -m ruff check {posargs:.} {envpython} -m pylint {posargs:cloudinit/ tests/ tools/} {envpython} -m black --check {posargs:.} {envpython} -m isort --check-only --diff {posargs:.} {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} + sh -c "{envpython} -m json.tool --indent 2 {[files]schema} | diff -q - {[files]schema} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]version} | diff -q - {[files]version} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]network_v1} | diff -q - {[files]network_v1} > /dev/null" + sh -c "{envpython} -m json.tool --indent 2 {[files]network_v2} | diff -q - {[files]network_v2} > /dev/null" + shfmt --list --case-indent --indent=4 --space-redirects --diff {posargs:./tools/} [testenv:check_format_tip] deps = -r{toxinidir}/integration-requirements.txt {[testenv]deps} {[latest_versions]deps} +allowlist_externals = + shfmt commands = {envpython} -m ruff check {posargs:.} {envpython} -m pylint {posargs:.} {envpython} -m black --check {posargs:.} {envpython} -m isort --check-only --diff {posargs:.} {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} + shfmt --list --case-indent --indent=4 --space-redirects --diff {posargs:./tools/} [testenv:do_format] deps = {[pinned_versions]deps} +allowlist_externals = + shfmt commands = {envpython} -m isort . {envpython} -m black . @@ -122,9 +135,12 @@ commands = {envpython} -m json.tool --indent 2 {[files]version} {[files]version} {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} + shfmt --list --case-indent --indent=4 --space-redirects -w {posargs:./tools/} [testenv:do_format_tip] deps = {[latest_versions]deps} +allowlist_externals = + sh commands = {envpython} -m isort . {envpython} -m black . @@ -132,6 +148,7 @@ commands = {envpython} -m json.tool --indent 2 {[files]version} {[files]version} {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} + shfmt --list --case-indent --indent=4 --space-redirects -w {posargs:./tools/} [testenv:py3] commands = {envpython} -m pytest -vv --cov=cloudinit --cov-branch {posargs:tests/unittests} From 4e410a273852942c3450851b091d0df8944cb7e3 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 25 Mar 2026 12:37:51 -0600 Subject: [PATCH 11/72] ci: fix integration test CLOUD_INIT_LOCAL_LOG_PATH. drop hashFiles (#6804) hashFiles OOMs github actions while attempting to md5sum large sets files under cloudinit_logs. Use `if-no-files-found: ignore` in upload-artifact to avoid errors in absence of cloudinit_logs due to unexpected errors which prevent writing integration test logs. --- .github/workflows/24-pr-integration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/24-pr-integration.yml b/.github/workflows/24-pr-integration.yml index fda996dd63e..9c1c40e8cf0 100644 --- a/.github/workflows/24-pr-integration.yml +++ b/.github/workflows/24-pr-integration.yml @@ -67,11 +67,12 @@ jobs: echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - name: Run integration Tests run: | - CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} LOCAL_LOG_PATH=./cloudinit_logs tox -e integration-tests-ci -- --color=yes tests/integration_tests/ + CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} CLOUD_INIT_LOCAL_LOG_PATH=./cloudinit_logs tox -e integration-tests-ci -- --color=yes tests/integration_tests/ - name: Upload cloudinit logs on failure - if: failure() && hashFiles('./cloudinit_logs/**') != '' + if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: 'cloudinit-logs' path: './cloudinit_logs' retention-days: 3 + if-no-files-found: ignore From 473fa33502ca19afe8644c22ff093098f5c9aeb7 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 26 Mar 2026 16:42:48 -0600 Subject: [PATCH 12/72] chore: drop outdated code (#6798) The biggest change is dropping jinja2 import handling code and testing handlers. The jinja2 module has been imported directly in templater.py since 24.1. The unit test helper has some additional helpers that can be dropped as well as overridden mock imports. --- cloudinit/handlers/jinja_template.py | 17 +--- cloudinit/templater.py | 29 +----- tests/unittests/analyze/test_boot.py | 2 +- tests/unittests/analyze/test_dump.py | 2 +- tests/unittests/cmd/devel/test_net_convert.py | 2 +- tests/unittests/cmd/devel/test_render.py | 8 +- tests/unittests/cmd/test_cloud_id.py | 3 +- tests/unittests/cmd/test_query.py | 2 +- .../config/test_cc_apt_pipelining.py | 3 +- .../config/test_cc_disable_ec2_metadata.py | 4 +- tests/unittests/config/test_cc_disk_setup.py | 3 +- .../config/test_cc_keys_to_console.py | 3 +- tests/unittests/config/test_cc_landscape.py | 3 +- tests/unittests/config/test_cc_ntp.py | 3 +- .../config/test_cc_power_state_change.py | 3 +- tests/unittests/config/test_cc_puppet.py | 3 +- .../unittests/config/test_cc_raspberry_pi.py | 4 +- .../config/test_cc_ubuntu_drivers.py | 3 +- .../config/test_cc_update_etc_hosts.py | 2 - tests/unittests/config/test_cc_wireguard.py | 3 +- tests/unittests/config/test_modules.py | 2 +- tests/unittests/distros/test_bsd_utils.py | 3 +- .../unittests/distros/test_raspberry_pi_os.py | 2 +- tests/unittests/helpers.py | 94 ------------------- tests/unittests/net/test_net_rendering.py | 2 +- .../unittests/sources/helpers/test_akamai.py | 2 +- .../unittests/sources/helpers/test_netlink.py | 2 +- tests/unittests/sources/test___init__.py | 3 +- tests/unittests/sources/test_akamai.py | 2 +- tests/unittests/sources/test_altcloud.py | 2 +- tests/unittests/sources/test_bigstep.py | 2 +- tests/unittests/sources/test_cloudstack.py | 2 +- tests/unittests/sources/test_configdrive.py | 3 +- tests/unittests/sources/test_digitalocean.py | 2 +- tests/unittests/sources/test_exoscale.py | 2 +- tests/unittests/sources/test_hetzner.py | 3 +- tests/unittests/sources/test_ibmcloud.py | 3 +- tests/unittests/sources/test_nocloud.py | 3 +- tests/unittests/sources/test_nwcs.py | 3 +- tests/unittests/sources/test_opennebula.py | 3 +- tests/unittests/sources/test_ovf.py | 2 +- tests/unittests/sources/test_scaleway.py | 3 +- tests/unittests/sources/test_smartos.py | 2 +- tests/unittests/sources/test_vmware.py | 3 +- tests/unittests/test__init__.py | 2 +- tests/unittests/test_apport.py | 2 +- tests/unittests/test_builtin_handlers.py | 9 +- tests/unittests/test_cli.py | 3 +- tests/unittests/test_netinfo.py | 3 +- tests/unittests/test_render_template.py | 2 - tests/unittests/test_stages.py | 2 +- tests/unittests/test_templating.py | 23 ----- tests/unittests/test_util.py | 6 +- 53 files changed, 81 insertions(+), 223 deletions(-) diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index 388588d8029..b4400304786 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -5,7 +5,9 @@ import os import re from errno import EACCES -from typing import Optional, Type +from typing import Optional + +from jinja2 import exceptions, lexer from cloudinit import handlers from cloudinit.atomic_helper import b64d, json_dumps @@ -19,15 +21,6 @@ ) from cloudinit.util import load_json, load_text_file -JUndefinedError: Type[Exception] -try: - from jinja2.exceptions import UndefinedError as JUndefinedError - from jinja2.lexer import operator_re -except ImportError: - # No jinja2 dependency - JUndefinedError = Exception - operator_re = re.compile(r"[-.]") - LOG = logging.getLogger(__name__) @@ -147,7 +140,7 @@ def render_jinja_payload(payload, payload_fn, instance_data, debug=False): ) try: rendered_payload = render_string(payload, instance_jinja_vars) - except (TypeError, JUndefinedError) as e: + except (TypeError, exceptions.UndefinedError) as e: LOG.warning("Ignoring jinja template for %s: %s", payload_fn, str(e)) return None warnings = [ @@ -180,7 +173,7 @@ def get_jinja_variable_alias(orig_name: str) -> Optional[str]: :return: A string with any jinja operators replaced if needed. Otherwise, none if no alias required. """ - alias_name = re.sub(operator_re, "_", orig_name) + alias_name = re.sub(lexer.operator_re, "_", orig_name) if alias_name != orig_name: return alias_name return None diff --git a/cloudinit/templater.py b/cloudinit/templater.py index b33f0c95a2e..d9853932bc3 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -16,29 +16,14 @@ import logging import re import sys -from typing import Any -from jinja2 import TemplateSyntaxError +from jinja2 import DebugUndefined, Template, TemplateSyntaxError from cloudinit import performance from cloudinit import type_utils as tu from cloudinit import util from cloudinit.atomic_helper import write_file -# After bionic EOL, mypy==1.0.0 will be able to type-analyse dynamic -# base types, substitute this by: -# JUndefined: typing.Type -JUndefined: Any -try: - from jinja2 import DebugUndefined as _DebugUndefined - from jinja2 import Template as JTemplate - - JINJA_AVAILABLE = True - JUndefined = _DebugUndefined -except (ImportError, AttributeError): - JINJA_AVAILABLE = False - JUndefined = object - LOG = logging.getLogger(__name__) MISSING_JINJA_PREFIX = "CI_MISSING_JINJA_VAR/" @@ -86,7 +71,7 @@ def format_error_message( # Mypy, and the PEP 484 ecosystem in general, does not support creating # classes with dynamic base types: https://stackoverflow.com/a/59636248 -class UndefinedJinjaVariable(JUndefined): +class UndefinedJinjaVariable(DebugUndefined): """Class used to represent any undefined jinja template variable.""" def __str__(self): @@ -150,7 +135,7 @@ def jinja_render(content, params): try: with performance.Timed("Rendering jinja2 template"): return ( - JTemplate( + Template( content, undefined=UndefinedJinjaVariable, trim_blocks=True, @@ -181,13 +166,7 @@ def jinja_render(content, params): "Unknown template rendering type '%s' requested" % template_type ) - if template_type == "jinja" and not JINJA_AVAILABLE: - LOG.warning( - "Jinja not available as the selected renderer for" - " desired template, reverting to the basic renderer." - ) - return ("basic", basic_render, rest) - elif template_type == "jinja" and JINJA_AVAILABLE: + elif template_type == "jinja": return ("jinja", jinja_render, rest) # Only thing left over is the basic renderer (it is always available). return ("basic", basic_render, rest) diff --git a/tests/unittests/analyze/test_boot.py b/tests/unittests/analyze/test_boot.py index ffe147e5110..6487790ddc6 100644 --- a/tests/unittests/analyze/test_boot.py +++ b/tests/unittests/analyze/test_boot.py @@ -1,4 +1,5 @@ import os +from unittest import mock import pytest @@ -11,7 +12,6 @@ dist_check_timestamp, gather_timestamps_using_systemd, ) -from tests.unittests.helpers import mock err_code = (FAIL_CODE, -1, -1, -1) diff --git a/tests/unittests/analyze/test_dump.py b/tests/unittests/analyze/test_dump.py index ebf717e088c..45bf1c743e4 100644 --- a/tests/unittests/analyze/test_dump.py +++ b/tests/unittests/analyze/test_dump.py @@ -4,6 +4,7 @@ from contextlib import suppress from datetime import datetime, timezone from textwrap import dedent +from unittest import mock import pytest @@ -14,7 +15,6 @@ parse_timestamp, ) from cloudinit.util import write_file -from tests.unittests.helpers import mock class TestParseTimestamp: diff --git a/tests/unittests/cmd/devel/test_net_convert.py b/tests/unittests/cmd/devel/test_net_convert.py index 9328150d5bc..584741dbe3b 100644 --- a/tests/unittests/cmd/devel/test_net_convert.py +++ b/tests/unittests/cmd/devel/test_net_convert.py @@ -1,13 +1,13 @@ # This file is part of cloud-init. See LICENSE file for license information. import itertools +from unittest import mock import pytest import yaml from cloudinit.cmd.devel import net_convert from cloudinit.distros.debian import NETWORK_FILE_HEADER -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.devel.net_convert." diff --git a/tests/unittests/cmd/devel/test_render.py b/tests/unittests/cmd/devel/test_render.py index 90557dcb49b..d6abb28aa39 100644 --- a/tests/unittests/cmd/devel/test_render.py +++ b/tests/unittests/cmd/devel/test_render.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from io import StringIO +from unittest import mock import pytest @@ -8,7 +9,6 @@ from cloudinit.helpers import Paths from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import ensure_dir, write_file -from tests.unittests.helpers import mock, skipUnlessJinja M_PATH = "cloudinit.cmd.devel.render." @@ -87,7 +87,6 @@ def test_root_uses_sensitive_instance_data(self, m_paths, tmpdir): assert render.render_template(user_data, None, False) == 0 assert "rendering: jinja worked" in m_stdout.getvalue() - @skipUnlessJinja() def test_renders_instance_data_vars_in_template(self, caplog, tmpdir): """If user_data file is a jinja template render instance-data vars.""" user_data = tmpdir.join("user-data") @@ -103,7 +102,6 @@ def test_renders_instance_data_vars_in_template(self, caplog, tmpdir): ) assert "rendering: jinja worked" == m_stdout.getvalue() - @skipUnlessJinja() def test_render_warns_and_gives_up_on_invalid_jinja_operation( self, caplog, tmpdir ): @@ -119,7 +117,6 @@ def test_render_warns_and_gives_up_on_invalid_jinja_operation( ' "my_var"?' % user_data ) in caplog.text - @skipUnlessJinja() def test_jinja_load_error(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja\nrendering: {{ my-var }}") @@ -130,7 +127,6 @@ def test_jinja_load_error(self, caplog, tmpdir): "Cannot render from instance data due to exception" in caplog.text ) - @skipUnlessJinja() def test_not_jinja_error(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "{{ my-var }}") @@ -141,7 +137,6 @@ def test_not_jinja_error(self, caplog, tmpdir): "Cannot render from instance data due to exception" in caplog.text ) - @skipUnlessJinja() def test_no_user_data(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja") @@ -150,7 +145,6 @@ def test_no_user_data(self, caplog, tmpdir): render.render_template(user_data, instance_data, False) assert "Unable to render user-data file" in caplog.text - @skipUnlessJinja() def test_invalid_jinja_syntax(self, caplog, tmpdir): user_data = tmpdir.join("user-data") write_file(user_data, "##template: jinja\nrendering: {{ my_var } }") diff --git a/tests/unittests/cmd/test_cloud_id.py b/tests/unittests/cmd/test_cloud_id.py index d01c6efa283..671a945de62 100644 --- a/tests/unittests/cmd/test_cloud_id.py +++ b/tests/unittests/cmd/test_cloud_id.py @@ -2,12 +2,13 @@ """Tests for cloud-id command line utility.""" +from unittest import mock + import pytest from cloudinit import atomic_helper from cloudinit.cmd import cloud_id, status from cloudinit.helpers import Paths -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.cloud_id." diff --git a/tests/unittests/cmd/test_query.py b/tests/unittests/cmd/test_query.py index d552f8bc399..e8843d4d585 100644 --- a/tests/unittests/cmd/test_query.py +++ b/tests/unittests/cmd/test_query.py @@ -8,6 +8,7 @@ from io import BytesIO from pathlib import Path from textwrap import dedent +from unittest import mock import pytest @@ -17,7 +18,6 @@ from cloudinit.sources import REDACT_SENSITIVE_VALUE from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import write_file -from tests.unittests.helpers import mock M_PATH = "cloudinit.cmd.query." diff --git a/tests/unittests/config/test_cc_apt_pipelining.py b/tests/unittests/config/test_cc_apt_pipelining.py index 2085a81f3df..6fb2926aa31 100644 --- a/tests/unittests/config/test_cc_apt_pipelining.py +++ b/tests/unittests/config/test_cc_apt_pipelining.py @@ -3,6 +3,7 @@ """Tests cc_apt_pipelining handler""" import re +from unittest import mock import pytest @@ -12,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestAptPipelining: diff --git a/tests/unittests/config/test_cc_disable_ec2_metadata.py b/tests/unittests/config/test_cc_disable_ec2_metadata.py index 55a23060a26..03afcf0ca18 100644 --- a/tests/unittests/config/test_cc_disable_ec2_metadata.py +++ b/tests/unittests/config/test_cc_disable_ec2_metadata.py @@ -3,6 +3,8 @@ """Tests cc_disable_ec2_metadata handler""" +from unittest import mock + import pytest import cloudinit.config.cc_disable_ec2_metadata as ec2_meta @@ -11,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema DISABLE_CFG = {"disable_ec2_metadata": "true"} diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index b81a3a97e92..d0f3c26dc47 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -4,6 +4,7 @@ import random import tempfile from contextlib import ExitStack +from unittest import mock import pytest @@ -13,7 +14,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestIsDiskUsed: diff --git a/tests/unittests/config/test_cc_keys_to_console.py b/tests/unittests/config/test_cc_keys_to_console.py index caa2fa7bba1..64ca6367e40 100644 --- a/tests/unittests/config/test_cc_keys_to_console.py +++ b/tests/unittests/config/test_cc_keys_to_console.py @@ -1,6 +1,7 @@ """Tests for cc_keys_to_console.""" import re +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema class TestHandle: diff --git a/tests/unittests/config/test_cc_landscape.py b/tests/unittests/config/test_cc_landscape.py index 559702a33a8..07edc426718 100644 --- a/tests/unittests/config/test_cc_landscape.py +++ b/tests/unittests/config/test_cc_landscape.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema, wrap_and_call +from tests.unittests.helpers import skipUnlessJsonSchema, wrap_and_call from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index aea0c40243c..5cfe9f302bc 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -5,6 +5,7 @@ import shutil from os.path import dirname from typing import Any, Dict, List +from unittest import mock import pytest @@ -15,7 +16,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud NTP_TEMPLATE = """\ diff --git a/tests/unittests/config/test_cc_power_state_change.py b/tests/unittests/config/test_cc_power_state_change.py index 37901baaab7..6c159b3bfd4 100644 --- a/tests/unittests/config/test_cc_power_state_change.py +++ b/tests/unittests/config/test_cc_power_state_change.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import sys +from unittest import mock import pytest @@ -11,7 +12,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema @pytest.fixture diff --git a/tests/unittests/config/test_cc_puppet.py b/tests/unittests/config/test_cc_puppet.py index 22911542d2a..637e78a5c82 100644 --- a/tests/unittests/config/test_cc_puppet.py +++ b/tests/unittests/config/test_cc_puppet.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import textwrap +from unittest import mock import pytest import responses @@ -13,7 +14,7 @@ ) from cloudinit.distros import PackageInstallerError from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index cdfe97c6a70..6c2e74a6c80 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest import cloudinit.config.cc_raspberry_pi as cc_rpi @@ -15,7 +17,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud M_PATH = "cloudinit.config.cc_raspberry_pi." diff --git a/tests/unittests/config/test_cc_ubuntu_drivers.py b/tests/unittests/config/test_cc_ubuntu_drivers.py index 02854585ba5..8344dd1ab71 100644 --- a/tests/unittests/config/test_cc_ubuntu_drivers.py +++ b/tests/unittests/config/test_cc_ubuntu_drivers.py @@ -5,6 +5,7 @@ import os import re from typing import Any, Dict +from unittest import mock import pytest @@ -15,7 +16,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py index ebf6f599e1e..080c92157db 100644 --- a/tests/unittests/config/test_cc_update_etc_hosts.py +++ b/tests/unittests/config/test_cc_update_etc_hosts.py @@ -61,7 +61,6 @@ def test_write_etc_hosts_suse_localhost(self, tmp_path): "192.168.1.1\tblah.blah.us\tblah" in contents ), "Default etc/hosts content modified" - @t_help.skipUnlessJinja() def test_write_etc_hosts_suse_template(self, tmp_path): cfg = { "manage_etc_hosts": "template", @@ -116,7 +115,6 @@ class TestUpdateEtcHosts: ), ], ) - @t_help.skipUnlessJsonSchema() def test_schema_validation(self, config, expectation): with expectation: validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_wireguard.py b/tests/unittests/config/test_cc_wireguard.py index 6702eb9385a..b87851aaa7e 100644 --- a/tests/unittests/config/test_cc_wireguard.py +++ b/tests/unittests/config/test_cc_wireguard.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock import pytest @@ -10,7 +11,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema NL = "\n" # Module path used in mocks diff --git a/tests/unittests/config/test_modules.py b/tests/unittests/config/test_modules.py index d986442e250..5dc542b64fe 100644 --- a/tests/unittests/config/test_modules.py +++ b/tests/unittests/config/test_modules.py @@ -6,6 +6,7 @@ import logging from pathlib import Path from typing import List +from unittest import mock import pytest @@ -16,7 +17,6 @@ from cloudinit.settings import FREQUENCIES from cloudinit.stages import Init from tests.helpers import cloud_init_project_dir -from tests.unittests.helpers import mock M_PATH = "cloudinit.config.modules." diff --git a/tests/unittests/distros/test_bsd_utils.py b/tests/unittests/distros/test_bsd_utils.py index f5a7efd8fc2..44351762f93 100644 --- a/tests/unittests/distros/test_bsd_utils.py +++ b/tests/unittests/distros/test_bsd_utils.py @@ -1,8 +1,9 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest import cloudinit.distros.bsd_utils as bsd_utils -from tests.unittests.helpers import mock RC_FILE = """ if something; then diff --git a/tests/unittests/distros/test_raspberry_pi_os.py b/tests/unittests/distros/test_raspberry_pi_os.py index 2ef36acf117..ba54cee2f10 100644 --- a/tests/unittests/distros/test_raspberry_pi_os.py +++ b/tests/unittests/distros/test_raspberry_pi_os.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from unittest import mock from cloudinit.distros import fetch from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock M_PATH = "cloudinit.distros.raspberry_pi_os." diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 429b999d339..e4acf61b23a 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -10,15 +10,12 @@ import unittest from contextlib import contextmanager from unittest import mock -from unittest.util import strclass from urllib.parse import urlsplit, urlunsplit import pytest import responses from cloudinit import distros, helpers, settings, util -from cloudinit.helpers import Paths -from cloudinit.templater import JINJA_AVAILABLE from tests.helpers import cloud_init_project_dir try: @@ -114,42 +111,6 @@ def random_string(length=8): ) -# Note: The use of this class and unittests.TestCase is discouraged. Use pytest -# instead. See development docs on testing. -class TestCase(unittest.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util._DNS_REDIRECT_IP = None - - def setUp(self): - super(TestCase, self).setUp() - self.reset_global_state() - - def shortDescription(self): - return strclass(self.__class__) + "." + self._testMethodName - - def add_patch(self, target, attr, *args, **kwargs): - """Patches specified target object and sets it as attr on test - instance also schedules cleanup""" - if "autospec" not in kwargs: - kwargs["autospec"] = True - m = mock.patch(target, *args, **kwargs) - p = m.start() - self.addCleanup(m.stop) - setattr(self, attr, p) - - def replicate_test_root(example_root, target_root): real_root = resourceLocation() real_root = os.path.join(real_root, "roots", example_root) @@ -195,24 +156,6 @@ def _ensure_url_default_path(url): ) -def get_mock_paths(temp_dir): - class MockPaths(Paths): - def __init__(self, path_cfgs: dict, ds=None): - super().__init__(path_cfgs=path_cfgs, ds=ds) - - self.cloud_dir: str = path_cfgs.get( - "cloud_dir", f"{temp_dir}/var/lib/cloud" - ) - self.run_dir: str = path_cfgs.get( - "run_dir", f"{temp_dir}/run/cloud/" - ) - self.template_dir: str = path_cfgs.get( - "templates_dir", f"{temp_dir}/etc/cloud/templates/" - ) - - return MockPaths - - def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) @@ -340,43 +283,6 @@ def skipUnlessJsonSchema(): ) -def skipUnlessJinja(): - return pytest.mark.skipif( - not JINJA_AVAILABLE, reason="No jinja dependency present." - ) - - -@skipUnlessJinja() -def skipUnlessJinjaVersionGreaterThan(version=(0, 0, 0)): - import jinja2 - - return pytest.mark.skipif( - tuple(map(int, jinja2.__version__.split("."))) < version, - reason=f"jinj2 version is less than {version}", - ) - - -def skipIfJinja(): - return pytest.mark.skipif( - JINJA_AVAILABLE, reason="Jinja dependency present." - ) - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, "assert_not_called"): - - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ( - "[citest] Expected '%s' to not have been called. " - "Called %s times." - % (mmock._mock_name or "mock", mmock.call_count) - ) - raise AssertionError(msg) - - mock.Mock.assert_not_called = __mock_assert_not_called # type: ignore - - @contextmanager def does_not_raise(): """Context manager to parametrize tests raising and not raising exceptions diff --git a/tests/unittests/net/test_net_rendering.py b/tests/unittests/net/test_net_rendering.py index 7698462b3bc..ded5795ec1e 100644 --- a/tests/unittests/net/test_net_rendering.py +++ b/tests/unittests/net/test_net_rendering.py @@ -28,6 +28,7 @@ import glob from enum import Flag, auto from pathlib import Path +from unittest import mock import pytest import yaml @@ -36,7 +37,6 @@ from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer from cloudinit.net.network_state import NetworkState, parse_net_config_data from cloudinit.net.networkd import Renderer as NetworkdRenderer -from tests.unittests.helpers import mock ARTIFACT_DIR = Path(__file__).parent.absolute() / "artifacts" diff --git a/tests/unittests/sources/helpers/test_akamai.py b/tests/unittests/sources/helpers/test_akamai.py index 3cc0e9190b3..9ca83c8ccf0 100644 --- a/tests/unittests/sources/helpers/test_akamai.py +++ b/tests/unittests/sources/helpers/test_akamai.py @@ -1,9 +1,9 @@ from typing import Any, Dict +from unittest import mock import pytest from cloudinit.sources.helpers.akamai import get_dmi_config, is_on_akamai -from tests.unittests.helpers import mock class TestAkamaiHelper: diff --git a/tests/unittests/sources/helpers/test_netlink.py b/tests/unittests/sources/helpers/test_netlink.py index 208ca7831d3..28197b49d90 100644 --- a/tests/unittests/sources/helpers/test_netlink.py +++ b/tests/unittests/sources/helpers/test_netlink.py @@ -5,6 +5,7 @@ import codecs import socket import struct +from unittest import mock import pytest @@ -31,7 +32,6 @@ wait_for_nic_attach_event, wait_for_nic_detach_event, ) -from tests.unittests.helpers import mock def int_to_bytes(i): diff --git a/tests/unittests/sources/test___init__.py b/tests/unittests/sources/test___init__.py index 2c214aeed66..67924d6a517 100644 --- a/tests/unittests/sources/test___init__.py +++ b/tests/unittests/sources/test___init__.py @@ -1,8 +1,9 @@ +from unittest import mock + import pytest from cloudinit import sources from cloudinit.sources import DataSourceOpenStack as ds -from tests.unittests.helpers import mock openstack_ds_name = ds.DataSourceOpenStack.dsname.lower() diff --git a/tests/unittests/sources/test_akamai.py b/tests/unittests/sources/test_akamai.py index 2a68099753a..05ad49847c3 100644 --- a/tests/unittests/sources/test_akamai.py +++ b/tests/unittests/sources/test_akamai.py @@ -1,5 +1,6 @@ from contextlib import suppress from typing import Any, Dict, List, Optional, Union +from unittest import mock import pytest @@ -8,7 +9,6 @@ DataSourceAkamaiLocal, MetadataAvailabilityResult, ) -from tests.unittests.helpers import mock class TestDataSourceAkamai: diff --git a/tests/unittests/sources/test_altcloud.py b/tests/unittests/sources/test_altcloud.py index 0a42602c6ab..24ef12ee7e5 100644 --- a/tests/unittests/sources/test_altcloud.py +++ b/tests/unittests/sources/test_altcloud.py @@ -12,12 +12,12 @@ import os import shutil +from unittest import mock import pytest import cloudinit.sources.DataSourceAltCloud as dsac from cloudinit import subp, util -from tests.unittests.helpers import mock OS_UNAME_ORIG = getattr(os, "uname") diff --git a/tests/unittests/sources/test_bigstep.py b/tests/unittests/sources/test_bigstep.py index 100b6fc63a5..0427dee6712 100644 --- a/tests/unittests/sources/test_bigstep.py +++ b/tests/unittests/sources/test_bigstep.py @@ -1,12 +1,12 @@ import json import os +from unittest import mock import pytest import responses from cloudinit import helpers from cloudinit.sources import DataSourceBigstep as bigstep -from tests.unittests.helpers import mock M_PATH = "cloudinit.sources.DataSourceBigstep." diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index 90c56291f5b..f276f274e64 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -2,6 +2,7 @@ # pylint: disable=attribute-defined-outside-init from socket import gaierror from textwrap import dedent +from unittest import mock from unittest.mock import patch import pytest @@ -16,7 +17,6 @@ get_data_server, get_vr_address, ) -from tests.unittests.helpers import mock from tests.unittests.util import MockDistro SOURCES_PATH = "cloudinit.sources" diff --git a/tests/unittests/sources/test_configdrive.py b/tests/unittests/sources/test_configdrive.py index 43ea8be3d4d..0f3e0c76439 100644 --- a/tests/unittests/sources/test_configdrive.py +++ b/tests/unittests/sources/test_configdrive.py @@ -4,6 +4,7 @@ import os from contextlib import ExitStack from copy import copy, deepcopy +from unittest import mock import pytest @@ -11,7 +12,7 @@ from cloudinit.net import eni, network_state from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir PUBKEY = "ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n" EC2_META = { diff --git a/tests/unittests/sources/test_digitalocean.py b/tests/unittests/sources/test_digitalocean.py index b5bc526ed37..587c767f7e2 100644 --- a/tests/unittests/sources/test_digitalocean.py +++ b/tests/unittests/sources/test_digitalocean.py @@ -8,13 +8,13 @@ # pylint: disable=attribute-defined-outside-init import json +from unittest import mock import pytest from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from tests.unittests.helpers import mock DO_MULTIPLE_KEYS = [ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", diff --git a/tests/unittests/sources/test_exoscale.py b/tests/unittests/sources/test_exoscale.py index 08cbeeb913d..667f42cd85e 100644 --- a/tests/unittests/sources/test_exoscale.py +++ b/tests/unittests/sources/test_exoscale.py @@ -3,6 +3,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. import os +from unittest import mock import requests import responses @@ -16,7 +17,6 @@ get_password, read_metadata, ) -from tests.unittests.helpers import mock TEST_PASSWORD_URL = "{}:{}/{}/".format( METADATA_URL, PASSWORD_SERVER_PORT, API_VERSION diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index af0a893cd0f..d7fe082028f 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -4,11 +4,12 @@ # # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest from cloudinit import settings, util from cloudinit.sources import DataSourceHetzner -from tests.unittests.helpers import mock METADATA = b""" hostname: cloudinit-test diff --git a/tests/unittests/sources/test_ibmcloud.py b/tests/unittests/sources/test_ibmcloud.py index f6b81908edf..c604ac6b922 100644 --- a/tests/unittests/sources/test_ibmcloud.py +++ b/tests/unittests/sources/test_ibmcloud.py @@ -5,6 +5,7 @@ import copy import json from textwrap import dedent +from unittest import mock import pytest @@ -12,8 +13,6 @@ from cloudinit.sources import DataSourceIBMCloud as ibm from tests.unittests import helpers as test_helpers -mock = test_helpers.mock - D_PATH = "cloudinit.sources.DataSourceIBMCloud." diff --git a/tests/unittests/sources/test_nocloud.py b/tests/unittests/sources/test_nocloud.py index a401c6292ed..383160e52b3 100644 --- a/tests/unittests/sources/test_nocloud.py +++ b/tests/unittests/sources/test_nocloud.py @@ -2,6 +2,7 @@ import os import textwrap +from unittest import mock import pytest import yaml @@ -11,7 +12,7 @@ DataSourceNoCloudNet, parse_cmdline_data, ) -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir @pytest.fixture(autouse=True) diff --git a/tests/unittests/sources/test_nwcs.py b/tests/unittests/sources/test_nwcs.py index c208c875974..219f322b950 100644 --- a/tests/unittests/sources/test_nwcs.py +++ b/tests/unittests/sources/test_nwcs.py @@ -1,10 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + import pytest from cloudinit import settings, util from cloudinit.sources import DataSourceNWCS -from tests.unittests.helpers import mock METADATA = util.load_yaml( """ diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index c6f4066abbe..f5db58c3592 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -3,12 +3,13 @@ import os import pwd +from unittest import mock import pytest from cloudinit import atomic_helper, util from cloudinit.sources import DataSourceOpenNebula as ds -from tests.unittests.helpers import mock, populate_dir +from tests.unittests.helpers import populate_dir TEST_VARS = { "VAR1": "single", diff --git a/tests/unittests/sources/test_ovf.py b/tests/unittests/sources/test_ovf.py index 5b771df1220..1d348bc378b 100644 --- a/tests/unittests/sources/test_ovf.py +++ b/tests/unittests/sources/test_ovf.py @@ -9,12 +9,12 @@ import os from collections import OrderedDict from textwrap import dedent +from unittest import mock import pytest from cloudinit import subp, util from cloudinit.sources import DataSourceOVF as dsovf -from tests.unittests.helpers import mock MPATH = "cloudinit.sources.DataSourceOVF." diff --git a/tests/unittests/sources/test_scaleway.py b/tests/unittests/sources/test_scaleway.py index 57983be08de..50d76f94831 100644 --- a/tests/unittests/sources/test_scaleway.py +++ b/tests/unittests/sources/test_scaleway.py @@ -3,6 +3,7 @@ import json import socket +from unittest import mock from urllib.parse import SplitResult, urlsplit import pytest @@ -13,7 +14,7 @@ from cloudinit import settings from cloudinit.distros import ubuntu from cloudinit.sources import DataSourceScaleway -from tests.unittests.helpers import mock, responses_assert_call_count +from tests.unittests.helpers import responses_assert_call_count class DataResponses: diff --git a/tests/unittests/sources/test_smartos.py b/tests/unittests/sources/test_smartos.py index e25e7818189..b1bc03d4c02 100644 --- a/tests/unittests/sources/test_smartos.py +++ b/tests/unittests/sources/test_smartos.py @@ -24,6 +24,7 @@ import uuid from binascii import crc32 from collections import namedtuple +from unittest import mock import pytest import serial @@ -44,7 +45,6 @@ ) from cloudinit.subp import ProcessExecutionError, subp, which from cloudinit.util import write_file -from tests.unittests.helpers import mock DSMOS = "cloudinit.sources.DataSourceSmartOS" SDC_NICS = json.loads( diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py index e4bd639b04d..12b2ec14332 100644 --- a/tests/unittests/sources/test_vmware.py +++ b/tests/unittests/sources/test_vmware.py @@ -11,6 +11,7 @@ from contextlib import ExitStack from logging import DEBUG from textwrap import dedent +from unittest import mock import pytest @@ -19,7 +20,7 @@ from cloudinit.sources import DataSourceVMware from cloudinit.sources.helpers.vmware.imc import guestcust_util from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import mock, populate_dir, wrap_and_call +from tests.unittests.helpers import populate_dir, wrap_and_call MPATH = "cloudinit.sources.DataSourceVMware." PRODUCT_NAME_FILE_PATH = "/sys/class/dmi/id/product_name" diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 43e38fd3d96..e9fbf5897cf 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -3,12 +3,12 @@ import logging import os from types import SimpleNamespace +from unittest import mock import pytest from cloudinit import handlers, helpers, settings, url_helper, util from cloudinit.cmd import main -from tests.unittests.helpers import mock @pytest.fixture diff --git a/tests/unittests/test_apport.py b/tests/unittests/test_apport.py index 7bbf40d6f68..c3fedcaf36e 100644 --- a/tests/unittests/test_apport.py +++ b/tests/unittests/test_apport.py @@ -1,11 +1,11 @@ import os import sys from importlib import reload +from unittest import mock import pytest from cloudinit import apport -from tests.unittests.helpers import mock M_PATH = "cloudinit.apport." diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 14edb44cde2..cd545479729 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -6,6 +6,7 @@ import errno import os import re +from unittest import mock import pytest @@ -25,7 +26,6 @@ path_map, ) from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE -from tests.unittests.helpers import mock, skipUnlessJinja from tests.unittests.util import FakeDataSource INSTANCE_DATA_FILE = "instance-data-sensitive.json" @@ -95,7 +95,6 @@ def test_jinja_template_handle_noop_on_content_signals(self, paths): ) m_handle_part.assert_not_called() - @skipUnlessJinja() def test_jinja_template_handle_subhandler_v2_with_clean_payload( self, paths ): @@ -122,7 +121,6 @@ def test_jinja_template_handle_subhandler_v2_with_clean_payload( "data", "!__begin__", "part01", "#!/bin/bash\necho himom", "freq" ) - @skipUnlessJinja() def test_jinja_template_handle_subhandler_v3_with_clean_payload( self, paths ): @@ -208,7 +206,6 @@ def test_jinja_template_handle_errors_on_unreadable_instance_data( "Unexpected file created %s" % script_file ) - @skipUnlessJinja() def test_jinja_template_handle_renders_jinja_content(self, paths, caplog): """When present, render jinja variables from instance data""" script_handler = ShellScriptPartHandler(paths) @@ -237,7 +234,6 @@ def test_jinja_template_handle_renders_jinja_content(self, paths, caplog): ) assert "#!/bin/bash\necho himom" == util.load_text_file(script_file) - @skipUnlessJinja() def test_jinja_template_handle_renders_jinja_content_missing_keys( self, paths, caplog ): @@ -349,7 +345,7 @@ def test_convert_instance_data_decodes_decode_paths(self): class TestRenderJinjaPayload: - @skipUnlessJinja() + def test_render_jinja_payload_logs_jinja_vars_on_debug(self, caplog): """When debug is True, log jinja variables available.""" payload = ( @@ -372,7 +368,6 @@ def test_render_jinja_payload_logs_jinja_vars_on_debug(self, caplog): ) assert re.match(expected_log, caplog.text, re.DOTALL) - @skipUnlessJinja() def test_render_jinja_payload_replaces_missing_variables_and_warns( self, caplog ): diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 77b1cc244d0..ff06cfe567c 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -7,6 +7,7 @@ import os import sys from collections import namedtuple +from unittest import mock import pytest @@ -14,8 +15,6 @@ from cloudinit.cmd import main as cli from tests.unittests import helpers as test_helpers -mock = test_helpers.mock - M_PATH = "cloudinit.cmd.main." Tmpdir = namedtuple("Tmpdir", ["tmpdir", "link_d", "data_d"]) FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) diff --git a/tests/unittests/test_netinfo.py b/tests/unittests/test_netinfo.py index 90f78356a9d..35b064678c0 100644 --- a/tests/unittests/test_netinfo.py +++ b/tests/unittests/test_netinfo.py @@ -4,6 +4,7 @@ import json from copy import copy +from unittest import mock import pytest @@ -14,7 +15,7 @@ netdev_pformat, route_pformat, ) -from tests.unittests.helpers import mock, readResource +from tests.unittests.helpers import readResource # Example ifconfig and route output SAMPLE_OLD_IFCONFIG_OUT = readResource("netinfo/old-ifconfig-output") diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 7f8fc944429..793786fa95d 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -6,7 +6,6 @@ from cloudinit import subp, templater, util from tests.helpers import cloud_init_project_dir -from tests.unittests.helpers import skipUnlessJinjaVersionGreaterThan # TODO(Look to align with tools.render-template or cloudinit.distos.OSFAMILIES) DISTRO_VARIANTS = [ @@ -204,7 +203,6 @@ class TestRenderSourcesList: ), ], ) - @skipUnlessJinjaVersionGreaterThan((3, 0, 0)) def test_render_sources_list_templates( self, tmpdir, template_path, expected ): diff --git a/tests/unittests/test_stages.py b/tests/unittests/test_stages.py index 85846352be6..b217272b1df 100644 --- a/tests/unittests/test_stages.py +++ b/tests/unittests/test_stages.py @@ -5,6 +5,7 @@ import json import os import stat +from unittest import mock import pytest @@ -13,7 +14,6 @@ from cloudinit.helpers import Paths from cloudinit.sources import DataSource, NetworkConfigSource from cloudinit.util import sym_link, write_file -from tests.unittests.helpers import mock from tests.unittests.util import TEST_INSTANCE_ID, FakeDataSource M_PATH = "cloudinit.stages." diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 83c1ba4fc69..a5aab1f9a03 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -4,16 +4,13 @@ # # This file is part of cloud-init. See LICENSE file for license information. -import logging import textwrap -from unittest import mock import pytest from cloudinit import templater from cloudinit.templater import JinjaSyntaxParsingException from cloudinit.util import load_binary_file, write_file -from tests.unittests import helpers as test_helpers class TestTemplates: @@ -137,26 +134,6 @@ def test_jinja_nonascii_render_from_file(self, tmp_path): result = templater.render_from_file(tmpl_fn, {"name": "bob"}) assert result == self.jinja_utf8_rbob - @test_helpers.skipIfJinja() - def test_jinja_warns_on_missing_dep_and_uses_basic_renderer( - self, caplog, tmp_path - ): - """Test jinja render_from_file will fallback to basic renderer.""" - tmpl_fn = str(tmp_path / "j-render-from-file.template") - write_file( - tmpl_fn, - omode="wb", - content=self.add_header("jinja", self.jinja_utf8).encode("utf-8"), - ) - result = templater.render_from_file(tmpl_fn, {"name": "bob"}) - assert result == self.jinja_utf8.decode() - assert ( - mock.ANY, - logging.WARNING, - "Jinja not available as the selected renderer for desired" - " template, reverting to the basic renderer.", - ) in caplog.record_tuples - def test_jinja_do_extension_render_to_string(self): """Test jinja render_to_string using do extension.""" expected_result = "[1, 2, 3]" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 789897504ab..88be28ecdbd 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -35,7 +35,7 @@ from cloudinit.sources import DataSourceHostname from cloudinit.subp import SubpResult from tests.unittests import helpers -from tests.unittests.helpers import random_string, skipUnlessJinja +from tests.unittests.helpers import random_string LOG = logging.getLogger(__name__) M_PATH = "cloudinit.util." @@ -470,7 +470,6 @@ def test_read_conf(self, mocker): ) assert util.read_conf("any") == {"a": "b"} - @skipUnlessJinja() def test_read_conf_with_template(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -489,7 +488,6 @@ def test_read_conf_with_template(self, mocker, caplog): "from 'cfg_path'" ) in caplog.text - @skipUnlessJinja() def test_read_conf_with_failed_config_json(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -504,7 +502,6 @@ def test_read_conf_with_failed_config_json(self, mocker, caplog): assert "Failed loading yaml blob" in caplog.text assert conf == {} - @skipUnlessJinja() def test_read_conf_with_failed_instance_data_json(self, mocker, caplog): mocker.patch("os.path.exists", return_value=True) mocker.patch( @@ -527,7 +524,6 @@ def test_read_conf_with_failed_instance_data_json(self, mocker, caplog): "{% if c %} C is present {% else % } C is NOT present {% endif %}", ], ) - @skipUnlessJinja() def test_read_conf_with_config_invalid_jinja_syntax( self, mocker, caplog, template ): From a8705449b35afab83899621f9f6538f3cd1e926e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 27 Mar 2026 18:12:28 -0600 Subject: [PATCH 13/72] test: fix missing mocker of rmtree for redhat unittests (#6808) Fixes leaked rmtree call during package builds for rocky and redhat. --- tests/unittests/config/test_cc_rh_subscription.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index a9b95072afc..b8323ed6449 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -116,7 +116,7 @@ def test_update_repos_disable_with_none( mock.call(["repos", "--enable=repo1"]) ] - def test_full_registration(self, m_sman_cli, caplog): + def test_full_registration(self, m_sman_cli, caplog, mocker): """ Registration with auto_attach, service_level, adding pools, enabling and disabling yum repos and setting release_version @@ -147,7 +147,7 @@ def test_full_registration(self, m_sman_cli, caplog): # to avoid deleting the actual cache files # (triggered by the presence of the release_version key) # on the host running the tests - mock.patch("shutil.rmtree") + mocker.patch("shutil.rmtree") cc_rh_subscription.handle(NAME, self.CONFIG_FULL, None, []) assert m_sman_cli.call_count == 10 From 35f5948e4b4ec9889654570f5ce2f59ecdf25e00 Mon Sep 17 00:00:00 2001 From: Hyacinthe Cartiaux Date: Mon, 30 Mar 2026 20:25:44 +0200 Subject: [PATCH 14/72] fix(openbsd): specify the correct usr_lib_exec path (#6793) --- cloudinit/distros/openbsd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index 14cf3be2b8e..cce1073c408 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -13,6 +13,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD): hostname_conf_fn = "/etc/myname" init_cmd = ["rcctl"] + usr_lib_exec = "/usr/local/libexec" # For OpenBSD (from https://man.openbsd.org/passwd.5) a password field # of "" indicates no password, and password field values of either From 17d28677d6db5fd924f1ac6b03b645eb48e1be6c Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 31 Mar 2026 10:02:54 -0600 Subject: [PATCH 15/72] ci: disable daily jobs on forks (#6821) --- .github/workflows/10-daily-unit-lint.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/10-daily-unit-lint.yml b/.github/workflows/10-daily-unit-lint.yml index 898c86ccc20..5e2452df679 100644 --- a/.github/workflows/10-daily-unit-lint.yml +++ b/.github/workflows/10-daily-unit-lint.yml @@ -28,6 +28,7 @@ jobs: run: tox -e hypothesis-slow devel_tests: name: "Tip: Python" + if: github.repository == 'canonical/cloud-init' runs-on: ubuntu-latest steps: - name: Checkout @@ -44,14 +45,15 @@ jobs: - name: Run unittest run: tox -e py3 -- --color=yes format_tip: + name: "Tip: Lint" + if: github.repository == 'canonical/cloud-init' + runs-on: ubuntu-latest env: FORCE_COLOR: 1 strategy: fail-fast: false matrix: env: [tip-ruff, tip-mypy, tip-pylint, tip-black, tip-isort, py3-fast] - name: "Tip: Lint" - runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -65,6 +67,7 @@ jobs: run: tox macos: name: Python ${{matrix.python-version}} ${{ matrix.slug }} + if: github.repository == 'canonical/cloud-init' runs-on: macos-latest strategy: matrix: From 0b7dc7e26db19ca6a8e557288c94608e5ab5c9c0 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 31 Mar 2026 14:44:48 -0600 Subject: [PATCH 16/72] chore: drop unnecessary pylint ignore settings, use pytest --strict-markers (#6817) --- .pylintrc | 29 ----------------------------- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/.pylintrc b/.pylintrc index 028e4ae73e7..2a3b18226b5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -32,32 +32,3 @@ output-format=parseable # Just the errors please, no full report reports=no - - -[TYPECHECK] - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - http.client, - httplib, - pkg_resources, - # cloud_tests requirements. - boto3, - botocore, - paramiko, - pylxd, - simplestreams - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413 -ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,ContainerManager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members=types,http.client,command_handlers,m_.*,enter_context diff --git a/tox.ini b/tox.ini index 58e1207caf8..a1e1e16972e 100644 --- a/tox.ini +++ b/tox.ini @@ -304,9 +304,8 @@ setenv = ON_JENKINS="1" [pytest] -# TODO: s/--strict/--strict-markers/ once pytest version is high enough testpaths = tools tests/unittests -addopts = --strict +addopts = --strict --strict-markers log_format = %(asctime)s %(levelname)-9s %(name)s:%(filename)s:%(lineno)d %(message)s log_date_format = %Y-%m-%d %H:%M:%S markers = From 65bacbedac25063553f153ea8e957605d1fdaa36 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 31 Mar 2026 14:46:17 -0600 Subject: [PATCH 17/72] tools: fix brpm script to use correct builddir in temp directory (#6809) RockyLinux 9 has Jinja 2.11. Template-based unit tests break due to unsupported closure: +%}. Revert portion of 473fa335 to retain skip decorator avoiding unit test when Jinja is < 3.0.0 to support rockylinux developer rpm builds. Update ruff extra-includes to validate read-dependencies, bddeb and brpm. --- packages/brpm | 4 ++-- pyproject.toml | 1 + tests/unittests/helpers.py | 9 +++++++++ tests/unittests/test_render_template.py | 2 ++ tools/read-dependencies | 20 +++++++++++++++++--- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/brpm b/packages/brpm index 7a1cf3d1d96..e24db2b7a92 100755 --- a/packages/brpm +++ b/packages/brpm @@ -29,7 +29,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import subp, templater, util + from cloudinit import subp, templater, temp_utils, util # Subdirectories of the ~/rpmbuild dir @@ -83,7 +83,7 @@ def read_version_from_meson(): [ "meson", "setup", - "builddir", + builddir, "-Dinit_system=systemd", "-Ddisable_sshd_keygen=true", ] diff --git a/pyproject.toml b/pyproject.toml index b9e25e083c4..4657424c946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,6 +211,7 @@ lint.ignore = [ "D403", # docstring: capitalized first line "E731", # Do not assign a `lambda` expression, use a `def` ] +extend-include = ["*read-dependencies", "*bddeb", "*brpm"] [tool.ruff.lint.pydocstyle] convention = "pep257" diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index e4acf61b23a..63aeae955b1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -283,6 +283,15 @@ def skipUnlessJsonSchema(): ) +def skipUnlessJinjaVersionGreaterThan(version=(0, 0, 0)): + import jinja2 + + return pytest.mark.skipif( + tuple(map(int, jinja2.__version__.split("."))) < version, + reason=f"jinj2 version is less than {version}", + ) + + @contextmanager def does_not_raise(): """Context manager to parametrize tests raising and not raising exceptions diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 793786fa95d..7f8fc944429 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -6,6 +6,7 @@ from cloudinit import subp, templater, util from tests.helpers import cloud_init_project_dir +from tests.unittests.helpers import skipUnlessJinjaVersionGreaterThan # TODO(Look to align with tools.render-template or cloudinit.distos.OSFAMILIES) DISTRO_VARIANTS = [ @@ -203,6 +204,7 @@ class TestRenderSourcesList: ), ], ) + @skipUnlessJinjaVersionGreaterThan((3, 0, 0)) def test_render_sources_list_templates( self, tmpdir, template_path, expected ): diff --git a/tools/read-dependencies b/tools/read-dependencies index f0e21124c63..bed826f3b79 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -140,8 +140,22 @@ CI_SYSTEM_BASE_PKGS = { "freebsd": [f"{FREEBSD_PKG_PREFIX}tox"], "redhat": ["python3-tox"], "centos": ["python3-tox"], - "ubuntu": ["make", "devscripts", "python3-dev", "libssl-dev", "tox", "sbuild"], - "debian": ["make", "devscripts", "python3-dev", "libssl-dev", "tox", "sbuild"], + "ubuntu": [ + "make", + "devscripts", + "python3-dev", + "libssl-dev", + "tox", + "sbuild", + ], + "debian": [ + "make", + "devscripts", + "python3-dev", + "libssl-dev", + "tox", + "sbuild", + ], } @@ -363,7 +377,7 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): if test_distro: pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS["common"] if distro not in ("freebsd",): - pkg_list += CI_SYSTEM_BASE_PKGS["linux_common"] + pkg_list += CI_SYSTEM_BASE_PKGS["linux_common"] distro_base_pkgs = CI_SYSTEM_BASE_PKGS.get(distro, []) pkg_list += distro_base_pkgs print( From ffd26f17fadfa023097ab3d975675f4707b24e19 Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Wed, 1 Apr 2026 23:25:01 +0530 Subject: [PATCH 18/72] test: fix test_schema when the schema validation takes longer (#6822) When the schema validation takes more than 0.01 seconds, a log is emitted: "Validating schema took 0.070 seconds" When this happens, the tuple length is 2 and not one and the test test_validateconfig_schema_non_strict_emits_warnings fails. Fix it by checking if the length of the tuple is either one or two, not just one. Signed-off-by: Ani Sinha --- tests/unittests/config/test_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 797403bc39f..a3b94eaa98f 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -451,8 +451,8 @@ def test_validateconfig_schema_non_strict_emits_warnings(self, caplog): """When strict is False validate_cloudconfig_schema emits warnings.""" schema = {"properties": {"p1": {"type": "string"}}} validate_cloudconfig_schema({"p1": -1}, schema=schema, strict=False) - assert ( - caplog.record_tuples and len(caplog.record_tuples) == 1 + assert caplog.record_tuples and ( + len(caplog.record_tuples) == 1 or len(caplog.record_tuples) == 2 ), caplog.record_tuples [(module, log_level, log_msg)] = caplog.record_tuples assert "cloudinit.config.schema" == module From a8ccc57b0bdcde71a24a9c6e4789b1396fedf7bb Mon Sep 17 00:00:00 2001 From: Amy Chen <66719270+xiachen-rh@users.noreply.github.com> Date: Thu, 2 Apr 2026 22:13:50 +0800 Subject: [PATCH 19/72] test: fix integration test_combined for rhel (#6791) integraton tests: add RHEL distro support - add default usename for RHEL distro and platform specific. - add optional LAUNCH_USERNAME in integration_settings to override default username per run. - use distro-specific USER_DATA in test_combined and adjust rsyslog and locale config so they work on RHEL, and removing the modules which not support RHEL. - skip the tests in test_combined which do not surport RHEL. Signed-off-by: Amy Chen --- tests/integration_tests/clouds.py | 23 +++- .../integration_tests/integration_settings.py | 4 + .../modules/test_combined.py | 127 +++++++++++++++--- tests/integration_tests/releases.py | 4 + 4 files changed, 136 insertions(+), 22 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index e72213a1002..3ccb06e104b 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -39,8 +39,24 @@ DISTRO_TO_USERNAME = { "ubuntu": "ubuntu", + "rhel": "cloud-user", + "centos": "cloud-user", } +# Platform specific username overrides: (distro, platform) -> username +# Only list entries where the platform differs from the distro default. +DISTRO_PLATFORM_TO_USERNAME = { + ("rhel", "ec2"): integration_settings.LAUNCH_USERNAME or "ec2-user", + ("rhel", "azure"): integration_settings.LAUNCH_USERNAME or "azureuser", +} + + +def get_launch_username(os: str, platform: str) -> str: + key = (os, platform) + if key in DISTRO_PLATFORM_TO_USERNAME: + return DISTRO_PLATFORM_TO_USERNAME[key] + return DISTRO_TO_USERNAME[os] + def _get_ubuntu_series() -> list: """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series""" @@ -135,7 +151,9 @@ def launch( default_launch_kwargs = { "image_id": self.image_id, "user_data": user_data, - "username": DISTRO_TO_USERNAME[CURRENT_RELEASE.os], + "username": get_launch_username( + CURRENT_RELEASE.os, self.datasource + ), } if self.settings.INSTANCE_TYPE: default_launch_kwargs["instance_type"] = ( @@ -259,9 +277,10 @@ def _get_initial_image(self, **kwargs) -> str: class AzureCloud(IntegrationCloud): datasource = "azure" cloud_instance: Azure + username = get_launch_username(CURRENT_RELEASE.os, datasource) def _get_cloud_instance(self) -> Azure: - return Azure(tag="azure-integration-test") + return Azure(tag="azure-integration-test", username=self.username) def _get_initial_image(self, **kwargs) -> str: return super()._get_initial_image( diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index cb4bae3a84e..6164b4e7332 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -56,6 +56,10 @@ # creating a new one. The exact contents will be platform dependent EXISTING_INSTANCE_ID: Optional[str] = None +# Username to use when launching the instance. +# If not set, the default username for the platform will be used. +LAUNCH_USERNAME: Optional[str] = None + ################################################################## # IMAGE GENERATION SETTINGS ################################################################## diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 857023780d1..ec3962f0738 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -26,7 +26,12 @@ OS_IMAGE_TYPE, PLATFORM, ) -from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, NOBLE +from tests.integration_tests.releases import ( + CURRENT_RELEASE, + IS_RHEL, + IS_UBUNTU, + NOBLE, +) from tests.integration_tests.util import ( get_feature_flag_value, get_inactive_modules, @@ -37,7 +42,7 @@ verify_ordered_items_in_text, ) -USER_DATA = """\ +USER_DATA_UBUNTU = """\ #cloud-config users: - default @@ -90,6 +95,61 @@ timezone: Europe/Madrid """ +USER_DATA_RHEL = """\ +#cloud-config +users: +- default +- name: craig + sudo: false # make sure craig doesn't get elevated perms +final_message: | + This is my final message! + $version + $timestamp + $datasource + $uptime +locale: en_GB.UTF-8 +locale_configfile: /etc/locale.conf +package_update: true +random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed +rsyslog: + configs: + - "*.* @@127.0.0.1" + - filename: 0-basic-config.conf + content: | + module(load="imtcp") + input(type="imtcp" port="514") + $template RemoteLogs,"/var/log/rsyslog-cloudinit.log" + *.* ?RemoteLogs + & ~ + remotes: + me: "127.0.0.1" +runcmd: + - echo 'hello world' > /var/tmp/runcmd_output + - echo '💩' > /var/tmp/unicode_data + + - # + - logger "My test log" + +timezone: Europe/Madrid +""" +# Update this dict with proper user data to support new distros +USER_DATA_BY_DISTRO = { + "ubuntu": USER_DATA_UBUNTU, + "rhel": USER_DATA_RHEL, + "centos": USER_DATA_RHEL, +} + +if CURRENT_RELEASE.os not in USER_DATA_BY_DISTRO: + raise KeyError( + f"No USER_DATA for distro {CURRENT_RELEASE.os!r}. " + f"Add an entry to USER_DATA_BY_DISTRO for this distro." + ) + +USER_DATA = USER_DATA_BY_DISTRO[CURRENT_RELEASE.os] + @pytest.mark.ci @pytest.mark.user_data(USER_DATA) @@ -163,6 +223,7 @@ def test_deprecated_message(self, class_client: IntegrationInstance): ignore_warnings=True, ) + @pytest.mark.skipif(IS_RHEL, reason="rhel does not support ntp module") def test_ntp_with_apt(self, class_client: IntegrationInstance): """LP #1628337. @@ -175,6 +236,9 @@ def test_ntp_with_apt(self, class_client: IntegrationInstance): assert "W: Some index files failed to download" not in log assert "E: Unable to locate package ntp" not in log + @pytest.mark.skipif( + IS_RHEL, reason="rhel does not enable byobu by default" + ) def test_byobu(self, class_client: IntegrationInstance): """Test byobu configured as enabled by default.""" client = class_client @@ -183,9 +247,13 @@ def test_byobu(self, class_client: IntegrationInstance): def test_configured_locale(self, class_client: IntegrationInstance): """Test locale can be configured correctly.""" client = class_client - default_locale = client.read_from_file("/etc/default/locale") + default_locale_file = ( + "/etc/locale.conf" if IS_RHEL else "/etc/default/locale" + ) + default_locale = client.read_from_file(default_locale_file) assert "LANG=en_GB.UTF-8" in default_locale - + if IS_RHEL: + return locale_a = client.execute("locale -a") locale_gen = client.execute("grep -v '^#' /etc/locale.gen | uniq") if OS_IMAGE_TYPE == "minimal": @@ -214,16 +282,21 @@ def test_random_seed_data(self, class_client: IntegrationInstance): def test_rsyslog(self, class_client: IntegrationInstance): """Test rsyslog is configured correctly when applicable.""" + # /var/spool/rsylog is not created on rhel by default + log_file = ( + "/var/log/rsyslog-cloudinit.log" + if IS_RHEL + else "/var/spool/rsyslog/cloudinit.log" + ) if class_client.execute("command -v rsyslogd").ok: - assert "My test log" in class_client.read_from_file( - "/var/spool/rsyslog/cloudinit.log" - ) + assert "My test log" in class_client.read_from_file(log_file) def test_runcmd(self, class_client: IntegrationInstance): """Test runcmd works as expected""" client = class_client assert "hello world" == client.read_from_file("/var/tmp/runcmd_output") + @pytest.mark.skipif(IS_RHEL, reason="rhel does not support snap module") def test_snap(self, class_client: IntegrationInstance): """Integration test for the snap module. @@ -276,19 +349,32 @@ def test_no_problems(self, class_client: IntegrationInstance): verify_clean_boot( client, ignore_deprecations=True, require_warnings=require_warnings ) - requested_modules = { - "apt_configure", - "byobu", - "final_message", - "locale", - "ntp", - "seed_random", - "rsyslog", - "runcmd", - "snap", - "ssh_import_id", - "timezone", - } + # remove modules that are not supported on rhel + requested_modules = ( + { + "byobu", + "final_message", + "locale", + "seed_random", + "rsyslog", + "runcmd", + "timezone", + } + if IS_RHEL + else { + "apt_configure", + "byobu", + "final_message", + "locale", + "ntp", + "seed_random", + "rsyslog", + "runcmd", + "snap", + "ssh_import_id", + "timezone", + } + ) inactive_modules = get_inactive_modules(log) assert not requested_modules.intersection(inactive_modules), ( f"Expected active modules:" @@ -578,6 +664,7 @@ def test_networkd_wait_online(self, class_client: IntegrationInstance): @pytest.mark.user_data(USER_DATA) class TestCombinedNoCI: @retry(tries=30, delay=1) + @pytest.mark.skipif(IS_RHEL, reason="rhel skips ssh_import_id module") def test_ssh_import_id(self, class_client: IntegrationInstance): """Integration test for the ssh_import_id module. diff --git a/tests/integration_tests/releases.py b/tests/integration_tests/releases.py index 3c3c1fd3df0..25ca759f8e3 100644 --- a/tests/integration_tests/releases.py +++ b/tests/integration_tests/releases.py @@ -105,3 +105,7 @@ def from_os_image( CURRENT_RELEASE = Release.from_os_image() IS_UBUNTU = CURRENT_RELEASE.os == "ubuntu" +IS_RHEL = CURRENT_RELEASE.os in ( + "rhel", + "centos", +) # will add other RHEL-like distros later From d4566b1aa35951e6c32da330e627c023785026ea Mon Sep 17 00:00:00 2001 From: Dave Jones Date: Fri, 3 Apr 2026 02:31:16 +0100 Subject: [PATCH 20/72] fix(ubuntu): Configure arm64 to use archive.ubuntu.com (#6826) Fixes GH-6825 LP: #2147101 --- config/cloud.cfg.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 558eab2433b..b0e8dd32794 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -359,7 +359,7 @@ system_info: security: https://deb.debian.org/debian-security {% elif variant in ["ubuntu", "unknown"] %} package_mirrors: - - arches: [i386, amd64] + - arches: [arm64, i386, amd64] failsafe: primary: http://archive.ubuntu.com/ubuntu security: http://security.ubuntu.com/ubuntu @@ -369,7 +369,7 @@ system_info: - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ security: [] - - arches: [arm64, armel, armhf] + - arches: [armel, armhf] failsafe: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports From c9f5427e80d841df18a91109a187f064457dbb4b Mon Sep 17 00:00:00 2001 From: Paul <44974737+paulober@users.noreply.github.com> Date: Sat, 4 Apr 2026 05:38:58 +0200 Subject: [PATCH 21/72] fix(debian): Fix locale generation (#6472) The current debian apply_locale implementation depends on a modified locale-gen script that only exists on ubuntu systems. This patch fixes the generation of new locales and should also work on ubuntu with out the reliance on the custom locale-gen patch. Fixes GH-6471 Signed-off-by: paulober --- cloudinit/distros/debian.py | 143 +++++++++++++++++++++-- tests/unittests/config/test_cc_locale.py | 12 +- tests/unittests/distros/test_debian.py | 85 ++++++++++++-- 3 files changed, 217 insertions(+), 23 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1936c34dccd..5422d4b1c33 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -1,14 +1,17 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2025 Raspberry Pi Ltd. # # Author: Scott Moser # Author: Juerg Haefliger # Author: Joshua Harlow +# Author: Paul Oberosler # # This file is part of cloud-init. See LICENSE file for license information. import logging import os +import re from typing import List from cloudinit import distros, subp, util @@ -28,6 +31,8 @@ """ LOCALE_CONF_FN = "/etc/default/locale" +LOCALE_GEN_FN = "/etc/locale.gen" +SUPPORTED_FN = "/usr/share/i18n/SUPPORTED" class Distro(distros.Distro): @@ -111,7 +116,7 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): if need_regen: regenerate_locale( locale, - out_fn, + self.default_locale, keyname=keyname, install_function=self.install_packages, ) @@ -127,6 +132,7 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): update_locale_conf( locale, out_fn, + default_locale=self.default_locale, keyname=keyname, install_function=self.install_packages, ) @@ -278,13 +284,13 @@ def read_system_locale(sys_path=LOCALE_CONF_FN, keyname="LANG"): def update_locale_conf( - locale, sys_path, keyname="LANG", install_function=None + locale, sys_path, default_locale, keyname="LANG", install_function=None ): """Update system locale config""" LOG.debug( "Updating %s with locale setting %s=%s", sys_path, keyname, locale ) - if not subp.which("update-locale"): + if not subp.which("update-locale") and install_function: install_function(["locales"]) subp.subp( [ @@ -292,27 +298,144 @@ def update_locale_conf( "--locale-file=" + sys_path, "%s=%s" % (keyname, locale), ], + update_env={ + "LANGUAGE": default_locale, + "LANG": default_locale, + "LC_ALL": default_locale, + }, capture=False, ) -def regenerate_locale(locale, sys_path, keyname="LANG", install_function=None): +def _lookup_supported_i18n_value(requested: str) -> str: """ - Run locale-gen for the provided locale and set the default - system variable `keyname` appropriately in the provided `sys_path`. + Return the canonical line from /usr/share/i18n/SUPPORTED for `requested`. + + Accepts: + - bare language_region: "fi_FI" + - with charset: "fi_FI.ISO-8859-1" or "fi_FI.UTF-8" + - with modifier: "fi_FI@euro" (works with/without charset) + Prefers UTF-8 only when the request didn’t specify a charset and multiple + candidates exist; otherwise returns the first match. + """ + try: + supported_lines = util.load_text_file(SUPPORTED_FN).splitlines() + except OSError: + supported_lines = [] + + # Parse requested into locale[.charset][@mod] + m = re.match( + r"^([A-Za-z_]+)(?:\.([A-Za-z0-9\-]+))?(?:@([A-Za-z0-9_\-]+))?$", + requested, + ) + if not m: + # fallback: treat whole string as a prefix + prefix = requested + wanted_charset = None + wanted_mod = None + else: + base, wanted_charset, wanted_mod = m.group(1), m.group(2), m.group(3) + prefix = base + (f"@{wanted_mod}" if wanted_mod else "") + + # Collect candidates that start with requested locale (+modifier), + # each SUPPORTED line is "locale[.charset][@mod] CHARMAP" + candidates = [] + rx = re.compile( + rf"^\s*{re.escape(prefix)}(?:\.[^\s@]+)?(?:@[^\s]+)?\s+[^\s]+$" + ) + for line in supported_lines: + if rx.match(line): + candidates.append(line.strip()) + + if not candidates: + # As a last resort, construct a reasonable default (don’t force UTF-8) + # If user gave a charset, use it; else use UTF-8. + if not wanted_charset: + wanted_charset = "UTF-8" + return f"{prefix}.{wanted_charset} {wanted_charset}" + + if wanted_charset: + # Find exact charset match on first field (before space) + rx_exact = re.compile( + rf"^\s*{re.escape(prefix)}\.{re.escape(wanted_charset)}(?:@[^\s]+)?\s+" + ) + for line in candidates: + if rx_exact.match(line): + return line + + # No explicit charset requested: prefer UTF-8 if + # present, else first candidate + for line in candidates: + if re.search(r"\sUTF-8$", line, re.IGNORECASE): + return line + return candidates[0] + + +def regenerate_locale( + locale, default_locale, keyname="LANG", install_function=None +): """ + Ensure `locale` is enabled in /etc/locale.gen, then run locale-gen. + Debian's locale-gen reads /etc/locale.gen and ignores positional args. + """ + # special case for locales which do not require regen # % locale -a # C # C.UTF-8 # POSIX - if locale.lower() in ["c", "c.utf-8", "posix"]: + if locale.lower() in ["c", "c.utf-8", "c.utf8", "posix"]: LOG.debug("%s=%s does not require rengeneration", keyname, locale) return - # finally, trigger regeneration - if not subp.which("locale-gen"): + if not subp.which("locale-gen") and install_function: install_function(["locales"]) + + # compute canonical line and NEW_LANG (first field) + line = _lookup_supported_i18n_value(locale) + + # ensure /etc/locale.gen contains the + # line (uncomment if present; append if absent) + existing = "" + try: + existing = util.load_text_file(LOCALE_GEN_FN) + except OSError: + existing = "" + + out_lines = [] + found_enabled = False + target_re = re.compile(rf"^#?\s*{re.escape(line)}\s*$") + + for raw in existing.splitlines(): + s = raw.strip() + if not s: + out_lines.append(raw) + continue + + if target_re.match(s.lstrip("# ")): + # enable target locale + out_lines.append(line) + found_enabled = True + else: + # keep all other locales as-is (don't disable them) + out_lines.append(raw) + + if not found_enabled: + out_lines.append(line) + + util.ensure_dir(os.path.dirname(LOCALE_GEN_FN)) + util.write_file(LOCALE_GEN_FN, "\n".join(out_lines).rstrip() + "\n") + + # finally, generate locales listed in /etc/locale.gen LOG.debug("Generating locales for %s", locale) - subp.subp(["locale-gen", locale], capture=False) + # Using --keep-existing to avoid removing already generated locales + subp.subp( + ["locale-gen", "--keep-existing"], + capture=False, + update_env={ + "LANGUAGE": default_locale, + "LANG": default_locale, + "LC_ALL": default_locale, + }, + ) diff --git a/tests/unittests/config/test_cc_locale.py b/tests/unittests/config/test_cc_locale.py index fb564323adc..22df4fb3f25 100644 --- a/tests/unittests/config/test_cc_locale.py +++ b/tests/unittests/config/test_cc_locale.py @@ -1,7 +1,3 @@ -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. -# -# Author: Juerg Haefliger -# # This file is part of cloud-init. See LICENSE file for license information. import logging from io import BytesIO @@ -48,7 +44,8 @@ def test_set_locale_arch(self): contents = util.load_text_file(cc.distro.locale_gen_fn) assert "%s UTF-8" % locale in contents m_subp.assert_called_with( - ["localectl", "set-locale", locale], capture=False + ["localectl", "set-locale", locale], + capture=False, ) @pytest.mark.parametrize( @@ -122,6 +119,11 @@ def test_locale_update_config_if_different_than_default(self, tmpdir): "--locale-file=%s" % locale_conf.strpath, "LANG=C.UTF-8", ], + update_env={ + "LANG": "C.UTF-8", + "LANGUAGE": "C.UTF-8", + "LC_ALL": "C.UTF-8", + }, capture=False, ) m_which.assert_called_once_with("update-locale") diff --git a/tests/unittests/distros/test_debian.py b/tests/unittests/distros/test_debian.py index 9634d185ce2..1262615d3a4 100644 --- a/tests/unittests/distros/test_debian.py +++ b/tests/unittests/distros/test_debian.py @@ -67,7 +67,7 @@ def test_rerun_if_different(self, distro, m_subp, caplog): util.write_file(LOCALE_PATH, "LANG=fr_FR.UTF-8", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -103,17 +103,20 @@ def test_rerun_if_no_file( with mock.patch.object(distro, "install_packages") as m_install: distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", f"LANG={locale}", ], ] == [p[0][0] for p in m_subp.call_args_list] - assert [ - mock.call("locale-gen"), - mock.call("update-locale"), - ] == m_which.call_args_list + calls = [c.args[0] for c in m_which.call_args_list] + # collapse consecutive duplicates from _ensure_tool re-check + uniq: list[str] = [] + for name in calls: + if not uniq or uniq[-1] != name: + uniq.append(name) + assert uniq == ["locale-gen", "update-locale"] if install_pkgs: m_install.assert_called_with(install_pkgs) else: @@ -125,7 +128,7 @@ def test_rerun_on_unset_system_locale(self, distro, m_subp, caplog): util.write_file(LOCALE_PATH, "LANG=", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH) assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -148,7 +151,7 @@ def test_rerun_on_mismatched_keys(self, distro, m_subp): util.write_file(LOCALE_PATH, "LANG=", omode="w") distro.apply_locale(locale, out_fn=LOCALE_PATH, keyname="LC_ALL") assert [ - ["locale-gen", locale], + ["locale-gen", "--keep-existing"], [ "update-locale", f"--locale-file={LOCALE_PATH}", @@ -170,3 +173,69 @@ def test_falseish_locale_raises_valueerror(self, distro, m_subp): ): distro.apply_locale("") m_subp.assert_not_called() + + +@pytest.mark.usefixtures("fake_filesystem") +class TestLookupSupportedI18nValue: + """Test _lookup_supported_i18n_value function.""" + + def test_no_match_constructs_default(self, mocker): + """When no match is found in SUPPORTED, construct a default line.""" + from cloudinit.distros.debian import _lookup_supported_i18n_value + + # Mock SUPPORTED file to be empty + mocker.patch( + "cloudinit.distros.debian.util.load_text_file", + side_effect=OSError("File not found"), + ) + + # Request a locale that won't be found + result = _lookup_supported_i18n_value("xyz_XY.UTF-8") + assert result == "xyz_XY.UTF-8 UTF-8" + + # Without charset, should default to UTF-8 + result = _lookup_supported_i18n_value("abc_AB") + assert result == "abc_AB.UTF-8 UTF-8" + + # With explicit charset + result = _lookup_supported_i18n_value("def_DE.ISO-8859-1") + assert result == "def_DE.ISO-8859-1 ISO-8859-1" + + def test_formats_with_charset_and_modifier(self, mocker): + """Test various locale formats: prefix, charset, and modifier.""" + from cloudinit.distros.debian import _lookup_supported_i18n_value + + # Mock SUPPORTED file with various locale formats + supported_content = """# Supported locales +en_US.UTF-8 UTF-8 +fi_FI.ISO-8859-1 ISO-8859-1 +fi_FI.UTF-8 UTF-8 +it_IT@euro ISO-8859-15 +ca_ES@valencia.UTF-8 UTF-8 +de_DE.UTF-8 UTF-8 + fr_FR.UTF-8 UTF-8 +""" + mocker.patch( + "cloudinit.distros.debian.util.load_text_file", + return_value=supported_content, + ) + + # Test modifier without explicit charset: it_IT@euro + result = _lookup_supported_i18n_value("it_IT@euro") + assert result == "it_IT@euro ISO-8859-15" + + # Test charset without modifier: fi_FI.ISO-8859-1 + result = _lookup_supported_i18n_value("fi_FI.ISO-8859-1") + assert result == "fi_FI.ISO-8859-1 ISO-8859-1" + + # Test both charset and modifier: ca_ES@valencia.UTF-8 + result = _lookup_supported_i18n_value("ca_ES@valencia.UTF-8") + assert result == "ca_ES@valencia.UTF-8 UTF-8" + + # Test bare locale preferring UTF-8 + result = _lookup_supported_i18n_value("fi_FI") + assert result == "fi_FI.UTF-8 UTF-8" + + # Test that lines with leading whitespace are matched (then stripped) + result = _lookup_supported_i18n_value("fr_FR.UTF-8") + assert result == "fr_FR.UTF-8 UTF-8" From 1d48a14460d41a31e90eed8e6f3897f0fa189fd2 Mon Sep 17 00:00:00 2001 From: Nick Brown Date: Mon, 6 Apr 2026 19:55:36 +0200 Subject: [PATCH 22/72] fix: increase raspberry pi usb timeout (#6823) Running on `rpi-usb-gadget` on a Raspberry Pi Zero 2 W takes more than 15 seconds. Even running it manually on a settled system is longer than the allowed timeout: Fixes: raspberrypi/rpi-imager#1403 --- cloudinit/config/cc_raspberry_pi.py | 2 +- tests/unittests/config/test_cc_raspberry_pi.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 56f3ebbcd98..42113ae8d4d 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -63,7 +63,7 @@ def configure_usb_gadget(enable: bool) -> bool: "-f", ], capture=False, - timeout=15, + timeout=30, ) return True diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 6c2e74a6c80..b32695743d3 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -113,7 +113,7 @@ def test_configure_usb_gadget_enable(self, m_subp): with mock.patch("os.path.exists", return_value=True): cc_rpi.configure_usb_gadget(True) m_subp.assert_called_once_with( - [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=15 + [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=30 ) @mock.patch("cloudinit.subp.subp") @@ -152,7 +152,7 @@ def test_configure_usb_gadget_script_failure(self, m_subp, caplog): # Subprocess should have been invoked once m_subp.assert_called_once_with( - [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=15 + [RPI_USB_GADGET_SCRIPT, "on", "-f"], capture=False, timeout=30 ) # Error log should contain failure message From 59b73939a2f14da26e2527b0e5e42a282c2ab671 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 7 Apr 2026 12:32:25 -0600 Subject: [PATCH 23/72] ci: move ssh-import-id tests to github instead of launchpad keys (#6830) ssh-import-id has been experiencing high failure rates retrieving Launchpad-based SSH keys resulting in exit 1 which raises errors from cloud-init status. Use Github based keys instead as this backend doesn't appear to cause problems. --- tests/integration_tests/modules/test_combined.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index ec3962f0738..7c04a183064 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -90,7 +90,7 @@ commands: - snap install hello-world ssh_import_id: - - lp:smoser + - gh:blackboxsw timezone: Europe/Madrid """ @@ -678,4 +678,4 @@ def test_ssh_import_id(self, class_client: IntegrationInstance): client = class_client ssh_output = client.read_from_file("/home/ubuntu/.ssh/authorized_keys") - assert "# ssh-import-id lp:smoser" in ssh_output + assert "# ssh-import-id gh:blackboxsw" in ssh_output From 6aba62a914e69c1ff4533c8653847e91cbc7be39 Mon Sep 17 00:00:00 2001 From: Hyacinthe Cartiaux Date: Fri, 10 Apr 2026 16:36:46 +0200 Subject: [PATCH 24/72] feat(bsd): add OpenBSD support to Meson build (#6789) --- meson.build | 21 ++++++++++++++++++ meson_options.txt | 2 +- packages/pkg-deps.json | 19 +++++++++++++++++ tools/build-on-openbsd | 47 +++++++++-------------------------------- tools/read-dependencies | 10 ++++++++- 5 files changed, 60 insertions(+), 39 deletions(-) diff --git a/meson.build b/meson.build index 1068a0a2a85..0e06d8de292 100644 --- a/meson.build +++ b/meson.build @@ -221,6 +221,27 @@ elif init_system == 'sysvinit_freebsd' endforeach # Enable cloud-init on reboot meson.add_install_script('sh', '-c', '/usr/sbin/sysrc cloudinit_enable=YES') +elif init_system == 'sysvinit_openbsd' + rcd_templates = run_command(find, 'sysvinit/openbsd', '-type', 'f', check: true) + foreach template : rcd_templates.stdout().strip().split('\n') + custom_target( + input: template, + output: '@BASENAME@', + command: [ + render_tmpl, + '@INPUT@', + meson.current_build_dir() / '@OUTPUT@', + ], + install: true, + install_dir: sysconfdir / 'rc.d', + install_mode: 'r-xr-xr-x', + install_tag: 'sysvinit', + ) + endforeach + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudinitlocal') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudinit') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudconfig') + meson.add_install_script('/usr/sbin/rcctl', 'enable', 'cloudfinal') endif custom_target( diff --git a/meson_options.txt b/meson_options.txt index 8802c8ef3ba..dcb0198afa5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,4 +1,4 @@ -option('init_system', type: 'combo', value: 'systemd', choices: ['systemd', 'sysvinit_openrc', 'sysvinit_freebsd'], description: 'Set target init system.') +option('init_system', type: 'combo', value: 'systemd', choices: ['systemd', 'sysvinit_openrc', 'sysvinit_freebsd', 'sysvinit_openbsd'], description: 'Set target init system.') option('distro_templates', type: 'array', value: [], description: 'Distro template files to install. WARNING: Templates may change in the future. If using this option, be sure to check new releases for template file changes.') option('disable_sshd_keygen', type: 'boolean', value: false, description: 'Provide systemd service to disable sshd-keygen if present to avoid races with cloud-init.') option('bash_completion', type: 'boolean', value: true, description: 'Bash completion for cloud-init.') diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 2d7b7947307..5af11a7b732 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -99,6 +99,25 @@ "sudo" ] }, + "openbsd": { + "renames" : { + "pyserial" : "py3-serial", + "pyyaml" : "py3-yaml", + "pytest" : "py3-test", + "pytest-cov" : "py3-test-cov", + "pytest-mock" : "py3-test-mock", + "pytest-xdist": "py3-test-xdist" + }, + "build-requires" : [ + "bash", + "bash-completion", + "meson" + ], + "requires" : [ + "e2fsprogs", + "sudo--" + ] + }, "suse" : { "renames" : { "jinja2" : "python3-Jinja2", diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd index 47c6677e8a2..1665f6b918c 100755 --- a/tools/build-on-openbsd +++ b/tools/build-on-openbsd @@ -1,4 +1,8 @@ #!/bin/sh +# This script provides a quick and dirty way of building and installing +# cloud-init on OpenBSD. + +set -eux fail() { echo "FAILED:" "$@" 1>&2 @@ -13,42 +17,11 @@ fi # Check dependencies: depschecked=/tmp/c-i.dependencieschecked -pkgs=" - bash - dmidecode - py3-configobj - py3-jinja2 - py3-jsonschema - py3-jsonpatch - py3-jsonpointer - py3-oauthlib - py3-requests - py3-setuptools - py3-serial - py3-yaml - sudo-- - wget -" - -[ -f $depschecked ] || echo "Installing the following packages: $pkgs" -output=$(pkg_add -zI $pkgs 2>&1) - -if echo "$output" | grep -q -e "Can't find" -e "Ambiguous"; then - echo "Failed to find or install one or more packages" - echo "Failed Package(s):" - echo "$output" - exit 1 -else - echo Successfully installed packages - touch $depschecked - - python3 setup.py build - python3 setup.py install -O1 --distro openbsd --skip-build --init-system sysvinit_openbsd +[ -f "$depschecked" ] || ./tools/read-dependencies --distro openbsd -t || fail "install packages" +touch "$depschecked" - echo "Installation completed." +# Build the code and install in /usr/local/: +meson setup builddir -Dinit_system=sysvinit_openbsd -Dsysconfdir=/etc +meson install -C builddir - rcctl enable cloudinitlocal - rcctl enable cloudinit - rcctl enable cloudconfig - rcctl enable cloudfinal -fi +echo "Installation completed." diff --git a/tools/read-dependencies b/tools/read-dependencies index bed826f3b79..ca58efd3b9c 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -23,6 +23,7 @@ DEFAULT_REQUIREMENTS = "requirements.txt" DEFAULT_PKG_PREFIX = "python3-" FREEBSD_PKG_PREFIX = f"py{sys.version_info.major}{sys.version_info.minor}-" +OPENBSD_PKG_PREFIX = "py3-" # Map the appropriate package dir needed for each distro choice DISTRO_PKG_TYPE_MAP = { @@ -30,6 +31,7 @@ DISTRO_PKG_TYPE_MAP = { "eurolinux": "redhat", "miraclelinux": "redhat", "freebsd": "freebsd", + "openbsd": "openbsd", "fedora": "fedora", "rocky": "redhat", "redhat": "redhat", @@ -113,12 +115,14 @@ DRYRUN_DISTRO_INSTALL_PKG_CMD = { "redhat": ["yum", "install", "--assumeyes"], "fedora": ["yum", "install", "--assumeyes"], "freebsd": ["pkg", "install", "--yes"], + "openbsd": ["pkg_add", "-n"], } DISTRO_INSTALL_PKG_CMD = { "redhat": MAYBE_RELIABLE_YUM_INSTALL, "fedora": MAYBE_RELIABLE_YUM_INSTALL, "freebsd": ["pkg", "install", "--yes"], + "openbsd": ["pkg_add", "-I"], "debian": ["apt", "install", "-y"], "suse": ZYPPER_INSTALL, } @@ -126,6 +130,7 @@ DISTRO_INSTALL_PKG_CMD = { DISTRO_UPDATE_PKG_CMD = { "redhat": ["yum", "update"], "freebsd": ["pkg", "update"], + "openbsd": ["pkg_add", "-I", "-u"], "debian": ["apt", "update", "-q"], "suse": ["zypper", "update"], } @@ -138,6 +143,7 @@ CI_SYSTEM_BASE_PKGS = { "miraclelinux": ["python3-tox"], "fedora": ["python3-tox"], "freebsd": [f"{FREEBSD_PKG_PREFIX}tox"], + "openbsd": [f"{OPENBSD_PKG_PREFIX}tox"], "redhat": ["python3-tox"], "centos": ["python3-tox"], "ubuntu": [ @@ -284,6 +290,8 @@ def translate_pip_to_system_pkg(distro, pip_requires, renames): """ if distro in ("freebsd",): prefix = FREEBSD_PKG_PREFIX + elif distro in ("openbsd",): + prefix = OPENBSD_PKG_PREFIX else: prefix = DEFAULT_PKG_PREFIX standard_pkg_name = "{0}{1}" @@ -376,7 +384,7 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" if test_distro: pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS["common"] - if distro not in ("freebsd",): + if distro not in ("freebsd", "openbsd"): pkg_list += CI_SYSTEM_BASE_PKGS["linux_common"] distro_base_pkgs = CI_SYSTEM_BASE_PKGS.get(distro, []) pkg_list += distro_base_pkgs From 275e0e0323839fa25f9a26835188412bb2d0625f Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 13 Apr 2026 10:26:54 -0500 Subject: [PATCH 25/72] test: Add new `fake_fs` fixture backed by pyfakefs (#6785) Also convert a few fake_filesystem tests to prove its usage: - tests/unittests/config/test_cc_keyboard.py - tests/unittests/config/test_cc_mounts.py - tests/unittests/config/test_cc_update_etc_hosts.py - tests/unittests/config/test_cc_puppet.py --- test-requirements.txt | 1 + tests/unittests/config/test_cc_keyboard.py | 15 +-- tests/unittests/config/test_cc_mounts.py | 115 ++++++------------ tests/unittests/config/test_cc_puppet.py | 19 +-- .../config/test_cc_update_etc_hosts.py | 56 +++------ tests/unittests/conftest.py | 13 +- tox.ini | 1 + 7 files changed, 83 insertions(+), 137 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9467f3d9328..4d89bf76946 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,6 +15,7 @@ jsonschema responses packaging passlib +pyfakefs # This one is currently used only by the CloudSigma and SmartOS datasources. # If these datasources are removed, this is no longer needed. diff --git a/tests/unittests/config/test_cc_keyboard.py b/tests/unittests/config/test_cc_keyboard.py index d0731a54075..39ae14eea61 100644 --- a/tests/unittests/config/test_cc_keyboard.py +++ b/tests/unittests/config/test_cc_keyboard.py @@ -2,7 +2,6 @@ """Tests cc_keyboard module""" -import os import re from unittest import mock @@ -14,7 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import populate_dir, skipUnlessJsonSchema +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -78,7 +77,7 @@ def test_schema_validation(self, config, error_msg): validate_cloudconfig_schema(config, schema, strict=True) -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("fake_fs") class TestKeyboard: @mock.patch("cloudinit.distros.Distro.uses_systemd") @mock.patch("cloudinit.distros.subp.subp") @@ -122,7 +121,7 @@ def test_debian_linux_cmd(self, m_subp, m_write_file): ) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_cmd(self, m_subp, tmpdir): + def test_alpine_linux_cmd(self, m_subp, fake_fs): """Alpine Linux runs setup-keymap""" cfg = {"keyboard": {"layout": "us", "variant": "us"}} layout = "us" @@ -133,14 +132,13 @@ def test_alpine_linux_cmd(self, m_subp, tmpdir): keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs(tmpdir.join(keymap_dir)) - populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) + fake_fs.create_file(keymap_file) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) m_subp.assert_called_once_with(["setup-keymap", layout, variant]) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_ignore_model(self, m_subp, caplog, tmpdir): + def test_alpine_linux_ignore_model(self, m_subp, caplog, fake_fs): """Alpine Linux ignores model setting""" cfg = { "keyboard": { @@ -155,8 +153,7 @@ def test_alpine_linux_ignore_model(self, m_subp, caplog, tmpdir): keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs(tmpdir.join(keymap_dir)) - populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) + fake_fs.create_file(keymap_file) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) assert "Keyboard model is ignored for Alpine Linux." in caplog.text diff --git a/tests/unittests/config/test_cc_mounts.py b/tests/unittests/config/test_cc_mounts.py index 549c59cf0cf..80dbf5c6e11 100644 --- a/tests/unittests/config/test_cc_mounts.py +++ b/tests/unittests/config/test_cc_mounts.py @@ -30,107 +30,87 @@ class TestSanitizeDevname: - def _touch(self, path, new_root): - path = os.path.join(new_root, path.lstrip("/")) - basedir = os.path.dirname(path) - if not os.path.exists(basedir): - os.makedirs(basedir) - open(path, "a").close() - - def _makedirs(self, directory, new_root): - directory = os.path.join(new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def mock_existence_of_disk(self, disk_path, new_root): - self._touch(disk_path, new_root) - self._makedirs( - os.path.join("/sys/block", disk_path.split("/")[-1]), new_root - ) + # E.g., /sys/block/sda + path = "/sys/block/" + disk_path.split("/")[-1] + new_root.create_dir(path) def mock_existence_of_partition( self, disk_path, partition_number, new_root ): self.mock_existence_of_disk(disk_path, new_root) - self._touch(disk_path + str(partition_number), new_root) + # E.g., /dev/sda1 + dev_path = disk_path + str(partition_number) + new_root.create_file(dev_path) disk_name = disk_path.split("/")[-1] - self._makedirs( - os.path.join( - "/sys/block", disk_name, disk_name + str(partition_number) - ), - new_root, + # E.g., /sys/block/sda/sda1 + block_path = os.path.join( + "/sys/block", disk_name, disk_name + str(partition_number) ) + new_root.create_dir(block_path) - def test_existent_full_disk_path_is_returned(self, fake_filesystem): + def test_existent_full_disk_path_is_returned(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( disk_path, lambda x: None ) - def test_existent_disk_name_returns_full_path(self, fake_filesystem): + def test_existent_disk_name_returns_full_path(self, fake_fs): disk_name = "sda" disk_path = "/dev/" + disk_name - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( disk_name, lambda x: None ) - def test_existent_meta_disk_is_returned(self, fake_filesystem): + def test_existent_meta_disk_is_returned(self, fake_fs): actual_disk_path = "/dev/sda" - self.mock_existence_of_disk(actual_disk_path, fake_filesystem) + self.mock_existence_of_disk(actual_disk_path, fake_fs) assert actual_disk_path == cc_mounts.sanitize_devname( "ephemeral0", lambda x: actual_disk_path, ) - def test_existent_meta_partition_is_returned(self, fake_filesystem): + def test_existent_meta_partition_is_returned(self, fake_fs): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.1", lambda x: disk_name, ) - def test_existent_meta_partition_with_p_is_returned(self, fake_filesystem): + def test_existent_meta_partition_with_p_is_returned(self, fake_fs): disk_name, partition_part = "/dev/sda", "p1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.1", lambda x: disk_name, ) def test_first_partition_returned_if_existent_disk_is_partitioned( - self, fake_filesystem + self, fake_fs ): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0", lambda x: disk_name, ) - def test_nth_partition_returned_if_requested(self, fake_filesystem): + def test_nth_partition_returned_if_requested(self, fake_fs): disk_name, partition_part = "/dev/sda", "3" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition( - disk_name, partition_part, fake_filesystem - ) + self.mock_existence_of_partition(disk_name, partition_part, fake_fs) assert actual_partition_path == cc_mounts.sanitize_devname( "ephemeral0.3", lambda x: disk_name, ) - def test_transformer_returning_none_returns_none(self, fake_filesystem): + def test_transformer_returning_none_returns_none(self, fake_fs): assert ( cc_mounts.sanitize_devname( "ephemeral0", @@ -139,7 +119,7 @@ def test_transformer_returning_none_returns_none(self, fake_filesystem): is None ) - def test_missing_device_returns_none(self, fake_filesystem): + def test_missing_device_returns_none(self, fake_fs): assert ( cc_mounts.sanitize_devname( "/dev/sda", @@ -148,9 +128,9 @@ def test_missing_device_returns_none(self, fake_filesystem): is None ) - def test_missing_sys_returns_none(self, fake_filesystem): + def test_missing_sys_returns_none(self, fake_fs): disk_path = "/dev/sda" - self._makedirs(disk_path, fake_filesystem) + fake_fs.create_dir(disk_path) assert ( cc_mounts.sanitize_devname( disk_path, @@ -159,11 +139,9 @@ def test_missing_sys_returns_none(self, fake_filesystem): is None ) - def test_existent_disk_but_missing_partition_returns_none( - self, fake_filesystem - ): + def test_existent_disk_but_missing_partition_returns_none(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert ( cc_mounts.sanitize_devname( "ephemeral0.1", @@ -172,16 +150,16 @@ def test_existent_disk_but_missing_partition_returns_none( is None ) - def test_network_device_returns_network_device(self, fake_filesystem): + def test_network_device_returns_network_device(self, fake_fs): disk_path = "netdevice:/path" assert disk_path == cc_mounts.sanitize_devname( disk_path, None, ) - def test_device_aliases_remapping(self, fake_filesystem): + def test_device_aliases_remapping(self, fake_fs): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path, fake_filesystem) + self.mock_existence_of_disk(disk_path, fake_fs) assert disk_path == cc_mounts.sanitize_devname( "mydata", lambda x: None, {"mydata": disk_path} ) @@ -189,13 +167,10 @@ def test_device_aliases_remapping(self, fake_filesystem): class TestSwapFileCreation: @pytest.fixture(autouse=True) - def setup(self, mocker, fake_filesystem: str): - self.new_root = fake_filesystem - self.swap_path = os.path.join(fake_filesystem, "swap.img") - fstab_path = os.path.join(fake_filesystem, "etc/fstab") - self._makedirs("/etc") + def setup(self, mocker, fake_fs): + self.swap_path = "/swap.img" + fake_fs.create_dir("/etc") - self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", fstab_path) self.m_subp = mocker.patch(f"{M_PATH}subp.subp") self.m_mounts = mocker.patch( f"{M_PATH}util.mounts", @@ -220,11 +195,6 @@ def setup(self, mocker, fake_filesystem: str): } } - def _makedirs(self, directory): - directory = os.path.join(self.new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def device_name_to_device(self, path): if path == "swap": return self.swap_path @@ -324,13 +294,9 @@ class TestFstabHandling: swap_path = "/dev/sdb1" @pytest.fixture(autouse=True) - def setup(self, mocker, fake_filesystem: str): - self.new_root = fake_filesystem + def setup(self, mocker, fake_fs): + fake_fs.create_dir("/etc") - self.fstab_path = os.path.join(self.new_root, "etc/fstab") - self._makedirs("/etc") - - self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", self.fstab_path) self.m_subp = mocker.patch(f"{M_PATH}subp.subp") self.m_mounts = mocker.patch( f"{M_PATH}util.mounts", @@ -351,11 +317,6 @@ def setup(self, mocker, fake_filesystem: str): self.mock_log = mock.Mock() self.mock_cloud.device_name_to_device = self.device_name_to_device - def _makedirs(self, directory): - directory = os.path.join(self.new_root, directory.lstrip("/")) - if not os.path.exists(directory): - os.makedirs(directory) - def device_name_to_device(self, path): if path == "swap": return self.swap_path diff --git a/tests/unittests/config/test_cc_puppet.py b/tests/unittests/config/test_cc_puppet.py index 637e78a5c82..a3f04fb8fe7 100644 --- a/tests/unittests/config/test_cc_puppet.py +++ b/tests/unittests/config/test_cc_puppet.py @@ -73,11 +73,11 @@ def test_enable_fallback_on_failure(self, m_subp): assert expected_calls == m_subp.call_args_list -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("fake_fs") @mock.patch("cloudinit.config.cc_puppet._manage_puppet_services") class TestPuppetHandle: - CONF = "puppet.conf" - CSR_ATTRIBUTES_PATH = "csr_attributes.yaml" + CONF = "/etc/puppet/puppet.conf" + CSR_ATTRIBUTES_PATH = "/etc/puppet/csr_attributes.yaml" def test_skips_missing_puppet_key_in_cloudconfig( self, m_man_puppet, caplog @@ -238,21 +238,14 @@ def test_puppet_config_installs_puppet_version(self, m_subp, _): mock.call([["puppet-agent", "3.8"]]) ] == cloud.distro.install_packages.call_args_list - @mock.patch("cloudinit.config.cc_puppet.get_config_value") @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) - def test_puppet_config_updates_puppet_conf( - self, m_subp, m_default, m_man_puppet - ): + def test_puppet_config_updates_puppet_conf(self, m_subp, m_man_puppet): """When 'conf' is provided update values in PUPPET_CONF_PATH.""" - def _fake_get_config_value(puppet_bin, setting): - return self.CONF - - m_default.side_effect = _fake_get_config_value - cfg = { "puppet": { - "conf": {"agent": {"server": "puppetserver.example.org"}} + "conf": {"agent": {"server": "puppetserver.example.org"}}, + "conf_file": self.CONF, } } util.write_file(self.CONF, "[agent]\nserver = origpuppet\nother = 3") diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py index 080c92157db..48f985cc217 100644 --- a/tests/unittests/config/test_cc_update_etc_hosts.py +++ b/tests/unittests/config/test_cc_update_etc_hosts.py @@ -1,13 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging -import os import re -import shutil import pytest +from pyfakefs.fake_filesystem import FakeFilesystem -from cloudinit import cloud, distros, helpers, util +from cloudinit import util from cloudinit.config import cc_update_etc_hosts from cloudinit.config.schema import ( SchemaValidationError, @@ -16,44 +15,34 @@ ) from tests.helpers import cloud_init_project_dir from tests.unittests import helpers as t_help +from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -@pytest.fixture(autouse=True) -def with_templates(tmp_path, fake_filesystem_hook): - shutil.copytree( - str(cloud_init_project_dir("templates")), - str(tmp_path / "templates"), - dirs_exist_ok=True, +@pytest.fixture +def with_templates(fake_fs: FakeFilesystem): + fake_fs.add_real_directory( + cloud_init_project_dir("templates"), + target_path="/etc/cloud/templates", + read_only=True, ) -@pytest.mark.usefixtures("fake_filesystem") +@pytest.mark.usefixtures("with_templates") class TestHostsFile: - def _fetch_distro(self, kind): - cls = distros.fetch(kind) - paths = helpers.Paths({}) - return cls(kind, {}, paths) - - def test_write_etc_hosts_suse_localhost(self, tmp_path): + def test_write_etc_hosts_suse_localhost(self, fake_fs: FakeFilesystem): cfg = { "manage_etc_hosts": "localhost", "hostname": "cloud-init.test.us", } - os.makedirs(tmp_path / "etc/") + hosts_path = "/etc/hosts" hosts_content = "192.168.1.1 blah.blah.us blah\n" - etc_hosts = str(tmp_path / "etc/hosts") - fout = open(etc_hosts, "w") - fout.write(hosts_content) - fout.close() - distro = self._fetch_distro("sles") - distro.hosts_fn = etc_hosts - paths = helpers.Paths({}) - ds = None - cc = cloud.Cloud(ds, paths, {}, distro, None) + fake_fs.create_file(hosts_path, contents=hosts_content) + cc = get_cloud("sles") + cc.distro.hosts_fn = hosts_path cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file(etc_hosts) + contents = util.load_text_file(hosts_path) assert ( "127.0.1.1\tcloud-init.test.us\tcloud-init" in contents ), "No entry for 127.0.1.1 in etc/hosts" @@ -61,21 +50,14 @@ def test_write_etc_hosts_suse_localhost(self, tmp_path): "192.168.1.1\tblah.blah.us\tblah" in contents ), "Default etc/hosts content modified" - def test_write_etc_hosts_suse_template(self, tmp_path): + def test_write_etc_hosts_suse_template(self, fake_fs): cfg = { "manage_etc_hosts": "template", "hostname": "cloud-init.test.us", } - shutil.copytree( - tmp_path / "templates", str(tmp_path / "etc/cloud/templates") - ) - distro = self._fetch_distro("sles") - paths = helpers.Paths({}) - paths.template_tpl = str(tmp_path / "etc/cloud/templates/%s.tmpl") - ds = None - cc = cloud.Cloud(ds, paths, {}, distro, None) + cc = get_cloud("sles") cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file(tmp_path / "etc/hosts") + contents = util.load_text_file("/etc/hosts") assert ( "127.0.1.1 cloud-init.test.us cloud-init" in contents ), "No entry for 127.0.1.1 in etc/hosts" diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index 07d50ce1a20..76be658b384 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -6,6 +6,7 @@ from unittest import mock import pytest +from pyfakefs.fake_filesystem import FakeFilesystem from cloudinit import ( atomic_helper, @@ -119,7 +120,9 @@ def fake_filesystem_hook(): @pytest.fixture def fake_filesystem(mocker, tmpdir, fake_filesystem_hook): - """Mocks fs functions to operate under `tmpdir` + """This fixture is DEPRECATED for new tests. Use fake_fs instead. + + Mocks fs functions to operate under `tmpdir`. This fixture is sorted after fix_cloud_init_hook to allow fixtures sorted before fake_cloud_init_hook to access the real filesystem. @@ -148,6 +151,14 @@ def fake_filesystem(mocker, tmpdir, fake_filesystem_hook): yield str(tmpdir) +@pytest.fixture +def fake_fs(fs: FakeFilesystem): + """Mocks fs and pathlib functions to operate under a fake filesystem. + + See https://pytest-pyfakefs.readthedocs.io""" + yield fs + + @pytest.fixture(scope="session", autouse=True) def disable_sysfs_net(tmpdir_factory): """Avoid tests which read the underlying host's /syc/class/net.""" diff --git a/tox.ini b/tox.ini index a1e1e16972e..7565be59722 100644 --- a/tox.ini +++ b/tox.ini @@ -203,6 +203,7 @@ deps = pytest-mock==3.6.1 responses==0.18.0 passlib==1.7.4 + pyfakefs==4.5.4 commands = {envpython} -m pytest --cov=cloud-init --cov-branch {posargs:tests/unittests} [testenv:doc] From dd28a36e9f4a555595222774f8335dd47487eaa7 Mon Sep 17 00:00:00 2001 From: Orukaria Ndukiye <110042947+Ndukiye@users.noreply.github.com> Date: Fri, 17 Apr 2026 01:42:37 +0100 Subject: [PATCH 26/72] fix(net): do not resolve IPs when netloc contains a port (#6841) --- cloudinit/util.py | 2 +- tests/unittests/test_util.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index fa258456b81..2cdd73655ef 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1298,7 +1298,7 @@ def is_resolvable(url) -> bool: # Early return for IP addresses - no DNS resolution needed with suppress(ValueError): - if net.is_ip_address(parsed_url.netloc.strip("[]")): + if net.is_ip_address(name): return True try: hostname_result = socket.getaddrinfo(name, None) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 88be28ecdbd..d75fbc4a8ab 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -3185,6 +3185,8 @@ def test_ips_need_not_be_resolved(self, m_getaddr, m_dns): """ assert util.is_resolvable("http://169.254.169.254/") is True assert util.is_resolvable("http://[fd00:ec2::254]/") is True + assert util.is_resolvable("http://169.254.169.254:80") is True + assert util.is_resolvable("http://[fd00:ec2::254]:80/") is True assert not m_getaddr.called @mock.patch.object(util.net, "is_ip_address") @@ -3206,6 +3208,7 @@ def mock_getaddrinfo(host, port, *args, **kwargs): m_getaddr.side_effect = mock_getaddrinfo assert util.is_resolvable("http://example.com/") is True + assert util.is_resolvable("http://example.com/:80") is True assert m_getaddr.called assert m_getaddr.call_args_list[0] == mock.call("example.com", None) From 87cc1ae50616c651887122f1c465a605d89a53ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Can=C3=A9vet?= Date: Tue, 21 Apr 2026 19:55:16 +0200 Subject: [PATCH 27/72] chore(opennebula): enable mypy type checking (#6827) Remove DataSourceOpenNebula and its unit tests from the mypy check_untyped_defs=false override list and add type annotations to satisfy stricter checking. Annotate parse_shell_config, OpenNebulaNetwork.__init__, get_field (with @overload to distinguish str vs Optional[str] return), all get_* methods, get_physical_nics_by_mac, and read_context_disk_dir (fixing its stale docstring). Remove test_get_field_nonecontext: shell variables cannot hold None values so the scenario is impossible; test_get_field_emptycontext covers the realistic equivalent. Remove leftover bare `util.find_devs_with` assignment in test_find_candidates, a refactoring artifact from d482353a5 that leaked state between tests. Refs GH-6810 --- cloudinit/sources/DataSourceOpenNebula.py | 119 +++++++++++++-------- pyproject.toml | 2 - tests/unittests/sources/test_opennebula.py | 13 +-- 3 files changed, 74 insertions(+), 60 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 5ad3b6bc6c1..4c1401a952c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -20,6 +20,7 @@ import re import shlex import textwrap +from typing import Any, Dict, List, Optional, overload from cloudinit import atomic_helper, net, sources, subp, util @@ -56,8 +57,8 @@ def __str__(self): def _get_data(self): defaults = {"instance-id": DEFAULT_IID} - results = None - seed = None + results: Optional[Dict[str, Any]] = None + seed: Optional[str] = None # decide parseuser for context.sh shell reader parseuser = DEFAULT_PARSEUSER @@ -94,7 +95,7 @@ def _get_data(self): LOG.debug("found datasource in %s", cdev) break - if not seed: + if not seed or results is None: return False # merge fetched metadata with datasource defaults @@ -115,8 +116,10 @@ def _get_data(self): self.userdata_raw = results.get("userdata") return True - def _get_subplatform(self): + def _get_subplatform(self) -> str: """Return the subplatform metadata source details.""" + if self.seed is None: + return sources.METADATA_UNKNOWN if self.seed_dir in self.seed: subplatform_type = "seed-dir" else: @@ -148,33 +151,40 @@ class BrokenContextDiskDir(Exception): class OpenNebulaNetwork: - def __init__(self, context, distro, system_nics_by_mac=None): + def __init__( + self, + context: Dict[str, str], + distro: Any, + system_nics_by_mac: Optional[Dict[str, str]] = None, + ) -> None: self.context = context if system_nics_by_mac is None: system_nics_by_mac = get_physical_nics_by_mac(distro) - self.ifaces = collections.OrderedDict( - [ - k - for k in sorted( - system_nics_by_mac.items(), - key=lambda k: net.natural_sort_key(k[1]), - ) - ] + self.ifaces: collections.OrderedDict[str, str] = ( + collections.OrderedDict( + [ + k + for k in sorted( + system_nics_by_mac.items(), + key=lambda k: net.natural_sort_key(k[1]), + ) + ] + ) ) # OpenNebula 4.14+ provide macaddr for ETHX in variable ETH_MAC. # context_devname provides {mac.lower():ETHX, mac2.lower():ETHX} - self.context_devname = {} + self.context_devname: Dict[str, str] = {} for k, v in context.items(): m = re.match(r"^(.+)_MAC$", k) if m: self.context_devname[v.lower()] = m.group(1) - def mac2ip(self, mac): + def mac2ip(self, mac: str) -> str: return ".".join([str(int(c, 16)) for c in mac.split(":")[2:]]) - def get_nameservers(self, dev): - nameservers = {} + def get_nameservers(self, dev: str) -> Dict[str, List[str]]: + nameservers: Dict[str, List[str]] = {} dns = self.get_field(dev, "dns", "").split() dns.extend(self.context.get("DNS", "").split()) if dns: @@ -184,14 +194,14 @@ def get_nameservers(self, dev): nameservers["search"] = search_domain return nameservers - def get_mtu(self, dev): + def get_mtu(self, dev: str) -> Optional[str]: return self.get_field(dev, "mtu") - def get_ip(self, dev, mac): + def get_ip(self, dev: str, mac: str) -> str: return self.get_field(dev, "ip", self.mac2ip(mac)) - def get_ip6(self, dev): - addresses6 = [] + def get_ip6(self, dev: str) -> List[str]: + addresses6: List[str] = [] ip6 = self.get_field(dev, "ip6") if ip6: addresses6.append(ip6) @@ -200,24 +210,35 @@ def get_ip6(self, dev): addresses6.append(ip6_ula) return addresses6 - def get_ip6_prefix(self, dev): + def get_ip6_prefix(self, dev: str) -> str: return self.get_field(dev, "ip6_prefix_length", "64") - def get_gateway(self, dev): + def get_gateway(self, dev: str) -> Optional[str]: return self.get_field(dev, "gateway") - def get_gateway6(self, dev): + def get_gateway6(self, dev: str) -> Optional[str]: # OpenNebula 6.1.80 introduced new context parameter ETHx_IP6_GATEWAY # to replace old ETHx_GATEWAY6. Old ETHx_GATEWAY6 will be removed in # OpenNebula 6.4.0 (https://github.com/OpenNebula/one/issues/5536). - return self.get_field( - dev, "ip6_gateway", self.get_field(dev, "gateway6") - ) + ip6_gateway = self.get_field(dev, "ip6_gateway") + if ip6_gateway is not None: + return ip6_gateway + return self.get_field(dev, "gateway6") - def get_mask(self, dev): + def get_mask(self, dev: str) -> str: return self.get_field(dev, "mask", "255.255.255.0") - def get_field(self, dev, name, default=None): + @overload + def get_field(self, dev: str, name: str) -> Optional[str]: ... + @overload + def get_field( + self, dev: str, name: str, default: None + ) -> Optional[str]: ... + @overload + def get_field(self, dev: str, name: str, default: str) -> str: ... + def get_field( + self, dev: str, name: str, default: Optional[str] = None + ) -> Optional[str]: """return the field name in context for device dev. context stores _ (example: eth0_DOMAIN). @@ -233,12 +254,10 @@ def get_field(self, dev, name, default=None): # allow empty string to return the default. return default if val in (None, "") else val - def gen_conf(self): - netconf = {} - netconf["version"] = 2 - netconf["ethernets"] = {} + def gen_conf(self) -> Dict[str, Any]: + netconf: Dict[str, Any] = {"version": 2, "ethernets": {}} - ethernets = {} + ethernets: Dict[str, Dict[str, Any]] = {} for mac, dev in self.ifaces.items(): mac = mac.lower() @@ -246,7 +265,7 @@ def gen_conf(self): # dev stores the current system name. c_dev = self.context_devname.get(mac, dev) - devconf = {} + devconf: Dict[str, Any] = {} # Set MAC address devconf["match"] = {"macaddress": mac} @@ -323,7 +342,9 @@ def varprinter(): ) -def parse_shell_config(content, asuser=None): +def parse_shell_config( + content: str, asuser: Optional[str] = None +) -> Dict[str, str]: """run content and return environment variables which changed WARNING: the special variable _start_ is used to delimit content @@ -394,13 +415,19 @@ def parse_shell_config(content, asuser=None): return ret -def read_context_disk_dir(source_dir, distro, asuser=None): - """ - read_context_disk_dir(source_dir): - read source_dir and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonContextDiskDir +def read_context_disk_dir( + source_dir: str, distro: Any, asuser: Optional[str] = None +) -> Dict[str, Any]: + """Read ``source_dir`` and return a dictionary containing context data. + + The returned dictionary always includes ``"metadata"`` and + ``"userdata"`` keys, and may also include ``"network-interfaces"`` + when network configuration can be generated from the context. + + If ``source_dir`` is not a valid context directory, raise + ``NonContextDiskDir``. """ - found = {} + found: Dict[str, str] = {} for af in CONTEXT_DISK_FILES: fn = os.path.join(source_dir, af) if os.path.isfile(fn): @@ -409,8 +436,8 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if not found: raise NonContextDiskDir("%s: %s" % (source_dir, "no files found")) - context = {} - results = {"userdata": None, "metadata": {}} + context: Dict[str, str] = {} + results: Dict[str, Any] = {"userdata": None, "metadata": {}} if "context.sh" in found: if asuser is not None: @@ -450,7 +477,7 @@ def read_context_disk_dir(source_dir, distro, asuser=None): ssh_key_var = "SSH_PUBLIC_KEY" if ssh_key_var: - lines = context.get(ssh_key_var).splitlines() + lines = context[ssh_key_var].splitlines() results["metadata"]["public-keys"] = [ line for line in lines if len(line) and not line.startswith("#") ] @@ -490,7 +517,7 @@ def read_context_disk_dir(source_dir, distro, asuser=None): return results -def get_physical_nics_by_mac(distro): +def get_physical_nics_by_mac(distro: Any) -> Dict[str, str]: devs = net.get_interfaces_by_mac() return dict( [(m, n) for m, n in devs.items() if distro.networking.is_physical(n)] diff --git a/pyproject.toml b/pyproject.toml index 4657424c946..9a3373709ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,6 @@ module = [ "cloudinit.sources.DataSourceHetzner", "cloudinit.sources.DataSourceNoCloud", "cloudinit.sources.DataSourceOVF", - "cloudinit.sources.DataSourceOpenNebula", "cloudinit.sources.DataSourceOpenStack", "cloudinit.sources.DataSourceOracle", "cloudinit.sources.DataSourceRbxCloud", @@ -146,7 +145,6 @@ module = [ "tests.unittests.sources.test_gce", "tests.unittests.sources.test_init", "tests.unittests.sources.test_nocloud", - "tests.unittests.sources.test_opennebula", "tests.unittests.sources.test_openstack", "tests.unittests.sources.test_oracle", "tests.unittests.sources.test_scaleway", diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index f5db58c3592..a2f07baf27e 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -7,7 +7,7 @@ import pytest -from cloudinit import atomic_helper, util +from cloudinit import atomic_helper from cloudinit.sources import DataSourceOpenNebula as ds from tests.unittests.helpers import populate_dir @@ -365,7 +365,6 @@ def my_devs_with(criteria): }.get(criteria, []) m_find_devs_with.side_effect = my_devs_with - util.find_devs_with = my_devs_with assert ["/dev/sdb", "/dev/sr0", "/dev/vdb"] == ds.find_candidate_devs() @@ -571,16 +570,6 @@ def test_get_field_emptycontext(self): val = net.get_field("eth9", "dummy") assert None is val - def test_get_field_nonecontext(self): - """ - Verify get_field('device', 'name') returns None if context value is - None. - """ - context = {"ETH9_DUMMY": None} - net = ds.OpenNebulaNetwork(context, mock.Mock()) - val = net.get_field("eth9", "dummy") - assert None is val - @mock.patch(DS_PATH + ".get_physical_nics_by_mac") def test_gen_conf_gateway(self, m_get_phys_by_mac): """Test rendering with/without IPv4 gateway""" From c7ae6a4cc0dad5a411945e0644dab6926d41a997 Mon Sep 17 00:00:00 2001 From: Amy Chen Date: Sat, 25 Apr 2026 04:18:55 +0800 Subject: [PATCH 28/72] fix:(disk_setup):handle empty disk in check_partition_gpt_layout_sfdisk (#6728) When checking GPT partition layout using sfdisk, handle the case where a disk has no partition table yet. Previously, this would raise an exception. Now it returns an empty list. Add unit test to cover this empty disk scenario. Fixes GH-6682 Signed-off-by: Amy Chen --- cloudinit/config/cc_disk_setup.py | 6 +++++- tests/unittests/config/test_cc_disk_setup.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e82ea36b4d3..26c22838365 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -805,7 +805,11 @@ def check_partition_gpt_layout_sfdisk(device, layout): # Use sfdisk's JSON output for reliability prt_cmd = ["sfdisk", "-l", "-J", device] try: - out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV, rcs=[0, 1]) + # Device has no partition table or other error, return empty list + if not out: + return [] + # Try to parse JSON output ptable = json.loads(out)["partitiontable"] if "partitions" in ptable: partitions = ptable["partitions"] diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index d0f3c26dc47..4951848a199 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -138,6 +138,25 @@ def test_simple1_gpt(self, *args): "gpt", "/dev/xvdb1", [(100, Linux_GUID)] ) + @mock.patch( + "cloudinit.config.cc_disk_setup.subp.subp", + return_value=( + "", + "/dev/sdb: does not contain a recognized partition table", + ), + ) + def test_empty_disk_no_partition_table(self, m_subp): + """Test that empty disk (no partition table) returns empty list.""" + result = cc_disk_setup.check_partition_gpt_layout_sfdisk( + "/dev/sdb", [] + ) + assert result == [] + m_subp.assert_called_once_with( + ["sfdisk", "-l", "-J", "/dev/sdb"], + update_env={"LANG": "C"}, + rcs=[0, 1], + ) + class TestUpdateFsSetupDevices: def test_regression_1634678(self): From 207d38fa067a1dd265ab076bbc2f32e3b8bbe899 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 28 Apr 2026 14:44:55 +0000 Subject: [PATCH 29/72] fix: add single retry of ssh-import-id on exit 1 (#6805) Add a single retry on ssh-import-id to mitigate intermittent errors from remote service avoiding boot failures. Seeing 10-20% failure rate of exit 1's from ssh-import-id during instance launches in github action runners. --- cloudinit/config/cc_ssh_import_id.py | 15 +++-- .../unittests/config/test_cc_ssh_import_id.py | 65 +++++++++++++++---- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index dc6adcf0482..2c55308cecf 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -9,6 +9,8 @@ import logging import pwd +import time +from contextlib import suppress from cloudinit import subp, util from cloudinit.cloud import Cloud @@ -148,13 +150,16 @@ def import_ssh_ids(ids, user): else: LOG.error("Neither sudo nor doas available! Unable to import SSH ids.") return - LOG.debug("Importing SSH ids for user %s.", user) + retry_ssh_import(cmd, 0.5) - try: + +def retry_ssh_import(cmd: list, delay: float) -> None: + """Retry ssh-import-id once if it exits in error.""" + with suppress(subp.ProcessExecutionError): subp.subp(cmd, capture=False) - except subp.ProcessExecutionError as exc: - util.logexc(LOG, "Failed to run command to import %s SSH ids", user) - raise exc + return + time.sleep(delay) + subp.subp(cmd, capture=False) def is_key_in_nested_dict(config: dict, search_key: str) -> bool: diff --git a/tests/unittests/config/test_cc_ssh_import_id.py b/tests/unittests/config/test_cc_ssh_import_id.py index 2d5608a6464..2daffdd9997 100644 --- a/tests/unittests/config/test_cc_ssh_import_id.py +++ b/tests/unittests/config/test_cc_ssh_import_id.py @@ -6,11 +6,12 @@ import pytest from cloudinit.config import cc_ssh_import_id +from cloudinit.subp import ProcessExecutionError from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -MODPATH = "cloudinit.config.cc_ssh_import_ids." +MODPATH = "cloudinit.config.cc_ssh_import_id." class TestIsKeyInNestedDict: @@ -69,7 +70,7 @@ class TestHandleSshImportIDs: ({"ssh_import_id": ["bobkey"]}, "ssh-import-id is not installed"), ), ) - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "subp.which") def test_skip_inapplicable_configs(self, m_which, cfg, log, caplog): """Skip config without ssh_import_id""" m_which.return_value = None @@ -77,9 +78,9 @@ def test_skip_inapplicable_configs(self, m_which, cfg, log, caplog): cc_ssh_import_id.handle("name", cfg, cloud, []) assert log in caplog.text - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_sudo(self, m_which, m_subp, m_getpwnam): """Check that sudo is available and use that""" m_which.return_value = "/usr/bin/ssh-import-id" @@ -98,9 +99,9 @@ def test_use_sudo(self, m_which, m_subp, m_getpwnam): capture=False, ) - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_doas(self, m_which, m_subp, m_getpwnam): """Check that doas is available and use that""" m_which.side_effect = [None, "/usr/bin/doas"] @@ -111,9 +112,9 @@ def test_use_doas(self, m_which, m_subp, m_getpwnam): ["doas", "-u", user, "ssh-import-id"] + ids, capture=False ) - @mock.patch("cloudinit.ssh_util.pwd.getpwnam") - @mock.patch("cloudinit.config.cc_ssh_import_id.subp.subp") - @mock.patch("cloudinit.subp.which") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.subp") + @mock.patch(MODPATH + "subp.which") def test_use_neither_sudo_nor_doas( self, m_which, m_subp, m_getpwnam, caplog ): @@ -125,3 +126,45 @@ def test_use_neither_sudo_nor_doas( assert ( "Neither sudo nor doas available! Unable to import SSH ids" ) in caplog.text + + @mock.patch(MODPATH + "time.sleep") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.which") + def test_retry_once_on_exit_code_1( + self, m_which, m_getpwnam, m_sleep, mocker + ): + """Only attempt one retry when ssh-import-id exits with code 1.""" + m_subp = mocker.patch( + "cloudinit.config.cc_ssh_import_id.subp.subp", + side_effect=[ + ProcessExecutionError(exit_code=1, stderr="try1"), + ProcessExecutionError(exit_code=1, stderr="try2"), + ], + ) + m_which.return_value = "/usr/bin/ssh-import-id" + with pytest.raises( + ProcessExecutionError, + match=r"(?s)Unexpected error while running command.*try2", + ): + cc_ssh_import_id.import_ssh_ids(["waffle"], "bob") + + assert m_subp.call_count == 2 + assert m_sleep.call_count == 1 + + @mock.patch(MODPATH + "time.sleep") + @mock.patch(MODPATH + "pwd.getpwnam") + @mock.patch(MODPATH + "subp.which") + def test_retry_with_success(self, m_which, m_getpwnam, m_sleep, mocker): + """Retry succeeds on ssh-import-id with a retry.""" + m_subp = mocker.patch( + "cloudinit.config.cc_ssh_import_id.subp.subp", + side_effect=[ + ProcessExecutionError(exit_code=1, stderr="try1"), + None, + ], + ) + m_which.return_value = "/usr/bin/ssh-import-id" + cc_ssh_import_id.import_ssh_ids(["waffle"], "bob") + + assert m_subp.call_count == 2 + assert m_sleep.call_count == 1 From 18c88305810ed68cd8a651ffa599a028cf8067a6 Mon Sep 17 00:00:00 2001 From: joe-usa Date: Thu, 30 Apr 2026 20:54:38 -0700 Subject: [PATCH 30/72] Enable Amazon Linux to yum_add_repo and ca_certs. (#6767) Adding amazon linux variant to cloud config. And Refreshing/Updating module values for Amazon Linux distro only. Namely cc_ca_cert and cc_yum_add_repo. --- cloudinit/config/cc_ca_certs.py | 57 ++++++++++------------------- cloudinit/config/cc_yum_add_repo.py | 1 + config/cloud.cfg.tmpl | 11 +----- 3 files changed, 22 insertions(+), 47 deletions(-) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 3fce70f62bb..939b62d071e 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -30,13 +30,6 @@ "ca_cert_config": "/etc/ca-certificates/conf.d/cloud-init.conf", "ca_cert_update_cmd": ["update-ca-bundle"], }, - "fedora": { - "ca_cert_path": "/etc/pki/ca-trust/", - "ca_cert_local_path": "/usr/share/pki/ca-trust-source/", - "ca_cert_filename": "anchors/cloud-init-ca-cert-{cert_index}.crt", - "ca_cert_config": None, - "ca_cert_update_cmd": ["update-ca-trust"], - }, "rhel": { "ca_cert_path": "/etc/pki/ca-trust/", "ca_cert_local_path": "/usr/share/pki/ca-trust-source/", @@ -60,42 +53,28 @@ }, } -for distro in ( - "opensuse-microos", - "opensuse-tumbleweed", - "opensuse-leap", - "sle_hpc", - "sle-micro", - "sles", -): - DISTRO_OVERRIDES[distro] = DISTRO_OVERRIDES["opensuse"] - -for distro in ( - "almalinux", - "centos", - "cloudlinux", - "rocky", -): - DISTRO_OVERRIDES[distro] = DISTRO_OVERRIDES["rhel"] - -distros = [ - "almalinux", +DISTRO_FAMILY = { + "almalinux": "rhel", + "amazon": "rhel", + "centos": "rhel", + "cloudlinux": "rhel", + "fedora": "rhel", + "opensuse-microos": "opensuse", + "opensuse-tumbleweed": "opensuse", + "opensuse-leap": "opensuse", + "rocky": "rhel", + "sle_hpc": "opensuse", + "sle-micro": "opensuse", + "sles": "opensuse", +} + +distros = list(DISTRO_FAMILY.keys()) + [ "aosc", - "centos", - "cloudlinux", "alpine", "debian", - "fedora", "raspberry-pi-os", "rhel", - "rocky", "opensuse", - "opensuse-microos", - "opensuse-tumbleweed", - "opensuse-leap", - "sle_hpc", - "sle-micro", - "sles", "ubuntu", "photon", ] @@ -114,6 +93,8 @@ def _distro_ca_certs_configs(distro_name): @param distro_name: String providing the distro class name. @returns: Dict of distro configurations for ca_cert. """ + if distro_name in DISTRO_FAMILY: + distro_name = DISTRO_FAMILY[distro_name] cfg = DISTRO_OVERRIDES.get(distro_name, DEFAULT_CONFIG) cfg["ca_cert_full_path"] = os.path.join( cfg["ca_cert_local_path"], cfg["ca_cert_filename"] @@ -160,7 +141,7 @@ def disable_default_ca_certs(distro_name, distro_cfg): @param distro_name: String providing the distro class name. @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - if distro_name in ["rhel", "photon"]: + if distro_name in ["rhel", "photon", "amazon"]: remove_default_ca_certs(distro_cfg) elif distro_name in [ "alpine", diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 9bfebbfd129..66e71801589 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -21,6 +21,7 @@ "id": "cc_yum_add_repo", "distros": [ "almalinux", + "amazon", "azurelinux", "centos", "cloudlinux", diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index b0e8dd32794..762d1502a52 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -97,13 +97,6 @@ apt: # timeout: 5 # (defaults to 50 seconds) # max_wait: 10 # (defaults to 120 seconds) -{% if variant == "amazon" %} -# Amazon Linux relies on ec2-net-utils for network configuration -network: - config: disabled - -{% endif -%} - {% if is_rhel %} # Default redhat settings: ssh_deletekeys: true @@ -168,12 +161,12 @@ cloud_config_modules: {% if variant == "ubuntu" %} - ubuntu_pro {% endif %} -{% elif variant in ["azurelinux", "fedora", "mariner", "openeuler", +{% elif variant in ["amazon", "azurelinux", "fedora", "mariner", "openeuler", "openmandriva", "photon"] or is_rhel %} {% if is_rhel %} - rh_subscription {% endif %} -{% if variant not in ["azurelinux", "mariner", "photon"] %} +{% if variant not in ["amazon", "azurelinux", "mariner", "photon"] %} - spacewalk {% endif %} - yum_add_repo From d7eb828f8e232a1ce6d2fa66f8028a3b10adab1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Can=C3=A9vet?= Date: Fri, 1 May 2026 22:17:53 +0200 Subject: [PATCH 31/72] fix(opennebula): coerce MTU to int in gen_conf() (#6856) ONE_CONTEXT passes ETHx_MTU as a string, but Netplan's network-config-v2 schema requires mtu to be an integer. Without this, cloud-init schema validation fails with "'1500' is not of type 'integer'". --- cloudinit/sources/DataSourceOpenNebula.py | 2 +- tests/unittests/sources/test_opennebula.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 4c1401a952c..68db118a51d 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -302,7 +302,7 @@ def gen_conf(self) -> Dict[str, Any]: # Set MTU size mtu = self.get_mtu(c_dev) if mtu: - devconf["mtu"] = mtu + devconf["mtu"] = int(mtu) ethernets[dev] = devconf diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index a2f07baf27e..c7b4ea6b1db 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -789,7 +789,7 @@ def test_gen_conf_mtu(self, m_get_phys_by_mac): "version": 2, "ethernets": { nic: { - "mtu": "1280", + "mtu": 1280, "match": {"macaddress": MACADDR}, "addresses": [IP_BY_MACADDR + "/" + IP4_PREFIX], } @@ -898,7 +898,7 @@ def test_eth0_v4v6_override(self): "addresses": ["1.2.3.6", "1.2.3.7", "1.2.3.8"], "search": ["example.com", "example.org"], }, - "mtu": "1280", + "mtu": 1280, } }, } @@ -959,7 +959,7 @@ def test_multiple_nics(self): "addresses": ["1.2.3.6", "1.2.3.7", "1.2.3.8"], "search": ["example.com"], }, - "mtu": "1280", + "mtu": 1280, }, "enp0s25": { "match": {"macaddress": MAC_1}, From e9e10ab66a696540f657b7631640e1c971989e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Can=C3=A9vet?= Date: Tue, 5 May 2026 15:38:29 +0200 Subject: [PATCH 32/72] feat(opennebula): support ETHx_ROUTES static routes in network config (#6810) Add `get_routes()` to `OpenNebulaNetwork` to parse the `ETHx_ROUTES` context variable (format: "NETWORK via GATEWAY, ...") and emit the resulting routes into the Netplan v2 `routes:` list in `gen_conf()`. Malformed entries are skipped with a warning. No `routes` key is emitted when the variable is absent or empty, preserving backward compatibility. --- cloudinit/sources/DataSourceOpenNebula.py | 29 ++++++++ doc/rtd/reference/datasources/opennebula.rst | 8 ++- tests/unittests/sources/test_opennebula.py | 70 ++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 68db118a51d..54ee0ec6d66 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -228,6 +228,30 @@ def get_gateway6(self, dev: str) -> Optional[str]: def get_mask(self, dev: str) -> str: return self.get_field(dev, "mask", "255.255.255.0") + def get_routes(self, dev: str) -> List[Dict[str, str]]: + """Parse ETHx_ROUTES into a list of Netplan route dicts. + + Expected format: "NETWORK via GATEWAY[, NETWORK via GATEWAY, ...]" + e.g. "10.0.0.0/8 via 192.168.1.1, 192.168.100.0/24 via 10.0.0.1" + Returns an empty list when the variable is absent or empty. + """ + routes: List[Dict[str, str]] = [] + for entry in self.get_field(dev, "routes", "").split(","): + entry = entry.strip() + if not entry: + continue + m = re.match( + r"\s*(?P\S+)\s+via\s+(?P\S+)\s*$", + entry, + ) + if m: + routes.append({"to": m["route_to"], "via": m["route_via"]}) + else: + LOG.warning( + "Unparseable ETHx_ROUTES entry for %s: %r", dev, entry + ) + return routes + @overload def get_field(self, dev: str, name: str) -> Optional[str]: ... @overload @@ -304,6 +328,11 @@ def gen_conf(self) -> Dict[str, Any]: if mtu: devconf["mtu"] = int(mtu) + # Set static routes + extra_routes: List[Dict[str, str]] = self.get_routes(c_dev) + if extra_routes: + devconf["routes"] = extra_routes + ethernets[dev] = devconf netconf["ethernets"] = ethernets diff --git a/doc/rtd/reference/datasources/opennebula.rst b/doc/rtd/reference/datasources/opennebula.rst index f4136c668f8..3e31b20065b 100644 --- a/doc/rtd/reference/datasources/opennebula.rst +++ b/doc/rtd/reference/datasources/opennebula.rst @@ -72,9 +72,15 @@ the OpenNebula documentation. ETH_IP6_ULA ETH_IP6_PREFIX_LENGTH ETH_IP6_GATEWAY + ETH_ROUTES Static `network configuration`_. +``ETH_ROUTES`` is a comma-separated list of static routes in the form +``NETWORK via GATEWAY``. For example:: + + ETH0_ROUTES="10.0.0.0/8 via 192.168.1.1, 172.16.0.0/12 via 192.168.1.254" + :: SET_HOSTNAME @@ -146,5 +152,5 @@ Example VM's context section .. _OpenNebula: http://opennebula.org/ .. _contextualization overview: http://opennebula.org/documentation:documentation:context_overview .. _contextualizing VMs: http://opennebula.org/documentation:documentation:cong -.. _network configuration: https://docs.opennebula.io/ +.. _network configuration: https://docs.opennebula.io/7.2/product/operation_references/configuration_references/template/#context-section .. _iso9660: https://en.wikipedia.org/wiki/ISO_9660 diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index c7b4ea6b1db..10ccdea2ed3 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -975,6 +975,76 @@ def test_multiple_nics(self): assert expected == net.gen_conf() + # ------------------------------------------------------------------ # + # ETHx_ROUTES # + # ------------------------------------------------------------------ # + + @pytest.mark.parametrize( + "context,expected", + [ + pytest.param({}, [], id="absent"), + pytest.param({"ETH0_ROUTES": ""}, [], id="empty_string"), + pytest.param( + {"ETH0_ROUTES": "10.0.0.0/8 via 192.168.1.1"}, + [{"to": "10.0.0.0/8", "via": "192.168.1.1"}], + id="single_entry", + ), + pytest.param( + { + "ETH0_ROUTES": ( + "10.0.0.0/8 via 192.168.1.1," + " 172.16.0.0/12 via 192.168.1.254" + ) + }, + [ + {"to": "10.0.0.0/8", "via": "192.168.1.1"}, + {"to": "172.16.0.0/12", "via": "192.168.1.254"}, + ], + id="multiple_comma_separated_entries", + ), + pytest.param( + {"ETH0_ROUTES": "bad-entry, 10.0.0.0/8 via 192.168.1.1"}, + [{"to": "10.0.0.0/8", "via": "192.168.1.1"}], + id="malformed_entry_skipped", + ), + ], + ) + def test_get_routes(self, context, expected): + net = ds.OpenNebulaNetwork(context, mock.Mock()) + assert net.get_routes("eth0") == expected + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_routes(self, m_get_phys_by_mac): + """Routes from ETHx_ROUTES appear in gen_conf() output.""" + self.maxDiff = None + context = { + "ETH0_MAC": "02:00:0a:12:01:01", + "ETH0_IP": "10.0.0.5", + "ETH0_MASK": "255.255.255.0", + "ETH0_GATEWAY": "10.0.0.1", + "ETH0_ROUTES": ( + "192.168.0.0/16 via 10.0.0.1, 172.16.0.0/12 via 10.0.0.1" + ), + } + for nic in self.system_nics: + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + routes = conf["ethernets"][nic].get("routes", []) + assert {"to": "192.168.0.0/16", "via": "10.0.0.1"} in routes + assert {"to": "172.16.0.0/12", "via": "10.0.0.1"} in routes + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_no_routes_key_when_absent(self, m_get_phys_by_mac): + """gen_conf() does not emit 'routes' key when ETHx_ROUTES is unset.""" + context = { + "ETH0_MAC": "02:00:0a:12:01:01", + } + m_get_phys_by_mac.return_value = {MACADDR: "eth0"} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + assert "routes" not in conf["ethernets"]["eth0"] + class TestParseShellConfig: @pytest.mark.allow_subp_for("bash", "sh") From 912aa555266ef5694c5088081cf0507fe33eaa71 Mon Sep 17 00:00:00 2001 From: Bastien Traverse Date: Wed, 6 May 2026 15:51:57 +0200 Subject: [PATCH 33/72] fix(docs): correct users and groups examples 3, 4 and 7 (#6858) Explanations did not match the code (or the opposite). Fixes GH-6857 --- doc/module-docs/cc_users_groups/data.yaml | 7 +++---- doc/module-docs/cc_users_groups/example3.yaml | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/module-docs/cc_users_groups/data.yaml b/doc/module-docs/cc_users_groups/data.yaml index 03bb32594d7..abb2359b44b 100644 --- a/doc/module-docs/cc_users_groups/data.yaml +++ b/doc/module-docs/cc_users_groups/data.yaml @@ -77,8 +77,6 @@ cc_users_groups: file: cc_users_groups/example3.yaml - comment: > Example 4: Skip creation of the ``default`` user and only create - ``newsuper``. Password-based login is rejected, but the GitHub user - ``TheRealFalcon`` and the Launchpad user ``falcojr`` can SSH as ``newsuper``. ``doas``/``opendoas`` is configured to permit this user to run commands as other users (without being prompted for a password) except not as root. @@ -97,8 +95,9 @@ cc_users_groups: - comment: > Example 7: Override any ``default_user`` config in ``/etc/cloud/cloud.cfg`` with supplemental config options. This config - will make the default user ``mynewdefault`` and change the user to not - have ``sudo`` rights. + will make the default user ``mynewdefault``, change the user to not + have ``sudo`` rights and allow the Launchpad user ``chad.smith`` to + SSH as ``mynewdefault``. file: cc_users_groups/example7.yaml - comment: > Example 8: Avoid creating any ``default_user``. diff --git a/doc/module-docs/cc_users_groups/example3.yaml b/doc/module-docs/cc_users_groups/example3.yaml index b39665b1b1a..d9b178ae934 100644 --- a/doc/module-docs/cc_users_groups/example3.yaml +++ b/doc/module-docs/cc_users_groups/example3.yaml @@ -1,4 +1,5 @@ #cloud-config users: - name: newsuper + ssh_import_id: [ gh:TheRealFalcon, lp:falcojr ] shell: /bin/bash From 9c7d85d82c16c2379b1f004623c9d43507f31d3e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 6 May 2026 10:24:18 -0600 Subject: [PATCH 34/72] ci: fix alpine edge python3.14 tox missing dep py3-python-discovery (#6864) --- .github/workflows/23-pr-unit-distro.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/23-pr-unit-distro.yml b/.github/workflows/23-pr-unit-distro.yml index e2fe03ba414..cef37e14e25 100644 --- a/.github/workflows/23-pr-unit-distro.yml +++ b/.github/workflows/23-pr-unit-distro.yml @@ -42,7 +42,7 @@ jobs: lxc exec alpine -- ping -c 1 dl-cdn.alpinelinux.org || true - name: Install dependencies - run: lxc exec alpine -- apk add py3-tox git tzdata + run: lxc exec alpine -- apk add py3-tox py3-python-discovery git tzdata - name: Mount source into container directory run: lxc config device add alpine gitdir disk source=$(pwd) path=/root/cloud-init-ro From 9cbd55fa5fdcb295e4a7e9a849b9c1defabb74f8 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelwahab <46541787+mostafaCamel@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:31:39 +0300 Subject: [PATCH 35/72] fix(ug_util): prioritize user-data.users over the default user config (#6860) Fixes GH-6703 --- cloudinit/distros/ug_util.py | 13 ++-- .../modules/test_set_password.py | 40 ++++++------ .../modules/test_users_groups.py | 65 ++++++++++++++++++- tests/integration_tests/util.py | 20 +++++- .../distros/test_user_data_normalize.py | 24 +++++++ 5 files changed, 133 insertions(+), 29 deletions(-) diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py index 2d0a887e7c4..cd50d3b45dd 100644 --- a/cloudinit/distros/ug_util.py +++ b/cloudinit/distros/ug_util.py @@ -134,13 +134,14 @@ def _normalize_users(u_cfg, def_user_cfg=None): # Now merge the extracted groups with the default config provided users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) parsed_config["groups"] = ",".join(users_groups) - # The real config for the default user is the combination of the - # default user config provided by the distro, the default user - # config provided by the above merging for the user 'default' and - # then the parsed config from the user's 'real name' which does not - # have to be 'default' (but could be) + # The real config for the default user is the combination of: + # - the parsed config from the user's 'real name' which does + # not have to be 'default' (but could be) + # - then the default user config provided by the above merging + # for the user 'default' + # - then the default user config provided by the distro users[def_user] = util.mergemanydict( - [def_user_cfg, def_config, parsed_config] + [parsed_config, def_config, def_user_cfg] ) # Ensure that only the default user that we found (if any) is actually diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py index ec6df783250..e04783520e3 100644 --- a/tests/integration_tests/modules/test_set_password.py +++ b/tests/integration_tests/modules/test_set_password.py @@ -14,7 +14,10 @@ from tests.integration_tests.decorators import retry from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU -from tests.integration_tests.util import get_console_log +from tests.integration_tests.util import ( + fetch_and_parse_etc_shadow, + get_console_log, +) COMMON_USER_DATA = """\ #cloud-config @@ -38,6 +41,9 @@ # sha256 gojanego passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." lock_passwd: false + - name: sally + # sha256 gosallygo + passwd: "$5$bA$KBMTe8lXf0e8lE4f4hYPU0h6h0HzQX4vHpnq6xHn9Q2" - name: "mikey" lock_passwd: false """ @@ -94,41 +100,33 @@ class Mixin: """Shared test definitions.""" - def _fetch_and_parse_etc_shadow(self, class_client): - """Fetch /etc/shadow and parse it into Python data structures - - Returns: ({user: password}, [duplicate, users]) - """ - shadow_content = class_client.read_from_file("/etc/shadow") - users = {} - dupes = [] - for line in shadow_content.splitlines(): - user, encpw = line.split(":")[0:2] - if user in users: - dupes.append(user) - users[user] = encpw - return users, dupes - def test_no_duplicate_users_in_shadow(self, class_client): """Confirm that set_passwords has not added duplicate shadow entries""" - _, dupes = self._fetch_and_parse_etc_shadow(class_client) + _, dupes = fetch_and_parse_etc_shadow(class_client) assert [] == dupes def test_password_in_users_dict_set_correctly(self, class_client): """Test that the password specified in the users dict is set.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) assert USERS_PASSWD_VALUES["jane"] == shadow_users["jane"] + def test_hashed_password_without_lock_passwd_override_is_locked( + self, class_client + ): + """Hashed passwords are locked when lock_passwd is not set.""" + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) + assert f"!{USERS_PASSWD_VALUES['sally']}" == shadow_users["sally"] + def test_password_in_chpasswd_list_set_correctly(self, class_client): """Test that a chpasswd password overrides one in the users dict.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) mikey_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" assert mikey_hash == shadow_users["mikey"] def test_random_passwords_set_correctly(self, class_client): """Test that RANDOM chpasswd entries replace users dict passwords.""" - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) # These should have been changed assert shadow_users["harry"] != USERS_PASSWD_VALUES["harry"] @@ -170,7 +168,7 @@ def test_explicit_password_set_correctly(self, class_client): ) if minor_version > 12: pytest.xfail("Instance under test doesn't have 'crypt' in stdlib") - shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) + shadow_users, _ = fetch_and_parse_etc_shadow(class_client) fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0] diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 55813220fc2..1ceb2587c99 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -10,13 +10,17 @@ import pytest from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.integration_settings import CLOUD_INIT_SOURCE from tests.integration_tests.releases import ( CURRENT_RELEASE, IS_UBUNTU, JAMMY, NOBLE, ) -from tests.integration_tests.util import verify_clean_boot +from tests.integration_tests.util import ( + fetch_and_parse_etc_shadow, + verify_clean_boot, +) USER_DATA = """\ #cloud-config @@ -192,3 +196,62 @@ def test_sudoers_includedir(client: IntegrationInstance): "/etc/sudoers.d/90-cloud-init-users" ).splitlines()[1:] assert sudoers_content_before == sudoers_content_after + + +USER_DATA_OVERRIDE = """\ +#cloud-config +users: + - default + - name: ubuntu + shell: /bin/sh + lock_passwd: false + hashed_passwd: $5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +""" + + +@pytest.mark.ci +@pytest.mark.skipif(not IS_UBUNTU, reason="Test is Ubuntu specific") +@pytest.mark.user_data(USER_DATA_OVERRIDE) +def test_default_user_settings_override(client: IntegrationInstance): + """ + Test that the default user settings are correctly overridden. + """ + # Check shell + shell_set = ( + client.execute(["getent", "passwd", "ubuntu"]) + .stdout.strip() + .split(":")[-1] + ) + if CLOUD_INIT_SOURCE in ["NONE", "IN_PLACE"]: + assert ( + "/bin/sh" == shell_set + ), "Shell setting not overriden even though the user is new" + else: + assert ( + "/bin/bash" == shell_set + ), "Shell setting overriden even though user already exists" + # Check password is not locked + passwd_status = client.execute(["passwd", "-S", "ubuntu"]).stdout + assert re.search(r"^ubuntu\s+P\b", passwd_status) + expected_passwd_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89" + parsed_shadow, _ = fetch_and_parse_etc_shadow(client) + assert parsed_shadow["ubuntu"] == expected_passwd_hash + + +@pytest.mark.skipif(not IS_UBUNTU, reason="Test is Ubuntu specific") +def test_default_user_settings(client: IntegrationInstance): + """ + This test serves as a "negative control" for + test_default_user_settings_override, confirming the default + user settings are as expected when not overridden by user-data. + """ + # Check shel + shell_set = ( + client.execute(["getent", "passwd", "ubuntu"]) + .stdout.strip() + .split(":")[-1] + ) + assert "/bin/bash" == shell_set + # Check password is not locked + passwd_status = client.execute(["passwd", "-S", "ubuntu"]).stdout + assert re.search(r"^ubuntu\s+L\b", passwd_status) diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 6e86b677552..9a2329eab33 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -10,7 +10,7 @@ from functools import lru_cache from itertools import chain from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Set, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union import pytest @@ -692,3 +692,21 @@ def get_datetime_from_string( ) ) return converted_datetime + + +def fetch_and_parse_etc_shadow( + client: "IntegrationInstance", +) -> Tuple[Dict[str, str], List[str]]: + """Fetch /etc/shadow and parse it into Python data structures + + Returns: ({user: password}, [duplicate, users]) + """ + shadow_content = client.read_from_file("/etc/shadow") + users = {} + dupes = [] + for line in shadow_content.splitlines(): + user, encpw = line.split(":")[0:2] + if user in users: + dupes.append(user) + users[user] = encpw + return users, dupes diff --git a/tests/unittests/distros/test_user_data_normalize.py b/tests/unittests/distros/test_user_data_normalize.py index a1a77d1a0e9..925a5a93855 100644 --- a/tests/unittests/distros/test_user_data_normalize.py +++ b/tests/unittests/distros/test_user_data_normalize.py @@ -196,6 +196,30 @@ def test_users_dict_default_additional(self): assert users["bob"]["blah"] is True assert users["bob"]["default"] is True + def test_users_dict_override_default_attribute(self): + distro = self._make_distro("ubuntu", bcfg) + ug_cfg = { + "users": ["default", {"name": "bob", "lock_passwd": False}], + } + users, _ = self._norm(ug_cfg, distro) + + assert "bob" in users + assert "name" not in users["bob"] + + for key, val in bcfg.items(): + if key == "lock_passwd": + # Assert that the default user config is True + assert val is True + # Assert that the resolved value + # matches the passed config: False + assert users["bob"][key] is False + elif key == "groups": + assert users["bob"][key] == ",".join(val) + elif key != "name": + assert users["bob"][key] == val + + assert users["bob"]["default"] is True + def test_users_dict_extract(self): distro = self._make_distro("ubuntu", bcfg) ug_cfg = { From 8174dd62aaf93267d00ec2bf916d9ab3eb3a1a9b Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jun 2026 08:33:13 -0600 Subject: [PATCH 36/72] test: logger specifically configure localhost:514 destination (#6897) logger from bsdutils v2.39 defaults to UDP unless --tcp is otherwise specified. Since our integration test is only grabbing imtcp events, udp is ignored on Noble and newer. Drop deprecated `~` in favor of explicit `stop` --- tests/integration_tests/modules/test_combined.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 7c04a183064..286aa5a704a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -77,7 +77,7 @@ input(type="imtcp" port="514") $template RemoteLogs,"/var/spool/rsyslog/cloudinit.log" *.* ?RemoteLogs - & ~ + & stop remotes: me: "127.0.0.1" runcmd: @@ -85,7 +85,7 @@ - echo '💩' > /var/tmp/unicode_data - # - - logger "My test log" + - logger --server localhost --tcp --port 514 "My test log" snap: commands: - snap install hello-world From 1d5d8f0ca2accff1571403550076b8cdfe3b5920 Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Wed, 1 Apr 2026 12:50:48 +0530 Subject: [PATCH 37/72] feat(dhcp): Add network manager lease parsing capability (#6829) DHCP leases can be directly obtained from network manager through appropriate command to the network manager cli. Add a couple of helper functions to get the lease information from network manager. In a subsequent patch, we will use the helper function from cloud stack datasource to get the lease information from network manager. Signed-off-by: Ani Sinha Co-authored-by: Chad Smith --- cloudinit/net/dhcp.py | 75 ++++++++++++++++++++++++++++ tests/unittests/net/test_dhcp.py | 84 ++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 6620b15da26..95a94fcfa5e 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -29,6 +29,7 @@ DHCLIENT_FALLBACK_LEASE_DIR = "/var/lib/dhclient" # Match something.lease or something.leases DHCLIENT_FALLBACK_LEASE_REGEX = r".+\.leases?$" +NMCLI = "nmcli" UDHCPC_SCRIPT = """#!/bin/sh log() { echo "udhcpc[$PPID]" "$interface: $2" @@ -147,6 +148,80 @@ def networkd_get_option_from_leases(keyname, leases_d=None): return None +def run_nmcli(nmcli_opts: List[str]) -> str: + nmcli_path = subp.which(NMCLI) + if not nmcli_path: + raise NoDHCPLeaseMissingDhclientError() + + command = [nmcli_path] + nmcli_opts + try: + out, _ = subp.subp( + command, + ) + except subp.ProcessExecutionError as error: + LOG.debug( + "nmcli command exited with code: %s stderr: %r stdout: %r", + error.exit_code, + error.stderr, + error.stdout, + ) + raise NoDHCPLeaseError from error + return out + + +def network_manager_load_leases(device: str) -> Dict[str, str]: + """Return a dictionary of lease options obtained from NM cli""" + + opts = ["--fields", "DHCP4", "device", "show", device] + nmcli_out = run_nmcli(opts) + + content = [] + for line in nmcli_out.splitlines(): + line = line.partition(":")[2].strip() + content.append(line) + + return dict(configobj.ConfigObj(content, list_values=False)) + + +def find_correct_device_nmcli() -> Optional[str]: + """Return the lease value for + the first connected device as returned by 'nmcli'""" + + device_list_opts = ["--terse", "device"] + nmcli_out = run_nmcli(device_list_opts) + + for line in nmcli_out.splitlines(): + if line == "": + continue + try: + dev_name, _, state, _ = line.split(":", 3) + except ValueError: + LOG.warning( + "Unexpected nmcli format: expected 4 colon-delimited" + " values, found %s", + line, + ) + continue + + # skip devices that are not connected + if state != "connected": + continue + # skip loopback + if dev_name == "lo": + continue + return dev_name + return None + + +def network_manager_get_option_from_leases(keyname: str) -> Optional[str]: + leases = None + dev = find_correct_device_nmcli() + if dev: + leases = network_manager_load_leases(dev) + + return leases.get(keyname) if leases else None + + class DhcpClient(abc.ABC): client_name = "" timeout = 10 diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py index ec1b8ef83ac..b1f07a4d8af 100644 --- a/tests/unittests/net/test_dhcp.py +++ b/tests/unittests/net/test_dhcp.py @@ -20,8 +20,11 @@ NoDHCPLeaseInterfaceError, NoDHCPLeaseMissingDhclientError, Udhcpc, + find_correct_device_nmcli, maybe_perform_dhcp_discovery, + network_manager_load_leases, networkd_load_leases, + run_nmcli, ) from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.subp import SubpResult @@ -1392,3 +1395,84 @@ def test_none_and_missing_fallback(self): with pytest.raises(NoDHCPLeaseInterfaceError): distro = mock.Mock(fallback_interface=None) maybe_perform_dhcp_discovery(distro, None) + + +class TestNMDhcpLeases: + def test_find_correct_connected_device_first_match(self): + with mock.patch( + "cloudinit.net.dhcp.run_nmcli", + return_value=dedent( + """ + ens150:ethernet:unavailable: + p2p-dev-wlp:wifi-p2p:disconnected: + ens160:ethernet:connected:Wired connection 2 + ens256:ethernet:connected:Wired connection 3 + lo:loopback:connected (externally):lo + """ + ), + ): + ret = find_correct_device_nmcli() + assert ret == "ens160" + + def test_network_manager_load_leases(self): + expected_return = { + "broadcast_address": "172.16.127.255", + "dhcp_client_identifier": "01:00:0c:29:bf:c5:56", + "dhcp_lease_time": "1800", + "dhcp_server_identifier": "172.16.127.254", + } + with mock.patch( + "cloudinit.net.dhcp.run_nmcli", + return_value=dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + ), + ): + ret = network_manager_load_leases("ens10") + assert ret == expected_return + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/usr/bin/nmcli") + @mock.patch("cloudinit.net.dhcp.subp.subp") + def test_run_nmcli(self, m_subp, m_which): + expected_return = dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + ) + m_subp.return_value = (expected_return, "") + ret = run_nmcli(["show"]) + assert ret == expected_return + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value=None) + def test_run_nmcli_missing_nmcli(self, m_which): + """verify that absence of nmcli binary can result in + raising NoDHCPLeaseMissingDhclientError""" + + with pytest.raises(NoDHCPLeaseMissingDhclientError): + run_nmcli(["show"]) + + @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/usr/bin/nmcli") + @mock.patch("cloudinit.net.dhcp.subp.subp") + def test_run_nmcli_subp_err(self, m_subp, m_which): + """verify that when subp.ProcessExecutionError is raised while + running nmcli, it results in run_nmcli raising NoDHCPLeaseError""" + + m_subp.side_effect = subp.ProcessExecutionError(exit_code=-5) + + with pytest.raises(NoDHCPLeaseError): + run_nmcli(["--fields", "all", "device", "show"]) + + m_subp.assert_has_calls( + [ + mock.call( + ["/usr/bin/nmcli", "--fields", "all", "device", "show"], + ), + ] + ) From deef04e02d949a626d282110cd9f3a008108fc4f Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Wed, 1 Apr 2026 18:22:47 +0530 Subject: [PATCH 38/72] test: fix test_get_domainname_isc_dhclient mock leak (#6829) Without mocking out dhcp.IscDhclient.get_newest_lease_file_from_distro, the functions on some platforms returns None. This means get_key_from_latest_lease() bails out early without calling parse_leases() to parse the lease file. Fix it. Signed-off-by: Ani Sinha --- tests/unittests/sources/test_cloudstack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index f276f274e64..9e524cd2e95 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -108,6 +108,10 @@ def test_get_domainname_isc_dhclient(self, cloudstack_ds, mocker): DHCP_MOD_PATH + ".networkd_get_option_from_leases", get_networkd_domain, ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) with patch( MOD_PATH + ".util.load_text_file", From a96aac48abbffe4da8d8965d54e7198d888e06cb Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Wed, 1 Apr 2026 13:14:05 +0530 Subject: [PATCH 39/72] fix(cloudstack): get domain name information from network manager leases (#6829) Red Hat uses network manager as the supported dhcp client. If network manager cli is available, we should try to get domain name information directly from the network manager leases before asking distro specific dhcp client (dhcpcd by default). Signed-off-by: Ani Sinha Co-authored-by: Chad Smith --- cloudinit/sources/DataSourceCloudStack.py | 15 +- tests/unittests/sources/test_cloudstack.py | 202 +++++++++++++++++++++ 2 files changed, 216 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 265727cd60c..bf63605c62b 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -90,6 +90,7 @@ def _get_domainname(self): """Try obtaining a "domain-name" DHCP lease parameter: - From systemd-networkd lease (case-insensitive) - From ISC dhclient + - From network manager dhcp client - From dhcpcd (ephemeral) - Return empty string if not found (non-fatal) """ @@ -113,7 +114,19 @@ def _get_domainname(self): LOG.debug( "Could not obtain FQDN from ISC dhclient leases. Falling back to " - "%s", + "Network Manager leases" + ) + with suppress( + dhcp.NoDHCPLeaseMissingDhclientError, dhcp.NoDHCPLeaseError + ): + domain_name = dhcp.network_manager_get_option_from_leases( + "domain_name" + ) + if domain_name: + return domain_name.strip() + + LOG.debug( + "Could not obtain FQDN from NM leases. Falling back to %s", self.distro.dhcp_client.client_name, ) try: diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index 9e524cd2e95..2e959f90271 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -67,6 +67,7 @@ def setup(self, mocker, tmp_path): self.hostname = "vm-hostname" self.networkd_domainname = "networkd.local" self.isc_dhclient_domainname = "dhclient.local" + self.nm_domainname = "nm.local" get_hostname_parent = mock.MagicMock( return_value=DataSourceHostname(self.hostname, True) @@ -145,6 +146,201 @@ def test_get_domainname_isc_dhclient(self, cloudstack_ds, mocker): result = cloudstack_ds._get_domainname() assert self.isc_dhclient_domainname == result + def test_get_domainname_network_manager(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + gets domain name from nmcli lease information + """ + nmcliop = f"DHCP4.OPTION[5]: domain_name = {self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".find_correct_device_nmcli", + return_value="ens120", + ) + + with patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + DHCP4.OPTION[1]: broadcast_address = 172.16.127.255 + DHCP4.OPTION[2]: dhcp_client_identifier = 01:00:0c:29:bf:c5:56 + DHCP4.OPTION[3]: dhcp_lease_time = 1800 + DHCP4.OPTION[4]: dhcp_server_identifier = 172.16.127.254 + """ + + nmcliop + + """ + DHCP4.OPTION[6]: domain_name_servers = 172.16.127.2 + DHCP4.OPTION[7]: expiry = 1775029195 + DHCP4.OPTION[8]: ip_address = 172.16.127.135 + DHCP4.OPTION[9]: next_server = 172.16.127.254 + """ + ), + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_multi_conn_nic(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + can handle multi connected nic environments. + """ + domain_name = f"{self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + ens160:ethernet:connected:Wired connection 1 + ens256:ethernet:connected:Wired connection 2 + lo:loopback:connected (externally):lo + """ + ), + ) + + with patch( + DHCP_MOD_PATH + ".network_manager_load_leases", + return_value={"domain_name": domain_name}, + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_single_conn_nic(self, cloudstack_ds, mocker): + """ + Test if DataSourceCloudStack._get_domainname() + can handle one connected nic environments. + """ + domain_name = f"{self.nm_domainname}" + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + ens160:ethernet:connected:Wired connection 1 + lo:loopback:connected (externally):lo + ens256:ethernet:unavailable: + """ + ), + ) + + with patch( + DHCP_MOD_PATH + ".network_manager_load_leases", + return_value={"domain_name": domain_name}, + ): + result = cloudstack_ds._get_domainname() + assert self.nm_domainname == result + + def test_get_domainname_nm_no_conn_nic( + self, cloudstack_ds, mocker, caplog + ): + """ + Test if DataSourceCloudStack._get_domainname() + can handle one connected nic environments. + """ + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".run_nmcli", + return_value=dedent( + """ + lo:loopback:connected (externally):lo + ens256:ethernet:unavailable: + """ + ), + ) + + result = cloudstack_ds._get_domainname() + assert "Could not obtain FQDN from NM leases" in caplog.text + assert ( + "No domain name found in any DHCP lease; returning empty" + in caplog.text + ) + assert result == "" + + def test_get_domainname_nm_nodhcp_lease_err( + self, cloudstack_ds, mocker, caplog + ): + """ + Test if DataSourceCloudStack._get_domainname() + can handle NoDHCPLeaseError. + """ + get_networkd_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, + ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, + ) + + mocker.patch( + MOD_PATH + ".util.load_text_file", + return_value=None, + ) + + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + side_effect=NoDHCPLeaseError, + ) + + cloudstack_ds._get_domainname() + assert "Could not obtain FQDN from NM leases" in caplog.text + def test_get_hostname_non_fqdn(self, cloudstack_ds): """ Test get_hostname() method implementation @@ -190,6 +386,12 @@ def test_get_hostname_fqdn_fallback(self, cloudstack_ds, mocker): get_networkd_domain, ) + get_nm_domain = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + get_nm_domain, + ) + mocker.patch( "cloudinit.distros.net.find_fallback_nic", return_value="eth0", From 8adfc0e7f418b2190b5936c93bd8ac236213b65d Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Thu, 23 Apr 2026 13:28:35 +0530 Subject: [PATCH 40/72] fix(cloudstack): get vr information from network manager leases (#6829) Red Hat distributions uses network manager to manage DHCP leases. This change adds support for obtaining vr address information from DHCP leases maintained by network manager. Co-authored-by: Chad Smith Signed-off-by: Ani Sinha --- cloudinit/sources/DataSourceCloudStack.py | 9 ++++ tests/unittests/sources/test_cloudstack.py | 58 +++++++++++++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index bf63605c62b..f5fa4c8e443 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -351,6 +351,15 @@ def get_vr_address(distro): LOG.debug("Found SERVER_ADDRESS '%s' via dhclient", latest_address) return latest_address + # try network manager DHCP lease information + with suppress(dhcp.NoDHCPLeaseMissingDhclientError, dhcp.NoDHCPLeaseError): + latest_address = dhcp.network_manager_get_option_from_leases( + "dhcp_server_identifier" + ) + if latest_address: + LOG.debug("Found SERVER_ADDRESS '%s' via nmcli", latest_address) + return latest_address + with suppress(FileNotFoundError): latest_lease = distro.dhcp_client.get_newest_lease( distro.fallback_interface diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index 2e959f90271..42544e42e05 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -492,9 +492,17 @@ def test_data_server_from_dns( MOD_PATH + ".dhcp.networkd_get_option_from_leases", return_value="10.1.37.132", ) +@mock.patch( + MOD_PATH + ".dhcp.network_manager_get_option_from_leases", + return_value="10.1.37.135", +) class TestGetVrAddress: def test_get_vr_addr_from_dns( - self, m_networkd_option_from_leases, m_get_data_server, caplog + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + caplog, ): """cloud-init first obtains data-server if resolved by DNS""" assert "10.1.37.131" == get_vr_address(MockDistro()) @@ -505,7 +513,12 @@ def test_get_vr_addr_from_dns( assert 0 == m_networkd_option_from_leases.call_count def test_get_vr_addr_from_networkd_leases( - self, m_networkd_option_from_leases, m_get_data_server, mocker, caplog + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + mocker, + caplog, ): """When no DNS for data-server use networkd dhcp-server-identifier""" mocker.patch(MOD_PATH + ".get_data_server", return_value=None) @@ -516,6 +529,36 @@ def test_get_vr_addr_from_networkd_leases( ) m_networkd_option_from_leases.assert_called_once_with("SERVER_ADDRESS") + def test_get_vr_addr_from_network_manager_leases( + self, + m_nm_get_option_from_leases, + m_networkd_option_from_leases, + m_get_data_server, + mocker, + caplog, + ): + """When no DNS for data-server or networkd or IscDhclient, + use dhcp_server_identifier from network manager lease""" + mocker.patch(MOD_PATH + ".get_data_server", return_value=None) + mocker.patch( + MOD_PATH + ".dhcp.networkd_get_option_from_leases", + return_value=None, + ) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", + return_value=None, + ) + assert "10.1.37.135" == get_vr_address(MockDistro()) + m_which = mocker.patch( + "cloudinit.net.dhcp.subp.which", + return_value=None, + ) + assert "10.1.37.135" == get_vr_address(MockDistro()) + m_which.assert_called_once_with("dhclient") + m_nm_get_option_from_leases.assert_called_with( + "dhcp_server_identifier" + ) + @pytest.mark.usefixtures("dhclient_exists") @mock.patch(MOD_PATH + ".dmi.read_dmi_data", return_value=CLOUD_STACK_DMI_NAME) @@ -537,6 +580,10 @@ def setup(self, mocker, tmp_path): "dhcp-server-identifier": "168.63.129.16", }, ) + mocker.patch( + DHCP_MOD_PATH + ".network_manager_get_option_from_leases", + return_value=None, + ) get_newest_lease_file_from_distro = mock.MagicMock(return_value=None) mocker.patch( DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", @@ -702,11 +749,11 @@ def test_local_datasource_fails_ephemeral_dhcp( MOD_PATH + ".EphemeralIPNetwork", autospec=True, ) - @mock.patch(MOD_PATH + ".net.find_fallback_nic") - # @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") + @mock.patch(MOD_PATH + ".net.find_fallback_nic", return_value="enp0s1") + @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") def test_local_datasource_success( self, - # m_get_vr_address, + m_get_vr_address, m_find_fallback_nic, m_dhcp, m_wait_for_mds, @@ -720,7 +767,6 @@ def test_local_datasource_success( {}, distro, helpers.Paths({"run_dir": tmpdir}) ) - m_find_fallback_nic.return_value = "enp0s1" m_dhcp.return_value.__enter__.side_effect = (None,) m_wait_for_mds.return_value = (True,) m_get_userdata.return_value = "ud" From 777cef758213591fa7c2603ada75c3e6ba902c60 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 21 May 2026 16:29:56 +0000 Subject: [PATCH 41/72] tests: expect intermittent warning accessing Azure IMDS. Avoid client.destroy Avoid client.destroy within the cloud_session.launch context manager as it already performs cloud.destroy on context exit which calls instance.destroy. This avoids the following error seen by pycloudlib on Azure instances: Date: Tue, 9 Jun 2026 13:19:24 -0400 Subject: [PATCH 42/72] fix(azurelinux): support Azure Linux 4.0 (#6874) Azure Linux 4 is now Fedora-based. Therefore: Simplify the config template. Remove the custom group config. Set the default user to "azureuser". Use fedora mount_default_fields and ssh_pwauth settings. Use network renderers: netplan, systemd-networkd, and NetworkManager. Use libexec path for ds-identify. Remove overridden package_command() method Undo various distro-specific overrides. --- cloudinit/distros/azurelinux.py | 30 ------------------------- config/cloud.cfg.tmpl | 16 +++++-------- systemd/cloud-init-generator.tmpl | 2 +- systemd/cloud-init-local.service.tmpl | 6 ++--- systemd/cloud-init-main.service.tmpl | 6 ++--- systemd/cloud-init-network.service.tmpl | 4 ++-- templates/hosts.azurelinux.tmpl | 1 + tests/unittests/test_render_template.py | 1 + tests/unittests/test_util.py | 22 ++++++++++++------ 9 files changed, 32 insertions(+), 56 deletions(-) diff --git a/cloudinit/distros/azurelinux.py b/cloudinit/distros/azurelinux.py index 591b870020e..e3880548de8 100644 --- a/cloudinit/distros/azurelinux.py +++ b/cloudinit/distros/azurelinux.py @@ -6,7 +6,6 @@ import logging -from cloudinit import subp, util from cloudinit.distros import rhel from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE @@ -22,8 +21,6 @@ class Distro(rhel.Distro): - usr_lib_exec = "/usr/lib" - def __init__(self, name, cfg, paths): super().__init__(name, cfg, paths) self.osfamily = "azurelinux" @@ -45,30 +42,3 @@ def __init__(self, name, cfg, paths): "postcmds": "True", }, } - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - if subp.which("dnf"): - LOG.debug("Using DNF for package management") - cmd = ["dnf"] - else: - LOG.debug("Using TDNF for package management") - cmd = ["tdnf"] - # Determines whether or not dnf/tdnf prompts for confirmation - # of critical actions. We don't want to prompt... - cmd.append("-y") - - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - cmd.append(command) - - pkglist = util.expand_package_list("%s-%s", pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - subp.subp(cmd, capture=False) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 762d1502a52..3926ea8b4d5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -13,7 +13,6 @@ "raspberry-pi-os": "Raspberry Pi OS", "ubuntu": "Ubuntu", "unknown": "Ubuntu"}) %} {% set groups = ({"alpine": "adm, wheel", "aosc": "wheel", "arch": "wheel, users", - "azurelinux": "wheel", "debian": "adm, audio, cdrom, dialout, dip, floppy, netdev, plugdev, sudo, video", "gentoo": "users, wheel", "mariner": "wheel", "photon": "wheel", @@ -25,7 +24,7 @@ {% set shells = ({"alpine": "/bin/ash", "dragonfly": "/bin/sh", "freebsd": "/bin/tcsh", "netbsd": "/bin/sh", "openbsd": "/bin/ksh"}) %} -{% set usernames = ({"amazon": "ec2-user", "centos": "cloud-user", +{% set usernames = ({"amazon": "ec2-user", "azurelinux": "azureuser", "centos": "cloud-user", "openmandriva": "omv", "raspberry-pi-os": "pi", "rhel": "cloud-user", "unknown": "ubuntu"}) %} @@ -61,7 +60,7 @@ disable_root: false disable_root: true {% endif %} -{%- if variant in ["alpine", "amazon", "fedora", "OpenCloudOS", "openeuler", +{%- if variant in ["alpine", "amazon", "azurelinux", "fedora", "OpenCloudOS", "openeuler", "openmandriva", "photon", "TencentOS"] or is_rhel %} {% if is_rhel %} @@ -178,9 +177,7 @@ cloud_config_modules: {% if variant == "raspberry-pi-os" %} - raspberry_pi {% endif %} -{% if variant not in ["azurelinux"] %} - disable_ec2_metadata -{% endif %} - runcmd {% if variant in ["debian", "ubuntu", "unknown"] %} - byobu @@ -198,17 +195,13 @@ cloud_final_modules: - ubuntu_drivers {% endif %} - write_files_deferred -{% if variant not in ["azurelinux"] %} - puppet - chef -{% endif %} - ansible -{% if variant not in ["azurelinux"] %} - mcollective - salt_minion {% if variant not in ["alpine"] %} - reset_rmc -{% endif %} {% endif %} - scripts_vendor - scripts_per_once @@ -287,6 +280,9 @@ system_info: {% if variant == "alpine" %} network: renderers: ['eni'] +{% elif variant == "azurelinux" %} + network: + renderers: ['netplan', 'networkd', 'network-manager'] {% elif variant == "debian" %} network: renderers: ['netplan', 'eni', 'networkd'] @@ -300,7 +296,7 @@ system_info: {% elif variant in ["freebsd", "netbsd", "openbsd"] %} network: renderers: ['{{ variant }}'] -{% elif variant in ["azurelinux", "mariner", "photon"] %} +{% elif variant in ["mariner", "photon"] %} network: renderers: ['networkd'] {% elif variant == "openmandriva" %} diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index e55ece6d276..888bad535ea 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -20,7 +20,7 @@ CLOUD_SYSTEM_TARGET="/usr/lib/systemd/system/cloud-init.target" {% else %} CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" {% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", +{% if variant in ["almalinux", "azurelinux", "centos", "cloudlinux", "eurolinux", "fedora", "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", "TencentOS", "virtuozzo"] %} dsidentify="/usr/libexec/cloud-init/ds-identify" {% elif variant == "benchmark" %} diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index e88b15ca246..ca6dbe0a159 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -2,14 +2,14 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Local Stage (pre-network) -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=network-pre.target After=hv_kvp_daemon.service Before=network-pre.target Before=shutdown.target -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} Before=firewalld.target {% endif %} {% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} @@ -22,7 +22,7 @@ ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled [Service] Type=oneshot -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} # This service is a shim which preserves systemd ordering while allowing a diff --git a/systemd/cloud-init-main.service.tmpl b/systemd/cloud-init-main.service.tmpl index 2ca6220c24c..a38b2e817cf 100644 --- a/systemd/cloud-init-main.service.tmpl +++ b/systemd/cloud-init-main.service.tmpl @@ -8,10 +8,10 @@ # https://www.freedesktop.org/software/systemd/man/latest/systemd-remount-fs.service.html [Unit] Description=Cloud-init: Single Process -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} Requires=dbus.socket After=dbus.socket {% endif %} @@ -31,7 +31,7 @@ ExecStart=/usr/bin/cloud-init --all-stages KillMode=process TasksMax=infinity TimeoutStartSec=infinity -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} +{% if variant in ["almalinux", "azurelinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} diff --git a/systemd/cloud-init-network.service.tmpl b/systemd/cloud-init-network.service.tmpl index c284024be76..58a36f5dfe0 100644 --- a/systemd/cloud-init-network.service.tmpl +++ b/systemd/cloud-init-network.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Network Stage -{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +{% if variant not in ["almalinux", "azurelinux", "cloudlinux", "photon", "rhel"] %} DefaultDependencies=no {% endif %} Wants=cloud-init-local.service @@ -15,7 +15,7 @@ After=systemd-networkd-wait-online.service {% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service {% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", +{% if variant in ["almalinux", "azurelinux", "centos", "cloudlinux", "eurolinux", "fedora", "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} After=NetworkManager.service diff --git a/templates/hosts.azurelinux.tmpl b/templates/hosts.azurelinux.tmpl index 8e3c23f6f12..9e64e26916e 100644 --- a/templates/hosts.azurelinux.tmpl +++ b/templates/hosts.azurelinux.tmpl @@ -19,4 +19,5 @@ you need to add the following to config: # The following lines are desirable for IPv6 capable hosts ::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost ::1 localhost6.localdomain6 localhost6 diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 7f8fc944429..32926da1dd2 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -93,6 +93,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): default_user_exceptions = { "amazon": "ec2-user", + "azurelinux": "azureuser", "rhel": "cloud-user", "centos": "cloud-user", "raspberry-pi-os": "pi", diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index d75fbc4a8ab..fe24962e19b 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -409,15 +409,23 @@ OS_RELEASE_AZURELINUX = dedent( """\ - NAME="Microsoft Azure Linux" - VERSION="3.0.20240206" + NAME="Azure Linux" + VERSION="4.0 (Cloud Variant)" + RELEASE_TYPE=stable ID=azurelinux - VERSION_ID="3.0" - PRETTY_NAME="Microsoft Azure Linux 3.0" - ANSI_COLOR="1;34" + ID_LIKE=fedora + VERSION_ID=4.0 + VERSION_CODENAME="" + PRETTY_NAME="Azure Linux 4.0 (Cloud Variant)" + ANSI_COLOR="0;38;2;60;110;180" + LOGO=azurelinux-logo-icon + CPE_NAME="cpe:/o:microsoft:azurelinux:4.0" HOME_URL="https://aka.ms/azurelinux" - BUG_REPORT_URL="https://aka.ms/azurelinux" + DOCUMENTATION_URL="https://aka.ms/azurelinux" SUPPORT_URL="https://aka.ms/azurelinux" + BUG_REPORT_URL="https://aka.ms/azurelinux" + VARIANT="Cloud Variant" + VARIANT_ID=cloud """ ) @@ -1276,7 +1284,7 @@ def test_get_linux_azurelinux_os_release( m_os_release.return_value = OS_RELEASE_AZURELINUX m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists dist = util.get_linux_distro() - assert ("azurelinux", "3.0", "") == dist + assert ("azurelinux", "4.0", "Cloud Variant") == dist @mock.patch(M_PATH + "load_text_file") def test_get_linux_openmandriva(self, m_os_release, m_path_exists): From 908f59982034eccbe08c20aee35d5c2576e9228d Mon Sep 17 00:00:00 2001 From: Richard Coker Date: Tue, 9 Jun 2026 18:20:53 +0100 Subject: [PATCH 43/72] docs: fix invalid badge URLs (#6900) Fixes GH-6899 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cfbb470b6d0..039d1f15b26 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # cloud-init -![Unit Tests](https://github.com/canonical/cloud-init/actions/workflows/unit.yml/badge.svg?branch=main) -![Integration Tests](https://github.com/canonical/cloud-init/actions/workflows/integration.yml/badge.svg?branch=main) -![Documentation](https://github.com/canonical/cloud-init/actions/workflows/check_format.yml/badge.svg?branch=main) +![Unit Tests](https://github.com/canonical/cloud-init/actions/workflows/22-pr-unit-python.yml/badge.svg?branch=main) +![Integration Tests](https://github.com/canonical/cloud-init/actions/workflows/24-pr-integration.yml/badge.svg?branch=main) +![Documentation](https://github.com/canonical/cloud-init/actions/workflows/21-pr-check-format.yml/badge.svg?branch=main) Cloud-init is the *industry standard* multi-distribution method for cross-platform cloud instance initialization. It is supported across all From 08251e544aff8b8327e2b51b23f02904af7b3c8e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jun 2026 08:15:39 -0600 Subject: [PATCH 44/72] test: prefer reading journalctl transport=syslog over /var/log/syslog Recent versions of journalctl and rsyslog avoid mirroring /dev/console messages to /var/log/syslog. Prefer to ask journalctl directly about _TRANSPORT=syslog type messages to track what cloud-init writes to console as this approach works in both legacy and current rsyslog/journald behavior. --- tests/integration_tests/util.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 8de8c943723..8906221276c 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -589,6 +589,13 @@ def get_syslog_or_console(client: "IntegrationInstance") -> str: if OS_IMAGE_TYPE == "minimal" and HAS_CONSOLE_LOG: return get_console_log(client) else: + # Prefer syslog transport categorized messages over presence of + # /var/log/syslog as recent versions of rsyslog will avoid mirroring + # /dev/console to syslog. + if client.execute(["which", "journalctl"]).ok: + return client.execute( + ["journalctl", "_TRANSPORT=syslog", "-b", "0"] + ) return client.read_from_file("/var/log/syslog") From 4694cde73ee594805f7baf09a3b358a8a40a205c Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jun 2026 12:54:40 -0600 Subject: [PATCH 45/72] tests: simplify get_syslog_or_console to get_journal_syslog for systemd-v255 --- .../modules/test_keys_to_console.py | 20 ++++++-------- .../modules/test_ssh_auth_key_fingerprints.py | 4 +-- tests/integration_tests/util.py | 26 +++++++------------ 3 files changed, 19 insertions(+), 31 deletions(-) diff --git a/tests/integration_tests/modules/test_keys_to_console.py b/tests/integration_tests/modules/test_keys_to_console.py index b8e74f57ba8..f20578ffd44 100644 --- a/tests/integration_tests/modules/test_keys_to_console.py +++ b/tests/integration_tests/modules/test_keys_to_console.py @@ -12,7 +12,7 @@ from tests.integration_tests.util import ( HAS_CONSOLE_LOG, get_console_log, - get_syslog_or_console, + get_journal_syslog, ) BLACKLIST_USER_DATA = """\ @@ -52,16 +52,14 @@ class TestKeysToConsoleBlacklist: @pytest.mark.parametrize("key_type", ["ECDSA"]) def test_excluded_keys(self, class_client, key_type): - assert "({})".format(key_type) not in get_syslog_or_console( - class_client - ) + assert "({})".format(key_type) not in get_journal_syslog(class_client) # retry decorator here because it can take some time to be reflected # in syslog @retry(tries=60, delay=1) @pytest.mark.parametrize("key_type", ["ED25519", "RSA"]) def test_included_keys(self, class_client, key_type): - assert "({})".format(key_type) in get_syslog_or_console(class_client) + assert "({})".format(key_type) in get_journal_syslog(class_client) @pytest.mark.user_data(BLACKLIST_ALL_KEYS_USER_DATA) @@ -75,12 +73,12 @@ class TestAllKeysToConsoleBlacklist: """ def test_header_excluded(self, class_client): - assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) def test_footer_excluded(self, class_client): - assert "END SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "END SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) @@ -95,17 +93,15 @@ class TestKeysToConsoleDisabled: @pytest.mark.parametrize("key_type", ["ECDSA", "ED25519", "RSA"]) def test_keys_excluded(self, class_client, key_type): - assert "({})".format(key_type) not in get_syslog_or_console( - class_client - ) + assert "({})".format(key_type) not in get_journal_syslog(class_client) def test_header_excluded(self, class_client): - assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) def test_footer_excluded(self, class_client): - assert "END SSH HOST KEY FINGERPRINTS" not in get_syslog_or_console( + assert "END SSH HOST KEY FINGERPRINTS" not in get_journal_syslog( class_client ) diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py index d55cba91e70..c43fb13ade5 100644 --- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -19,7 +19,7 @@ OS_IMAGE_TYPE, PLATFORM, ) -from tests.integration_tests.util import HAS_CONSOLE_LOG, get_syslog_or_console +from tests.integration_tests.util import HAS_CONSOLE_LOG, get_journal_syslog USER_DATA_SSH_AUTHKEY_DISABLE = """\ #cloud-config @@ -55,7 +55,7 @@ def test_ssh_authkey_fingerprints_disable(self, client): reason=f"No console_log available for minimal images on {PLATFORM}", ) def test_ssh_authkey_fingerprints_enable(self, client): - syslog_output = get_syslog_or_console(client) + syslog_output = get_journal_syslog(client) assert re.search(r"256 SHA256:.*(ECDSA)", syslog_output) is not None assert re.search(r"256 SHA256:.*(ED25519)", syslog_output) is not None assert re.search(r"2048 SHA256:.*(RSA)", syslog_output) is None diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 8906221276c..42715afea40 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -16,10 +16,7 @@ from cloudinit.subp import subp from tests.integration_tests.decorators import retry -from tests.integration_tests.integration_settings import ( - OS_IMAGE_TYPE, - PLATFORM, -) +from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, NOBLE LOG = logging.getLogger("integration_testing.util") @@ -584,19 +581,14 @@ def get_console_log(client: "IntegrationInstance"): @retry(tries=5, delay=1) # Retry on get_console_log failures -def get_syslog_or_console(client: "IntegrationInstance") -> str: - """minimal OS_IMAGE_TYPE does not contain rsyslog""" - if OS_IMAGE_TYPE == "minimal" and HAS_CONSOLE_LOG: - return get_console_log(client) - else: - # Prefer syslog transport categorized messages over presence of - # /var/log/syslog as recent versions of rsyslog will avoid mirroring - # /dev/console to syslog. - if client.execute(["which", "journalctl"]).ok: - return client.execute( - ["journalctl", "_TRANSPORT=syslog", "-b", "0"] - ) - return client.read_from_file("/var/log/syslog") +def get_journal_syslog(client: "IntegrationInstance") -> str: + """Syslog events are categorized _TRANSPORT=syslog from systemd v205.""" + # Prefer syslog transport categorized messages over presence of + # /var/log/syslog as systemd v255 introduced systemd-executor + # which sandboxes unit processes resulting in direct writes to + # /dev/console being logged directly to journal binary instead + # of mirrored as rsyslog events. + return client.execute(["journalctl", "_TRANSPORT=syslog", "-b", "0"]) @lru_cache() From 5a5bca1507078bb564c3db723607922f0255b67c Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 4 Jun 2026 17:18:58 +0000 Subject: [PATCH 46/72] ai: copilot review comment resolution --- tests/integration_tests/modules/test_keys_to_console.py | 2 +- .../modules/test_ssh_auth_key_fingerprints.py | 8 ++++---- tests/integration_tests/util.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/modules/test_keys_to_console.py b/tests/integration_tests/modules/test_keys_to_console.py index f20578ffd44..2f53fd7f4df 100644 --- a/tests/integration_tests/modules/test_keys_to_console.py +++ b/tests/integration_tests/modules/test_keys_to_console.py @@ -55,7 +55,7 @@ def test_excluded_keys(self, class_client, key_type): assert "({})".format(key_type) not in get_journal_syslog(class_client) # retry decorator here because it can take some time to be reflected - # in syslog + # in the journal @retry(tries=60, delay=1) @pytest.mark.parametrize("key_type", ["ED25519", "RSA"]) def test_included_keys(self, class_client, key_type): diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py index c43fb13ade5..8cc68ae7880 100644 --- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -55,10 +55,10 @@ def test_ssh_authkey_fingerprints_disable(self, client): reason=f"No console_log available for minimal images on {PLATFORM}", ) def test_ssh_authkey_fingerprints_enable(self, client): - syslog_output = get_journal_syslog(client) - assert re.search(r"256 SHA256:.*(ECDSA)", syslog_output) is not None - assert re.search(r"256 SHA256:.*(ED25519)", syslog_output) is not None - assert re.search(r"2048 SHA256:.*(RSA)", syslog_output) is None + log_output = get_journal_syslog(client) + assert re.search(r"256 SHA256:.*(ECDSA)", log_output) is not None + assert re.search(r"256 SHA256:.*(ED25519)", log_output) is not None + assert re.search(r"2048 SHA256:.*(RSA)", log_output) is None @pytest.mark.user_data( diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 42715afea40..0906d3dbb10 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -580,7 +580,7 @@ def get_console_log(client: "IntegrationInstance"): return console_log -@retry(tries=5, delay=1) # Retry on get_console_log failures +@retry(tries=5, delay=1) # Retry on transient journalctl failures def get_journal_syslog(client: "IntegrationInstance") -> str: """Syslog events are categorized _TRANSPORT=syslog from systemd v205.""" # Prefer syslog transport categorized messages over presence of From e9011868eb08b8600122402267f3953bedeaee11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20Velasco=20Santamar=C3=ADa?= <58155462+jaborvs@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:26:28 +0200 Subject: [PATCH 47/72] fix(cmd): enable mypy strict checking for cloudinit.cmd (#6861) Refs GH-5445 --- cloudinit/cmd/devel/make_mime.py | 8 +++--- cloudinit/cmd/devel/net_convert.py | 6 ++-- cloudinit/cmd/main.py | 45 ++++++++++++++++++++++-------- cloudinit/net/eni.py | 4 +-- cloudinit/stages.py | 6 ++-- pyproject.toml | 3 -- 6 files changed, 45 insertions(+), 27 deletions(-) diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 5411ad602d1..e233df04c65 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -32,11 +32,11 @@ def create_mime_message(files): ) content_type = sub_message.get_content_type().lower() if content_type not in get_content_types(): - msg = ("content type %r for attachment %s may be incorrect!") % ( - content_type, - i + 1, + err_msg = ( + f"content type {content_type!r} for attachment" + f" {i + 1} may be incorrect!" ) - errors.append(msg) + errors.append(err_msg) sub_messages.append(sub_message) combined_message = MIMEMultipart() for msg in sub_messages: diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index eafb11f16e9..e64f0de5444 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -18,6 +18,7 @@ network_manager, network_state, networkd, + renderer, sysconfig, ) @@ -158,13 +159,14 @@ def handle_args(name, args): apply_network_config_for_secondary_ips=True, ) elif args.kind == "vmware-imc": - config = guestcust_util.Config( + vmware_config = guestcust_util.Config( guestcust_util.ConfigFile(args.network_data.name) ) pre_ns = guestcust_util.get_network_data_from_vmware_cust_cfg( - config, False + vmware_config, False ) + r_cls: type[renderer.Renderer] distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) if args.output_kind == "eni": diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 042d89420e2..89b6e79458a 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -19,7 +19,7 @@ import traceback import logging import yaml -from typing import Optional, Tuple, Callable, Union +from typing import Any, Optional, Tuple, Callable, Union from cloudinit import features, netinfo from cloudinit import signal_handler @@ -98,18 +98,20 @@ def error(self, message): if not self._raw_args: self._raw_args = sys.argv[1:] subcommand = None + subparsers_action = ( + self._subparsers._group_actions[0] if self._subparsers else None + ) + choices = getattr(subparsers_action, "choices", None) or {} if self._raw_args: for arg in self._raw_args: - if arg in self._subparsers._group_actions[0].choices: + if arg in choices: subcommand = arg break # Check if the subcommand exists and show its help if subcommand: - subparser = self._subparsers._group_actions[0].choices[subcommand] - subparser.print_help( - file=sys.stderr - ) # Print subcommand help to stderr + subparser = choices[subcommand] + subparser.print_help(file=sys.stderr) else: self.print_help(file=sys.stderr) sys.exit(2) @@ -546,6 +548,13 @@ def main_init(name, args): bring_up_interfaces = _should_bring_up_interfaces(init, args) try: init.fetch(existing=existing) + if init.datasource is None: + LOG.debug( + "[%s] Exiting. datasource is None after fetch," + " cannot continue.", + mode, + ) + return (None, []) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: @@ -613,6 +622,13 @@ def main_init(name, args): ) util.write_file(init.paths.get_runpath(".skip-network"), "") + if init.datasource is None: + LOG.debug( + "[%s] Exiting. datasource is None in local mode," + " cannot check dsmode.", + mode, + ) + return (None, []) if init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s not in local mode.", @@ -912,13 +928,13 @@ def status_wrapper(name, args): "Invalid cloud init mode specified '{0}'".format(mode) ) - nullstatus = { + nullstatus: dict[str, Union[list[Any], dict[str, Any], float, None]] = { "errors": [], "recoverable_errors": {}, "start": None, "finished": None, } - status = { + status: dict[str, Any] = { "v1": { "datasource": None, "init": nullstatus.copy(), @@ -951,10 +967,15 @@ def status_wrapper(name, args): v1[mode]["start"] = float(util.uptime()) handler = next( - filter( - lambda h: isinstance(h, loggers.LogExporter), root_logger.handlers - ) + ( + h + for h in root_logger.handlers + if isinstance(h, loggers.LogExporter) + ), + None, ) + if not isinstance(handler, loggers.LogExporter): + raise RuntimeError("LogExporter handler not found in root logger") preexisting_recoverable_errors = handler.export_logs() # Write status.json prior to running init / module code @@ -1052,7 +1073,7 @@ def _maybe_set_hostname(init, stage, retry_stage): ) if hostname: # meta-data or user-data hostname content try: - cc_set_hostname.handle("set_hostname", init.cfg, cloud, None) + cc_set_hostname.handle("set_hostname", init.cfg, cloud, []) except cc_set_hostname.SetHostnameError as e: LOG.debug( "Failed setting hostname in %s stage. Will" diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 89292597145..fb8bc1dfb69 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -7,7 +7,7 @@ import os import re from contextlib import suppress -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Mapping, Optional from cloudinit import performance, subp, util from cloudinit.net import ( @@ -453,7 +453,7 @@ def has_same_ip_version(addr_or_net: str, is_ipv6: bool) -> bool: class Renderer(renderer.Renderer): """Renders network information in a /etc/network/interfaces format.""" - def __init__(self, config: Optional[dict] = None): + def __init__(self, config: Optional[Mapping[str, Any]] = None): if not config: config = {} self.eni_path = config.get("eni_path", "etc/network/interfaces") diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5abb19ab95a..69283dd83a0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -558,10 +558,8 @@ def is_new_instance(self): or previous != self.ds.get_instance_id() ) - def fetch(self, existing="check"): - """optionally load datasource from cache, otherwise discover - datasource - """ + def fetch(self, existing="check") -> sources.DataSource: + """Load datasource from cache, otherwise discover datasource""" return self._get_data_source(existing=existing) def instancify(self): diff --git a/pyproject.toml b/pyproject.toml index 9a3373709ed..b4839e7385a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,6 @@ no_implicit_optional = true # See GH-5445 [[tool.mypy.overrides]] module = [ - "cloudinit.cmd.devel.make_mime", - "cloudinit.cmd.devel.net_convert", - "cloudinit.cmd.main", "cloudinit.config.cc_apt_configure", "cloudinit.config.cc_ca_certs", "cloudinit.config.cc_growpart", From cef1d0b160348b90b613608856548ba5859f8bb4 Mon Sep 17 00:00:00 2001 From: alan747271363-art Date: Thu, 11 Jun 2026 00:26:18 +0700 Subject: [PATCH 48/72] docs: clarify default user entry in users examples (#6894) Fixes GH-6702 Signed-off-by: Alan --- doc/module-docs/cc_users_groups/data.yaml | 13 +++++++++---- doc/rtd/reference/yaml_examples/user_groups.rst | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/doc/module-docs/cc_users_groups/data.yaml b/doc/module-docs/cc_users_groups/data.yaml index abb2359b44b..a0185bb36af 100644 --- a/doc/module-docs/cc_users_groups/data.yaml +++ b/doc/module-docs/cc_users_groups/data.yaml @@ -23,16 +23,21 @@ cc_users_groups: ``default_user key``. Each ``users`` dictionary item must contain either a ``name`` or - ``snapuser`` key, otherwise it will be ignored. Omission of ``default`` as - the first item in the ``users`` list skips creation the default user. If - no ``users`` key is provided, the default behavior is to create the - default user via this config: + ``snapuser`` key, otherwise it will be ignored. If no ``users`` key is + provided, the default behavior is to create the default user via this + config: .. code-block:: yaml users: - default + .. note:: + If you provide a ``users`` list and still want cloud-init to create the + default distribution user, keep ``default`` as the first item in the + list. Omitting ``default`` skips creation of the default user and can + prevent cloud-provided SSH keys from being installed for that user. + .. note:: Specifying a hash of a user's password with ``passwd`` is a security risk if the cloud-config can be intercepted. SSH authentication is diff --git a/doc/rtd/reference/yaml_examples/user_groups.rst b/doc/rtd/reference/yaml_examples/user_groups.rst index c1b2b74cd4e..2b1a0bb115c 100644 --- a/doc/rtd/reference/yaml_examples/user_groups.rst +++ b/doc/rtd/reference/yaml_examples/user_groups.rst @@ -35,9 +35,14 @@ and ``'sys'``, and the empty group ``cloud-users``. Add users to the system ======================= -Users are added after groups. Note that most of these configuration options -will not be honored if the user already exists. The following options are -exceptions and can be applied to already-existing users: +Users are added after groups. If you provide a ``users`` list and still want +cloud-init to create the default distribution user, keep ``default`` as the +first item in the list. Omitting ``default`` skips creation of the default user +and can prevent cloud-provided SSH keys from being installed for that user. + +Note that most of these configuration options will not be honored if the user +already exists. The following options are exceptions and can be applied to +already-existing users: - ``plain_text_passwd`` - ``hashed_passwd`` From d0bd056e727895499eeb88e68026ce8e60227056 Mon Sep 17 00:00:00 2001 From: Clinton Phillips Date: Thu, 11 Jun 2026 21:19:44 -0400 Subject: [PATCH 49/72] docs: make password lock explicit in users-groups example 3 (#6869) The example3 description states 'Password-based login is rejected', but the YAML did not actually set lock_passwd. While true is the default, adding it explicitly removes the documentation gap that arose during review of #6865 / #6858. Suggested by @neitsab in GH-6865 --- doc/module-docs/cc_users_groups/example3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/module-docs/cc_users_groups/example3.yaml b/doc/module-docs/cc_users_groups/example3.yaml index d9b178ae934..49ef8108d63 100644 --- a/doc/module-docs/cc_users_groups/example3.yaml +++ b/doc/module-docs/cc_users_groups/example3.yaml @@ -1,5 +1,6 @@ #cloud-config users: - name: newsuper + lock_passwd: true ssh_import_id: [ gh:TheRealFalcon, lp:falcojr ] shell: /bin/bash From da4ab09ee3dd2fded21c3a7e2c76e9c893fcd7a0 Mon Sep 17 00:00:00 2001 From: Cade Jacobson <91996442+cadejacobson@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:41:25 -0400 Subject: [PATCH 50/72] feat(azure): report failure if missing customdata (#6779) Add experimental Azure datasource config key experimental_fail_on_missing_customdata with a default of false and use it to control reporting when OVF custom data is missing despite IMDS indicating that hasCustomData=true. --- cloudinit/sources/DataSourceAzure.py | 50 +++++++++++--- cloudinit/sources/azure/errors.py | 13 ++++ tests/unittests/sources/test_azure.py | 95 +++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 535268d9f01..f6c91f4dfc3 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -295,6 +295,7 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, "apply_network_config": True, # Use IMDS published network configuration "apply_network_config_for_secondary_ips": True, # Configure secondary ips + "experimental_fail_on_missing_customdata": False, "experimental_skip_ready_report": False, # Skip final ready report } @@ -662,7 +663,6 @@ def crawl_metadata(self): # it determines the value of ret. More specifically, the first one in # the candidate list determines the path to take in order to get the # metadata we need. - ovf_source = None md = {"local-hostname": ""} cfg = {"system_info": {"default_user": {"name": ""}}} userdata_raw = "" @@ -684,9 +684,9 @@ def crawl_metadata(self): else: md, userdata_raw, cfg, files = load_azure_ds_dir(src) - ovf_source = src + self.seed = src report_diagnostic_event( - "Found provisioning metadata in %s" % ovf_source, + "Found provisioning metadata in %s" % self.seed, logger_func=LOG.debug, ) break @@ -714,7 +714,7 @@ def crawl_metadata(self): # not have UDF support. In either case, require IMDS metadata. # If we require IMDS metadata, try harder to obtain networking, waiting # for at least 20 minutes. Otherwise only wait 5 minutes. - requires_imds_metadata = bool(self._iso_dev) or ovf_source is None + requires_imds_metadata = bool(self._iso_dev) or self.seed is None timeout_minutes = 20 if requires_imds_metadata else 5 try: self._setup_ephemeral_networking(timeout_minutes=timeout_minutes) @@ -730,14 +730,18 @@ def crawl_metadata(self): imds_md = self.get_metadata_from_imds(report_failure=True) - if not imds_md and ovf_source is None: + if not imds_md and self.seed is None: msg = "No OVF or IMDS available" report_diagnostic_event(msg) raise sources.InvalidMetaDataException(msg) + self.seed = self.seed or "IMDS" + # Refresh PPS type using metadata. pps_type = self._determine_pps_type(cfg, imds_md) if pps_type != PPSType.NONE: + self.seed = "IMDS" + if util.is_FreeBSD(): msg = "Free BSD is not supported for PPS VMs" report_diagnostic_event(msg, logger_func=LOG.error) @@ -777,7 +781,6 @@ def crawl_metadata(self): # Report errors if IMDS network configuration is missing data. self.validate_imds_network_metadata(imds_md=imds_md) - self.seed = ovf_source or "IMDS" crawled_data.update( { "cfg": cfg, @@ -816,9 +819,31 @@ def crawl_metadata(self): logger_func=LOG.debug, ) - # only use userdata from imds if OVF did not provide custom data - # userdata provided by IMDS is always base64 encoded + # Only use userdata from IMDS if OVF did not provide custom data. + # Userdata provided by IMDS is always base64 encoded. if not userdata_raw: + # First, check to see if the OVF was supposed to provide custom + # data. If it was supposed to and did not, we report failure. + has_custom_data = _hascustomdata_from_imds(imds_md) + if has_custom_data: + if self.ds_cfg.get("experimental_fail_on_missing_customdata"): + self._report_failure( + errors.ReportableErrorMissingCustomData( + pps_type=pps_type.value, + provisioning_media=self.seed, + ) + ) + else: + report_diagnostic_event( + "Did not find custom data in %s, IMDS returned" + " extended.compute.hasCustomData=%r" + % ( + self.seed, + has_custom_data, + ), + logger_func=LOG.error, + ) + imds_userdata = _userdata_from_imds(imds_md) if imds_userdata: LOG.debug("Retrieved userdata from IMDS") @@ -831,7 +856,7 @@ def crawl_metadata(self): "Bad userdata in IMDS", logger_func=LOG.warning ) - if ovf_source == ddir: + if self.seed == ddir: report_diagnostic_event( "using files cached in %s" % ddir, logger_func=LOG.debug ) @@ -1724,6 +1749,13 @@ def _userdata_from_imds(imds_data): return None +def _hascustomdata_from_imds(imds_data: Dict) -> Optional[bool]: + try: + return imds_data["extended"]["compute"]["hasCustomData"] + except KeyError: + return None + + def _hostname_from_imds(imds_data): try: return imds_data["compute"]["osProfile"]["computerName"] diff --git a/cloudinit/sources/azure/errors.py b/cloudinit/sources/azure/errors.py index 8ef9515304b..a752e37468a 100644 --- a/cloudinit/sources/azure/errors.py +++ b/cloudinit/sources/azure/errors.py @@ -161,6 +161,19 @@ def __init__(self, *, key: str, value: Any) -> None: self.supporting_data["type"] = type(value).__name__ +class ReportableErrorMissingCustomData(ReportableError): + def __init__( + self, + *, + pps_type: str, + provisioning_media: str, + ) -> None: + super().__init__("failure to read customData while hasCustomData=true") + + self.supporting_data["pps_type"] = pps_type + self.supporting_data["provisioning_media"] = provisioning_media + + class ReportableErrorImdsMetadataParsingException(ReportableError): def __init__(self, *, exception: ValueError) -> None: super().__init__("error parsing IMDS metadata") diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index e032c25f279..b097509fb7d 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -5481,6 +5481,84 @@ def test_imds_failure_results_in_provisioning_failure(self): assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert not self.mock_kvp_report_success_to_host.mock_calls + @pytest.mark.parametrize( + "flag_enabled", + [False, True], + ) + @pytest.mark.parametrize( + "has_custom_data,custom_data", + [ + (True, None), + (True, "myCustomData"), + (True, ""), + (False, None), + ], + ) + def test_missing_customdata_reporting( + self, + caplog, + flag_enabled, + has_custom_data, + custom_data, + ): + """Test failure reporting behavior based on custom data fields. + + Failure is reported only when + experimental_fail_on_missing_customdata is True, + IMDS reports hasCustomData=True, and OVF has no custom data. + When the flag is not enabled but IMDS reports custom data + should be present, a diagnostic event is logged. + """ + self.azure_ds.ds_cfg["experimental_fail_on_missing_customdata"] = ( + flag_enabled + ) + + imds_md = copy.deepcopy(self.imds_md) + imds_md["extended"]["compute"]["hasCustomData"] = has_custom_data + + ovf = construct_ovf_env( + custom_data=custom_data, + provision_guest_proxy_agent=False, + ) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(imds_md).encode()), + ] + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() + + expect_failure = flag_enabled and has_custom_data and not custom_data + if expect_failure: + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 + assert ( + len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 + ) + assert not self.mock_kvp_report_success_to_host.mock_calls + else: + assert not self.mock_kvp_report_via_kvp.mock_calls + assert not self.mock_azure_report_failure_to_fabric.mock_calls + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + + if custom_data: + assert self.azure_ds.userdata_raw == custom_data.encode("utf-8") + else: + assert self.azure_ds.userdata_raw == "" + + # Verify diagnostic event for missing custom data when + # the experimental flag is not enabled. + expect_diagnostic = ( + not flag_enabled and has_custom_data and not custom_data + ) + if expect_diagnostic: + assert ( + "Did not find custom data in /dev/sr0, IMDS returned" + " extended.compute.hasCustomData=True" + ) in caplog.text + else: + assert "Did not find custom data in" not in caplog.text + class TestCheckAzureProxyAgent: @pytest.fixture(autouse=True) @@ -5920,6 +5998,23 @@ def test_query_vm_id_vm_id_conversion_failure( mock_convert_uuid.assert_called_once_with("test-system-uuid") +class TestHasCustomDataFromImds: + """Unit tests for the _hascustomdata_from_imds helper.""" + + @pytest.mark.parametrize( + "imds_data,expected", + [ + ({"extended": {"compute": {"hasCustomData": True}}}, True), + ({"extended": {"compute": {"hasCustomData": False}}}, False), + ({}, None), + ({"extended": {}}, None), + ({"extended": {"compute": {}}}, None), + ], + ) + def test_hascustomdata_from_imds(self, imds_data, expected): + assert dsaz._hascustomdata_from_imds(imds_data) is expected + + class TestHashPassword: """Tests for the hash_password function.""" From 9598a6070f48bbe081128bed84b6637cac297141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Can=C3=A9vet?= Date: Fri, 12 Jun 2026 03:49:30 +0200 Subject: [PATCH 51/72] feat(opennebula): support global SEARCH_DOMAIN for all interfaces (#6813) netbox-network-config sets a global SEARCH_DOMAIN context variable (never per-interface ETHx_SEARCH_DOMAIN). The existing get_nameservers() already handled the global DNS key but silently ignored SEARCH_DOMAIN, leaving VMs with no DNS search domain configured. Extend get_nameservers() to append entries from the global SEARCH_DOMAIN key, matching the existing pattern for global DNS. Per-interface ETHx_SEARCH_DOMAIN entries take precedence and duplicates are suppressed. --- cloudinit/sources/DataSourceOpenNebula.py | 17 ++++- doc/rtd/reference/datasources/opennebula.rst | 8 +- tests/unittests/sources/test_opennebula.py | 78 ++++++++++++++++++++ 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 54ee0ec6d66..16b5f82c07c 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -185,11 +185,22 @@ def mac2ip(self, mac: str) -> str: def get_nameservers(self, dev: str) -> Dict[str, List[str]]: nameservers: Dict[str, List[str]] = {} - dns = self.get_field(dev, "dns", "").split() - dns.extend(self.context.get("DNS", "").split()) + dns: List[str] = [] + for server in ( + self.get_field(dev, "dns", "").split() + + self.context.get("DNS", "").split() + ): + if server not in dns: + dns.append(server) if dns: nameservers["addresses"] = dns - search_domain = self.get_field(dev, "search_domain", "").split() + search_domain: List[str] = [] + for domain in ( + self.get_field(dev, "search_domain", "").split() + + self.context.get("SEARCH_DOMAIN", "").split() + ): + if domain not in search_domain: + search_domain.append(domain) if search_domain: nameservers["search"] = search_domain return nameservers diff --git a/doc/rtd/reference/datasources/opennebula.rst b/doc/rtd/reference/datasources/opennebula.rst index 3e31b20065b..5171704d48c 100644 --- a/doc/rtd/reference/datasources/opennebula.rst +++ b/doc/rtd/reference/datasources/opennebula.rst @@ -59,6 +59,7 @@ the OpenNebula documentation. :: DNS + SEARCH_DOMAIN ETH_IP ETH_NETWORK ETH_MASK @@ -74,7 +75,12 @@ the OpenNebula documentation. ETH_IP6_GATEWAY ETH_ROUTES -Static `network configuration`_. +Static `network configuration`_. ``DNS`` and ``SEARCH_DOMAIN`` are global +values applied to every interface. Per-interface ``ETH_DNS`` and +``ETH_SEARCH_DOMAIN`` (defined in `context-linux`_) take precedence; +duplicate entries across both levels are suppressed. + +.. _context-linux: https://github.com/OpenNebula/one-apps/blob/v7.0.0/context-linux/src/etc/one-context.d/loc-10-network.d/functions#L463-L466 ``ETH_ROUTES`` is a comma-separated list of static routes in the form ``NETWORK via GATEWAY``. For example:: diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index 10ccdea2ed3..1c7e48691dd 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -975,6 +975,84 @@ def test_multiple_nics(self): assert expected == net.gen_conf() + @pytest.mark.parametrize( + "context,expected_search", + [ + pytest.param( + {"SEARCH_DOMAIN": "global.example.com global.example.org"}, + ["global.example.com", "global.example.org"], + id="global_only", + ), + pytest.param( + { + "ETH0_SEARCH_DOMAIN": "iface.example.com", + "SEARCH_DOMAIN": "global.example.com", + }, + ["iface.example.com", "global.example.com"], + id="per_interface_and_global", + ), + pytest.param( + {"ETH0_SEARCH_DOMAIN": "iface.example.com"}, + ["iface.example.com"], + id="per_interface_only", + ), + pytest.param( + { + "ETH0_SEARCH_DOMAIN": "shared.example.com", + # extra precedes shared in global; shared must still come + # first because per-interface ordering takes precedence + "SEARCH_DOMAIN": "extra.example.com shared.example.com", + }, + ["shared.example.com", "extra.example.com"], + id="dedup_iface_order_preferred", + ), + ], + ) + def test_get_nameservers_search_domain(self, context, expected_search): + """get_nameservers merges and deduplicates SEARCH_DOMAIN correctly.""" + net = ds.OpenNebulaNetwork(context, mock.Mock()) + val = net.get_nameservers("eth0") + assert val["search"] == expected_search + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_global_search_domain(self, m_get_phys_by_mac): + """gen_conf includes global SEARCH_DOMAIN in nameservers.search.""" + context = { + "ETH0_MAC": MACADDR, + "SEARCH_DOMAIN": "global.example.com", + } + for nic in self.system_nics: + m_get_phys_by_mac.return_value = {MACADDR: nic} + net = ds.OpenNebulaNetwork(context, mock.Mock()) + conf = net.gen_conf() + assert conf["ethernets"][nic]["nameservers"]["search"] == [ + "global.example.com" + ] + + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_gen_conf_global_search_domain_multiple_nics( + self, m_get_phys_by_mac + ): + """Global SEARCH_DOMAIN appears on every NIC.""" + MAC_1 = "02:00:0a:12:01:01" + MAC_2 = "02:00:0a:12:01:02" + context = { + "ETH0_MAC": MAC_1, + "ETH1_MAC": MAC_2, + "SEARCH_DOMAIN": "global.example.com", + } + net = ds.OpenNebulaNetwork( + context, + mock.Mock(), + system_nics_by_mac={MAC_1: "eth0", MAC_2: "eth1"}, + ) + conf = net.gen_conf() + for nic in ("eth0", "eth1"): + assert ( + "global.example.com" + in conf["ethernets"][nic]["nameservers"]["search"] + ) + # ------------------------------------------------------------------ # # ETHx_ROUTES # # ------------------------------------------------------------------ # From 1e3cbdd135305527bc681648bd19251adcc5cb5e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 17 Apr 2026 03:24:55 +0000 Subject: [PATCH 52/72] chore(ci): rename and renumber workflow files to 1xx scheme Rename the reusable dispatch workflow to 100-dispatch-common.yml to establish a convention where 1xx = reusable/callable workflows. Workflow indexes 11x/12x/13x are caller workflows grouped by platform: lxd_container, lxd_vm and ec2. Move scheduled daily-integration-*-lxd-container.yml files to 11x-daily-integration-*-lxd_container.yml equivalent. Each platform group of scheduled workflows will be ordered by increasing Ubuntu series: 22.04, 24.04, 25.10, 26.04. --- .../{11-dispatch-common.yml => 100-dispatch-common.yml} | 0 ...iner.yml => 110-daily-integration-22.04-lxd_container.yml} | 4 ++-- ...iner.yml => 111-daily-integration-24.04-lxd_container.yml} | 4 ++-- ...iner.yml => 112-daily-integration-25.10-lxd_container.yml} | 4 ++-- ...iner.yml => 113-daily-integration-26.04-lxd_container.yml} | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{11-dispatch-common.yml => 100-dispatch-common.yml} (100%) rename .github/workflows/{03-daily-integration-22.04-lxd-container.yml => 110-daily-integration-22.04-lxd_container.yml} (63%) rename .github/workflows/{02-daily-integration-24.04-lxd-container.yml => 111-daily-integration-24.04-lxd_container.yml} (63%) rename .github/workflows/{01-daily-integration-25.10-lxd-container.yml => 112-daily-integration-25.10-lxd_container.yml} (64%) rename .github/workflows/{00-daily-integration-26.04-lxd-container.yml => 113-daily-integration-26.04-lxd_container.yml} (64%) diff --git a/.github/workflows/11-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml similarity index 100% rename from .github/workflows/11-dispatch-common.yml rename to .github/workflows/100-dispatch-common.yml diff --git a/.github/workflows/03-daily-integration-22.04-lxd-container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml similarity index 63% rename from .github/workflows/03-daily-integration-22.04-lxd-container.yml rename to .github/workflows/110-daily-integration-22.04-lxd_container.yml index d17a36f39c5..1c3746754c7 100644 --- a/.github/workflows/03-daily-integration-22.04-lxd-container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -1,4 +1,4 @@ -name: "Daily(22.04): lxd_container" +name: "Daily 22.04: lxd_container" on: workflow_dispatch: @@ -7,7 +7,7 @@ on: jobs: jammy-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml + uses: ./.github/workflows/100-dispatch-common.yml with: release: jammy platform: lxd_container diff --git a/.github/workflows/02-daily-integration-24.04-lxd-container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml similarity index 63% rename from .github/workflows/02-daily-integration-24.04-lxd-container.yml rename to .github/workflows/111-daily-integration-24.04-lxd_container.yml index acfd4c9e0f9..cbb76e1889a 100644 --- a/.github/workflows/02-daily-integration-24.04-lxd-container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -1,4 +1,4 @@ -name: "Daily(24.04): lxd_container" +name: "Daily 24.04: lxd_container" on: workflow_dispatch: @@ -7,7 +7,7 @@ on: jobs: noble-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml + uses: ./.github/workflows/100-dispatch-common.yml with: release: noble platform: lxd_container diff --git a/.github/workflows/01-daily-integration-25.10-lxd-container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml similarity index 64% rename from .github/workflows/01-daily-integration-25.10-lxd-container.yml rename to .github/workflows/112-daily-integration-25.10-lxd_container.yml index b2eb301cbee..73d2c450f29 100644 --- a/.github/workflows/01-daily-integration-25.10-lxd-container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -1,4 +1,4 @@ -name: "Daily(25.10): lxd_container" +name: "Daily 25.10: lxd_container" on: workflow_dispatch: @@ -7,7 +7,7 @@ on: jobs: questing-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml + uses: ./.github/workflows/100-dispatch-common.yml with: release: questing platform: lxd_container diff --git a/.github/workflows/00-daily-integration-26.04-lxd-container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml similarity index 64% rename from .github/workflows/00-daily-integration-26.04-lxd-container.yml rename to .github/workflows/113-daily-integration-26.04-lxd_container.yml index 0c4193bd1b5..b503d2d3e02 100644 --- a/.github/workflows/00-daily-integration-26.04-lxd-container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -1,4 +1,4 @@ -name: "Daily(26.04): lxd_container" +name: "Daily 26.04: lxd_container" on: workflow_dispatch: @@ -7,7 +7,7 @@ on: jobs: resolute-lxd_container: - uses: ./.github/workflows/11-dispatch-common.yml + uses: ./.github/workflows/100-dispatch-common.yml with: release: resolute platform: lxd_container From 624c05aa38018e962d4905395decc41fcc9bc81f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 17 Apr 2026 03:26:13 +0000 Subject: [PATCH 53/72] ci: require PYCLOUDLIB_TOML and SSH key secrets, write config to runner temp - Declare PYCLOUDLIB_TOML, SSH_PRIVATE_KEY, and SSH_PUBLIC_KEY as required secrets on the workflow_call trigger. - Add an early assertion step that exits immediately if PYCLOUDLIB_TOML is empty. - Replace the ssh-keygen throwaway key with a new 'Setup SSH' step that writes the injected public/private key pair to ~/.ssh, for use in authentication to instances under test - base64-decode PYCLOUDLIB_TOML secret and write it to $RUNNER_TEMP/pycloudlib.toml to keep secrets off the persistent filesystem. Add cleanup to remove the file unconditionally on exit. --- .github/workflows/100-dispatch-common.yml | 48 +++++++++++++++---- ...-daily-integration-22.04-lxd_container.yml | 4 ++ ...-daily-integration-24.04-lxd_container.yml | 4 ++ ...-daily-integration-25.10-lxd_container.yml | 4 ++ ...-daily-integration-26.04-lxd_container.yml | 4 ++ 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index 9a463e4d872..de8a4930545 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -1,3 +1,6 @@ +# WARNING: This workflow handles repository secrets (PYCLOUDLIB_TOML, SSH keys). +# It MUST NOT be triggered by PR-driven workflows to prevent secret exposure to +# untrusted code submitted via pull requests. name: "Dispatch: Integration" on: @@ -46,6 +49,13 @@ on: filter_tests: required: false type: string + secrets: + PYCLOUDLIB_TOML: + required: true + SSH_PRIVATE_KEY: + required: true + SSH_PUBLIC_KEY: + required: true jobs: lxc: @@ -53,14 +63,21 @@ jobs: if: github.repository == 'canonical/cloud-init' env: - CLOUD_INIT_PLATFORM: ${{ inputs.platform || 'lxd_container' }} - CLOUD_INIT_OS_IMAGE: ${{ inputs.release || 'questing' }} - CLOUD_INIT_OS_IMAGE_TYPE: ${{ inputs.image_type || 'generic' }} - CLOUD_INIT_CLOUD_INIT_SOURCE: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs - + REQUIRED_SECRET: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + CLOUD_INIT_PLATFORM: ${{ inputs.platform }} + CLOUD_INIT_OS_IMAGE: ${{ inputs.release }} + CLOUD_INIT_OS_IMAGE_TYPE: ${{ inputs.image_type }} + CLOUD_INIT_CLOUD_INIT_SOURCE: ${{ inputs.install_source }} steps: + - name: Assert repo secrets.PYCLOUDLIB_TOML is set + run: | + if [ -z "${{ env.REQUIRED_SECRET }}" ]; then + echo "ERROR: Missing required repo secrets.PYCLOUDLIB_TOML non-empty value." + exit 1 + fi - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup LXD @@ -70,15 +87,26 @@ jobs: - name: Clean workspace run: | rm -rf ${{ github.workspace }}/cloud_init_test_logs/ + - name: Setup SSH + run: | + mkdir -p ~/.ssh + # Dump secrets using a subprocess to avoid accidental leaks while debugging. + sh -c 'printf "%s\n" "$SSH_PUBLIC_KEY" > ~/.ssh/cloudinit_id_rsa.pub' + sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/cloudinit_id_rsa' + chmod 600 ~/.ssh/cloudinit_id_rsa - name: Setup pycloudlib + env: + PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | - ssh-keygen -P "" -q -f ~/.ssh/id_rsa - echo "[lxd]" > /home/$USER/.config/pycloudlib.toml + sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' - name: Install Dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data devscripts - name: Run integration Tests + env: + PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml + CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs run: | tox -e integration-tests -- --junitxml="${{ github.workspace }}/reports/junit-report-${{ inputs.platform }}-${{ inputs.release }}.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} - name: Publish Test Report with Insights @@ -117,3 +145,7 @@ jobs: name: failure-${{ github.job }} path: ${{ github.workspace }}/cloud_init_test_logs/ retention-days: 2 + - name: Clean pycloudlib + if: ${{ always() }} + run: | + rm -f ${{ runner.temp }}/pycloudlib.toml diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml index 1c3746754c7..62031eae83b 100644 --- a/.github/workflows/110-daily-integration-22.04-lxd_container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -11,3 +11,7 @@ jobs: with: release: jammy platform: lxd_container + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml index cbb76e1889a..b1c3e69a2ad 100644 --- a/.github/workflows/111-daily-integration-24.04-lxd_container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -11,3 +11,7 @@ jobs: with: release: noble platform: lxd_container + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml index 73d2c450f29..06ee39a3e83 100644 --- a/.github/workflows/112-daily-integration-25.10-lxd_container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -11,3 +11,7 @@ jobs: with: release: questing platform: lxd_container + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml index b503d2d3e02..430d8f5c76c 100644 --- a/.github/workflows/113-daily-integration-26.04-lxd_container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -11,3 +11,7 @@ jobs: with: release: resolute platform: lxd_container + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} From dd2cd3645aca1a017486b557ee2c7fc85a8cbee1 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 17 Apr 2026 03:27:25 +0000 Subject: [PATCH 54/72] ci: add lxd_vm and EC2 integration test workflows Add workflow files 120-123 for lxd_vm, daily, on all supported releases. Add workflow files 130-133 for EC2, twice weekly, all supported releases All new caller workflows expose an install_source workflow_dispatch input, defaulting to ppa:cloud-init-dev/daily, and forward the three required secrets to the common dispatch workflow. --- .github/workflows/100-dispatch-common.yml | 8 +++--- ...-daily-integration-22.04-lxd_container.yml | 2 ++ ...-daily-integration-24.04-lxd_container.yml | 2 ++ ...-daily-integration-25.10-lxd_container.yml | 2 ++ ...-daily-integration-26.04-lxd_container.yml | 2 ++ .../120-daily-integration-22.04-lxd_vm.yml | 24 ++++++++++++++++++ .../121-daily-integration-24.04-lxd_vm.yml | 24 ++++++++++++++++++ .../122-daily-integration-25.10-lxd_vm.yml | 24 ++++++++++++++++++ .../123-daily-integration-26.04-lxd_vm.yml | 24 ++++++++++++++++++ .../130-daily-integration-22.04-ec2.yml | 25 +++++++++++++++++++ .../131-daily-integration-24.04-ec2.yml | 25 +++++++++++++++++++ .../132-daily-integration-25.10-ec2.yml | 25 +++++++++++++++++++ .../133-daily-integration-26.04-ec2.yml | 24 ++++++++++++++++++ 13 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/120-daily-integration-22.04-lxd_vm.yml create mode 100644 .github/workflows/121-daily-integration-24.04-lxd_vm.yml create mode 100644 .github/workflows/122-daily-integration-25.10-lxd_vm.yml create mode 100644 .github/workflows/123-daily-integration-26.04-lxd_vm.yml create mode 100644 .github/workflows/130-daily-integration-22.04-ec2.yml create mode 100644 .github/workflows/131-daily-integration-24.04-ec2.yml create mode 100644 .github/workflows/132-daily-integration-25.10-ec2.yml create mode 100644 .github/workflows/133-daily-integration-26.04-ec2.yml diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index de8a4930545..b292e642dcc 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -10,16 +10,17 @@ on: required: true type: choice options: + - resolute - questing - jammy - noble - - resolute platform: required: true type: choice options: - lxd_container - lxd_vm + - ec2 image_type: type: choice options: @@ -41,10 +42,10 @@ on: required: true type: string image_type: - required: false + required: true type: string install_source: - required: false + required: true type: string filter_tests: required: false @@ -81,6 +82,7 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup LXD + if: ${{ contains(fromJSON('["lxd_vm", "lxd_container"]'), env.CLOUD_INIT_PLATFORM ) }} uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v0.1.2 with: channel: 6/stable diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml index 62031eae83b..834cf319fb6 100644 --- a/.github/workflows/110-daily-integration-22.04-lxd_container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -11,6 +11,8 @@ jobs: with: release: jammy platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic secrets: PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml index b1c3e69a2ad..265f2a6d14d 100644 --- a/.github/workflows/111-daily-integration-24.04-lxd_container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -11,6 +11,8 @@ jobs: with: release: noble platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic secrets: PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml index 06ee39a3e83..e52dbe6a693 100644 --- a/.github/workflows/112-daily-integration-25.10-lxd_container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -11,6 +11,8 @@ jobs: with: release: questing platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic secrets: PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml index 430d8f5c76c..16842bf4f6f 100644 --- a/.github/workflows/113-daily-integration-26.04-lxd_container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -11,6 +11,8 @@ jobs: with: release: resolute platform: lxd_container + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic secrets: PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml new file mode 100644 index 00000000000..0f0ed71d0d7 --- /dev/null +++ b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml @@ -0,0 +1,24 @@ +name: "Daily 22.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + - cron: '2 23 * * *' + +jobs: + jammy-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: jammy + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml new file mode 100644 index 00000000000..efd38a6e97e --- /dev/null +++ b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml @@ -0,0 +1,24 @@ +name: "Daily 24.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + - cron: '2 23 * * *' + +jobs: + noble-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: noble + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml new file mode 100644 index 00000000000..0d39b2d4d79 --- /dev/null +++ b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml @@ -0,0 +1,24 @@ +name: "Daily 25.10: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + - cron: '2 23 * * *' + +jobs: + questing-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: questing + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml new file mode 100644 index 00000000000..46d55c7b3c1 --- /dev/null +++ b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml @@ -0,0 +1,24 @@ +name: "Daily 26.04: lxd_vm" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + - cron: '2 23 * * *' + +jobs: + resolute-lxd_vm: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: resolute + platform: lxd_vm + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/130-daily-integration-22.04-ec2.yml b/.github/workflows/130-daily-integration-22.04-ec2.yml new file mode 100644 index 00000000000..98ae5b4a574 --- /dev/null +++ b/.github/workflows/130-daily-integration-22.04-ec2.yml @@ -0,0 +1,25 @@ +name: "Twice weekly 22.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + # Run Mon & Thurs for high-cost test runs + - cron: '2 22 * * 1,4' + +jobs: + jammy-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: jammy + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/131-daily-integration-24.04-ec2.yml b/.github/workflows/131-daily-integration-24.04-ec2.yml new file mode 100644 index 00000000000..d29da095113 --- /dev/null +++ b/.github/workflows/131-daily-integration-24.04-ec2.yml @@ -0,0 +1,25 @@ +name: "Twice weekly 24.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + # Run Mon & Thurs for high-cost test runs + - cron: '2 22 * * 1,4' + +jobs: + noble-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: noble + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/132-daily-integration-25.10-ec2.yml b/.github/workflows/132-daily-integration-25.10-ec2.yml new file mode 100644 index 00000000000..6feb85c8b43 --- /dev/null +++ b/.github/workflows/132-daily-integration-25.10-ec2.yml @@ -0,0 +1,25 @@ +name: "Twice weekly 25.10: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + # Run Mon & Thurs for high-cost test runs + - cron: '2 22 * * 1,4' + +jobs: + questing-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: questing + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml new file mode 100644 index 00000000000..f9362b6190d --- /dev/null +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -0,0 +1,24 @@ +name: "Twice weekly 26.04: ec2" + +on: + workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' + schedule: + # Run Mon & Thurs for high-cost test runs + - cron: '2 22 * * 1,4' + +jobs: + resolute-ec2: + uses: ./.github/workflows/100-dispatch-common.yml + with: + release: resolute + platform: ec2 + install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + secrets: + PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} From f9f156c2d334a3cf845247d65be1d4af6a728c64 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 17 Apr 2026 03:28:50 +0000 Subject: [PATCH 55/72] ci: switch to pytest-json-ctrf and refactor job env/defaults Avoid dependency on ctrf.io integrations and shift to pytest-json-ctrf for report formatting needs. - Add pytest-json-ctrf to integration-requirements.txt; this plugin writes a native CTRF JSON report file and provide --ctrf to pytest. - Set a descriptive title that includes platform, release, image_type, and install_source. Job and environment cleanup: - Move CLOUD_INIT_OS_IMAGE, CLOUD_INIT_OS_IMAGE_TYPE, CLOUD_INIT_CLOUD_INIT_SOURCE, and CLOUD_INIT_LOCAL_LOG_PATH from the job-level env block into the 'Run integration Tests' step env, keeping the job env focused on credentials and config that steps other than the test runner need. - Add a default of 'ppa:cloud-init-dev/daily' to the workflow_call install_source input so callers that omit it still get a sensible value. - Add install_source workflow_dispatch input with the same default to the lxd_container caller workflows (110-113) and pass it through. --- .github/workflows/100-dispatch-common.yml | 29 +++++-------------- ...-daily-integration-22.04-lxd_container.yml | 5 ++++ ...-daily-integration-24.04-lxd_container.yml | 5 ++++ ...-daily-integration-25.10-lxd_container.yml | 5 ++++ ...-daily-integration-26.04-lxd_container.yml | 5 ++++ integration-requirements.txt | 1 + 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index b292e642dcc..58eb9a6106f 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -30,6 +30,7 @@ on: install_source: required: false type: string + default: 'ppa:cloud-init-dev/daily' filter_tests: required: false type: string @@ -59,7 +60,7 @@ on: required: true jobs: - lxc: + integration-test: runs-on: ubuntu-latest if: github.repository == 'canonical/cloud-init' @@ -110,38 +111,24 @@ jobs: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs run: | - tox -e integration-tests -- --junitxml="${{ github.workspace }}/reports/junit-report-${{ inputs.platform }}-${{ inputs.release }}.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} + tox -e integration-tests -- --ctrf=${{ github.workspace }}/report.json --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} - name: Publish Test Report with Insights - if: always() + if: ${{ always() }} uses: ctrf-io/github-test-reporter@024bc4b64d997ca9da86833c6b9548c55c620e40 # v1.0.26 with: - report-path: '${{ github.workspace }}/reports/junit-report-${{ inputs.platform}}-${{ inputs.release }}.xml' - integrations-config: | - { - "junit-to-ctrf": { - "enabled": true, - "action": "convert", - "options": { - "output": "./reports/ctrf-report-${{ inputs.platform }}-${{ inputs.release }}-${{ inputs.image_type }}.json", - "toolname": "junit-to-ctrf", - "useSuiteName": false, - "env": { - "appName": "my-app" - } - } - } - } + report-path: '${{ github.workspace }}/report.json' + title: '${{ inputs.platform }}-${{ inputs.release }}-${{ inputs.image_type }}:${{ inputs.install_source }}' summary-delta-report: true insights-report: true flaky-rate-report: true slowest-report: true upload-artifact: true github-report: true - artifact-name: ctrf-report-${{ inputs.platform }}-${{inputs.release}} + artifact-name: ctrf-report-${{ inputs.platform }}-${{ inputs.release }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload failure artifacts - if: failure() + if: ${{ failure() }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: failure-${{ github.job }} diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml index 834cf319fb6..52f0966dd09 100644 --- a/.github/workflows/110-daily-integration-22.04-lxd_container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -2,6 +2,11 @@ name: "Daily 22.04: lxd_container" on: workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' schedule: - cron: '2 22 * * *' diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml index 265f2a6d14d..08f5c836890 100644 --- a/.github/workflows/111-daily-integration-24.04-lxd_container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -2,6 +2,11 @@ name: "Daily 24.04: lxd_container" on: workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' schedule: - cron: '2 22 * * *' diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml index e52dbe6a693..c72945188bf 100644 --- a/.github/workflows/112-daily-integration-25.10-lxd_container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -2,6 +2,11 @@ name: "Daily 25.10: lxd_container" on: workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' schedule: - cron: '2 22 * * *' diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml index 16842bf4f6f..56ed73872cc 100644 --- a/.github/workflows/113-daily-integration-26.04-lxd_container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -2,6 +2,11 @@ name: "Daily 26.04: lxd_container" on: workflow_dispatch: + inputs: + install_source: + required: false + type: string + default: 'ppa:cloud-init-dev/daily' schedule: - cron: '2 22 * * *' diff --git a/integration-requirements.txt b/integration-requirements.txt index d0db753dabc..cf84b474469 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,6 +5,7 @@ pycloudlib>=1!10.0.2,<1!11 pytest-timeout +pytest-json-ctrf # Used for GH ctrf.io test report dashboard # Even when xdist is not actively used, we have fixtures that require it pytest-xdist From eca37ba377897b2901b2164a0f5ccc2882a6653a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 6 May 2026 16:00:24 +0000 Subject: [PATCH 56/72] ci: update image_type defaults, validate required secrets set perms Address review comments from copilot and Brett: - update setup-lxd pinning - validate presence of non-empty required SSH_*_KEY secrets - simplifiy -z secret test conditionals to avoid multi-line issues - use install -m 600 before creating SSH_PRIVATE_KEY file - test and exit 1 on non-encoded PYCLOUDLIB_TOML repo secret - set image_type: generic as defaults in 100-dispatch-common and ec2 job. --- .github/workflows/100-dispatch-common.yml | 25 +++++++++++++++---- .../133-daily-integration-26.04-ec2.yml | 1 + 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index 58eb9a6106f..8dd3d08c358 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -27,6 +27,7 @@ on: - generic - minimal required: false + default: generic install_source: required: false type: string @@ -74,17 +75,26 @@ jobs: CLOUD_INIT_CLOUD_INIT_SOURCE: ${{ inputs.install_source }} steps: - - name: Assert repo secrets.PYCLOUDLIB_TOML is set + - name: Assert required repo secrets are set run: | - if [ -z "${{ env.REQUIRED_SECRET }}" ]; then + if [ -z "$REQUIRED_SECRET" ]; then echo "ERROR: Missing required repo secrets.PYCLOUDLIB_TOML non-empty value." exit 1 fi + if [ -z "$SSH_PUBLIC_KEY" ]; then + echo "ERROR: Missing required repo secrets.SSH_PUBLIC_KEY non-empty value." + exit 1 + fi + if [ -z "$SSH_PRIVATE_KEY" ]; then + echo "ERROR: Missing required repo secrets.SSH_PRIVATE_KEY non-empty value." + exit 1 + fi - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup LXD + # Avoid unnecessary snap install lxd on non-lxd platforms. if: ${{ contains(fromJSON('["lxd_vm", "lxd_container"]'), env.CLOUD_INIT_PLATFORM ) }} - uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v0.1.2 + uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v1 with: channel: 6/stable - name: Clean workspace @@ -95,13 +105,18 @@ jobs: mkdir -p ~/.ssh # Dump secrets using a subprocess to avoid accidental leaks while debugging. sh -c 'printf "%s\n" "$SSH_PUBLIC_KEY" > ~/.ssh/cloudinit_id_rsa.pub' + # Create empty cloudinit_id_rsa with file mode 600. + touch ~/.ssh/cloudinit_id_rsa && chmod 600 ~/.ssh/cloudinit_id_rsa sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/cloudinit_id_rsa' - chmod 600 ~/.ssh/cloudinit_id_rsa - name: Setup pycloudlib env: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | - sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' + sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' | true + if [ ! -s $PYCLOUDLIB_CONFIG ]; then + echo "PYCLOUDLIB_TOML repo secret is not a base64-encoded string" + exit 1 + fi - name: Install Dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml index f9362b6190d..553fe3ed5dc 100644 --- a/.github/workflows/133-daily-integration-26.04-ec2.yml +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -18,6 +18,7 @@ jobs: release: resolute platform: ec2 install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} + image_type: generic secrets: PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} From c7f16aba653a24cc35933bc6c77cc5af1d7a1602 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 6 May 2026 16:44:16 +0000 Subject: [PATCH 57/72] ci: rename PYCLOUDLIB_TOML_B64 to indicate base64-encoded --- .github/workflows/100-dispatch-common.yml | 12 ++++++------ .../110-daily-integration-22.04-lxd_container.yml | 2 +- .../111-daily-integration-24.04-lxd_container.yml | 2 +- .../112-daily-integration-25.10-lxd_container.yml | 2 +- .../113-daily-integration-26.04-lxd_container.yml | 2 +- .../workflows/120-daily-integration-22.04-lxd_vm.yml | 2 +- .../workflows/121-daily-integration-24.04-lxd_vm.yml | 2 +- .../workflows/122-daily-integration-25.10-lxd_vm.yml | 2 +- .../workflows/123-daily-integration-26.04-lxd_vm.yml | 2 +- .../workflows/130-daily-integration-22.04-ec2.yml | 2 +- .../workflows/131-daily-integration-24.04-ec2.yml | 2 +- .../workflows/132-daily-integration-25.10-ec2.yml | 2 +- .../workflows/133-daily-integration-26.04-ec2.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index 8dd3d08c358..1ad37f439b3 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -1,4 +1,4 @@ -# WARNING: This workflow handles repository secrets (PYCLOUDLIB_TOML, SSH keys). +# WARNING: This workflow handles repository secrets (PYCLOUDLIB_TOML_B64, SSH keys). # It MUST NOT be triggered by PR-driven workflows to prevent secret exposure to # untrusted code submitted via pull requests. name: "Dispatch: Integration" @@ -53,7 +53,7 @@ on: required: false type: string secrets: - PYCLOUDLIB_TOML: + PYCLOUDLIB_TOML_B64: required: true SSH_PRIVATE_KEY: required: true @@ -66,7 +66,7 @@ jobs: if: github.repository == 'canonical/cloud-init' env: - REQUIRED_SECRET: ${{ secrets.PYCLOUDLIB_TOML }} + REQUIRED_SECRET: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} CLOUD_INIT_PLATFORM: ${{ inputs.platform }} @@ -78,7 +78,7 @@ jobs: - name: Assert required repo secrets are set run: | if [ -z "$REQUIRED_SECRET" ]; then - echo "ERROR: Missing required repo secrets.PYCLOUDLIB_TOML non-empty value." + echo "ERROR: Missing required repo secrets.PYCLOUDLIB_TOML_B64 non-empty value." exit 1 fi if [ -z "$SSH_PUBLIC_KEY" ]; then @@ -112,9 +112,9 @@ jobs: env: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | - sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' | true + sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML_B64}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' | true if [ ! -s $PYCLOUDLIB_CONFIG ]; then - echo "PYCLOUDLIB_TOML repo secret is not a base64-encoded string" + echo "PYCLOUDLIB_TOML_B64 repo secret is not a base64-encoded string" exit 1 fi - name: Install Dependencies diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml index 52f0966dd09..6aea254ee4c 100644 --- a/.github/workflows/110-daily-integration-22.04-lxd_container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml index 08f5c836890..58eb7575cd9 100644 --- a/.github/workflows/111-daily-integration-24.04-lxd_container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml index c72945188bf..5eb45f4c7a9 100644 --- a/.github/workflows/112-daily-integration-25.10-lxd_container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml index 56ed73872cc..e99a7a2d626 100644 --- a/.github/workflows/113-daily-integration-26.04-lxd_container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml index 0f0ed71d0d7..ad9b530761d 100644 --- a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml +++ b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml index efd38a6e97e..7a27d46eb78 100644 --- a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml +++ b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml index 0d39b2d4d79..705b24d793f 100644 --- a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml +++ b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml index 46d55c7b3c1..5046fd7d910 100644 --- a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml +++ b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml @@ -19,6 +19,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/130-daily-integration-22.04-ec2.yml b/.github/workflows/130-daily-integration-22.04-ec2.yml index 98ae5b4a574..8b6bcbddad8 100644 --- a/.github/workflows/130-daily-integration-22.04-ec2.yml +++ b/.github/workflows/130-daily-integration-22.04-ec2.yml @@ -20,6 +20,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/131-daily-integration-24.04-ec2.yml b/.github/workflows/131-daily-integration-24.04-ec2.yml index d29da095113..30c5196c292 100644 --- a/.github/workflows/131-daily-integration-24.04-ec2.yml +++ b/.github/workflows/131-daily-integration-24.04-ec2.yml @@ -20,6 +20,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/132-daily-integration-25.10-ec2.yml b/.github/workflows/132-daily-integration-25.10-ec2.yml index 6feb85c8b43..2d3ae0f95d1 100644 --- a/.github/workflows/132-daily-integration-25.10-ec2.yml +++ b/.github/workflows/132-daily-integration-25.10-ec2.yml @@ -20,6 +20,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml index 553fe3ed5dc..f4117fbe7cf 100644 --- a/.github/workflows/133-daily-integration-26.04-ec2.yml +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -20,6 +20,6 @@ jobs: install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} image_type: generic secrets: - PYCLOUDLIB_TOML: ${{ secrets.PYCLOUDLIB_TOML }} + PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} From 18933979d6247004ea2e2573f4c5c924457f2a5e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 7 May 2026 01:27:26 +0000 Subject: [PATCH 58/72] ci: standardize input/dispatch defaults, errors for missing secrets Migrate away from deprecated pytest-json-ctrf reverting to use ctrf intrgration junit-to-ctrf. --- .github/workflows/100-dispatch-common.yml | 36 ++++++++++++------- ...-daily-integration-22.04-lxd_container.yml | 12 ++++++- ...-daily-integration-24.04-lxd_container.yml | 12 ++++++- ...-daily-integration-25.10-lxd_container.yml | 12 ++++++- ...-daily-integration-26.04-lxd_container.yml | 12 ++++++- .../120-daily-integration-22.04-lxd_vm.yml | 12 ++++++- .../121-daily-integration-24.04-lxd_vm.yml | 12 ++++++- .../122-daily-integration-25.10-lxd_vm.yml | 12 ++++++- .../123-daily-integration-26.04-lxd_vm.yml | 12 ++++++- .../130-daily-integration-22.04-ec2.yml | 12 ++++++- .../131-daily-integration-24.04-ec2.yml | 12 ++++++- .../132-daily-integration-25.10-ec2.yml | 12 ++++++- .../133-daily-integration-26.04-ec2.yml | 12 ++++++- integration-requirements.txt | 1 - 14 files changed, 155 insertions(+), 26 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index 1ad37f439b3..461f4eb801e 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -22,12 +22,11 @@ on: - lxd_vm - ec2 image_type: + required: true type: choice options: - generic - minimal - required: false - default: generic install_source: required: false type: string @@ -78,15 +77,15 @@ jobs: - name: Assert required repo secrets are set run: | if [ -z "$REQUIRED_SECRET" ]; then - echo "ERROR: Missing required repo secrets.PYCLOUDLIB_TOML_B64 non-empty value." + echo "ERROR: Missing required repo secret. Please provide the necessary repo secret at ${{ github.repository }}/settings/secrets/action." exit 1 fi if [ -z "$SSH_PUBLIC_KEY" ]; then - echo "ERROR: Missing required repo secrets.SSH_PUBLIC_KEY non-empty value." + echo "ERROR: Missing required repo secret. Please provide SSH_PUBLIC_KEY repo secret at ${{ github.repository }}/settings/secrets/actions." exit 1 fi if [ -z "$SSH_PRIVATE_KEY" ]; then - echo "ERROR: Missing required repo secrets.SSH_PRIVATE_KEY non-empty value." + echo "ERROR: Missing required repo secret. Please provide SSH_PRIVATE_KEY repo secret at ${{ github.repository }}/settings/secrets/actions." exit 1 fi - name: Checkout @@ -106,17 +105,13 @@ jobs: # Dump secrets using a subprocess to avoid accidental leaks while debugging. sh -c 'printf "%s\n" "$SSH_PUBLIC_KEY" > ~/.ssh/cloudinit_id_rsa.pub' # Create empty cloudinit_id_rsa with file mode 600. - touch ~/.ssh/cloudinit_id_rsa && chmod 600 ~/.ssh/cloudinit_id_rsa + install -m 600 /dev/null ~/.ssh/cloudinit_id_rsa sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/cloudinit_id_rsa' - name: Setup pycloudlib env: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | - sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML_B64}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' | true - if [ ! -s $PYCLOUDLIB_CONFIG ]; then - echo "PYCLOUDLIB_TOML_B64 repo secret is not a base64-encoded string" - exit 1 - fi + sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML_B64}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' - name: Install Dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update @@ -126,13 +121,28 @@ jobs: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs run: | - tox -e integration-tests -- --ctrf=${{ github.workspace }}/report.json --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} + tox -e integration-tests -- --junitxml="${{ github.workspace }}/junit-report.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} - name: Publish Test Report with Insights if: ${{ always() }} uses: ctrf-io/github-test-reporter@024bc4b64d997ca9da86833c6b9548c55c620e40 # v1.0.26 with: - report-path: '${{ github.workspace }}/report.json' + report-path: '${{ github.workspace }}/junit-report.xml' title: '${{ inputs.platform }}-${{ inputs.release }}-${{ inputs.image_type }}:${{ inputs.install_source }}' + integrations-config: | + { + "junit-to-ctrf": { + "enabled": true, + "action": "convert", + "options": { + "output": "./ctrf-report.json", + "toolname": "junit-to-ctrf", + "useSuiteName": false, + "env": { + "appName": "my-app" + } + } + } + } summary-delta-report: true insights-report: true flaky-rate-report: true diff --git a/.github/workflows/110-daily-integration-22.04-lxd_container.yml b/.github/workflows/110-daily-integration-22.04-lxd_container.yml index 6aea254ee4c..bf719312c78 100644 --- a/.github/workflows/110-daily-integration-22.04-lxd_container.yml +++ b/.github/workflows/110-daily-integration-22.04-lxd_container.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 22 * * *' @@ -17,7 +26,8 @@ jobs: release: jammy platform: lxd_container install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/111-daily-integration-24.04-lxd_container.yml b/.github/workflows/111-daily-integration-24.04-lxd_container.yml index 58eb7575cd9..73fcd9e302d 100644 --- a/.github/workflows/111-daily-integration-24.04-lxd_container.yml +++ b/.github/workflows/111-daily-integration-24.04-lxd_container.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 22 * * *' @@ -17,7 +26,8 @@ jobs: release: noble platform: lxd_container install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/112-daily-integration-25.10-lxd_container.yml b/.github/workflows/112-daily-integration-25.10-lxd_container.yml index 5eb45f4c7a9..e59de2c3feb 100644 --- a/.github/workflows/112-daily-integration-25.10-lxd_container.yml +++ b/.github/workflows/112-daily-integration-25.10-lxd_container.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 22 * * *' @@ -17,7 +26,8 @@ jobs: release: questing platform: lxd_container install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/113-daily-integration-26.04-lxd_container.yml b/.github/workflows/113-daily-integration-26.04-lxd_container.yml index e99a7a2d626..2e12f25ce03 100644 --- a/.github/workflows/113-daily-integration-26.04-lxd_container.yml +++ b/.github/workflows/113-daily-integration-26.04-lxd_container.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 22 * * *' @@ -17,7 +26,8 @@ jobs: release: resolute platform: lxd_container install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml index ad9b530761d..2b81dfca8e0 100644 --- a/.github/workflows/120-daily-integration-22.04-lxd_vm.yml +++ b/.github/workflows/120-daily-integration-22.04-lxd_vm.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 23 * * *' @@ -17,7 +26,8 @@ jobs: release: jammy platform: lxd_vm install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml index 7a27d46eb78..fa1b671223a 100644 --- a/.github/workflows/121-daily-integration-24.04-lxd_vm.yml +++ b/.github/workflows/121-daily-integration-24.04-lxd_vm.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 23 * * *' @@ -17,7 +26,8 @@ jobs: release: noble platform: lxd_vm install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml index 705b24d793f..c14c2abd9bb 100644 --- a/.github/workflows/122-daily-integration-25.10-lxd_vm.yml +++ b/.github/workflows/122-daily-integration-25.10-lxd_vm.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 23 * * *' @@ -17,7 +26,8 @@ jobs: release: questing platform: lxd_vm install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml index 5046fd7d910..54a5dc4413a 100644 --- a/.github/workflows/123-daily-integration-26.04-lxd_vm.yml +++ b/.github/workflows/123-daily-integration-26.04-lxd_vm.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: - cron: '2 23 * * *' @@ -17,7 +26,8 @@ jobs: release: resolute platform: lxd_vm install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/130-daily-integration-22.04-ec2.yml b/.github/workflows/130-daily-integration-22.04-ec2.yml index 8b6bcbddad8..499d6d7bdea 100644 --- a/.github/workflows/130-daily-integration-22.04-ec2.yml +++ b/.github/workflows/130-daily-integration-22.04-ec2.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: # Run Mon & Thurs for high-cost test runs - cron: '2 22 * * 1,4' @@ -18,7 +27,8 @@ jobs: release: jammy platform: ec2 install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/131-daily-integration-24.04-ec2.yml b/.github/workflows/131-daily-integration-24.04-ec2.yml index 30c5196c292..460cec7570b 100644 --- a/.github/workflows/131-daily-integration-24.04-ec2.yml +++ b/.github/workflows/131-daily-integration-24.04-ec2.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: # Run Mon & Thurs for high-cost test runs - cron: '2 22 * * 1,4' @@ -18,7 +27,8 @@ jobs: release: noble platform: ec2 install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/132-daily-integration-25.10-ec2.yml b/.github/workflows/132-daily-integration-25.10-ec2.yml index 2d3ae0f95d1..38a92882077 100644 --- a/.github/workflows/132-daily-integration-25.10-ec2.yml +++ b/.github/workflows/132-daily-integration-25.10-ec2.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: # Run Mon & Thurs for high-cost test runs - cron: '2 22 * * 1,4' @@ -18,7 +27,8 @@ jobs: release: questing platform: ec2 install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml index f4117fbe7cf..1bf1c757a30 100644 --- a/.github/workflows/133-daily-integration-26.04-ec2.yml +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -7,6 +7,15 @@ on: required: false type: string default: 'ppa:cloud-init-dev/daily' + image_type: + required: true + type: choice + options: + - generic + - minimal + filter_tests: + required: false + type: string schedule: # Run Mon & Thurs for high-cost test runs - cron: '2 22 * * 1,4' @@ -18,7 +27,8 @@ jobs: release: resolute platform: ec2 install_source: ${{ inputs.install_source || 'ppa:cloud-init-dev/daily' }} - image_type: generic + image_type: ${{ inputs.image_type || 'generic' }} + filter_tests: ${{ inputs.filter_tests }} secrets: PYCLOUDLIB_TOML_B64: ${{ secrets.PYCLOUDLIB_TOML_B64 }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} diff --git a/integration-requirements.txt b/integration-requirements.txt index cf84b474469..d0db753dabc 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,7 +5,6 @@ pycloudlib>=1!10.0.2,<1!11 pytest-timeout -pytest-json-ctrf # Used for GH ctrf.io test report dashboard # Even when xdist is not actively used, we have fixtures that require it pytest-xdist From d053bb0052bce0f65910c12eefb264f7bde87dc2 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 4 Jun 2026 17:49:44 +0000 Subject: [PATCH 59/72] chore(ci): address review comments, single operation PRIVATE_KEY creation --- .github/workflows/100-dispatch-common.yml | 16 ++++++++-------- .../130-daily-integration-22.04-ec2.yml | 2 +- .../131-daily-integration-24.04-ec2.yml | 2 +- .../132-daily-integration-25.10-ec2.yml | 2 +- .../133-daily-integration-26.04-ec2.yml | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/100-dispatch-common.yml b/.github/workflows/100-dispatch-common.yml index 461f4eb801e..92c628363b4 100644 --- a/.github/workflows/100-dispatch-common.yml +++ b/.github/workflows/100-dispatch-common.yml @@ -77,7 +77,7 @@ jobs: - name: Assert required repo secrets are set run: | if [ -z "$REQUIRED_SECRET" ]; then - echo "ERROR: Missing required repo secret. Please provide the necessary repo secret at ${{ github.repository }}/settings/secrets/action." + echo "ERROR: Missing required repo secret. Please provide the necessary repo secret at ${{ github.repository }}/settings/secrets/actions." exit 1 fi if [ -z "$SSH_PUBLIC_KEY" ]; then @@ -91,7 +91,9 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup LXD - # Avoid unnecessary snap install lxd on non-lxd platforms. + # This shared workflow supports manual dispatch with platform choices including + # lxd_vm and lxd_container. Only install the lxd snap for those platforms; + # cloud platforms such as EC2 do not need it. if: ${{ contains(fromJSON('["lxd_vm", "lxd_container"]'), env.CLOUD_INIT_PLATFORM ) }} uses: canonical/setup-lxd@8c6a87bfb56aa48f3fb9b830baa18562d8bfd4ee # v1 with: @@ -102,24 +104,22 @@ jobs: - name: Setup SSH run: | mkdir -p ~/.ssh - # Dump secrets using a subprocess to avoid accidental leaks while debugging. sh -c 'printf "%s\n" "$SSH_PUBLIC_KEY" > ~/.ssh/cloudinit_id_rsa.pub' - # Create empty cloudinit_id_rsa with file mode 600. - install -m 600 /dev/null ~/.ssh/cloudinit_id_rsa - sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/cloudinit_id_rsa' + # Dump secret using a subprocess to avoid accidental leaks when using set -x. + sh -c 'printf "%s\n" "$SSH_PRIVATE_KEY" | install -m 600 /dev/stdin ~/.ssh/cloudinit_id_rsa' - name: Setup pycloudlib env: PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | - sh -c 'echo "${{ secrets.PYCLOUDLIB_TOML_B64}}" | base64 -d > "$PYCLOUDLIB_CONFIG"' + sh -c 'echo "$REQUIRED_SECRET" | base64 -d > "$PYCLOUDLIB_CONFIG"' - name: Install Dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox distro-info-data devscripts - name: Run integration Tests env: - PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml CLOUD_INIT_LOCAL_LOG_PATH: ${{ github.workspace }}/cloud_init_test_logs + PYCLOUDLIB_CONFIG: ${{ runner.temp }}/pycloudlib.toml run: | tox -e integration-tests -- --junitxml="${{ github.workspace }}/junit-report.xml" --color=yes ${{ inputs.filter_tests || 'tests/integration_tests' }} - name: Publish Test Report with Insights diff --git a/.github/workflows/130-daily-integration-22.04-ec2.yml b/.github/workflows/130-daily-integration-22.04-ec2.yml index 499d6d7bdea..78d509cfea0 100644 --- a/.github/workflows/130-daily-integration-22.04-ec2.yml +++ b/.github/workflows/130-daily-integration-22.04-ec2.yml @@ -17,7 +17,7 @@ on: required: false type: string schedule: - # Run Mon & Thurs for high-cost test runs + # Run Mon & Thurs to reduce additional cost on shared external tokens - cron: '2 22 * * 1,4' jobs: diff --git a/.github/workflows/131-daily-integration-24.04-ec2.yml b/.github/workflows/131-daily-integration-24.04-ec2.yml index 460cec7570b..52d88e9ba8e 100644 --- a/.github/workflows/131-daily-integration-24.04-ec2.yml +++ b/.github/workflows/131-daily-integration-24.04-ec2.yml @@ -17,7 +17,7 @@ on: required: false type: string schedule: - # Run Mon & Thurs for high-cost test runs + # Run Mon & Thurs to reduce additional cost on shared external tokens - cron: '2 22 * * 1,4' jobs: diff --git a/.github/workflows/132-daily-integration-25.10-ec2.yml b/.github/workflows/132-daily-integration-25.10-ec2.yml index 38a92882077..aeb66287533 100644 --- a/.github/workflows/132-daily-integration-25.10-ec2.yml +++ b/.github/workflows/132-daily-integration-25.10-ec2.yml @@ -17,7 +17,7 @@ on: required: false type: string schedule: - # Run Mon & Thurs for high-cost test runs + # Run Mon & Thurs to reduce additional cost on shared external tokens - cron: '2 22 * * 1,4' jobs: diff --git a/.github/workflows/133-daily-integration-26.04-ec2.yml b/.github/workflows/133-daily-integration-26.04-ec2.yml index 1bf1c757a30..cabaf3e1dcb 100644 --- a/.github/workflows/133-daily-integration-26.04-ec2.yml +++ b/.github/workflows/133-daily-integration-26.04-ec2.yml @@ -17,7 +17,7 @@ on: required: false type: string schedule: - # Run Mon & Thurs for high-cost test runs + # Run Mon & Thurs to reduce additional cost on shared external tokens - cron: '2 22 * * 1,4' jobs: From 1ee6e652fee4f1d43c47815ff0f38bc7c3d25c08 Mon Sep 17 00:00:00 2001 From: Sandhya-d <132327887+Sandhya-d@users.noreply.github.com> Date: Tue, 16 Jun 2026 22:43:37 +0530 Subject: [PATCH 60/72] fix(distros): clarify misleading ssh_redirect_user warning (#6906) When ssh_redirect_user is set to true or default, cc_users_groups resolves it to the actual default username before create_user runs. The 'no cloud public-keys' warning then printed that resolved name (e.g. 'ubuntu'), making it look like the user had configured 'ubuntu'. Reword the warning so the value reads as the redirect target, not the configured value. Fixes GH-3924 --- cloudinit/distros/__init__.py | 5 +++-- tests/unittests/distros/test_create_users.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 9ed5540d7d7..2703ebd8357 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -1008,8 +1008,9 @@ def create_user(self, name, **kwargs): cloud_keys = kwargs.get("cloud_public_ssh_keys", []) if not cloud_keys: LOG.warning( - "Unable to disable SSH logins for %s given" - " ssh_redirect_user: %s. No cloud public-keys present.", + "Unable to disable SSH logins for %s." + " ssh_redirect_user was set to redirect logins to" + " %s, but no cloud public-keys are present.", name, kwargs["ssh_redirect_user"], ) diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 095924b106b..a875472cd3d 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -750,8 +750,9 @@ def test_create_user_with_ssh_redirect_user_no_cloud_keys( dist.create_user(USER, ssh_redirect_user="someuser") assert caplog.records[1].levelname in ["WARNING", "DEPRECATED"] assert ( - "Unable to disable SSH logins for foo_user given " - "ssh_redirect_user: someuser. No cloud public-keys present.\n" + "Unable to disable SSH logins for foo_user." + " ssh_redirect_user was set to redirect logins to" + " someuser, but no cloud public-keys are present.\n" ) in caplog.text m_setup_user_keys.assert_not_called() From afdd0c5b5b383ca0bbc875af2dc9993a73179aa9 Mon Sep 17 00:00:00 2001 From: Wai Hlyan Min Thein Date: Wed, 17 Jun 2026 10:28:21 -0400 Subject: [PATCH 61/72] docs: fix chpasswd user entries in cloud-config example (#6905) In the commented multi-user chpasswd example, user2 and user3 were written as bare list items rather than mappings with a name field, unlike user1 in the same block. The chpasswd.users schema requires each entry to be a mapping containing a name key. Fixes GH-6765 --- doc/examples/cloud-config.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index cba70b593a7..87e5299a6bc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -360,9 +360,9 @@ timezone: US/Eastern # - name: user1 # password: password1 # type: text -# - user2 +# - name: user2 # type: RANDOM -# - user3 +# - name: user3 # password: $5$eriogqzq$Dg7PxHsKGzziuEGkZgkLvacjuEFeljJ.rLf.hZqKQLA # type: hash # expire: True From 73de6365f8469f68793b3e724d0943bdc6b441a9 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 25 Jun 2026 20:09:14 -0600 Subject: [PATCH 62/72] test: pytest fix strict_parametrization_ids providing required id (#6908) Update test_net to avoid duplicated parametrized test values, drop fixed xfail decorated test cases. --- tests/unittests/cmd/test_query.py | 1 + tests/unittests/cmd/test_status.py | 1 - tests/unittests/config/test_cc_ansible.py | 6 ++--- tests/unittests/config/test_cc_phone_home.py | 4 +++- .../unittests/config/test_cc_set_passwords.py | 2 -- tests/unittests/config/test_schema.py | 2 +- tests/unittests/net/test_dhcp.py | 1 - tests/unittests/net/test_init.py | 7 ++++++ tests/unittests/sources/azure/test_errors.py | 1 + tests/unittests/sources/test_ec2.py | 2 +- tests/unittests/test_ds_identify.py | 2 +- tests/unittests/test_net.py | 20 ++++------------- tests/unittests/test_url_helper.py | 22 ++++++++++++++++++- 13 files changed, 43 insertions(+), 28 deletions(-) diff --git a/tests/unittests/cmd/test_query.py b/tests/unittests/cmd/test_query.py index e8843d4d585..9f22c504ee8 100644 --- a/tests/unittests/cmd/test_query.py +++ b/tests/unittests/cmd/test_query.py @@ -258,6 +258,7 @@ def test_handle_args_root_fallsback_to_instance_data(self, caplog, tmpdir): (_gzip_data(b"ud"), "ud", _gzip_data(b"vd"), "vd"), (_gzip_data("ud".encode("utf-8")), "ud", _gzip_data(b"vd"), "vd"), ), + ids=("plain-text", "bytes", "gzip-bytes", "gzip-encoded-bytes"), ) def test_handle_args_root_processes_user_data( self, ud_src, ud_expected, vd_src, vd_expected, capsys, tmpdir diff --git a/tests/unittests/cmd/test_status.py b/tests/unittests/cmd/test_status.py index 022e4034caa..d75e5735958 100644 --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py @@ -937,7 +937,6 @@ def common_mocks(self, mocker): # Because of this I'm only testing SubState combinations seen # in real-world testing (or using "any" string if we dont care). ("activating", "enabled", "start", "123", False), - ("activating", "enabled", "start", "123", False), ("active", "enabled-runtime", "exited", "0", False), ("active", "enabled", "exited", "0", False), ("active", "enabled", "running", "345", False), diff --git a/tests/unittests/config/test_cc_ansible.py b/tests/unittests/config/test_cc_ansible.py index 3f3353b25d7..9c6a6bca1e7 100644 --- a/tests/unittests/config/test_cc_ansible.py +++ b/tests/unittests/config/test_cc_ansible.py @@ -281,7 +281,7 @@ def test_schema_validation_deprecations(self, config, error_msg): } }, "'playbook_name' is a required property", - id="require-url-dict", + id="require-playbook-name-dict", ), param( CFG_MINIMAL_LIST, @@ -312,7 +312,7 @@ def test_schema_validation_deprecations(self, config, error_msg): param( CFG_CTRL, None, - id="ctrl-keys", + id="ctrl-keys-list", ), param( { @@ -355,7 +355,7 @@ def test_schema_validation_deprecations(self, config, error_msg): } }, "'playbook_name' is a required property", - id="require-url-list", + id="require-playbook-name-list", ), ), ) diff --git a/tests/unittests/config/test_cc_phone_home.py b/tests/unittests/config/test_cc_phone_home.py index 77428fcedc1..9c07d23b8dc 100644 --- a/tests/unittests/config/test_cc_phone_home.py +++ b/tests/unittests/config/test_cc_phone_home.py @@ -54,7 +54,9 @@ def test_no_url(self, m_readurl, caplog): (0, -1), (1, 0), (2, 1), - ("2", 1), + # override parametrized id to differentiate str "2" from int 2 in former test + # GH pytest-dev/pytest#14650. + pytest.param("2", 1, id="retries-as-int-str"), ("two", 9), (None, 9), ({}, 9), diff --git a/tests/unittests/config/test_cc_set_passwords.py b/tests/unittests/config/test_cc_set_passwords.py index a706917d731..ad112c95fbe 100644 --- a/tests/unittests/config/test_cc_set_passwords.py +++ b/tests/unittests/config/test_cc_set_passwords.py @@ -581,8 +581,6 @@ def _get_str_class_num(self, str): (3, ValueError), (4, 4), (5, 4), - (5, 4), - (6, 4), (20, 4), ], ) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index a3b94eaa98f..cc87d2ce825 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1758,7 +1758,7 @@ class TestNetworkSchema: SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", - id="config_key_required", + id="config_key_empty_valid", ), pytest.param( { diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py index b1f07a4d8af..c21f858a390 100644 --- a/tests/unittests/net/test_dhcp.py +++ b/tests/unittests/net/test_dhcp.py @@ -842,7 +842,6 @@ def test_ephemeral_dhcp_setup_network_if_url_connectivity( @pytest.mark.parametrize( "error_class", [ - NoDHCPLeaseInterfaceError, NoDHCPLeaseInterfaceError, NoDHCPLeaseMissingDhclientError, ], diff --git a/tests/unittests/net/test_init.py b/tests/unittests/net/test_init.py index 6aed3ecfa18..7a02bd4524d 100644 --- a/tests/unittests/net/test_init.py +++ b/tests/unittests/net/test_init.py @@ -1731,6 +1731,13 @@ class TestIsIpAddress: (lambda _: ipaddress.IPv6Address("2001:db8::"), True), (lambda _: ipaddress.IPv6Address("2001:db8::/48"), False), ), + ids=( + "value-error", + "ipv4-address", + "ipv4-network", + "ipv6-address", + "ipv6-network", + ), ) def test_is_ip_address(self, ip_address_side_effect, expected_return): with mock.patch( diff --git a/tests/unittests/sources/azure/test_errors.py b/tests/unittests/sources/azure/test_errors.py index 7b405368d39..61d16faa45c 100644 --- a/tests/unittests/sources/azure/test_errors.py +++ b/tests/unittests/sources/azure/test_errors.py @@ -266,6 +266,7 @@ def test_unhandled_exception(): "None", None, ], + ids=["running", "string-none", "none-value"], ) def test_imds_invalid_metadata(value): key = "compute" diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index 0cb990c1a31..27a9a034c8f 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -1411,7 +1411,7 @@ class TestBuildNicOrder: "0a:0d:dd:44:cd:7b": 1, "0a:f7:8d:96:f2:a2": 2, }, - id="no-device-number-info-subset-sort-by-nic-name", + id="no-device-number-info-extra-mac-sort-by-nic-name", ), ], ) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index ba5aae5ebb7..d9ef8ef441f 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -663,7 +663,7 @@ def test_wb_print_variables(self, tmp_path): # Launched by os code always has config-2 disk. pytest.param("IBMCloud-config-2", True, id="ibmcloud_os_code"), # Test that Aliyun cloud is identified by product id. - pytest.param("AliYun", True, id="ibmcloud_os_code"), + pytest.param("AliYun", True, id="aliyun_found_by_product_id"), # On Intel, openstack must be identified. pytest.param( "OpenStack", True, id="default_openstack_intel_is_found" diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5b92ba28041..81c2b8d612c 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2229,13 +2229,7 @@ def test_config_with_explicit_loopback(self): "expected_name,yaml_version", [ ("bond_v1", "yaml"), - pytest.param( - "bond_v2", - "yaml", - marks=pytest.mark.xfail( - reason="Bond MAC address not rendered" - ), - ), + ("bond_v2", "yaml"), ("vlan_v1", "yaml"), ("vlan_v2", "yaml"), ("bridge", "yaml_v1"), @@ -2884,13 +2878,7 @@ def test_config_with_explicit_loopback(self): "expected_name,yaml_name", [ ("bond_v1", "yaml"), - pytest.param( - "bond_v2", - "yaml", - marks=pytest.mark.xfail( - reason="Bond MAC address not rendered" - ), - ), + ("bond_v2", "yaml"), ("vlan_v1", "yaml"), ("vlan_v2", "yaml"), ("bridge", "yaml_v1"), @@ -5141,8 +5129,8 @@ class TestRenderersSelect: ("eni", False, True, False, False, False), # +netplan +ifupdown -sys -nm -networkd selects eni ("eni", True, True, False, False, False), - # +netplan -ifupdown -sys -nm -networkd selects netplan - ("netplan", True, False, False, False, False), + # +netplan -ifupdown -sys +nm -networkd selects netplan + ("netplan", True, False, False, True, False), # +netplan -ifupdown -sys -nm -networkd selects netplan ("netplan", True, False, False, False, False), # -netplan -ifupdown +sys -nm -networkd selects sysconfig diff --git a/tests/unittests/test_url_helper.py b/tests/unittests/test_url_helper.py index e441f85ca7f..4bf87f7600b 100644 --- a/tests/unittests/test_url_helper.py +++ b/tests/unittests/test_url_helper.py @@ -283,7 +283,16 @@ def request(cls, **kwargs): class TestReadFileOrUrlParameters: @mock.patch(M_PATH + "readurl") @pytest.mark.parametrize( - "timeout", [1, 1.2, "1", (1, None), (1, 1), (None, None)] + "timeout", + [1, 1.2, "1", (1, None), (1, 1), (None, None)], + ids=[ + "timeout-int", + "timeout-float", + "timeout-str-int", + "timeout-tuple-write-default", + "timeout-tuple-read-write", + "timeout-tuple-none", + ], ) def test_read_file_or_url_passes_params_to_readurl( self, m_readurl, timeout @@ -322,6 +331,17 @@ def test_read_file_or_url_passes_params_to_readurl( ((1, 1), (1, 1)), ((None, None), (None, None)), ], + ids=[ + "negative-int-defaults-to-zero", + "negative-str-defaults-to-zero", + "none-timeout", + "int-timeout", + "float-timeout", + "str-int-timeout", + "tuple-read-timeout-only", + "tuple-read-write-timeout", + "tuple-none-timeout", + ], ) def test_readurl_timeout(self, readurl_timeout, request_timeout): url = "http://hostname/path" From eb787d22ba777035ecde968436befdc826203353 Mon Sep 17 00:00:00 2001 From: Chris Patterson Date: Mon, 29 Jun 2026 17:09:25 -0400 Subject: [PATCH 63/72] feat(azure): add apply_network_config_set_name option to disable renames (#6807) Also upgrade test_upgrade.py to assert the pickle upgrade path initializes all relevant datasource config fields. --- cloudinit/cmd/devel/net_convert.py | 1 + cloudinit/sources/DataSourceAzure.py | 29 +++-- doc/rtd/reference/datasources/azure.rst | 17 +++ tests/unittests/sources/test_azure.py | 142 +++++++++++++++++++++--- tests/unittests/test_upgrade.py | 4 + 5 files changed, 170 insertions(+), 23 deletions(-) diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index e64f0de5444..d11ff64c024 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -157,6 +157,7 @@ def handle_args(name, args): pre_ns = azure.generate_network_config_from_instance_network_metadata( json.loads(net_data)["network"], apply_network_config_for_secondary_ips=True, + apply_network_config_set_name=True, ) elif args.kind == "vmware-imc": vmware_config = guestcust_util.Config( diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f6c91f4dfc3..87b4ab1fbe3 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -296,6 +296,7 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: "apply_network_config": True, # Use IMDS published network configuration "apply_network_config_for_secondary_ips": True, # Configure secondary ips "experimental_fail_on_missing_customdata": False, + "apply_network_config_set_name": True, # Use set-name for NICs "experimental_skip_ready_report": False, # Skip final ready report } @@ -363,6 +364,13 @@ def _unpickle(self, ci_pkl_version: int) -> None: self._system_uuid = None self._vm_id = None self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT + for key in ( + "apply_network_config_for_secondary_ips", + "experimental_fail_on_missing_customdata", + "apply_network_config_set_name", + "experimental_skip_ready_report", + ): + self.ds_cfg.setdefault(key, BUILTIN_DS_CONFIG[key]) def __str__(self): root = sources.DataSource.__str__(self) @@ -1641,9 +1649,12 @@ def _generate_network_config(self): try: return generate_network_config_from_instance_network_metadata( self._metadata_imds["network"], - apply_network_config_for_secondary_ips=self.ds_cfg.get( + apply_network_config_for_secondary_ips=self.ds_cfg[ "apply_network_config_for_secondary_ips" - ), + ], + apply_network_config_set_name=self.ds_cfg[ + "apply_network_config_set_name" + ], ) except Exception as e: LOG.error( @@ -2127,6 +2138,7 @@ def generate_network_config_from_instance_network_metadata( network_metadata: dict, *, apply_network_config_for_secondary_ips: bool, + apply_network_config_set_name: bool, ) -> dict: """Convert imds network metadata dictionary to network v2 configuration. @@ -2140,7 +2152,11 @@ def generate_network_config_from_instance_network_metadata( # First IPv4 and/or IPv6 address will be obtained via DHCP. # Any additional IPs of each type will be set as static # addresses. - nicname = "eth{idx}".format(idx=idx) + mac = normalize_mac_address(intf["macAddress"]) + if apply_network_config_set_name: + nicname = "eth{idx}".format(idx=idx) + else: + nicname = "enx{mac}".format(mac=mac.replace(":", "")) dhcp_override = {"route-metric": (idx + 1) * 100} # DNS resolution through secondary NICs is not supported, disable it. if idx > 0: @@ -2184,10 +2200,9 @@ def generate_network_config_from_instance_network_metadata( "{ip}/{prefix}".format(ip=privateIp, prefix=netPrefix) ) if dev_config and has_ip_address: - mac = normalize_mac_address(intf["macAddress"]) - dev_config.update( - {"match": {"macaddress": mac.lower()}, "set-name": nicname} - ) + dev_config["match"] = {"macaddress": mac.lower()} + if apply_network_config_set_name: + dev_config["set-name"] = nicname driver = determine_device_driver_for_mac(mac) if driver: dev_config["match"]["driver"] = driver diff --git a/doc/rtd/reference/datasources/azure.rst b/doc/rtd/reference/datasources/azure.rst index a3c6ffa0d98..61703b9344e 100644 --- a/doc/rtd/reference/datasources/azure.rst +++ b/doc/rtd/reference/datasources/azure.rst @@ -45,6 +45,22 @@ The settings that may be configured are: Boolean to configure secondary IP address(es) for each NIC per IMDS configuration. Default is True. + +* :command:`apply_network_config_set_name` + + Boolean to include ``set-name`` directives in the generated network + configuration, which renames interfaces to ``ethX`` naming. When set to + False, interfaces are matched by MAC address (and optionally driver) + without renaming, and retain kernel-assigned names. Default is True. + + Azure's IMDS does not guarantee the ordering of NICs in the network metadata + response (see `Azure IMDS documentation + `_). + Because cloud-init derives ``ethX`` names from the IMDS response order, + NIC names may change between reboots. Disabling this option avoids that + problem by matching interfaces on MAC address (and optionally driver), + allowing the kernel or udev to assign and retain stable names. + * :command:`data_dir` Path used to read meta-data files and write crawled data. @@ -67,6 +83,7 @@ An example configuration with the default values is provided below: Azure: apply_network_config: true apply_network_config_for_secondary_ips: true + apply_network_config_set_name: true data_dir: /var/lib/waagent disk_aliases: ephemeral0: /dev/disk/cloud/azure_resource diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index b097509fb7d..807acdbdd71 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -984,11 +984,98 @@ def test_parsing_scenarios( ): assert ( dsaz.generate_network_config_from_instance_network_metadata( - metadata, apply_network_config_for_secondary_ips=ip_config + metadata, + apply_network_config_for_secondary_ips=ip_config, + apply_network_config_set_name=True, ) == expected ) + @pytest.mark.parametrize( + "set_name,expected", + [ + ( + True, + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + "set-name": "eth0", + }, + "eth1": { + "dhcp4": True, + "dhcp4-overrides": { + "route-metric": 200, + "use-dns": False, + }, + "dhcp6": False, + "match": {"macaddress": "22:0d:3a:04:75:98"}, + "set-name": "eth1", + }, + }, + "version": 2, + }, + ), + ( + False, + { + "ethernets": { + "enx000d3a047598": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + }, + "enx220d3a047598": { + "dhcp4": True, + "dhcp4-overrides": { + "route-metric": 200, + "use-dns": False, + }, + "dhcp6": False, + "match": {"macaddress": "22:0d:3a:04:75:98"}, + }, + }, + "version": 2, + }, + ), + ], + ) + def test_set_name_config(self, mock_get_interfaces, set_name, expected): + """Verify set-name with two NICs (primary with IPv6, secondary).""" + two_nic_metadata = { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "subnet": [{"prefix": "64", "address": "fd00::"}], + "ipAddress": [{"privateIpAddress": "fd00::4"}], + }, + "ipv4": { + "subnet": [{"prefix": "24", "address": "10.0.0.0"}], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81", + } + ], + }, + }, + SECONDARY_INTERFACE, + ] + } + result = dsaz.generate_network_config_from_instance_network_metadata( + two_nic_metadata, + apply_network_config_for_secondary_ips=True, + apply_network_config_set_name=set_name, + ) + assert result == expected + class TestNetworkConfig: fallback_config = { @@ -1004,22 +1091,45 @@ class TestNetworkConfig: ], } - def test_single_ipv4_nic_configuration( - self, azure_ds, mock_get_interfaces - ): - """Network config emits dhcp on single nic with ipv4""" - expected = { - "ethernets": { - "eth0": { - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": False, - "match": {"macaddress": "00:0d:3a:04:75:98"}, - "set-name": "eth0", + @pytest.mark.parametrize( + "set_name,expected", + [ + ( + True, + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + "set-name": "eth0", + }, + }, + "version": 2, }, - }, - "version": 2, - } + ), + ( + False, + { + "ethernets": { + "enx000d3a047598": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"macaddress": "00:0d:3a:04:75:98"}, + }, + }, + "version": 2, + }, + ), + ], + ) + def test_network_config( + self, azure_ds, mock_get_interfaces, set_name, expected + ): + """Verify network_config via ds_cfg for set-name enabled/disabled.""" + azure_ds.ds_cfg["apply_network_config_set_name"] = set_name azure_ds._metadata_imds = NETWORK_METADATA assert azure_ds.network_config == expected diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 85209409ffc..5cdd3fbe9f4 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -294,6 +294,10 @@ def test_pkl_load_defines_all_init_side_effect_attributes( missing_attrs = ds.__dict__.keys() - previous_obj_pkl.__dict__.keys() for attr in missing_attrs: assert attr in expected + missing_ds_cfg_attrs = ( + ds.ds_cfg.keys() - previous_obj_pkl.ds_cfg.keys() + ) + assert set() == missing_ds_cfg_attrs def test_networking_set_on_distro(self, previous_obj_pkl): """We always expect to have ``.networking`` on ``Distro`` objects.""" From 71b5d36b440d084d7943c335adc779af1038b7cd Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 30 Jun 2026 08:17:42 -0600 Subject: [PATCH 64/72] test: apt resolute gpg remove accounts for gpg-from-sq (#6902) On resolute and newer, removal of gpg triggers the install of gpg-from-sq as an alternative for the metapkg if python3-software-properties or libgpgme45 are still installed. Ensure removal of the additional rdepends to avoid triggering installation of gpg-from-sq during the removal of gpg. --- .../modules/test_apt_functionality.py | 20 +++++++++++++------ tests/unittests/config/test_cc_phone_home.py | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index a2b7172e145..e4f7e708b1a 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -18,6 +18,7 @@ CURRENT_RELEASE, IS_UBUNTU, MANTIC, + QUESTING, ) from tests.integration_tests.util import ( get_feature_flag_value, @@ -523,11 +524,16 @@ def test_apt_proxy(client: IntegrationInstance): r"software-properties-common', 'gnupg)" ) -REMOVE_GPG_USERDATA = """ +GPG_PACKAGES = "gpg software-properties-common" +# On Ubuntu Resolute and newer, gpg-from-sq can replace the gpg metapackage +# when gpg is removed. Remove other packages which rdepend on gpg to avoid +# gpg-from-sq being installed as an alternative to gpg. +GPG_PACKAGES_SQ = f"{GPG_PACKAGES} python3-software-properties libgpgme45" + +REMOVE_GPG_USERDATA_TMPL = """ #cloud-config runcmd: - - DEBIAN_FRONTEND=noninteractive apt-get remove gpg -y - - DEBIAN_FRONTEND=noninteractive apt-get remove software-properties-common -y + - DEBIAN_FRONTEND=noninteractive apt-get remove {packages} -y """ @@ -570,9 +576,11 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): 'software-properties-common' are installed successfully. """ # Two stage install: First stage: remove gpg noninteractively from image - instance1 = session_cloud.launch( - user_data=_do_oci_customization(REMOVE_GPG_USERDATA) - ) + if CURRENT_RELEASE <= QUESTING: + userdata = REMOVE_GPG_USERDATA_TMPL.format(packages=GPG_PACKAGES) + else: + userdata = REMOVE_GPG_USERDATA_TMPL.format(packages=GPG_PACKAGES_SQ) + instance1 = session_cloud.launch(user_data=_do_oci_customization(userdata)) # look for r"un gpg" using regex ('un' means uninstalled) for package in ["gpg", "software-properties-common"]: diff --git a/tests/unittests/config/test_cc_phone_home.py b/tests/unittests/config/test_cc_phone_home.py index 9c07d23b8dc..d44f365cff2 100644 --- a/tests/unittests/config/test_cc_phone_home.py +++ b/tests/unittests/config/test_cc_phone_home.py @@ -54,8 +54,8 @@ def test_no_url(self, m_readurl, caplog): (0, -1), (1, 0), (2, 1), - # override parametrized id to differentiate str "2" from int 2 in former test - # GH pytest-dev/pytest#14650. + # override parametrized id to differentiate str "2" from int 2 in + # former test GH pytest-dev/pytest#14650. pytest.param("2", 1, id="retries-as-int-str"), ("two", 9), (None, 9), From 8217935f236ab2a265de1d548c0055c905a9d246 Mon Sep 17 00:00:00 2001 From: ph1n3y Date: Tue, 30 Jun 2026 18:57:06 +0000 Subject: [PATCH 65/72] chore: add type annotations to cloudinit.distros.parsers.hosts (#6916) --- cloudinit/distros/parsers/hosts.py | 47 ++++++++++++++------------- pyproject.toml | 2 -- tests/unittests/distros/test_hosts.py | 4 +-- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/cloudinit/distros/parsers/hosts.py b/cloudinit/distros/parsers/hosts.py index 8d2f73ac91f..2528264d1e8 100644 --- a/cloudinit/distros/parsers/hosts.py +++ b/cloudinit/distros/parsers/hosts.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from io import StringIO +from typing import Any, List, Tuple from cloudinit.distros.parsers import chop_comment @@ -13,69 +14,71 @@ # or https://linux.die.net/man/5/hosts # or https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/configtuning-configfiles.html # noqa class HostsConf: - def __init__(self, text): + def __init__(self, text: str) -> None: self._text = text - self._contents = None + self._contents: List[Tuple[str, List[Any]]] = [] - def parse(self): - if self._contents is None: + def parse(self) -> None: + if not self._contents: self._contents = self._parse(self._text) - def get_entry(self, ip): + def get_entry(self, ip: str) -> List[List[str]]: self.parse() - options = [] + options: List[List[str]] = [] for line_type, components in self._contents: if line_type == "option": - (pieces, _tail) = components + pieces, _tail = components if len(pieces) and pieces[0] == ip: options.append(pieces[1:]) return options - def del_entries(self, ip): + def del_entries(self, ip: str) -> None: self.parse() - n_entries = [] + n_entries: List[Tuple[str, List[Any]]] = [] for line_type, components in self._contents: if line_type != "option": n_entries.append((line_type, components)) continue else: - (pieces, _tail) = components + pieces, _tail = components if len(pieces) and pieces[0] == ip: pass elif len(pieces): n_entries.append((line_type, list(components))) self._contents = n_entries - def add_entry(self, ip, canonical_hostname, *aliases): + def add_entry( + self, ip: str, canonical_hostname: str, *aliases: str + ) -> None: self.parse() self._contents.append( - ("option", ([ip, canonical_hostname] + list(aliases), "")) + ("option", [[ip, canonical_hostname] + list(aliases), ""]) ) - def _parse(self, contents): - entries = [] + def _parse(self, contents: str) -> List[Tuple[str, List[Any]]]: + entries: List[Tuple[str, List[Any]]] = [] for line in contents.splitlines(): if not len(line.strip()): entries.append(("blank", [line])) continue - (head, tail) = chop_comment(line.strip(), "#") + head, tail = chop_comment(line.strip(), "#") if not len(head): entries.append(("all_comment", [line])) continue entries.append(("option", [head.split(None), tail])) return entries - def __str__(self): + def __str__(self) -> str: self.parse() contents = StringIO() for line_type, components in self._contents: if line_type == "blank": - contents.write("%s\n" % (components[0])) + contents.write("%s\n" % components[0]) elif line_type == "all_comment": - contents.write("%s\n" % (components[0])) + contents.write("%s\n" % components[0]) elif line_type == "option": - (pieces, tail) = components - pieces = [str(p) for p in pieces] - pieces = "\t".join(pieces) - contents.write("%s%s\n" % (pieces, tail)) + raw_pieces, tail = components + str_pieces = [str(p) for p in raw_pieces] + joined = "\t".join(str_pieces) + contents.write(f"{joined}{tail}\n") return contents.getvalue() diff --git a/pyproject.toml b/pyproject.toml index b4839e7385a..8f08187ca8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ module = [ "cloudinit.distros.bsd", "cloudinit.distros.opensuse", "cloudinit.distros.parsers.hostname", - "cloudinit.distros.parsers.hosts", "cloudinit.distros.parsers.resolv_conf", "cloudinit.distros.ug_util", "cloudinit.helpers", @@ -114,7 +113,6 @@ module = [ "tests.unittests.config.test_cc_zypper_add_repo", "tests.unittests.config.test_modules", "tests.unittests.config.test_schema", - "tests.unittests.distros.test_hosts", "tests.unittests.distros.test_ifconfig", "tests.unittests.distros.test_netbsd", "tests.unittests.distros.test_netconfig", diff --git a/tests/unittests/distros/test_hosts.py b/tests/unittests/distros/test_hosts.py index 7fd5abf2bd9..30c00d9c08a 100644 --- a/tests/unittests/distros/test_hosts.py +++ b/tests/unittests/distros/test_hosts.py @@ -21,8 +21,8 @@ def test_parse(self): ["foo.mydomain.org", "foo"], ["bar.mydomain.org", "bar"], ] - eh = str(eh) - assert eh.startswith("# Example") + eh_str = str(eh) + assert eh_str.startswith("# Example") def test_add(self): eh = hosts.HostsConf(BASE_ETC) From c9a520abefcc032dd46892a7dfe9f0553237601b Mon Sep 17 00:00:00 2001 From: nanookclaw Date: Tue, 30 Jun 2026 19:22:58 +0000 Subject: [PATCH 66/72] fix(analyze): return integer exit code from analyze_boot (#6863) Return 0 for success/container and 1 for failure. --- cloudinit/analyze/__init__.py | 4 ++-- tests/unittests/analyze/test_boot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/analyze/__init__.py b/cloudinit/analyze/__init__.py index ea2144cb387..ae978ba7a01 100644 --- a/cloudinit/analyze/__init__.py +++ b/cloudinit/analyze/__init__.py @@ -115,7 +115,7 @@ def get_parser( return parser -def analyze_boot(name: str, args: argparse.Namespace) -> str: +def analyze_boot(name: str, args: argparse.Namespace) -> int: """Report a list of how long different boot operations took. For Example: @@ -199,7 +199,7 @@ def analyze_boot(name: str, args: argparse.Namespace) -> str: outfh.write(status_map[status_code].format(**kwargs)) clean_io(infh, outfh) - return status_code + return 1 if status_code == show.FAIL_CODE else 0 def analyze_blame(name, args: argparse.Namespace) -> None: diff --git a/tests/unittests/analyze/test_boot.py b/tests/unittests/analyze/test_boot.py index 6487790ddc6..9e89ef8867e 100644 --- a/tests/unittests/analyze/test_boot.py +++ b/tests/unittests/analyze/test_boot.py @@ -160,7 +160,7 @@ def test_container_no_ci_log_line(self, m_is_container, m_subp): finish_code = analyze_boot(name_default, args) self.remove_dummy_file(path, log_path) - assert FAIL_CODE == finish_code + assert 1 == finish_code @mock.patch("cloudinit.util.is_container", return_value=True) @mock.patch("cloudinit.subp.subp", return_value=("U=1000000", None)) @@ -190,7 +190,7 @@ def test_container_ci_log_line(self, m_is_container, m_subp, m_get, m_g): finish_code = analyze_boot(name_default, args) self.remove_dummy_file(path, log_path) - assert CONTAINER_CODE == finish_code + assert 0 == finish_code @mock.patch("cloudinit.analyze.show.SystemctlReader") @pytest.mark.parametrize( From ba862bb9047bae35db90a9823b6db5808215311c Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 30 Jun 2026 14:05:15 -0600 Subject: [PATCH 67/72] chore: bump pylint version, dropping deprecated argparse.FileType lints (#6917) Update pinned pylint versions to avoid false positives for ProcessPoolExecutor, resolve any update pylint errors. Resolve python 3.14 deprecated argparse.FileType by converting --files param to str type, adapting extract_fns to use expected file paths instead of filehandle.name during config file processing. This retains original behavior for multi-file config processing on the CLI. --- cloudinit/cmd/main.py | 13 +++++-------- tests/unittests/cmd/test_main.py | 2 +- tox.ini | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 89b6e79458a..bc1fcdd87c4 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -161,15 +161,12 @@ def close_stdin(logger: Callable[[str], None] = LOG.debug): def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... fn_cfgs = [] if args.files: - for fh in args.files: + for filepath in args.files: # The realpath is more useful in logging # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) + fn_cfgs.append(os.path.realpath(filepath)) return fn_cfgs @@ -1151,7 +1148,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) # This is used so that we can know which action is selected + # the functor to use to run this subcommand @@ -1184,7 +1181,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) parser_mod.set_defaults(action=("modules", main_modules)) @@ -1224,7 +1221,7 @@ def main(sysv_args=None): action="append", dest="files", help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), + type=str, ) parser_single.set_defaults(action=("single", main_single)) diff --git a/tests/unittests/cmd/test_main.py b/tests/unittests/cmd/test_main.py index bcf93bec395..1dd09bc6a38 100644 --- a/tests/unittests/cmd/test_main.py +++ b/tests/unittests/cmd/test_main.py @@ -116,7 +116,7 @@ def test_main_init_run_net_runs_modules( supplemental_config_file.write( EXTRA_CLOUD_CONFIG.format(tmpdir=tmpdir) ) - files = [open(supplemental_config_file)] + files = [supplemental_config_file] else: files = None cmdargs = MyArgs( diff --git a/tox.ini b/tox.ini index 7565be59722..faf449bf114 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ deps = hypothesis_jsonschema==0.23.1 isort==6.0.1 mypy==1.19.1 - pylint==3.3.8 + pylint==4.0.6 ruff==0.12.9 [latest_versions] From 10831fac171dca4722ca64b81831f3e2ca04c6b3 Mon Sep 17 00:00:00 2001 From: Medha Mummigatti Date: Wed, 1 Jul 2026 03:15:18 +0530 Subject: [PATCH 68/72] fix(tests): make user_groups integration tests distro-aware (#6889) The integration tests for users_groups module had Ubuntu-specific assumptions that prevented them from running on RHEL-family systems. --- tests/integration_tests/modules/test_users_groups.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 1ceb2587c99..7edb5956048 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -119,6 +119,10 @@ def test_users_groups(self, regex, getent_args, class_client): ) ) + @pytest.mark.skipif( + not IS_UBUNTU, + reason="Warning expectations are Ubuntu-specific", + ) def test_initial_warnings(self, class_client): """Check for initial warnings.""" warnings = ( @@ -138,6 +142,10 @@ def test_user_root_in_secret(self, class_client): groups = groups_str.split() assert "secret" in groups + @pytest.mark.skipif( + not IS_UBUNTU, + reason="Password unlock warning behavior differs across distros", + ) def test_nopassword_unlock_warnings(self, class_client): """Verify warnings for empty passwords for new and existing users.""" # Fake admin clearing and unlocking and empty unlocked password foobar @@ -165,8 +173,8 @@ def test_nopassword_unlock_warnings(self, class_client): @pytest.mark.user_data(USER_DATA) @pytest.mark.skipif( - CURRENT_RELEASE < JAMMY, - reason="Requires version of sudo not available in older releases", + IS_UBUNTU and CURRENT_RELEASE < JAMMY, + reason="Requires version of sudo not available in older Ubuntu releases", ) def test_sudoers_includedir(client: IntegrationInstance): """Ensure we don't add additional #includedir to sudoers. From fa1bca7e03ed4c470b44669fe1e1707608525445 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 2 Jul 2026 12:47:55 -0600 Subject: [PATCH 69/72] ci: bump labeler ver add pull-request write permission for labeling (#6919) --- .github/workflows/45-ci-label.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/45-ci-label.yml b/.github/workflows/45-ci-label.yml index f76b8cf4acd..9372f1f3527 100644 --- a/.github/workflows/45-ci-label.yml +++ b/.github/workflows/45-ci-label.yml @@ -5,6 +5,9 @@ on: jobs: labeler: if: github.repository == 'canonical/cloud-init' + permissions: + contents: read + pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + - uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0 From db621d92fb2475cdeacf011e72160733a468718f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 2 Jul 2026 12:52:00 -0600 Subject: [PATCH 70/72] drop cherry picks included in upstream/main. drop the following cherry picks: cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com --- ...-Configure-arm64-to-use-archive.ubuntu.com | 32 ------------------- debian/patches/series | 1 - 2 files changed, 33 deletions(-) delete mode 100644 debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com delete mode 100644 debian/patches/series diff --git a/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com b/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com deleted file mode 100644 index cc441ab9bcf..00000000000 --- a/debian/patches/cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com +++ /dev/null @@ -1,32 +0,0 @@ -From d4566b1aa35951e6c32da330e627c023785026ea Mon Sep 17 00:00:00 2001 -From: Dave Jones -Date: Fri, 3 Apr 2026 02:31:16 +0100 -Subject: [PATCH] fix(ubuntu): Configure arm64 to use archive.ubuntu.com - (#6826) - -Fixes GH-6825 -LP: #2147101 ---- - config/cloud.cfg.tmpl | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - ---- a/config/cloud.cfg.tmpl -+++ b/config/cloud.cfg.tmpl -@@ -359,7 +359,7 @@ system_info: - security: https://deb.debian.org/debian-security - {% elif variant in ["ubuntu", "unknown"] %} - package_mirrors: -- - arches: [i386, amd64] -+ - arches: [arm64, i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu -@@ -369,7 +369,7 @@ system_info: - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] -- - arches: [arm64, armel, armhf] -+ - arches: [armel, armhf] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index db6ddb3ffad..00000000000 --- a/debian/patches/series +++ /dev/null @@ -1 +0,0 @@ -cpick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com From acfa741c0573733e52ec177c920d09c07c56ce3e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 2 Jul 2026 12:52:04 -0600 Subject: [PATCH 71/72] update changelog (new upstream snapshot) --- debian/changelog | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 473d76acf5d..66993cebf3e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -cloud-init (26.2~1g275e0e03-0ubuntu1) UNRELEASED; urgency=medium +cloud-init (26.2~2gfa1bca7e-0ubuntu1) UNRELEASED; urgency=medium + * drop the following cherry-picks now included: + + pick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com * d/control: add python3-pyfakefs for testing + * Upstream snapshot based on upstream/main at fa1bca7e. + - Bugs fixed in this snapshot: (LP: #2147101) - -- Chad Smith Tue, 14 Apr 2026 09:55:08 -0600 + -- Chad Smith Thu, 02 Jul 2026 12:52:04 -0600 cloud-init (26.1-0ubuntu2) resolute; urgency=medium From 551fca05c735b1aece20a732354cf0694ad71d60 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 2 Jul 2026 12:52:23 -0600 Subject: [PATCH 72/72] releasing cloud-init version 26.2~2gfa1bca7e-0ubuntu1 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 66993cebf3e..2ff9d571231 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -cloud-init (26.2~2gfa1bca7e-0ubuntu1) UNRELEASED; urgency=medium +cloud-init (26.2~2gfa1bca7e-0ubuntu1) stonking; urgency=medium * drop the following cherry-picks now included: + pick-d4566b1a-fix-ubuntu-Configure-arm64-to-use-archive.ubuntu.com @@ -6,7 +6,7 @@ cloud-init (26.2~2gfa1bca7e-0ubuntu1) UNRELEASED; urgency=medium * Upstream snapshot based on upstream/main at fa1bca7e. - Bugs fixed in this snapshot: (LP: #2147101) - -- Chad Smith Thu, 02 Jul 2026 12:52:04 -0600 + -- Chad Smith Thu, 02 Jul 2026 12:52:15 -0600 cloud-init (26.1-0ubuntu2) resolute; urgency=medium