From 5d86a5d4804f15450cab45610e4ba8edb5530a8e Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Tue, 14 Jul 2026 01:34:23 -0700 Subject: [PATCH] refactor: make tracing logs consistently structured and searchable NICo's tracing events now keep operational values in named fields and use stable messages, making logfmt output easier to search and aggregate. The sweep keeps presentation-oriented CLI and passthrough output intact where the rendered text is itself the payload. Primary callouts: - Convert roughly 1,400 direct tracing events across more than 50 crates to stable messages with semantic fields. - Keep IDs, endpoints, counts, states, and errors typed and searchable while preserving trace levels and intentional Display, Debug, and custom formatting. - Strengthen STYLE_GUIDE.md so new tracing events follow the same structured-logging convention. - Leave CLI output, child-process streams, aligned or multiline displays, and logging fixtures rendered where the text itself is the payload. This supports https://github.com/NVIDIA/infra-controller/issues/3476 Signed-off-by: Chet Nichols III --- STYLE_GUIDE.md | 41 +- crates/admin-cli/src/machine/show/cmd.rs | 5 +- crates/admin-cli/src/rpc.rs | 29 +- crates/agent/src/agent_platform.rs | 12 +- crates/agent/src/astra_weave.rs | 27 +- crates/agent/src/containerd/container.rs | 18 +- crates/agent/src/containerd/image.rs | 4 +- crates/agent/src/dhcp_server_grpc_client.rs | 4 +- crates/agent/src/dpu/interface.rs | 32 +- crates/agent/src/dpu/link.rs | 6 +- crates/agent/src/dpu/route.rs | 22 +- crates/agent/src/ethernet_virtualization.rs | 73 +- .../dpu_extension_service_observability.rs | 15 +- .../src/extension_services/k8s_pod_handler.rs | 59 +- crates/agent/src/hbn.rs | 17 +- crates/agent/src/health.rs | 65 +- crates/agent/src/health/bgp.rs | 4 +- crates/agent/src/host_machine_id.rs | 2 +- .../agent/src/instance_metadata_endpoint.rs | 8 +- crates/agent/src/instrumentation.rs | 8 +- crates/agent/src/lib.rs | 6 +- crates/agent/src/machine_inventory_updater.rs | 22 +- crates/agent/src/main_loop.rs | 76 +- crates/agent/src/managed_files.rs | 2 +- crates/agent/src/mtu.rs | 17 +- crates/agent/src/netlink.rs | 3 +- crates/agent/src/network_monitor.rs | 25 +- crates/agent/src/nvue.rs | 11 +- crates/agent/src/ovs.rs | 12 +- crates/agent/src/periodic_config_fetcher.rs | 20 +- crates/agent/src/tests/common/mod.rs | 2 +- crates/agent/src/tests/full.rs | 4 +- .../agent/src/tests/test_network_monitor.rs | 4 +- .../agent/src/traffic_intercept_bridging.rs | 25 +- crates/agent/src/upgrade.rs | 29 +- .../api-core/src/attestation/measured_boot.rs | 9 +- .../api-core/src/attestation/tpm_ca_cert.rs | 46 +- crates/api-core/src/db_init.rs | 41 +- crates/api-core/src/dhcp/discover.rs | 6 +- crates/api-core/src/dpa/handler.rs | 94 ++- crates/api-core/src/dynamic_settings.rs | 5 +- crates/api-core/src/handlers/api.rs | 21 +- crates/api-core/src/handlers/astra.rs | 59 +- crates/api-core/src/handlers/attestation.rs | 10 +- .../src/handlers/bmc_endpoint_explorer.rs | 72 +- .../src/handlers/component_manager.rs | 3 +- crates/api-core/src/handlers/dns.rs | 4 +- .../api-core/src/handlers/dpu_remediation.rs | 5 +- .../api-core/src/handlers/expected_machine.rs | 4 +- .../src/handlers/extension_service.rs | 20 +- crates/api-core/src/handlers/finder.rs | 16 +- crates/api-core/src/handlers/firmware.rs | 8 +- crates/api-core/src/handlers/instance.rs | 29 +- crates/api-core/src/handlers/instance_type.rs | 4 +- crates/api-core/src/handlers/machine.rs | 60 +- .../src/handlers/machine_discovery.rs | 8 +- crates/api-core/src/handlers/machine_scout.rs | 33 +- .../src/handlers/machine_validation.rs | 28 +- crates/api-core/src/handlers/rack.rs | 5 +- crates/api-core/src/handlers/redfish.rs | 13 +- crates/api-core/src/handlers/scout_stream.rs | 10 +- crates/api-core/src/handlers/site_explorer.rs | 5 +- crates/api-core/src/handlers/svpc.rs | 69 +- crates/api-core/src/handlers/uefi.rs | 15 +- crates/api-core/src/instance/mod.rs | 49 +- crates/api-core/src/ipxe.rs | 11 +- crates/api-core/src/listener.rs | 22 +- crates/api-core/src/logging/setup.rs | 5 +- .../dpu_nic_firmware.rs | 18 +- .../machine_update_manager/host_firmware.rs | 4 +- .../src/machine_update_manager/mod.rs | 10 +- crates/api-core/src/machine_validation/mod.rs | 8 +- .../measured_boot/metrics_collector/mod.rs | 5 +- .../src/mqtt_state_change_hook/hook.rs | 5 +- crates/api-core/src/run.rs | 19 +- crates/api-core/src/scout_stream.rs | 46 +- crates/api-core/src/setup.rs | 52 +- .../src/tests/common/api_fixtures/dpu.rs | 5 +- .../src/tests/common/api_fixtures/mod.rs | 13 +- .../common/api_fixtures/site_explorer.rs | 5 +- .../common/api_fixtures/test_machine/mod.rs | 11 +- crates/api-core/src/tests/dns.rs | 20 +- .../api-core/src/tests/dpu_reprovisioning.rs | 8 +- crates/api-core/src/tests/instance.rs | 5 +- crates/api-core/src/tests/machine_network.rs | 4 +- crates/api-core/src/tests/sku.rs | 25 +- .../tests/integration/machine_bmc_metadata.rs | 7 +- crates/api-db/src/bmc_metadata.rs | 20 +- crates/api-db/src/carbide_version.rs | 3 +- crates/api-db/src/dns/resource_record.rs | 2 +- crates/api-db/src/expected_power_shelf.rs | 4 +- crates/api-db/src/expected_switch.rs | 4 +- crates/api-db/src/instance.rs | 15 +- crates/api-db/src/lib.rs | 9 +- crates/api-db/src/machine.rs | 27 +- crates/api-db/src/machine_interface.rs | 6 +- crates/api-db/src/machine_topology.rs | 5 +- crates/api-db/src/measured_boot/bundle.rs | 4 +- crates/api-db/src/vpc_dpu_loopback.rs | 4 +- crates/api-db/src/work_lock_manager.rs | 89 ++- crates/api-integration-tests/tests/lib.rs | 91 ++- crates/api-model/src/dpa_interface/mod.rs | 18 +- crates/api-model/src/firmware.rs | 7 +- .../api-model/src/instance/status/network.rs | 24 +- .../api-model/src/instance/status/nvlink.rs | 4 +- crates/api-model/src/instance/status/spx.rs | 4 +- crates/api-model/src/machine/mod.rs | 16 +- crates/api-model/src/machine/nvlink.rs | 4 +- crates/api-model/src/machine/spx.rs | 4 +- crates/api-model/src/site_explorer/mod.rs | 16 +- crates/api-test-helper/src/domain.rs | 5 +- crates/api-test-helper/src/instance.rs | 63 +- crates/api-test-helper/src/machine.rs | 8 +- crates/api-test-helper/src/machine_a_tron.rs | 4 +- crates/api-test-helper/src/subnet.rs | 31 +- crates/api-test-helper/src/tenant.rs | 7 +- crates/api-test-helper/src/vpc.rs | 15 +- crates/api-test-helper/src/vpc_prefix.rs | 7 +- crates/api-web/src/auth.rs | 11 +- crates/api-web/src/interface.rs | 7 +- crates/api-web/src/lib.rs | 4 +- crates/api-web/src/machine.rs | 4 +- crates/api-web/src/managed_host.rs | 14 +- crates/api-web/src/search.rs | 10 +- crates/bmc-explorer/src/chassis.rs | 12 +- crates/bmc-explorer/src/computer_system.rs | 30 +- crates/bmc-explorer/src/manager.rs | 114 +-- crates/bmc-mock/src/combined_server.rs | 16 +- crates/bmc-mock/src/combined_service.rs | 7 +- crates/bmc-mock/src/main.rs | 33 +- .../bmc-mock/src/redfish/expander_router.rs | 9 +- crates/bmc-mock/src/tar_router.rs | 11 +- crates/bmc-mock/src/tls.rs | 6 +- crates/bmc-proxy/src/bmc_proxy.rs | 39 +- crates/bmc-proxy/src/setup.rs | 2 +- crates/dhcp-server/src/cache.rs | 4 +- crates/dhcp-server/src/grpc_server.rs | 6 +- crates/dhcp-server/src/main.rs | 48 +- crates/dhcp-server/src/modes/controller.rs | 12 +- crates/dhcp-server/src/packet_handler.rs | 2 +- crates/dhcp-server/src/util.rs | 14 +- crates/dns-record/src/lib.rs | 9 +- crates/dns/src/lib.rs | 16 +- crates/dns/src/main.rs | 10 +- crates/dpa-manager/src/card_handler/astra.rs | 51 +- crates/dpa-manager/src/card_handler/svpc.rs | 63 +- crates/dpa-manager/src/lib.rs | 16 +- crates/dpa/src/lib.rs | 8 +- crates/dpf/src/sdk.rs | 47 +- crates/dpu-otel-agent/src/lib.rs | 6 +- crates/dpu-remediation/src/remediation.rs | 24 +- .../dsx-exchange-consumer/src/api_client.rs | 6 +- crates/firmware/src/config.rs | 31 +- crates/firmware/src/downloader.rs | 15 +- crates/fmds/src/http_request_metrics.rs | 8 +- crates/fmds/src/main.rs | 4 +- crates/fmds/src/nic_init.rs | 9 +- crates/fmds/src/phone_home.rs | 6 +- crates/health/src/api_client.rs | 12 +- crates/health/src/collectors/logs/periodic.rs | 4 +- crates/health/src/discovery/cleanup.rs | 31 +- crates/health/src/discovery/context.rs | 21 - crates/health/src/discovery/spawn.rs | 32 +- crates/health/src/metrics.rs | 10 +- .../host-support/src/hardware_enumeration.rs | 93 ++- .../src/hardware_enumeration/dpu.rs | 12 +- .../src/hardware_enumeration/gpu.rs | 2 +- .../src/hardware_enumeration/tpm.rs | 6 +- crates/host-support/src/registration.rs | 38 +- crates/http-connector/src/connector.rs | 24 +- crates/http-connector/src/resolver.rs | 2 +- crates/ib-fabric/src/lib.rs | 88 ++- crates/ib-partition-controller/src/handler.rs | 3 +- crates/ipmi/src/tool.rs | 2 +- crates/kms-provider/src/providers/transit.rs | 7 +- crates/libmlx/src/device/discovery.rs | 30 +- crates/libnmxc/src/lib.rs | 2 +- crates/libnmxm/src/lib.rs | 6 +- crates/machine-a-tron/src/api_client.rs | 10 +- crates/machine-a-tron/src/api_throttler.rs | 5 +- crates/machine-a-tron/src/bmc_mock_wrapper.rs | 12 +- crates/machine-a-tron/src/dhcp_wrapper.rs | 20 +- crates/machine-a-tron/src/host_machine.rs | 19 +- crates/machine-a-tron/src/machine_a_tron.rs | 48 +- .../src/machine_state_machine.rs | 147 ++-- crates/machine-a-tron/src/machine_utils.rs | 25 +- crates/machine-a-tron/src/main.rs | 33 +- crates/machine-a-tron/src/mock_ssh_server.rs | 15 +- crates/machine-a-tron/src/subnet.rs | 12 +- crates/machine-a-tron/src/tui.rs | 10 +- crates/machine-a-tron/src/vpc.rs | 7 +- crates/machine-controller/src/dpf.rs | 6 +- crates/machine-controller/src/handler.rs | 657 ++++++++++------ .../src/handler/attestation.rs | 16 +- .../src/handler/bios_config.rs | 6 +- crates/machine-controller/src/handler/dpf.rs | 2 +- .../src/handler/machine_validation.rs | 21 +- .../machine-controller/src/handler/power.rs | 7 +- crates/machine-controller/src/handler/sku.rs | 17 +- .../src/machine_validation.rs | 78 +- crates/metrics-endpoint/src/lib.rs | 2 +- crates/metrics-utils/src/lib.rs | 6 +- crates/mqttea-example/src/main.rs | 30 +- crates/mqttea/src/client/core.rs | 45 +- crates/mqttea/src/client/messages.rs | 2 +- crates/mqttea/src/registry/core.rs | 15 +- crates/nvlink-manager/src/lib.rs | 128 ++-- .../nvlink-manager/src/switch_cert_monitor.rs | 8 +- .../power-shelf-controller/src/configuring.rs | 4 +- crates/power-shelf-controller/src/deleting.rs | 2 +- .../src/fetching_data.rs | 4 +- .../src/initializing.rs | 4 +- .../src/bfb_rshim_copier.rs | 14 +- crates/preingestion-manager/src/lib.rs | 700 ++++++++++++------ .../tests/integration/host_bmc_firmware.rs | 4 +- crates/rack-controller/src/created.rs | 6 +- crates/rack-controller/src/discovering.rs | 4 +- crates/rack-controller/src/error_state.rs | 28 +- crates/rack-controller/src/handler.rs | 12 +- crates/rack-controller/src/lib.rs | 11 +- crates/rack-controller/src/maintenance.rs | 19 +- crates/rack-controller/src/ready.rs | 52 +- crates/rack-controller/src/validating.rs | 58 +- crates/rpc-utils/src/managed_host_display.rs | 13 +- crates/rpc/src/errors.rs | 9 +- crates/rpc/src/forge_tls_client.rs | 28 +- crates/rpc/src/model/dpu_remediation.rs | 18 +- crates/scout/src/attestation.rs | 41 +- crates/scout/src/deprovision/scrabbing.rs | 123 +-- crates/scout/src/firmware_upgrade.rs | 32 +- crates/scout/src/main.rs | 107 ++- crates/scout/src/mlx_device.rs | 278 ++++--- crates/scout/src/register.rs | 12 +- crates/scout/src/stream.rs | 36 +- crates/scout/src/tpm.rs | 16 +- crates/secrets/src/forge_vault.rs | 25 +- crates/secrets/src/local_credentials/file.rs | 22 +- .../src/bmc_endpoint_explorer.rs | 387 ++++++---- .../src/explored_endpoint_index.rs | 24 +- crates/site-explorer/src/lib.rs | 262 ++++--- crates/site-explorer/src/machine_creator.rs | 20 +- crates/site-explorer/src/redfish.rs | 20 +- crates/site-explorer/src/switch_creator.rs | 9 +- .../test_support/mock_endpoint_explorer.rs | 2 +- .../tests/integration/power_shelf.rs | 54 +- .../tests/integration/site_explorer.rs | 6 +- .../site-explorer/tests/integration/switch.rs | 6 +- crates/spdm-controller/src/handler.rs | 11 +- crates/ssh-console-mock-api-server/src/lib.rs | 2 +- crates/ssh-console/src/bmc/client.rs | 32 +- crates/ssh-console/src/bmc/connection.rs | 9 +- .../src/bmc/connection_impl/ipmi.rs | 6 +- .../src/bmc/connection_impl/ssh.rs | 11 +- crates/ssh-console/src/bmc/message_proxy.rs | 2 +- crates/ssh-console/src/config.rs | 5 +- crates/ssh-console/src/console_logger.rs | 20 +- crates/ssh-console/src/frontend.rs | 20 +- crates/ssh-console/src/metrics.rs | 7 +- crates/ssh-console/src/ssh_cert_parsing.rs | 10 +- crates/ssh-console/src/ssh_server.rs | 2 +- crates/ssh-console/tests/util/ipmi_sim.rs | 8 +- crates/ssh-console/tests/util/mod.rs | 5 +- .../src/controller/periodic_enqueuer.rs | 6 +- .../src/controller/processor.rs | 4 +- crates/switch-controller/src/certificate.rs | 12 +- crates/switch-controller/src/configuring.rs | 42 +- crates/switch-controller/src/created.rs | 4 +- crates/switch-controller/src/deleting.rs | 5 +- crates/switch-controller/src/error_state.rs | 5 +- crates/switch-controller/src/initializing.rs | 54 +- crates/switch-controller/src/ready.rs | 4 +- crates/switch-controller/src/validating.rs | 5 +- 272 files changed, 4771 insertions(+), 2858 deletions(-) diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index d216af2c52..a96909fceb 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -162,26 +162,47 @@ count, like per-machine or per-instance attributes. All services should emit logs in "logfmt" syntax. This structured logging format allows administrators to efficiently search logs by certain attributes (key/value pairs). -When writing log messages, prefer placing common fields as attributes passed to tracing function, instead of using -string interpolation. For example: +Tracing events must use a stable, human-readable string literal for the message and record dynamic operational values +as structured fields whenever a meaningful field name can be derived. Do not interpolate such values into the message, +including when the event already contains other structured fields. + +Describe the event in the message; do not narrate fields that already describe themselves. For example, prefer +`info!(%domain_id, "Domain created")` over `info!(%domain_id, "Domain created with ID")`, and prefer +`info!(%instance_id, %ip_address, "Instance is terminating")` over a message ending in `"at address"`. Keep wording +that conveys a relationship, source, destination, or behavior that the field names alone do not express. + +Use native field shorthand for types that implement `tracing::Value`, `%field` for `Display`, and `?field` for `Debug`. +Use consistent semantic field names rather than incidental local variable names; for example, record an error held in +`e` as `error = %e`. Do not reuse formatter metadata keys such as `level`; choose a domain-specific key such as +`configured_log_level` instead. + +For example: ```rust -fn avoid(machine_id: MachineId) { +fn avoid(machine_id: MachineId, attempt_number: u32) { if let Err(e) = process_machine(machine_id) { - tracing::error!("process_machine failed for {machine_id}: {e}"); + tracing::error!( + "failed to process machine {machine_id} on attempt {attempt_number}: {e}" + ); } } -fn prefer(machine_id: MachineId) { +fn prefer(machine_id: MachineId, attempt_number: u32) { if let Err(e) = process_machine(machine_id) { - tracing::error!(%machine_id, error=%e, "process_machine failed"); + tracing::error!( + %machine_id, + attempt_number, + error = %e, + "failed to process machine", + ); } } - ``` -This helps in log parsing, especially when we want to find logs corresponding to a given machine_id. `error` and -`machine_id` are probably the two most important examples, but try to express other relevant data as fields instead of -using interpolation if it makes sense. +Keep intentional presentation formatting intact when the rendered text is itself the payload, such as CLI output, +aligned tables, or multi-line displays. Passthrough helpers and macros whose purpose is to forward caller-supplied text +unchanged are also exempt from the literal-message requirement. Preserve special or custom formatting when converting +it would change behavior; add structured fields alongside it when that can be done without changing the rendered +output. ## Core API handlers diff --git a/crates/admin-cli/src/machine/show/cmd.rs b/crates/admin-cli/src/machine/show/cmd.rs index d63f180bf2..fd6bbaf992 100644 --- a/crates/admin-cli/src/machine/show/cmd.rs +++ b/crates/admin-cli/src/machine/show/cmd.rs @@ -432,7 +432,10 @@ pub async fn get_next_free_machine( flat_vpc_id: Option, ) -> Option { while let Some(id) = machine_ids.pop_front() { - tracing::debug!("Checking {}", id); + tracing::debug!( + machine_id = %id, + "Checking machine", + ); if let Ok(machine) = api_client.get_machine(id).await { if machine.state != "Ready" { tracing::debug!("Machine is not ready"); diff --git a/crates/admin-cli/src/rpc.rs b/crates/admin-cli/src/rpc.rs index 43a3fb7d52..50fd019cc3 100644 --- a/crates/admin-cli/src/rpc.rs +++ b/crates/admin-cli/src/rpc.rs @@ -194,8 +194,9 @@ impl ApiClient { Ok(ot) => ot, Err(e) => { tracing::error!( - "Invalid UuidType from carbide api: {}", - uuid_details.object_type + object_type = uuid_details.object_type, + error = %e, + "Invalid UUID type from Carbide API", ); return Err(CarbideCliError::GenericError(e.to_string())); } @@ -226,8 +227,9 @@ impl ApiClient { Ok(ot) => ot, Err(e) => { tracing::error!( - "Invalid MachineOwner from carbide api: {}", - mac_details.object_type + object_type = mac_details.object_type, + error = %e, + "Invalid machine owner from Carbide API", ); return Err(CarbideCliError::GenericError(e.to_string())); } @@ -1585,7 +1587,7 @@ impl ApiClient { } else { vf_network_segment_ids.len() / pf_network_segment_ids.len() }; - tracing::debug!("VFs per PF: {vfs_per_pf}"); + tracing::debug!(vfs_per_pf, "VFs per PF",); let mut next_device_instance = HashMap::new(); @@ -1684,7 +1686,7 @@ impl ApiClient { // pf_vpc_prefix_ids is checked for empty above (len() cannot be 0) vf_vpc_prefix_ids.len() / pf_vpc_prefix_ids.len() }; - tracing::debug!("VFs per PF: {vfs_per_pf}"); + tracing::debug!(vfs_per_pf, "VFs per PF",); let mut vf_chunk_iter = vf_vpc_prefix_ids.chunks(vfs_per_pf); let mut ipv6_vf_chunk_iter = allocate_instance.ipv6_vf_prefix_id.chunks(vfs_per_pf); for (map_index, i) in discovery_info @@ -1729,7 +1731,10 @@ impl ApiClient { }), routing_profile: None, }; - tracing::debug!("Adding interface: {:?}", new_interface); + tracing::debug!( + new_interface = ?new_interface, + "Adding interface", + ); interface_configs.push(new_interface); @@ -1761,12 +1766,18 @@ impl ApiClient { routing_profile: None, }; vf_function_id += 1; - tracing::debug!("Adding interface: {:?}", new_interface); + tracing::debug!( + new_interface = ?new_interface, + "Adding interface", + ); interface_configs.push(new_interface); } } } else { - tracing::debug!("No pci device info for interface: {i:?}"); + tracing::debug!( + interface = ?i, + "No PCI device info", + ); } } diff --git a/crates/agent/src/agent_platform.rs b/crates/agent/src/agent_platform.rs index e94cfed4f0..062e84ea43 100644 --- a/crates/agent/src/agent_platform.rs +++ b/crates/agent/src/agent_platform.rs @@ -65,16 +65,16 @@ impl ManagedFile { if !destination_exists { safe_copy(self.path.as_path(), source_path).inspect(|_| { tracing::info!( - "Copied file contents from {src} to {dst}", - src = source_path.display(), - dst = self.path.display() + source_path = %source_path.display(), + destination_path = %self.path.display(), + "Copied file contents" ); }) } else { tracing::debug!( - "File {dst} already exists, will not be updated from {src}", - src = source_path.display(), - dst = self.path.display() + source_path = %source_path.display(), + destination_path = %self.path.display(), + "File already exists, will not be updated" ); Ok(()) } diff --git a/crates/agent/src/astra_weave.rs b/crates/agent/src/astra_weave.rs index b3f4c3b969..e7406255bd 100644 --- a/crates/agent/src/astra_weave.rs +++ b/crates/agent/src/astra_weave.rs @@ -292,10 +292,7 @@ async fn create_weave_ew_vpc_virtual_networks( continue; } - tracing::info!( - "Created virtual network from astra attachment status {:?}", - astra_attachment_status - ); + tracing::info!(?astra_attachment_status, "Created virtual network"); } Ok(()) @@ -340,8 +337,8 @@ async fn delete_stale_weave_ew_vpc_virtual_networks( match weave_ew_vpc_delete_virtual_network(socket_path, delete_vni_req).await { Ok(_) => { tracing::info!( - "Deleted stale virtual network {:?} from DOCA Weave server", - virtual_network + ?virtual_network, + "Deleted stale virtual network from DOCA Weave server" ); } Err(err) => { @@ -566,8 +563,8 @@ async fn create_or_recreate_weave_ew_vpc_astra_attachment( } tracing::info!( - "Created virtual network attachment for attachment status {:?}", - astra_attachment_status + ?astra_attachment_status, + "Created virtual network attachment" ); Ok(()) @@ -625,8 +622,8 @@ async fn delete_stale_weave_ew_vpc_astra_attachments( Ok(_) => { deleted_attachment_ids.insert(del_attachment_id); tracing::info!( - "Deleted stale virtual network attachment {:?} from DOCA Weave server", - virtual_network_attachment + ?virtual_network_attachment, + "Deleted stale virtual network attachment from DOCA Weave server" ); } Err(err) => { @@ -699,8 +696,8 @@ pub async fn delete_match_attachment_with_vni_changed( Ok(_) => { deleted_attachment_ids.insert(delete_attachment_id); tracing::info!( - "Deleted mismatched virtual network attachment {:?} from DOCA Weave server", - match_attachment.as_ref().unwrap() + ?match_attachment, + "Deleted mismatched virtual network attachment from DOCA Weave server" ); } Err(err) => { @@ -835,7 +832,8 @@ async fn build_synced_astra_config_status_if_version_unchanged( // the astra config despite the matching revision, fall back to a full // reconcile (return None) rather than erroring, so the drift is repaired. tracing::trace!( - "AstraConfig version {nico_spx_version} has no changes, copying attachment status" + spx_version = %nico_spx_version, + "AstraConfig version has no changes, copying attachment status" ); match sync_astra_config_status_from_weave_ew_vpc_attachments( astra_config, @@ -845,7 +843,8 @@ async fn build_synced_astra_config_status_if_version_unchanged( Err(err) => { tracing::warn!( error = format!("{err:#}"), - "AstraConfig version {nico_spx_version} matches, but DOCA Weave state drifted; running full reconcile" + spx_version = %nico_spx_version, + "AstraConfig version matches, but DOCA Weave state drifted; running full reconcile" ); Ok(None) } diff --git a/crates/agent/src/containerd/container.rs b/crates/agent/src/containerd/container.rs index 1067cbb555..33d9c5ad77 100644 --- a/crates/agent/src/containerd/container.rs +++ b/crates/agent/src/containerd/container.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use tracing::log::error; +use tracing::error; use crate::containerd::image::{Image, ImageNameComponent}; use crate::containerd::{BashCommand, Command}; @@ -226,7 +226,7 @@ async fn get_container_images() -> eyre::Result { let test_data_dir = PathBuf::from(TEST_DATA_DIR); std::fs::read_to_string(test_data_dir.join("container_images.json")).map_err(|e| { - error!("Could not read container_images.json: {e}"); + error!(error = %e, "Could not read container_images.json"); eyre::eyre!("Could not read container_images.json: {}", e) }) } else { @@ -235,7 +235,7 @@ async fn get_container_images() -> eyre::Result { .run() .await .map_err(|e| { - error!("Could not read container_images.json: {e}"); + error!(error = %e, "Could not read container_images.json"); eyre::eyre!("Could not read container_images.json: {}", e) })?; Ok(result) @@ -250,7 +250,7 @@ async fn get_containers() -> eyre::Result { println!("Path: {}", test_data_dir.join("containers.json").display()); std::fs::read_to_string(test_data_dir.join("containers.json")).map_err(|e| { - error!("Could not read containers.json: {e}"); + error!(error = %e, "Could not read containers.json"); eyre::eyre!("Could not read containers.json: {}", e) }) } else { @@ -259,7 +259,7 @@ async fn get_containers() -> eyre::Result { .run() .await .map_err(|e| { - error!("Could not read containers.json: {e}"); + error!(error = %e, "Could not read containers.json"); eyre::eyre!("Could not read containers.json: {}", e) })?; Ok(result) @@ -274,7 +274,7 @@ async fn get_pod_containers(pod_id: &str) -> eyre::Result { println!("Path: {}", test_data_dir.join("containers.json").display()); std::fs::read_to_string(test_data_dir.join("containers.json")).map_err(|e| { - error!("Could not read containers.json: {e}"); + error!(error = %e, "Could not read containers.json"); eyre::eyre!("Could not read containers.json: {}", e) }) } else { @@ -284,7 +284,7 @@ async fn get_pod_containers(pod_id: &str) -> eyre::Result { .run() .await .map_err(|e| { - error!("Could not read containers.json: {e}"); + error!(error = %e, "Could not read containers.json"); eyre::eyre!("Could not read containers.json: {}", e) })?; Ok(result) @@ -343,9 +343,9 @@ mod tests { #[tokio::test] async fn test_find_container_by_name() { let containers = Containers::list().await.expect("Could not get containers"); - tracing::info!("Container: {:?}", containers); + tracing::info!(?containers, "Listed containers"); let container = containers.find_by_name("doca-hbn").unwrap(); - tracing::info!("Container: {:?}", container); + tracing::info!(?container, "Found container by name"); assert_eq!(container.metadata.name, "doca-hbn"); assert_eq!(container.state, ContainerState::Running); } diff --git a/crates/agent/src/containerd/image.rs b/crates/agent/src/containerd/image.rs index 8200ce1779..90cad81c97 100644 --- a/crates/agent/src/containerd/image.rs +++ b/crates/agent/src/containerd/image.rs @@ -19,7 +19,7 @@ use std::fmt::{Display, Formatter}; use regex::Regex; use serde::{Deserialize, Serialize}; -use tracing::log::trace; +use tracing::trace; /// Represents the individual components of a container image name. /// e.g. @@ -76,7 +76,7 @@ where let vec: Vec = Vec::deserialize(deserializer)?; let initial: Vec = Vec::new(); - trace!("Container name component: {vec:?}"); + trace!(image_name_components = ?vec, "Container name component"); vec.iter().try_fold(initial, |mut accum, value| { let re = Regex::new(r#"(.+)\/(.+):(.+)"#).unwrap(); re.captures(value.as_str()) diff --git a/crates/agent/src/dhcp_server_grpc_client.rs b/crates/agent/src/dhcp_server_grpc_client.rs index 816f602daf..0b2611f6dc 100644 --- a/crates/agent/src/dhcp_server_grpc_client.rs +++ b/crates/agent/src/dhcp_server_grpc_client.rs @@ -132,7 +132,9 @@ pub async fn get_dhcp_timestamps( let id = e .host_interface_id .parse::() - .map_err(|err| tracing::warn!("Skipping unparseable host_interface_id: {err}")) + .map_err( + |err| tracing::warn!(error = %err, "Skipping unparseable host_interface_id"), + ) .ok()?; Some(::rpc::forge::LastDhcpRequest { host_interface_id: Some(id), diff --git a/crates/agent/src/dpu/interface.rs b/crates/agent/src/dpu/interface.rs index 073fba625b..b33d010b12 100644 --- a/crates/agent/src/dpu/interface.rs +++ b/crates/agent/src/dpu/interface.rs @@ -23,7 +23,7 @@ use carbide_utils::none_if_empty::NoneIfEmpty; use eyre::WrapErr; use ipnetwork::IpNetwork; use serde::{Deserialize, Serialize}; -use tracing::log::error; +use tracing::error; use crate::dpu::link::IpLink; use crate::dpu::{Action, DpuNetworkInterfaces}; @@ -68,11 +68,7 @@ impl Interface { if let Some(networks) = networks && !networks.is_empty() { - tracing::info!( - "Adding addresses {:?} to Interface {:?}", - networks, - interface - ); + tracing::info!(?networks, %interface, "Adding addresses"); for network in networks { Interface::ip_addrs_add(&interface, network).await?; @@ -113,8 +109,9 @@ impl Interface { Interface::find_common_addresses(¤t_addresses, &proposed_addresses) { tracing::trace!( - "Proposed addresses already present on interface {interface}: {:?}", - common_networks + %interface, + ?common_networks, + "Proposed addresses already present" ); // If the proposed addresses are already present on the interface @@ -124,8 +121,9 @@ impl Interface { proposed_addresses.retain(|x| !common_networks.contains(x)); } tracing::trace!( - "Proposed addresses needing to be added to {interface}: {:?}", - proposed_addresses + %interface, + ?proposed_addresses, + "Proposed addresses needing to be added" ); let entry = interface_plan.entry(Action::Add).or_default(); entry.insert(interface.to_string(), Some(proposed_addresses.clone())); @@ -146,7 +144,7 @@ impl Interface { let test_data_dir = PathBuf::from(crate::dpu::ARMOS_TEST_DATA_DIR); std::fs::read_to_string(test_data_dir.join("ipaddr.json")).map_err(|e| { - error!("Could not read ipaddr.json: {e}"); + error!(error = %e, "Could not read ipaddr.json"); eyre::eyre!("Could not read ipaddr.json: {}", e) }) } else { @@ -167,7 +165,7 @@ impl Interface { pub async fn current_addresses(interface: &str) -> eyre::Result> { let data = Self::ip_addrs(interface).await?; - tracing::trace!("interface data from ip addr show: {data:?}"); + tracing::trace!(ip_address_data = ?data, "interface data from ip addr show"); serde_json::from_str::>(&data).map_err(|err| eyre::eyre!(err)) } @@ -185,7 +183,7 @@ impl Interface { cmd.kill_on_drop(true); let cmd_str = pretty_cmd(cmd.as_std()); - tracing::trace!("Running command: {:?}", cmd_str); + tracing::trace!(command = ?cmd_str, "Running command"); let output = tokio::time::timeout(crate::dpu::COMMAND_TIMEOUT, cmd.output()) .await @@ -195,7 +193,7 @@ impl Interface { if output.status.success() { Ok(true) } else { - tracing::error!("Failed to add address: {:?}", fout); + tracing::error!(command_output = ?fout, "Failed to add address"); Ok(false) } } @@ -204,7 +202,7 @@ impl Interface { interface: &str, ) -> eyre::Result> { let data = Self::ip_addrs(interface).await?; - tracing::trace!("interfaces data from ip show: {:?}", data); + tracing::trace!(ip_link_data = ?data, "interfaces data from ip show"); let data = serde_json::from_str::>(&data).map_err(|err| eyre::eyre!(err))?; let filtered_list: Vec<_> = data @@ -326,7 +324,7 @@ mod tests { let interface = Interface::get_addresses_for_interface(HBNDeviceNames::hbn_23().sfs[0]) .await .unwrap(); - tracing::trace!("Interface: {:?}", interface); + tracing::trace!(?interface, "Interface"); assert_eq!(interface[0].prefixlen, 30); assert_eq!(interface[0].local, IpAddr::from([192, 168, 0, 1])); @@ -338,7 +336,7 @@ mod tests { .await .unwrap() .unwrap(); - tracing::trace!("Link: {:?}", link); + tracing::trace!(?link, "Link"); assert_eq!(link.ifindex, 16); } diff --git a/crates/agent/src/dpu/link.rs b/crates/agent/src/dpu/link.rs index 77fa698c74..c35f19c5b4 100644 --- a/crates/agent/src/dpu/link.rs +++ b/crates/agent/src/dpu/link.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; use eyre::Context; use serde::{Deserialize, Serialize}; -use tracing::log::error; +use tracing::error; use crate::pretty_cmd; @@ -42,7 +42,7 @@ pub struct IpLink { impl IpLink { pub async fn get_link_by_name(interface: &str) -> eyre::Result> { let data = Self::ip_links().await?; - tracing::trace!("interfaces data from ip show: {:?}", data); + tracing::trace!(ip_link_data = ?data, "interfaces data from ip show"); let data = serde_json::from_str::>(&data).map_err(|err| eyre::eyre!(err)); data.map(|i| { i.into_iter() @@ -54,7 +54,7 @@ impl IpLink { let test_data_dir = PathBuf::from(crate::dpu::ARMOS_TEST_DATA_DIR); std::fs::read_to_string(test_data_dir.join("iplink.json")).map_err(|e| { - error!("Could not read iplink.json: {e}"); + error!(error = %e, "Could not read iplink.json"); eyre::eyre!("Could not read iplink.json: {}", e) }) } else { diff --git a/crates/agent/src/dpu/route.rs b/crates/agent/src/dpu/route.rs index eee8582692..9a31b59760 100644 --- a/crates/agent/src/dpu/route.rs +++ b/crates/agent/src/dpu/route.rs @@ -24,7 +24,7 @@ use std::process::ExitStatus; use eyre::Context; use ipnetwork::IpNetwork; use serde::{Deserialize, Serialize}; -use tracing::log::error; +use tracing::error; use crate::dpu::Action; use crate::pretty_cmd; @@ -94,7 +94,7 @@ impl Route { entry.extend(routes); } - tracing::trace!("Route plan: {:?}", plan); + tracing::trace!(route_plan = ?plan, "Route plan"); Ok(plan) } @@ -104,7 +104,7 @@ impl Route { Action::Add => { if !routes.is_empty() { for r in routes { - tracing::info!("Adding route to Dpu: {:?}", r); + tracing::info!(route = ?r, "Adding route to Dpu"); Self::ip_route_add(r.dst, r.dev.as_deref(), r.prefsrc, r.gateway) .await?; } @@ -113,7 +113,7 @@ impl Route { Action::Remove => { if !routes.is_empty() { for r in routes { - tracing::info!("Removing route from Dpu: {:?}", r); + tracing::info!(route = ?r, "Removing route from Dpu"); Self::ip_route_del(r.dst, r.dev.as_deref(), r.prefsrc, r.gateway) .await?; } @@ -129,7 +129,7 @@ impl Route { let test_data_dir = PathBuf::from(crate::dpu::ARMOS_TEST_DATA_DIR); std::fs::read_to_string(test_data_dir.join("iproute.json")).map_err(|e| { - error!("Could not read iproute.json: {e}"); + error!(error = %e, "Could not read iproute.json"); eyre::eyre!("Could not read iproute.json: {}", e) }) } else { @@ -173,7 +173,7 @@ impl Route { cmd.kill_on_drop(true); let cmd_str = pretty_cmd(cmd.as_std()); - tracing::trace!("Running command: {:?}", cmd_str); + tracing::trace!(command = ?cmd_str, "Running command"); let output = tokio::time::timeout(crate::dpu::COMMAND_TIMEOUT, cmd.output()) .await @@ -183,7 +183,7 @@ impl Route { if output.status.success() { Ok(true) } else { - tracing::error!("Failed to add route: {:?}", fout); + tracing::error!(command_output = ?fout, "Failed to add route"); Ok(false) } } @@ -212,7 +212,7 @@ impl Route { cmd.kill_on_drop(true); let cmd_str = pretty_cmd(cmd.as_std()); - tracing::trace!("Running command: {:?}", cmd_str); + tracing::trace!(command = ?cmd_str, "Running command"); let output = tokio::time::timeout(crate::dpu::COMMAND_TIMEOUT, cmd.output()) .await @@ -222,14 +222,14 @@ impl Route { if output.status == ExitStatus::from_raw(0) { Ok(true) } else { - tracing::error!("Failed to remove route: {:?}", fout); + tracing::error!(command_output = ?fout, "Failed to remove route"); Ok(false) } } pub async fn current_routes(interface: &str) -> eyre::Result> { let data = Self::ip_route(interface).await?; - tracing::trace!("route data from ip route show: {:?}", data); + tracing::trace!(route_data = ?data, "route data from ip route show"); let mut data = serde_json::from_str::>(&data).map_err(|err| eyre::eyre!(err))?; @@ -362,6 +362,6 @@ mod tests { assert_eq!(add[0], new_proposed_route1); assert_eq!(remove[0], route_to_remove); - tracing::trace!("Full Plan: {:?}", plan); + tracing::trace!(route_plan = ?plan, "Full Plan"); } } diff --git a/crates/agent/src/ethernet_virtualization.rs b/crates/agent/src/ethernet_virtualization.rs index 89975ebe27..1456c25b51 100644 --- a/crates/agent/src/ethernet_virtualization.rs +++ b/crates/agent/src/ethernet_virtualization.rs @@ -92,10 +92,10 @@ impl InterfaceState { // Execute command only if interface state is changed. let mut cmd = needed_state.command(); tracing::info!( - "Updating interface state from {:?} to {:?} with command: {:?}", - current_state, - needed_state, - cmd + ?current_state, + ?needed_state, + ?cmd, + "Updating interface state" ); let result = cmd.output().await?; if !result.status.success() { @@ -684,7 +684,11 @@ pub async fn update_nvue( if !skip_post { let cmd = acl_rules::RELOAD_CMD; if let Err(err) = hbn::run_in_container_shell(cmd).await { - tracing::error!("running nvue extra acl post '{}': {err:#}", cmd); + tracing::error!( + %cmd, + error = format!("{err:#}"), + "running nvue extra acl post" + ); } path_acl.del("BAK"); } @@ -692,7 +696,7 @@ pub async fn update_nvue( // ACLs didn't need changing, should be always this except on first boot Ok(false) => {} // Log the error but continue so that we get network working - Err(err) => tracing::error!("write nvue extra ACL: {err:#}"), + Err(err) => tracing::error!(error = format!("{err:#}"), "write nvue extra ACL"), } // nvue can save a copy of the config here. If that exists nvue uses it on boot. @@ -702,8 +706,9 @@ pub async fn update_nvue( && let Err(err) = fs::remove_file(&saved_config) { tracing::warn!( - "Failed removing old startup.yaml at {}: {err:#}", - saved_config.display() + saved_config_path = %saved_config.display(), + error = format!("{err:#}"), + "Failed removing old startup.yaml" ); } @@ -827,7 +832,13 @@ pub async fn update_traffic_intercept_bridging( }; interface_to_bridge.get(&name).map(|bridging| { - tracing::debug!("update_traffic_intercept_bridging representor={name} bridge={} vni={} gateway={}", bridging.bridge, i.vni, i.gateway); + tracing::debug!( + representor = %name, + bridge = %bridging.bridge, + vni = i.vni, + gateway = %i.gateway, + "Created traffic-intercept bridge mapping" + ); traffic_intercept_bridging::TrafficInterceptBridgeMapping { bridge: bridging.bridge.clone(), patch_port: bridging.patch_port.clone(), @@ -1428,7 +1439,11 @@ fn write_dhcp_v4_server_config( dhcp_relay_path.del("BAK"); } Ok(false) => {} - Err(err) => tracing::warn!("Write blank DHCP relay {dhcp_relay_path}: {err:#}"), + Err(err) => tracing::warn!( + %dhcp_relay_path, + error = format!("{err:#}"), + "Write blank DHCP relay" + ), } let interfaces = if nc.use_admin_network { @@ -1515,7 +1530,11 @@ fn write_dhcp_v4_server_config( dhcp_server_path.server.del("BAK"); } Ok(false) => {} - Err(err) => tracing::error!("Write DHCP server {}: {err:#}", dhcp_server_path.server), + Err(err) => tracing::error!( + dhcp_server_path = %dhcp_server_path.server, + error = format!("{err:#}"), + "Write DHCP server" + ), } let next_contents = dhcp::build_server_config( @@ -1537,8 +1556,9 @@ fn write_dhcp_v4_server_config( } Ok(false) => {} Err(err) => tracing::error!( - "Write DHCP server config {}: {err:#}", - dhcp_server_path.config + dhcp_server_config_path = %dhcp_server_path.config, + error = format!("{err:#}"), + "Write DHCP server config" ), } @@ -1555,8 +1575,9 @@ fn write_dhcp_v4_server_config( } Ok(false) => {} Err(err) => tracing::error!( - "Write DHCP server host config {}: {err:#}", - dhcp_server_path.host_config + dhcp_server_host_config_path = %dhcp_server_path.host_config, + error = format!("{err:#}"), + "Write DHCP server host config" ), } @@ -1599,7 +1620,7 @@ fn write( if !has_changed { return Ok(false); } - tracing::debug!("Applying new {file_type} config"); + tracing::debug!(%file_type, "Applying new config"); let path_bak = path.backup(); if path.0.exists() { @@ -1730,9 +1751,9 @@ async fn tenant_vf_mac(vlan_fdb: &[Fdb]) -> eyre::Result<&str> { if !ip_out.status.success() { tracing::debug!( - "STDERR {}: {}", - super::pretty_cmd(cmd.as_std()), - String::from_utf8_lossy(&ip_out.stderr) + command = %super::pretty_cmd(cmd.as_std()), + stderr = %String::from_utf8_lossy(&ip_out.stderr), + "STDERR" ); return Err(eyre::eyre!( "{} for cmd '{}'", @@ -1843,7 +1864,11 @@ impl FPath { match fs::remove_file(&p) { Ok(_) => true, Err(err) => { - tracing::warn!("Failed removing {}: {err}.", p.display()); + tracing::warn!( + file_path = %p.display(), + error = %err, + "Failed to remove file" + ); false } } @@ -1896,10 +1921,14 @@ fn cleanup_old_acls(hbn_root: &Path) { if p.exists() { match fs::remove_file(p) { Ok(_) => { - tracing::info!("Cleaned up old ACL file {}", p.display()); + tracing::info!(acl_file_path = %p.display(), "Cleaned up old ACL file"); } Err(err) => { - tracing::warn!("Failed removing old ACL file {}: {err}.", p.display()); + tracing::warn!( + acl_file_path = %p.display(), + error = %err, + "Failed removing old ACL file." + ); } } } diff --git a/crates/agent/src/extension_services/dpu_extension_service_observability.rs b/crates/agent/src/extension_services/dpu_extension_service_observability.rs index edcb6f01d3..7adae5dcc2 100644 --- a/crates/agent/src/extension_services/dpu_extension_service_observability.rs +++ b/crates/agent/src/extension_services/dpu_extension_service_observability.rs @@ -146,7 +146,10 @@ pub fn build( pub async fn validate() -> eyre::Result { let mut cmd = tokio::process::Command::new(OTEL_CONTRIB_VALIDATE_BIN); let cmd_str = super::super::pretty_cmd(cmd.as_std()); - tracing::debug!("running otel validation commands: {cmd_str}"); + tracing::debug!( + command = cmd_str.as_str(), + "running otel validation commands" + ); // Invoke the validation command and allow a few seconds for completion. // The validation should be immediate (under a second), but a few seconds of buffer @@ -159,12 +162,10 @@ pub async fn validate() -> eyre::Result { if !out.status.success() { tracing::error!( - " STDOUT {cmd_str}: {}", - String::from_utf8_lossy(&out.stdout) - ); - tracing::error!( - " STDERR {cmd_str}: {}", - String::from_utf8_lossy(&out.stderr) + command = cmd_str.as_str(), + stdout = %String::from_utf8_lossy(&out.stdout), + stderr = %String::from_utf8_lossy(&out.stderr), + "OTel validation command failed" ); return Ok(false); diff --git a/crates/agent/src/extension_services/k8s_pod_handler.rs b/crates/agent/src/extension_services/k8s_pod_handler.rs index cc6cf42e9f..10e67c9478 100644 --- a/crates/agent/src/extension_services/k8s_pod_handler.rs +++ b/crates/agent/src/extension_services/k8s_pod_handler.rs @@ -188,8 +188,8 @@ impl KubernetesPodServicesHandler { /// Restart a systemd service and apply changes to the service configuration. async fn systemctl_restart(&self, service: &str) -> Result<()> { tracing::debug!( - "systemctl daemon-reload and restart {} to apply changes", - service + service, + "systemctl daemon-reload and restart to apply changes" ); // Run systemctl daemon-reload @@ -201,7 +201,7 @@ impl KubernetesPodServicesHandler { if !daemon_reload.status.success() { let stderr = String::from_utf8_lossy(&daemon_reload.stderr); - tracing::warn!("systemctl daemon-reload failed: {}", stderr); + tracing::warn!(%stderr, "systemctl daemon-reload failed"); } // Run systemctl restart @@ -216,7 +216,7 @@ impl KubernetesPodServicesHandler { return Err(eyre::eyre!("Failed to restart {}: {}", service, stderr)); } - tracing::debug!("Successfully restarted {}", service); + tracing::debug!(%service, "Successfully restarted"); Ok(()) } @@ -294,10 +294,10 @@ impl KubernetesPodServicesHandler { })?; tracing::debug!( - "Pod spec for service {} V{} written successfully at {}", - service.id, - service.version, - pod_spec_path.display() + service_id = %service.id, + service_version = %service.version, + pod_spec_path = %pod_spec_path.display(), + "Pod spec written successfully" ); Ok(()) @@ -310,18 +310,18 @@ impl KubernetesPodServicesHandler { match fs::remove_file(&pod_spec_path).await { Ok(()) => { tracing::debug!( - "Pod spec for service {} V{} removed successfully at {}", service_id, service_version, - pod_spec_path.display() + pod_spec_path = %pod_spec_path.display(), + "Pod spec removed successfully" ); } Err(e) if e.kind() == ErrorKind::NotFound => { tracing::debug!( - "Pod spec for service {} V{} already gone (nothing to remove) at {}", service_id, service_version, - pod_spec_path.display() + pod_spec_path = %pod_spec_path.display(), + "Pod spec already gone (nothing to remove)" ); } Err(e) => { @@ -478,9 +478,9 @@ impl KubernetesPodServicesHandler { } else { let stderr = String::from_utf8_lossy(&output.stderr); tracing::debug!( - "go-template inspect failed for container {}: {}", container_id, - stderr + %stderr, + "go-template inspect failed" ); } @@ -960,13 +960,13 @@ JSON std::fs::write(KUBELET_SYSTEMD_OVERRIDE_FILE, override_content) .wrap_err("Failed to write kubelet systemd override file")?; tracing::debug!( - "Written kubelet systemd override to {}", - KUBELET_SYSTEMD_OVERRIDE_FILE + kubelet_systemd_override_path = KUBELET_SYSTEMD_OVERRIDE_FILE, + "Written kubelet systemd override" ); } else { tracing::debug!( - "Kubelet systemd override already up to date at {}", - KUBELET_SYSTEMD_OVERRIDE_FILE + kubelet_systemd_override_path = KUBELET_SYSTEMD_OVERRIDE_FILE, + "Kubelet systemd override already up to date" ); } @@ -1131,17 +1131,17 @@ Environment="NO_PROXY=127.0.0.1,localhost,.svc,.svc.cluster.local" KUBELET_SYSTEMD_OVERRIDE_FILE, ] { match std::fs::remove_file(path) { - Ok(_) => tracing::debug!("Removed {}", path), + Ok(_) => tracing::debug!(path, "Removed"), Err(e) if e.kind() == ErrorKind::NotFound => { - tracing::debug!("{} already absent", path); + tracing::debug!(path, "already absent"); } Err(e) => return Err(eyre::eyre!("Failed to remove {}: {}", path, e)), } } } else { tracing::debug!( - "Configuring credential provider for {} registries or organizations", - credential_list.len() + registry_count = credential_list.len(), + "Configuring credential provider for registries or organizations" ); // Create credential directory structure @@ -1162,8 +1162,8 @@ Environment="NO_PROXY=127.0.0.1,localhost,.svc,.svc.cluster.local" .wrap_err("Failed to write credential provider config")?; tracing::debug!( - "Written credential provider config to {}", - KUBELET_POD_IMAGE_CRED_CONFIG_FILE + credential_provider_config_path = KUBELET_POD_IMAGE_CRED_CONFIG_FILE, + "Written credential provider config" ); // Write kubelet systemd override to configure image credential provider @@ -1185,8 +1185,8 @@ Environment="NO_PROXY=127.0.0.1,localhost,.svc,.svc.cluster.local" } tracing::debug!( - "Written credential provider script to {}", - KUBELET_POD_IMAGE_CRED_PROVIDER_FILE + credential_provider_script_path = KUBELET_POD_IMAGE_CRED_PROVIDER_FILE, + "Written credential provider script" ); } @@ -1222,8 +1222,9 @@ Environment="NO_PROXY=127.0.0.1,localhost,.svc,.svc.cluster.local" } } else if observability.configs.len() > MAX_OBSERVABILITY_CONFIG_PER_SERVICE { tracing::error!( - "number of observability configs for service `{}` exceeds the limit of {MAX_OBSERVABILITY_CONFIG_PER_SERVICE}", - service.id + service_id = %service.id, + limit = MAX_OBSERVABILITY_CONFIG_PER_SERVICE, + "Number of observability configs exceeds the limit" ); // We protect against this case in the API layer, so this case, @@ -1348,7 +1349,7 @@ impl ExtensionServiceHandler for KubernetesPodServicesHandler { /// This reconciles the /etc/kubelet.d directory with the desired services async fn update_active_services(&mut self, services: &[ServiceConfig]) -> Result<()> { if let Err(e) = self.update_services(services).await { - tracing::error!("Failed to update active services: {}", e); + tracing::error!(error = %e, "Failed to update active services"); } Ok(()) } diff --git a/crates/agent/src/hbn.rs b/crates/agent/src/hbn.rs index b71d8becea..8bddea89ca 100644 --- a/crates/agent/src/hbn.rs +++ b/crates/agent/src/hbn.rs @@ -49,7 +49,7 @@ impl<'a> RunCommandPredicate<'a> { /// ENV::VAR `IGNORE_MGMT_VRF` dictates which command predicate to use pub fn new(container_id: &'a str) -> Self { let ignore_mgmt_vrf = std::env::var("IGNORE_MGMT_VRF").is_ok(); - tracing::trace!("RunCommandPredicate: IGNORE_MGMT_VRF is {ignore_mgmt_vrf}"); + tracing::trace!(ignore_mgmt_vrf, "RunCommandPredicate: IGNORE_MGMT_VRF"); match ignore_mgmt_vrf { true => Self { @@ -117,7 +117,7 @@ pub async fn run_in_container( let cmd = crictl.args(args); cmd.kill_on_drop(true); let cmd_str = super::pretty_cmd(cmd.as_std()); - tracing::trace!("run_in_container: {cmd_str}"); + tracing::trace!(command = cmd_str.as_str(), "run_in_container"); let cmd_res = timeout(TIMEOUT_CONTAINER_CMD, cmd.output()) .await @@ -128,7 +128,11 @@ pub async fn run_in_container( let stdout = String::from_utf8_lossy(&out.stdout).to_string(); if need_success && !out.status.success() { - tracing::debug!("STDERR {cmd_str}: {}", stderr); + tracing::debug!( + command = cmd_str.as_str(), + stderr = stderr.as_str(), + "Container command failed" + ); return Err(eyre::eyre!( "cmd '{cmd_str}' failed with status: {}, stderr: {}, stdout: {}", out.status, // includes the string "exit status" @@ -200,10 +204,9 @@ impl HBNContainerFileConfigs { // replaced. l => { tracing::info!( - "HBN container ID {c} is new to us, updating its neighbor \ - learning config (previous container ID was {l})", - c = current_container_id.as_str(), - l = l.map_or("None", |s| s.as_str()) + container_id = current_container_id.as_str(), + previous_container_id = l.map_or("None", |s| s.as_str()), + "HBN container ID is new to us, updating its neighbor learning config" ); set_strict_neighbor_learning(current_container_id.as_str()) .await diff --git a/crates/agent/src/health.rs b/crates/agent/src/health.rs index 155c03d920..ee2b1b9f09 100644 --- a/crates/agent/src/health.rs +++ b/crates/agent/src/health.rs @@ -202,7 +202,10 @@ async fn check_hbn_services_running( { Ok(s) => s, Err(err) => { - tracing::warn!("check_hbn_services_running supervisorctl status: {err}"); + tracing::warn!( + error = %err, + "Failed to get supervisorctl status for HBN health check" + ); failed( hr, probe_ids::SupervisorctlStatus.clone(), @@ -215,7 +218,10 @@ async fn check_hbn_services_running( let st = match parse_status(&sctl) { Ok(s) => s, Err(err) => { - tracing::warn!("check_hbn_services_running supervisorctl status parse: {err}"); + tracing::warn!( + error = %err, + "Failed to parse supervisorctl status for HBN health check" + ); failed( hr, probe_ids::SupervisorctlStatus.clone(), @@ -231,7 +237,11 @@ async fn check_hbn_services_running( match st.status_of(&service) { SctlState::Running => passed(hr, probe_ids::ServiceRunning.clone(), Some(service)), status => { - tracing::warn!("check_hbn_services_running {service}: {status}"); + tracing::warn!( + service = service.as_str(), + %status, + "HBN service is not running" + ); failed( hr, probe_ids::ServiceRunning.clone(), @@ -255,7 +265,10 @@ async fn check_dhcp_server(hr: &mut health_report::HealthReport, container_id: & { Ok(s) => s, Err(err) => { - tracing::warn!("check_hbn_services_running supervisorctl status: {err}"); + tracing::warn!( + error = %err, + "Failed to get supervisorctl status for DHCP health check" + ); failed( hr, probe_ids::SupervisorctlStatus.clone(), @@ -268,7 +281,10 @@ async fn check_dhcp_server(hr: &mut health_report::HealthReport, container_id: & let st = match parse_status(&sctl) { Ok(s) => s, Err(err) => { - tracing::warn!("check_hbn_services_running supervisorctl status parse: {err}"); + tracing::warn!( + error = %err, + "Failed to parse supervisorctl status for DHCP health check" + ); failed( hr, probe_ids::SupervisorctlStatus.clone(), @@ -306,12 +322,12 @@ async fn check_ifreload(hr: &mut health_report::HealthReport, container_id: &str if stdout.is_empty() { passed(hr, probe_ids::Ifreload.clone(), None); } else { - tracing::warn!("check_ifreload: {stdout}"); + tracing::warn!(stdout = stdout.as_str(), "check_ifreload"); failed(hr, probe_ids::Ifreload.clone(), None, stdout); } } Err(err) => { - tracing::warn!("check_ifreload: {err}"); + tracing::warn!(error = %err, "check_ifreload"); failed(hr, probe_ids::Ifreload.clone(), None, err.to_string()); } } @@ -341,7 +357,7 @@ fn check_files(hr: &mut health_report::HealthReport, hbn_root: &Path, expected_f let stat = match std::fs::metadata(path) { Ok(s) => s, Err(err) => { - tracing::warn!("check_files {filename}: {err}"); + tracing::warn!(filename, error = %err, "check_files"); failed( hr, probe_ids::FileIsValid.clone(), @@ -355,8 +371,10 @@ fn check_files(hr: &mut health_report::HealthReport, hbn_root: &Path, expected_f dhcp_server_size = stat.len(); } else if stat.len() < MIN_SIZE { tracing::warn!( - "check_files {filename}: Too small {} < {MIN_SIZE} bytes", - stat.len() + filename, + size = stat.len(), + min_size = MIN_SIZE, + "check_files: Too small" ); failed( hr, @@ -373,7 +391,7 @@ fn check_files(hr: &mut health_report::HealthReport, hbn_root: &Path, expected_f } if dhcp_server_size < MIN_SIZE { - tracing::warn!("check_files {DHCP_SERVER_FILE}: Too small"); + tracing::warn!(filename = DHCP_SERVER_FILE, "check_files: Too small"); failed( hr, probe_ids::FileIsValid.clone(), @@ -415,9 +433,9 @@ async fn check_restricted_mode(hr: &mut health_report::HealthReport) { }; if !out.status.success() { tracing::debug!( - "STDERR {}: {}", - super::pretty_cmd(cmd.as_std()), - String::from_utf8_lossy(&out.stderr) + command = %super::pretty_cmd(cmd.as_std()), + stderr = %String::from_utf8_lossy(&out.stderr), + "STDERR" ); failed( hr, @@ -499,9 +517,9 @@ async fn check_disk_utilization(hr: &mut health_report::HealthReport) { }; if !out.status.success() { tracing::debug!( - "STDERR {}: {}", - super::pretty_cmd(cmd.as_std()), - String::from_utf8_lossy(&out.stderr) + command = %super::pretty_cmd(cmd.as_std()), + stderr = %String::from_utf8_lossy(&out.stderr), + "STDERR" ); failed( hr, @@ -576,7 +594,7 @@ fn parse_disk_utilization(df_out: &str) -> eyre::Result { for line in df_out.lines().skip(1) { let parts: Vec<&str> = line.split_ascii_whitespace().collect(); if parts.len() < 6 { - tracing::warn!("du status line too short: '{line}'"); + tracing::warn!(line, "df output line too short"); continue; } @@ -590,7 +608,7 @@ fn parse_disk_utilization(df_out: &str) -> eyre::Result { let utilization: u32 = match utilization.parse() { Ok(u) => u, Err(_) => { - tracing::warn!("Can not parse disk utilization in line '{line}'"); + tracing::warn!(line, "Can not parse disk utilization in line"); continue; } }; @@ -615,7 +633,7 @@ fn parse_status(status_out: &str) -> eyre::Result { for line in status_out.lines() { let parts: Vec<&str> = line.split_ascii_whitespace().collect(); if parts.len() < 2 { - tracing::warn!("supervisorctl status line too short: '{line}'"); + tracing::warn!(line, "supervisorctl status line too short"); continue; } let state: SctlState = match parts[1].parse() { @@ -623,8 +641,9 @@ fn parse_status(status_out: &str) -> eyre::Result { Err(_err) => { // unreachable but future proof. SctlState::from_str is currently infallible. tracing::warn!( - "supervisorctl status invalid state '{}' in line '{line}'", - parts[1] + state = parts[1], + line, + "supervisorctl status invalid state in line" ); continue; } @@ -657,7 +676,7 @@ impl FromStr for SctlState { "EXITED" => Self::Exited, "FATAL" => Self::Fatal, _ => { - tracing::warn!("Unknown supervisorctl status '{s}'"); + tracing::warn!(status = s, "Unknown supervisorctl status"); Self::Unknown } }) diff --git a/crates/agent/src/health/bgp.rs b/crates/agent/src/health/bgp.rs index 41cb5a08d8..5586e322c2 100644 --- a/crates/agent/src/health/bgp.rs +++ b/crates/agent/src/health/bgp.rs @@ -61,7 +61,7 @@ pub async fn check_bgp_stats( hbn_device_names, ), Err(err) => { - tracing::warn!("check_network_stats show bgp summary: {err}"); + tracing::warn!(error = %err, "Failed to fetch BGP summary"); health_data.other_errors.push(err.to_string()); } }; @@ -73,7 +73,7 @@ pub fn check_daemon_enabled(hr: &mut health_report::HealthReport, hbn_daemons_fi let daemons = match std::fs::read_to_string(hbn_daemons_file) { Ok(s) => s, Err(err) => { - tracing::warn!("check_bgp_daemon_enabled: {err}"); + tracing::warn!(error = %err, "Failed to read BGP daemon configuration"); failed( hr, probe_ids::BgpDaemonEnabled.clone(), diff --git a/crates/agent/src/host_machine_id.rs b/crates/agent/src/host_machine_id.rs index 9aa53e41b9..76c453c663 100644 --- a/crates/agent/src/host_machine_id.rs +++ b/crates/agent/src/host_machine_id.rs @@ -93,7 +93,7 @@ pub async fn get_host_machine_id_retry( ) .await .map_err(|e| { - tracing::warn!("get_host_machine_id() failed: {:?}", e); + tracing::warn!(error = ?e, "Failed to get host machine ID"); e })? .ok_or(eyre::eyre!("get_host_machine_id() got no value")) diff --git a/crates/agent/src/instance_metadata_endpoint.rs b/crates/agent/src/instance_metadata_endpoint.rs index 0de5c91feb..d6eba93c01 100644 --- a/crates/agent/src/instance_metadata_endpoint.rs +++ b/crates/agent/src/instance_metadata_endpoint.rs @@ -124,12 +124,12 @@ impl InstanceMetadataRouterState for InstanceMetadataRouterStateImpl { let mut client = create_forge_client(&self.forge_api, &self.forge_client_config).await?; - let timestamp = phone_home(&mut client, &self.machine_id).await?.to_string() + "\n"; + let timestamp = phone_home(&mut client, &self.machine_id).await?; tracing::info!( - "Successfully phoned home for Machine {} at {}", - self.machine_id, - timestamp + machine_id = %self.machine_id, + %timestamp, + "Successfully phoned home" ); Ok(()) diff --git a/crates/agent/src/instrumentation.rs b/crates/agent/src/instrumentation.rs index 46b9f5a71c..dd46e8838c 100644 --- a/crates/agent/src/instrumentation.rs +++ b/crates/agent/src/instrumentation.rs @@ -294,7 +294,11 @@ impl WithTracingLayer for Router { let layer = tower_http::trace::TraceLayer::new_for_http() .on_request(move |request: &Request, _span: &Span| { metrics.http_counter.add(1, &[]); - tracing::info!("started {} {}", request.method(), request.uri().path()) + tracing::info!( + method = %request.method(), + request_path = %request.uri().path(), + "HTTP request started" + ) }) .on_response( move |_response: &Response, latency: Duration, _span: &Span| { @@ -303,7 +307,7 @@ impl WithTracingLayer for Router { .http_req_latency_histogram .record(latency.as_secs_f64() * 1000.0, &[]); - tracing::info!("response generated in {:?}", latency) + tracing::info!(?latency, "HTTP response generated") }, ); diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index 97002e56ad..4135920a3b 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -129,7 +129,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { .machine_identity .validate() .map_err(|e| eyre::eyre!("invalid [machine-identity] in agent config: {e}"))?; - tracing::info!("Using configuration from {path}: {agent:?}"); + tracing::info!(config_path = path.as_str(), ?agent, "Using configuration"); if agent.machine.is_fake_dpu { tracing::warn!("Pretending local host is a DPU. Dev only."); @@ -243,7 +243,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { None => NetworkPingerType::OobNetBind, }; - tracing::info!("Using {}", pinger_type); + tracing::info!(%pinger_type, "Using pinger"); let pinger: Arc = Arc::from(pinger_type); let mut network_monitor = @@ -299,7 +299,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { { Ok(id) => id, Err(e) => { - tracing::error!("get_host_machine_id_retry() failed: {:?}", e); + tracing::error!(error = ?e, "Failed to get host machine ID after retries"); return Err(e); } }; diff --git a/crates/agent/src/machine_inventory_updater.rs b/crates/agent/src/machine_inventory_updater.rs index e0912f29ab..2263baca19 100644 --- a/crates/agent/src/machine_inventory_updater.rs +++ b/crates/agent/src/machine_inventory_updater.rs @@ -38,8 +38,8 @@ pub struct MachineInventoryUpdaterConfig { pub async fn single_run(config: &MachineInventoryUpdaterConfig) -> eyre::Result<()> { tracing::trace!( - "Updating machine inventory for machine: {}", - config.machine_id + machine_id = %config.machine_id, + "Updating machine inventory" ); let machine_id = config.machine_id; @@ -50,7 +50,7 @@ pub async fn single_run(config: &MachineInventoryUpdaterConfig) -> eyre::Result< let images = container::Images::list().await?; - tracing::trace!("Containers: {:?}", containers); + tracing::trace!(?containers, "Containers"); let mut result: Vec = Vec::new(); @@ -104,8 +104,8 @@ pub async fn single_run(config: &MachineInventoryUpdaterConfig) -> eyre::Result< .await { tracing::error!( - "Error while executing update_agent_reported_inventory: {:#}", - e + error = ?e, + "Error while executing update_agent_reported_inventory" ); } else { tracing::debug!("Successfully updated machine inventory"); @@ -134,12 +134,20 @@ async fn update_agent_reported_inventory( } }; - tracing::trace!("update_machine_inventory: {:?}", inventory_report); + let software_component_count = inventory_report + .inventory + .as_ref() + .map_or(0, |inventory| inventory.components.len()); + tracing::trace!( + machine_id = ?inventory_report.machine_id, + software_component_count, + "update_machine_inventory" + ); let request = tonic::Request::new(inventory_report); match client.update_agent_reported_inventory(request).await { Ok(response) => { - tracing::trace!("update_agent_reported_inventory response: {:?}", response); + tracing::trace!(?response, "update_agent_reported_inventory response"); Ok(()) } Err(err) => Err(eyre::eyre!( diff --git a/crates/agent/src/main_loop.rs b/crates/agent/src/main_loop.rs index eaebeabe69..4ee9fbc72c 100644 --- a/crates/agent/src/main_loop.rs +++ b/crates/agent/src/main_loop.rs @@ -41,7 +41,7 @@ use mac_address::MacAddress; use tokio::signal::unix::{SignalKind, signal}; use tokio::sync::watch; use tokio::task::JoinHandle; -use tracing::log::error; +use tracing::error; use version_compare::Version; use crate::command_line::HbnConfigMode; @@ -156,7 +156,8 @@ pub async fn setup_and_run( Ok(fmds_client) => FmdsUpdater::External(Box::new(fmds_client)), Err(e) => { tracing::warn!( - "Failed to connect to external FMDS service: {e:#}, falling back to embedded" + error = format!("{e:#}"), + "Failed to connect to external FMDS service, falling back to embedded" ); FmdsUpdater::Embedded(instance_metadata_state.clone()) } @@ -169,7 +170,7 @@ pub async fn setup_and_run( instance_metadata_state.clone(), ) .unwrap_or_else(|e| { - tracing::warn!("Failed to run metadata service: {:#}", e); + tracing::warn!(error = format!("{e:#}"), "Failed to run metadata service"); }); } tracing::info!("Using FmdsUpdater::Embedded FMDS service"); @@ -184,7 +185,10 @@ pub async fn setup_and_run( metrics.record_agent_start_time(timestamp); } Err(e) => { - tracing::warn!("Error calculating process start timestamp: {e:#}"); + tracing::warn!( + error = format!("{e:#}"), + "Error calculating process start timestamp" + ); } } @@ -193,7 +197,10 @@ pub async fn setup_and_run( metrics.record_machine_boot_time(timestamp); } Err(e) => { - tracing::warn!("Error getting host boot timestamp: {e:#}"); + tracing::warn!( + error = format!("{e:#}"), + "Error getting host boot timestamp" + ); } } @@ -213,7 +220,7 @@ pub async fn setup_and_run( // turn it into an unhealthy status that gets reported to the // API, so we're not going to do much more than log this for // now. - tracing::error!("Couldn't ensure DOCA pods: {e}"); + tracing::error!(error = %e, "Couldn't ensure DOCA pods"); } let fmds_minimum_hbn_version = Version::from(FMDS_MINIMUM_HBN_VERSION).ok_or(eyre::eyre!( @@ -263,7 +270,7 @@ pub async fn setup_and_run( { Ok(id) => id, Err(e) => { - tracing::error!("get_host_machine_id_retry() failed: {:?}", e); + tracing::error!(error = ?e, "Failed to get host machine ID after retries"); return Err(e); } }; @@ -282,7 +289,7 @@ pub async fn setup_and_run( if options.agent_platform_type.is_dpu_os() && let Err(e) = lldp::set_lldp_system_description(&machine_id) { - tracing::warn!("Couldn't update LLDP system description: {e}") + tracing::warn!(error = %e, "Couldn't update LLDP system description") } let periodic_config_reader = periodic_config_fetcher.reader(); @@ -319,7 +326,7 @@ pub async fn setup_and_run( let network_monitor_handle: Option> = match network_pinger_type { Some(pinger_type) => { - tracing::debug!("Starting network monitor with {} pinger", pinger_type); + tracing::debug!(%pinger_type, "Starting network monitor"); let mut network_monitor = network_monitor::NetworkMonitor::new( machine_id, Some(network_monitor_metrics_state), @@ -573,7 +580,10 @@ async fn fetch_last_dhcp_requests(dhcp_grpc_server: Option<&str>) -> Vec requests, Err(e) => { - tracing::warn!("Failed to fetch DHCP timestamps via gRPC: {e:#}"); + tracing::warn!( + error = format!("{e:#}"), + "Failed to fetch DHCP timestamps via gRPC" + ); vec![] } }; @@ -582,8 +592,9 @@ async fn fetch_last_dhcp_requests(dhcp_grpc_server: Option<&str>) -> Vec eyre::Result<( let test_data_dir = PathBuf::from(crate::dpu::ARMOS_TEST_DATA_DIR); std::fs::read_to_string(test_data_dir.join("bfvcheck.out")).map_err(|e| { - error!("Could not read bfvcheck.out: {e}"); + error!(error = %e, "Could not read bfvcheck.out"); eyre::eyre!("Could not read bfvcheck.out: {}", e) })? } else { diff --git a/crates/agent/src/managed_files.rs b/crates/agent/src/managed_files.rs index c373f0f466..3030b98bfc 100644 --- a/crates/agent/src/managed_files.rs +++ b/crates/agent/src/managed_files.rs @@ -88,7 +88,7 @@ pub fn main_sync(sync_options: SyncOptions, machine_id: &MachineId, host_machine ), ]); if let Err(e) = duppet::sync(duppet_files, sync_options) { - tracing::error!("error during duppet run: {}", e) + tracing::error!(error = %e, "error during duppet run") } } diff --git a/crates/agent/src/mtu.rs b/crates/agent/src/mtu.rs index 7559cfd691..3aa64f02ca 100644 --- a/crates/agent/src/mtu.rs +++ b/crates/agent/src/mtu.rs @@ -29,7 +29,10 @@ pub async fn ensure() -> eyre::Result<()> { let current = get_mtu(iface).await?; if current != CORRECT_MTU { tracing::info!( - "Interface {iface} has incorrect MTU {current}. Setting to {CORRECT_MTU}." + iface, + current_mtu = current, + target_mtu = CORRECT_MTU, + "Interface has incorrect MTU. Setting target MTU." ); set_mtu(iface, CORRECT_MTU).await?; } @@ -53,9 +56,9 @@ async fn get_mtu(iface: &str) -> eyre::Result { Ok(o[0].mtu) } else { tracing::debug!( - "STDERR {}: {}", - super::pretty_cmd(cmd.as_std()), - String::from_utf8_lossy(&out.stderr) + command = %super::pretty_cmd(cmd.as_std()), + stderr = %String::from_utf8_lossy(&out.stderr), + "MTU query command failed" ); Err(eyre::eyre!( "{} for cmd '{}'", @@ -73,9 +76,9 @@ async fn set_mtu(iface: &str, mtu: usize) -> eyre::Result<()> { Ok(()) } else { tracing::debug!( - "STDERR {}: {}", - super::pretty_cmd(cmd.as_std()), - String::from_utf8_lossy(&out.stderr) + command = %super::pretty_cmd(cmd.as_std()), + stderr = %String::from_utf8_lossy(&out.stderr), + "MTU update command failed" ); Err(eyre::eyre!( "{} for cmd '{}'", diff --git a/crates/agent/src/netlink.rs b/crates/agent/src/netlink.rs index 91fffa5aea..8345df770e 100644 --- a/crates/agent/src/netlink.rs +++ b/crates/agent/src/netlink.rs @@ -227,7 +227,8 @@ pub async fn get_all_interface_links() -> Result { let idx = link_message.header.index; tracing::warn!( - "Network interface with index {idx} doesn't have a name (no IfName attribute)" + interface_index = idx, + "Network interface doesn't have a name (no IfName attribute)" ); None } diff --git a/crates/agent/src/network_monitor.rs b/crates/agent/src/network_monitor.rs index bf991aebac..6eefa75667 100644 --- a/crates/agent/src/network_monitor.rs +++ b/crates/agent/src/network_monitor.rs @@ -131,7 +131,10 @@ impl NetworkMonitor { loopback_ip = Some(dpu_info.ip); } Err(e) => { - tracing::debug!("Network monitor failed to get dpu info list from API {}", e); + tracing::debug!( + error = %e, + "Network monitor failed to get dpu info list from API" + ); } } @@ -152,7 +155,10 @@ impl NetworkMonitor { loopback_ip = Some(dpu_info.ip); } Err(e) => { - tracing::debug!("Network monitor failed to get dpu info list from API {}", e); + tracing::debug!( + error = %e, + "Network monitor failed to get dpu info list from API" + ); peer_dpus = Vec::new(); loopback_ip = None; } @@ -201,7 +207,7 @@ impl NetworkMonitor { metrics.update_network_reachable_map(reachable_map); } } - Err(e) => tracing::error!("Failed to run network check: {}", e), + Err(e) => tracing::error!(error = %e, "Failed to run network check"), } elapsed_time = start_time.elapsed(); } @@ -223,14 +229,17 @@ impl NetworkMonitor { { Ok((dpu_info, new_peer_dpus)) => (dpu_info.ip, new_peer_dpus), Err(e) => { - tracing::error!("Network monitor failed to get dpu info list from API {}", e); + tracing::error!( + error = %e, + "Network monitor failed to get dpu info list from API" + ); return; } }; match self.monitor_concurrent(&peer_dpus, loopback_ip).await { Ok(results) => self.format_results(&results, loopback_ip.to_string()), - Err(e) => tracing::error!("Failed to run network check: {}", e), + Err(e) => tracing::error!(error = %e, "Failed to run network check"), } } @@ -284,8 +293,8 @@ impl NetworkMonitor { let results = recv_task.await.map_err(|err| { self.record_error_metrics(NetworkMonitorError::TaskJoinError, None); tracing::error!( - "Failed to join task spawned for collecting ping result: {}", - err + error = %err, + "Failed to join task spawned for collecting ping result" ); err })?; @@ -327,7 +336,7 @@ impl NetworkMonitor { match serde_json::to_string_pretty(&final_result) { Ok(json) => println!("{json}"), - Err(e) => tracing::error!("Failed to serialize results to JSON: {}", e), + Err(e) => tracing::error!(error = %e, "Failed to serialize results to JSON"), } } diff --git a/crates/agent/src/nvue.rs b/crates/agent/src/nvue.rs index f7f5307625..532eb9aa6a 100644 --- a/crates/agent/src/nvue.rs +++ b/crates/agent/src/nvue.rs @@ -938,7 +938,11 @@ pub async fn apply(hbn_root: &Path, config_path: &super::FPath) -> eyre::Result< Ok(applied) } Err(err) => { - tracing::error!("update_nvue post command failed: {err:#}"); + tracing::error!( + error = %err, + error_chain = format!("{err:#}").replace('\n', "; "), + "NVUE post command failed" + ); // If the config apply failed, we won't be using it, so move it out // of the way to an .error file for others to enjoy (while attempting @@ -948,8 +952,9 @@ pub async fn apply(hbn_root: &Path, config_path: &super::FPath) -> eyre::Result< && let Err(e) = fs::remove_file(path_error.clone()) { tracing::warn!( - "Failed to remove previous error file ({}): {e}", - path_error.display() + error_file_path = %path_error.display(), + error = %e, + "Failed to remove previous error file" ); } diff --git a/crates/agent/src/ovs.rs b/crates/agent/src/ovs.rs index 993b8ba1fa..4589ddd02c 100644 --- a/crates/agent/src/ovs.rs +++ b/crates/agent/src/ovs.rs @@ -36,7 +36,7 @@ pub async fn set_vswitchd_yield() -> eyre::Result<()> { .arg("other_config:pmd-sleep-max=100") .kill_on_drop(true); let cmd_str = super::pretty_cmd(cmd.as_std()); - tracing::trace!("set_ovs_vswitchd_yield running: {cmd_str}"); + tracing::trace!(command = cmd_str.as_str(), "set_ovs_vswitchd_yield running"); // It takes less than 1s, so allow up to 5 let out = tokio::time::timeout(std::time::Duration::from_secs(5), cmd.output()) @@ -45,12 +45,10 @@ pub async fn set_vswitchd_yield() -> eyre::Result<()> { .wrap_err("Error running command")?; if !out.status.success() { tracing::error!( - " STDOUT {cmd_str}: {}", - String::from_utf8_lossy(&out.stdout) - ); - tracing::error!( - " STDERR {cmd_str}: {}", - String::from_utf8_lossy(&out.stderr) + command = cmd_str.as_str(), + stdout = %String::from_utf8_lossy(&out.stdout), + stderr = %String::from_utf8_lossy(&out.stderr), + "OVS command failed" ); eyre::bail!("Failed running ovs-vsctl command. Check logs for stdout/stderr."); } diff --git a/crates/agent/src/periodic_config_fetcher.rs b/crates/agent/src/periodic_config_fetcher.rs index d796d80907..ab695fee50 100644 --- a/crates/agent/src/periodic_config_fetcher.rs +++ b/crates/agent/src/periodic_config_fetcher.rs @@ -111,7 +111,7 @@ impl PeriodicConfigFetcher { let sitename = match fetch_sitename(&forge_client_config, &config.forge_api).await { Ok(sn) => sn, Err(e) => { - warn!("Unable to fetch sitename. Error {}", e); + warn!(error = %e, "Unable to fetch sitename"); None } }; @@ -190,8 +190,8 @@ async fn single_fetch( } trace!( - "Fetching periodic configuration for Machine {}", - state.config.machine_id + machine_id = %state.config.machine_id, + "Fetching periodic configuration" ); match fetch( @@ -213,22 +213,24 @@ async fn single_fetch( } Err(err) => { error!( - "Failed to fetch the latest configuration: {err}.\n Will retry in {:?}", - state.config.config_fetch_interval + error = %err, + retry_interval = ?state.config.config_fetch_interval, + "Failed to fetch the latest configuration. Will retry" ); } }; } Err(err) => match err.downcast_ref::() { Some(grpc_status) if grpc_status.code() == tonic::Code::NotFound => { - warn!("DPU not found: {}", state.config.machine_id); + warn!(machine_id = %state.config.machine_id, "DPU not found"); state.netconf.store(None); state.instmeta.store(None); } _ => { error!( - "Failed to fetch the latest configuration. Will retry in {:?}. {err:#?}", - state.config.config_fetch_interval + retry_interval = ?state.config.config_fetch_interval, + error = ?err, + "Failed to fetch the latest configuration. Will retry" ); } }, @@ -289,7 +291,7 @@ pub fn instance_metadata_from_instance( let devices = match extract_instance_ib_config(&instance) { Ok(value) => Some(value), Err(e) => { - trace!("Failed to fetch IB config: {}", e.to_string()); + trace!(error = %e, "Failed to fetch IB config"); None } }; diff --git a/crates/agent/src/tests/common/mod.rs b/crates/agent/src/tests/common/mod.rs index 8088ebeb92..bc15f91bc8 100644 --- a/crates/agent/src/tests/common/mod.rs +++ b/crates/agent/src/tests/common/mod.rs @@ -97,7 +97,7 @@ pub fn setup_agent_run_env( } let hbn_root = td.path(); - tracing::info!("Using hbn_root: {:?}", hbn_root); + tracing::info!(?hbn_root, "Using HBN root"); fs::create_dir_all(hbn_root.join("etc/frr"))?; fs::create_dir_all(hbn_root.join("etc/network"))?; fs::create_dir_all(hbn_root.join("etc/supervisor/conf.d"))?; diff --git a/crates/agent/src/tests/full.rs b/crates/agent/src/tests/full.rs index 4526022a9d..b0a9999cc5 100644 --- a/crates/agent/src/tests/full.rs +++ b/crates/agent/src/tests/full.rs @@ -336,7 +336,7 @@ async fn run_common_parts( // Start forge-dpu-agent tokio::spawn(async move { if let Err(e) = crate::start(opts).await { - tracing::error!("Failed to start DPU agent: {:#}", e); + tracing::error!(error = ?e, "Failed to start DPU agent"); } }); @@ -1029,7 +1029,7 @@ async fn handle_find_interfaces() -> impl axum::response::IntoResponse { } async fn handler(uri: Uri) -> impl IntoResponse { - tracing::debug!("general handler: {:?}", uri); + tracing::debug!(?uri, "General request handler received request"); StatusCode::NOT_FOUND } diff --git a/crates/agent/src/tests/test_network_monitor.rs b/crates/agent/src/tests/test_network_monitor.rs index 251bdf99f3..37936bcf19 100644 --- a/crates/agent/src/tests/test_network_monitor.rs +++ b/crates/agent/src/tests/test_network_monitor.rs @@ -182,7 +182,7 @@ async fn handle_version() -> impl IntoResponse { } async fn handler(uri: Uri) -> impl IntoResponse { - tracing::debug!("general handler: {:?}", uri); + tracing::debug!(?uri, "General request handler received request"); StatusCode::NOT_FOUND } @@ -323,7 +323,7 @@ impl Ping for MockPinger { dpu_info: DpuInfo, _interface: IpAddr, ) -> Result { - info!("Received ping request for {}", dpu_info); + info!(%dpu_info, "Received ping request"); let ping_result = DpuPingResult { dpu_info, success_count: 1, diff --git a/crates/agent/src/traffic_intercept_bridging.rs b/crates/agent/src/traffic_intercept_bridging.rs index 8060d0b729..dd6df0075a 100644 --- a/crates/agent/src/traffic_intercept_bridging.rs +++ b/crates/agent/src/traffic_intercept_bridging.rs @@ -98,7 +98,10 @@ pub async fn apply(sh_path: &super::FPath) -> eyre::Result<()> { Ok(()) } Err(err) => { - tracing::error!("update_intercept_bridging command failed: {err:#}"); + tracing::error!( + error = format!("{err:#}"), + "update_intercept_bridging command failed" + ); // If the config apply failed, we won't be using it, so move it out // of the way to an .error file for others to enjoy (while attempting @@ -108,8 +111,9 @@ pub async fn apply(sh_path: &super::FPath) -> eyre::Result<()> { && let Err(e) = fs::remove_file(path_error.clone()) { tracing::warn!( - "Failed to remove previous error file ({}): {e}", - path_error.display() + error_file_path = %path_error.display(), + error = %e, + "Failed to remove previous error file" ); } @@ -141,7 +145,10 @@ pub async fn run_apply(sh_path: &super::FPath) -> eyre::Result<()> { let mut cmd = tokio::process::Command::new("bash"); cmd.arg(sh_path.to_string()).kill_on_drop(true); let cmd_str = super::pretty_cmd(cmd.as_std()); - tracing::debug!("running intercept bridging commands: {cmd_str}"); + tracing::debug!( + command = cmd_str.as_str(), + "running intercept bridging commands" + ); let out = tokio::time::timeout(std::time::Duration::from_secs(3), cmd.output()) .await @@ -150,12 +157,10 @@ pub async fn run_apply(sh_path: &super::FPath) -> eyre::Result<()> { if !out.status.success() { tracing::error!( - " STDOUT {cmd_str}: {}", - String::from_utf8_lossy(&out.stdout) - ); - tracing::error!( - " STDERR {cmd_str}: {}", - String::from_utf8_lossy(&out.stderr) + command = cmd_str.as_str(), + stdout = %String::from_utf8_lossy(&out.stdout), + stderr = %String::from_utf8_lossy(&out.stderr), + "Intercept bridging command failed" ); let path_error = sh_path.with_ext("error"); diff --git a/crates/agent/src/upgrade.rs b/crates/agent/src/upgrade.rs index 16616df285..7df5dfae62 100644 --- a/crates/agent/src/upgrade.rs +++ b/crates/agent/src/upgrade.rs @@ -69,7 +69,10 @@ pub async fn upgrade( Err(err) => match err.downcast_ref::() { Some(grpc_status) if grpc_status.code() == tonic::Code::Internal => { // If something is wrong on the server wait for that to be fixed - tracing::error!("Internal server error, will not upgrade. {err:#}"); + tracing::error!( + error = ?err, + "Internal server error, will not upgrade." + ); UpgradeCheckResult { should_upgrade: false, ..Default::default() @@ -77,7 +80,10 @@ pub async fn upgrade( } _ => { // If something is broken in dpu-agent we need to replace it - tracing::error!("Failed upgrade check, forcing upgrade: {err:#}"); + tracing::error!( + error = ?err, + "Failed upgrade check, forcing upgrade" + ); UpgradeCheckResult { should_upgrade: true, ..Default::default() @@ -107,9 +113,10 @@ pub async fn upgrade( && let Err(err) = fs::rename(&binary_path, &backup) { tracing::warn!( - "Failed backing up current binary: 'mv {} {}', {err}", - binary_path.display(), - backup.display() + source = %binary_path.display(), + destination = %backup.display(), + error = %err, + "Failed backing up current binary" ); // keep going - if the rename fails we still want the upgrade } @@ -136,7 +143,7 @@ pub async fn upgrade( Ok(true) } Err(err) => { - tracing::error!(upgrade_cmd, err = format!("{err:#}"), "Upgrade failed"); + tracing::error!(upgrade_cmd, error = ?err, "Upgrade failed"); if override_upgrade_cmd.is_none() { fs::rename(backup, binary_path)?; } @@ -244,8 +251,14 @@ async fn run_upgrade_cmd(upgrade_cmd: &str) -> eyre::Result<()> { .wrap_err("Timeout")? .wrap_err("Error running command")?; if !out.status.success() { - tracing::error!(" STDOUT: {}", String::from_utf8_lossy(&out.stdout)); - tracing::error!(" STDERR: {}", String::from_utf8_lossy(&out.stderr)); + tracing::error!( + stdout = %String::from_utf8_lossy(&out.stdout), + "Upgrade command stdout" + ); + tracing::error!( + stderr = %String::from_utf8_lossy(&out.stderr), + "Upgrade command stderr" + ); eyre::bail!("Failed running upgrade command. Check logs for stdout/stderr."); } Ok(()) diff --git a/crates/api-core/src/attestation/measured_boot.rs b/crates/api-core/src/attestation/measured_boot.rs index 40ca382c1d..567ee0ea7d 100644 --- a/crates/api-core/src/attestation/measured_boot.rs +++ b/crates/api-core/src/attestation/measured_boot.rs @@ -199,7 +199,10 @@ pub fn cli_make_cred( "tpm2 makecredential -u {ek_file_path_str} -s {session_key_path_str} -n {ak_name_hex} -o {cred_out_path_str} -G rsa -V --tcti=none" ); - tracing::debug!("make credential command is {}", cmd_str); + tracing::debug!( + command = %cmd_str, + "make credential command", + ); // execute the makecredential command let output = Command::new("sh") .arg("-c") @@ -213,8 +216,8 @@ pub fn cli_make_cred( if !output.stderr.is_empty() { tracing::error!( - "tpm2 makecredential returned error: {}", - String::from_utf8_lossy(output.stderr.as_slice()) + error = %String::from_utf8_lossy(output.stderr.as_slice()), + "tpm2 makecredential returned error", ); } diff --git a/crates/api-core/src/attestation/tpm_ca_cert.rs b/crates/api-core/src/attestation/tpm_ca_cert.rs index 7fe9421d4b..be13936c42 100644 --- a/crates/api-core/src/attestation/tpm_ca_cert.rs +++ b/crates/api-core/src/attestation/tpm_ca_cert.rs @@ -77,18 +77,18 @@ pub async fn match_insert_new_ek_cert_status_against_ca( ca_id = Some(ca_cert_db_entry.id); } Err(e) => tracing::error!( - "Could not verify signature for EK cert serial - {}, issuer - {}, supposedly signed by CA subject - {}, error: {}", - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string(), - ca_cert.subject.to_string(), - e + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + subject = %ca_cert.subject.to_string(), + error = %e, + "Could not verify signature for EK certificate", ), } } None => tracing::info!( - "No CA cert found for EK cert: serial - {}, issuer - {}", - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string() + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + "No CA certificate found for EK certificate", ), } @@ -111,10 +111,10 @@ pub async fn match_insert_new_ek_cert_status_against_ca( .await?; tracing::info!( - "Set CA verification status to {} for EK serial - {}, issuer - {}", found_signing_ca, - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string() + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + "Set CA verification status for EK certificate", ); } else { // we must insert the new entry entirely @@ -154,10 +154,10 @@ pub async fn match_insert_new_ek_cert_status_against_ca( .await?; tracing::info!( - "Added new CA verification status for EK serial - {}, issuer - {}, status is {}", - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string(), - found_signing_ca + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + found_signing_ca, + "Added new CA verification status for EK certificate", ); } @@ -188,11 +188,11 @@ pub async fn match_update_existing_ek_cert_status_against_ca( // verify signature if let Err(e) = ek_cert.verify_signature(Some(ca_cert.public_key())) { tracing::error!( - "Could not verify signature for EK cert serial - {}, issuer - {}, supposedly signed by CA subject - {}, error: {}", - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string(), - ca_cert.subject.to_string(), - e + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + subject = %ca_cert.subject.to_string(), + error = %e, + "Could not verify signature for EK certificate", ); return Ok(false); // nothing more to do here } @@ -215,9 +215,9 @@ pub async fn match_update_existing_ek_cert_status_against_ca( })?; tracing::info!( - "Set CA verification status to true for EK serial - {}, issuer - {}", - ek_cert.raw_serial_as_string(), - ek_cert.issuer.to_string() + ek_certificate_serial = %ek_cert.raw_serial_as_string(), + issuer = %ek_cert.issuer.to_string(), + "Set CA verification status for EK certificate", ); Ok(true) diff --git a/crates/api-core/src/db_init.rs b/crates/api-core/src/db_init.rs index 3d5b1ce9c9..9462b64c47 100644 --- a/crates/api-core/src/db_init.rs +++ b/crates/api-core/src/db_init.rs @@ -55,8 +55,9 @@ pub async fn create_initial_domain( let names: Vec = domains.into_iter().map(|d| d.name).collect(); if !names.iter().any(|n| n == domain_name) { tracing::warn!( - "Initial domain name '{domain_name}' in config file does not match existing database domains: {:?}", - names + domain_name, + domains = ?names, + "Initial domain name in config file does not match existing database domains", ); } Ok(false) @@ -96,7 +97,10 @@ pub async fn create_initial_networks( // Network segments are only created the first time we start carbide-api; // `reconcile_network_defs` above has already recorded the snapshot if // it was missing (the backfill path). - tracing::debug!("Network segment {name} exists"); + tracing::debug!( + network_segment_name = %name, + "Network segment exists", + ); continue; } @@ -129,12 +133,19 @@ pub async fn create_initial_networks( // the id because `network_def.segment_id` is FK-bound to it. let segment_id = ns.id; // update_network_segments_svi_ip will take care of allocating svi ip. - tracing::info!("Creating network segment {name} from config: {ns:?}"); + tracing::info!( + network_segment_name = %name, + network_segment = ?ns, + "Creating network segment from config", + ); crate::handlers::network_segment::save(api, &mut txn, ns, true, false).await?; // Snapshot the network definition in the same transaction as the network_segment row, // so the two stay consistent across restarts. db::network_segment::insert_network_def(&mut txn, name, segment_id, def).await?; - tracing::info!("Created network segment {name}"); + tracing::info!( + network_segment_name = %name, + "Created network segment", + ); } ensure_static_assignments_segment(api, &mut txn, Some(domain_id)).await?; @@ -154,7 +165,10 @@ pub async fn create_initial_vpcs( .await .is_ok_and(|v| !v.is_empty()) { - tracing::debug!("VPC {name} exists"); + tracing::debug!( + vpc_name = %name, + "VPC exists", + ); continue; } @@ -194,7 +208,10 @@ pub async fn create_initial_vpcs( } db::vpc::persist(vpc, VpcStatus { vni: Some(vni) }, &mut txn).await?; - tracing::info!("Created VPC {name}"); + tracing::info!( + vpc_name = %name, + "Created VPC", + ); } txn.commit().await?; @@ -246,7 +263,10 @@ pub async fn ensure_static_assignments_segment( allocation_strategy: model::network_segment::AllocationStrategy::Reserved, }; crate::handlers::network_segment::save(api, txn, ns, true, false).await?; - tracing::info!("Created internal {segment_name} segment for holding static assignments"); + tracing::info!( + segment_name, + "Created internal segment for holding static assignments", + ); Ok(()) } @@ -311,8 +331,9 @@ pub async fn update_network_segments_svi_ip(db_pool: &Pool) -> Result< } Err(err) => { tracing::error!( - "Updating SVI IP filed for segment: {} - Error: {err}", - segment.id + network_segment_id = %segment.id, + error = %err, + "Failed to update SVI IP", ); txn.rollback().await?; } diff --git a/crates/api-core/src/dhcp/discover.rs b/crates/api-core/src/dhcp/discover.rs index 7ab95fba2c..68e8929bf0 100644 --- a/crates/api-core/src/dhcp/discover.rs +++ b/crates/api-core/src/dhcp/discover.rs @@ -419,9 +419,9 @@ async fn handle_dhcp_from_dpa( // Log cases where len is neither 0 nor 1. if !dpa_ifs.is_empty() { tracing::error!( - "handle_dpa_message - invalid dpa_ifs len from find_by_mac_addr maddr: {} len: {}", - macaddr, - dpa_ifs.len() + mac_address = %macaddr, + dpa_interface_count = dpa_ifs.len(), + "Unexpected number of DPA interfaces found", ); } return Ok(None); diff --git a/crates/api-core/src/dpa/handler.rs b/crates/api-core/src/dpa/handler.rs index ec9dc3596b..44b5023f8a 100644 --- a/crates/api-core/src/dpa/handler.rs +++ b/crates/api-core/src/dpa/handler.rs @@ -48,19 +48,20 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) let tokens: Vec<&str> = topic.split("/").collect(); if tokens.len() < 3 { tracing::error!( - "handle_dpa_message: token len {} is unusable topic: {}", - tokens.len(), - topic + token_count = tokens.len(), + topic = %topic, + "DPA MQTT topic has too few path segments", ); return; } let macaddr = match MacAddress::from_str(tokens[2]) { Ok(m) => m, - Err(_e) => { + Err(error) => { tracing::error!( - "handle_dpa_message: Unable to parse mac addr: {}", - tokens[2] + mac_address = tokens[2], + error = %error, + "Failed to parse DPA MAC address from MQTT topic", ); return; } @@ -68,8 +69,8 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) if message.metadata.is_none() || message.pf_info.is_none() { tracing::error!( - "handle_dpa_message: message metadata or pf_info is empty: {:#?}", - message + dpa_message = ?message, + "DPA message is missing metadata or PF info", ); return; } @@ -79,7 +80,10 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) let mut txn = match services.database_connection.begin().await { Ok(t) => t, Err(e) => { - tracing::error!("handle_dpa_message: Unable to start txn: {:#?}", e); + tracing::error!( + error = ?e, + "Failed to start DPA message database transaction", + ); return; } }; @@ -88,8 +92,9 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) Ok(ifs) => ifs, Err(e) => { tracing::error!( - "handle_dpa_message: Error for mac {macaddr} from find_by_mac_addr {:#?}", - e + mac_address = %macaddr, + error = ?e, + "Failed to find DPA interface", ); return; } @@ -97,8 +102,9 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) if dpa_ifs.len() != 1 { tracing::error!( - "handle_dpa_message: invalid dpa_ifs len from find_by_mac_addr maddr {macaddr} len {:#?}", - dpa_ifs.len() + mac_address = %macaddr, + dpa_interface_count = dpa_ifs.len(), + "Found an invalid DPA interface count", ); return; } @@ -110,9 +116,9 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) Ok(ncv) => ncv, Err(e) => { tracing::error!( - "handle_dpa_message: Error parsing config version from DPA Ack msg {:#?} {:#?}", - message, - e + dpa_message = ?message, + error = ?e, + "Failed to parse DPA acknowledgment config version", ); ConfigVersion::invalid() } @@ -135,15 +141,25 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) Ok(p) => p, Err(e) => { tracing::error!( - "handle_dpa_message: Error for vni {vni} from find_by_vni {:#?}", - e + vni = vni, + error = ?e, + "Failed to find SPX partition", ); return; } }; - if partition.len() != 1 { - tracing::error!("handle_dpa_message: SPX partition with vni {vni} is not found"); + if partition.is_empty() { + tracing::error!(vni, "SPX partition not found"); + return; + } + + if partition.len() > 1 { + tracing::error!( + vni, + partition_count = partition.len(), + "Multiple SPX partitions found", + ); return; } @@ -151,13 +167,14 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) spx_partition_id = spx_partition.id; tracing::debug!( - "handle_dpa_message: SPX partition with vni {vni} found: {:#?}", - spx_partition + vni = vni, + spx_partition = ?spx_partition, + "SPX partition found", ); } else { tracing::debug!( - "handle_dpa_message: received vni 0 in DPA message {:#?}", - message + dpa_message = ?message, + "DPA message has zero VNI", ); } @@ -186,15 +203,21 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) { Ok(m) => m, Err(e) => { - tracing::error!("handle_dpa_message: Error for machine {:#?}", e); + tracing::error!( + machine_id = %dpa_if.machine_id, + dpa_interface_id = %dpa_if.id, + error = ?e, + "Failed to find machine", + ); return; } }; if machine.is_none() { tracing::error!( - "handle_dpa_message: Machine not found for DPA interface {:#?}", - dpa_if + machine_id = %dpa_if.machine_id, + dpa_interface_id = %dpa_if.id, + "Machine not found", ); return; } @@ -236,17 +259,17 @@ async fn handle_dpa_message(services: Arc, message: SetVni, topic: String) let res = txn.commit().await; if res.is_err() { tracing::error!( - "handle_dpa_message: txn commit error for msg {:#?} res {:#?}", - message, - res + dpa_message = ?message, + commit_result = ?res, + "Failed to commit DPA message transaction", ); } } Err(e) => { tracing::error!( - "handle_dpa_message: update_network_observation error for msg {:#?} {:#?}", - message, - e + dpa_message = ?message, + error = ?e, + "Failed to update DPA network observation", ); } } @@ -306,7 +329,10 @@ pub async fn start_dpa_handler( }) .await { - tracing::error!("handle_dpa_message failed: {e}"); + tracing::error!( + error = %e, + "Failed to handle DPA message", + ); } } }) diff --git a/crates/api-core/src/dynamic_settings.rs b/crates/api-core/src/dynamic_settings.rs index 00a8c4e6eb..678cbb7a3b 100644 --- a/crates/api-core/src/dynamic_settings.rs +++ b/crates/api-core/src/dynamic_settings.rs @@ -72,7 +72,10 @@ impl DynamicSettings { } if let Err(err) = log_filter.reset_if_expired() { - tracing::error!("Failed resetting log level: {err}"); + tracing::error!( + error = %err, + "Failed resetting log level", + ); } } }) diff --git a/crates/api-core/src/handlers/api.rs b/crates/api-core/src/handlers/api.rs index 64d918aa0d..b8d5afd3db 100644 --- a/crates/api-core/src/handlers/api.rs +++ b/crates/api-core/src/handlers/api.rs @@ -106,9 +106,9 @@ pub(crate) fn set_dynamic_config( )) })?; tracing::info!( - "Log filter updated to '{}'; global log level: {}", - req.value, - tracing_subscriber::filter::LevelFilter::current() + log_filter = %req.value, + global_log_level = %tracing_subscriber::filter::LevelFilter::current(), + "Log filter updated", ); } rpc::ConfigSetting::CreateMachines => { @@ -121,7 +121,10 @@ pub(crate) fn set_dynamic_config( api.dynamic_settings .create_machines .store(is_enabled, Ordering::Relaxed); - tracing::info!("site-explorer create_machines updated to '{}'", req.value); + tracing::info!( + create_machines = is_enabled, + "site-explorer create_machines setting updated", + ); } rpc::ConfigSetting::SiteExplorerEnabled => { let is_enabled = req.value.parse::().map_err(|err| { @@ -133,7 +136,10 @@ pub(crate) fn set_dynamic_config( api.dynamic_settings .site_explorer_enabled .store(is_enabled, Ordering::Relaxed); - tracing::info!("site-explorer enabled updated to '{}'", req.value); + tracing::info!( + site_explorer_enabled = is_enabled, + "site-explorer enabled setting updated", + ); } rpc::ConfigSetting::BmcProxy => { let Some(true) = api.runtime_config.site_explorer.allow_changing_bmc_proxy else { @@ -157,7 +163,10 @@ pub(crate) fn set_dynamic_config( .bmc_proxy .store(Arc::new(Some(host_port_pair))); } - tracing::info!("site-explorer create_machines updated to '{}'", req.value); + tracing::info!( + bmc_proxy = %req.value, + "BMC proxy setting updated", + ); } rpc::ConfigSetting::TracingEnabled => { if !api.runtime_config.tracing.allow_runtime_changes { diff --git a/crates/api-core/src/handlers/astra.rs b/crates/api-core/src/handlers/astra.rs index 003da89767..146ae9aebf 100644 --- a/crates/api-core/src/handlers/astra.rs +++ b/crates/api-core/src/handlers/astra.rs @@ -71,8 +71,8 @@ pub(crate) async fn get_astra_config( if dpa_interfaces.is_empty() { tracing::info!( - "No Astra NICs found in host {:#?}, skipping Astra config retrieval", - snapshot.host_snapshot.id + machine_id = %snapshot.host_snapshot.id, + "No Astra NICs found; skipping Astra config retrieval", ); return Ok(None); } @@ -102,8 +102,8 @@ pub(crate) async fn get_astra_config( let Some(instance) = instance else { // If use_admin_network is false, we expect an instance to be associated with the host. tracing::error!( - "DPA interface {:#?} is not associated with an instance", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "DPA interface is not associated with an instance", ); continue; }; @@ -121,9 +121,9 @@ pub(crate) async fn get_astra_config( }) else { tracing::info!( - "SPX attachment {:#?} is not found in instance id {:#?}", - dpa_interface.mac_address, - instance.id + mac_address = %dpa_interface.mac_address, + instance_id = %instance.id, + "SPX attachment was not found", ); continue; }; @@ -136,13 +136,19 @@ pub(crate) async fn get_astra_config( ) .await?; if dpa_vni.is_empty() { - tracing::error!("SPX partition {:#?} is not found", spx_partition_id); + tracing::error!( + %spx_partition_id, + "SPX partition is not found", + ); continue; } let dpa_vni = dpa_vni[0].vni.unwrap_or(0); if dpa_vni == 0 { - tracing::error!("SPX partition {:#?} has no DPA VNI", spx_partition_id); + tracing::error!( + %spx_partition_id, + "SPX partition has no DPA VNI", + ); continue; } @@ -243,8 +249,8 @@ pub(crate) async fn process_astra_config_status( .and_then(|status| AstraPhase::try_from(status.phase).ok()) else { tracing::info!( - "Astra status {:#?} is not READY or DELETING, skipping Astra config status processing", - obs + astra_status_observation = ?obs, + "Astra status is not READY or DELETING, skipping Astra config status processing", ); continue; }; @@ -256,9 +262,9 @@ pub(crate) async fn process_astra_config_status( Ok(ncv) => ncv, Err(e) => { tracing::error!( - "process_astra_config_status: Error parsing config version from DPA Ack msg {:#?} {:#?}", - obs, - e + astra_status_observation = ?obs, + error = ?e, + "Failed to parse Astra DPA acknowledgment config version", ); ConfigVersion::invalid() } @@ -280,8 +286,9 @@ pub(crate) async fn process_astra_config_status( Ok(p) => p, Err(e) => { tracing::error!( - "process_astra_config_status: Error for vni {vni} from find_byi {:#?}", - e + vni, + error = ?e, + "Failed to find SPX partition", ); continue; } @@ -290,8 +297,9 @@ pub(crate) async fn process_astra_config_status( if partition.len() != 1 { // Given a VNI, we expect exactly one partition to be found. tracing::error!( - "process_astra_config_status: multiple SPX partitions with vni {vni} found, len: {:#?}", - partition.len() + vni, + partition_count = partition.len(), + "Unexpected number of SPX partitions found", ); continue; } @@ -300,13 +308,14 @@ pub(crate) async fn process_astra_config_status( spx_partition_id = spx_partition.id; tracing::debug!( - "process_astra_config_status: SPX partition with vni {vni} found: {:#?}", - spx_partition + vni, + spx_partition = ?spx_partition, + "Found SPX partition", ); } else { tracing::debug!( - "process_astra_config_status: received vni 0 in DPA message {:#?}", - obs + astra_status_observation = ?obs, + "Received VNI zero in Astra status observation", ); } @@ -321,9 +330,9 @@ pub(crate) async fn process_astra_config_status( .find(|dpa_interface| dpa_interface.mac_address == obs_mac) else { tracing::info!( - "DPA interface {:#?} is not found in host {:#?}", - obs.mac_address, - snapshot.host_snapshot.id + mac_address = %obs.mac_address, + machine_id = %snapshot.host_snapshot.id, + "DPA interface was not found", ); continue; }; diff --git a/crates/api-core/src/handlers/attestation.rs b/crates/api-core/src/handlers/attestation.rs index f9e7c86080..a8aba81470 100644 --- a/crates/api-core/src/handlers/attestation.rs +++ b/crates/api-core/src/handlers/attestation.rs @@ -344,8 +344,8 @@ pub(crate) async fn attest_quote( if attestation_failed { tracing::info!( - "Attestation failed for machine with id {} - not vending any certs", - machine_id + machine_id = %machine_id, + "Attestation failed; not vending any certificates", ); return Ok(Response::new(rpc::AttestQuoteResponse { success: false, @@ -364,9 +364,9 @@ pub(crate) async fn attest_quote( }; tracing::info!( - "Attestation succeeded for machine with id {} - sending a cert back. Attestion_enabled is {}", - machine_id, - api.runtime_config.attestation_enabled + machine_id = %machine_id, + attestation_enabled = api.runtime_config.attestation_enabled, + "Attestation succeeded; sending a certificate", ); Ok(Response::new(rpc::AttestQuoteResponse { success: true, diff --git a/crates/api-core/src/handlers/bmc_endpoint_explorer.rs b/crates/api-core/src/handlers/bmc_endpoint_explorer.rs index ea710ed652..40a1901116 100644 --- a/crates/api-core/src/handlers/bmc_endpoint_explorer.rs +++ b/crates/api-core/src/handlers/bmc_endpoint_explorer.rs @@ -202,9 +202,9 @@ pub(crate) async fn admin_bmc_reset( let endpoint_address = bmc_endpoint_request.ip_address.clone(); tracing::info!( - "Resetting BMC (ipmi tool: {}): {}", - req.use_ipmitool, - endpoint_address + use_ipmitool = req.use_ipmitool, + endpoint_address = %endpoint_address, + "Resetting BMC", ); if req.use_ipmitool { @@ -214,9 +214,9 @@ pub(crate) async fn admin_bmc_reset( } tracing::info!( - "BMC Reset (ipmi tool: {}) request succeeded to {}", - req.use_ipmitool, - endpoint_address + use_ipmitool = req.use_ipmitool, + endpoint_address = %endpoint_address, + "BMC reset request succeeded", ); Ok(Response::new(rpc::AdminBmcResetResponse {})) @@ -246,8 +246,8 @@ pub(crate) async fn disable_secure_boot( let endpoint_address = bmc_endpoint_request.ip_address.clone(); tracing::info!( - "disable_secure_boot request succeeded to {}", - endpoint_address + endpoint_address = %endpoint_address, + "Disable secure boot request succeeded", ); Ok(Response::new(rpc::DisableSecureBootResponse {})) @@ -286,9 +286,9 @@ pub(crate) async fn lockdown( let endpoint_address = bmc_endpoint_request.ip_address.clone(); tracing::info!( - "lockdown {} request succeeded to {}", - action.to_string().to_lowercase(), - endpoint_address + action = %action.to_string().to_lowercase(), + endpoint_address = %endpoint_address, + "lockdown request succeeded", ); Ok(Response::new(rpc::LockdownResponse {})) @@ -356,8 +356,8 @@ pub(crate) async fn enable_infinite_boot( let endpoint_address = bmc_endpoint_request.ip_address.clone(); tracing::info!( - "enable_infinite_boot request succeeded to {}", - endpoint_address + endpoint_address = %endpoint_address, + "Enable infinite boot request succeeded", ); Ok(Response::new(rpc::EnableInfiniteBootResponse {})) @@ -395,9 +395,9 @@ pub(crate) async fn is_infinite_boot_enabled( .map_err(|e| CarbideError::internal(e.to_string()))?; tracing::info!( - "is_infinite_boot_enabled request succeeded to {}, result: {:?}", - bmc_endpoint_request.ip_address, - is_enabled + endpoint_address = %bmc_endpoint_request.ip_address, + is_enabled, + "Infinite boot status request succeeded", ); Ok(Response::new(rpc::IsInfiniteBootEnabledResponse { @@ -430,7 +430,10 @@ pub(crate) async fn machine_setup( let endpoint_address = &bmc_endpoint_request.ip_address; - tracing::info!("Starting Machine Setup for BMC: {}", endpoint_address); + tracing::info!( + endpoint_address = %endpoint_address, + "Starting machine setup", + ); let (bmc_addr, bmc_mac_address) = resolve_bmc_interface(api, &bmc_endpoint_request).await?; let machine_interface = MachineInterfaceSnapshot::mock_with_mac(bmc_mac_address); @@ -456,7 +459,10 @@ pub(crate) async fn machine_setup( .await .map_err(|e| CarbideError::internal(e.to_string()))?; - tracing::info!("Machine Setup request succeeded to {}", endpoint_address); + tracing::info!( + endpoint_address = %endpoint_address, + "Machine setup request succeeded", + ); Ok(Response::new(rpc::MachineSetupResponse {})) } @@ -487,8 +493,8 @@ pub(crate) async fn set_dpu_first_boot_order( let endpoint_address = &bmc_endpoint_request.ip_address; tracing::info!( - "Setting DPU first in boot order for BMC: {}", - endpoint_address + endpoint_address = %endpoint_address, + "Setting DPU first in boot order", ); let entered_mac = req @@ -524,8 +530,8 @@ pub(crate) async fn set_dpu_first_boot_order( .map_err(|e| CarbideError::internal(e.to_string()))?; tracing::info!( - "Set DPU first in boot order request succeeded to {}", - endpoint_address + endpoint_address = %endpoint_address, + "Set DPU first in boot order request succeeded", ); Ok(Response::new(rpc::SetDpuFirstBootOrderResponse {})) @@ -927,8 +933,10 @@ pub(crate) async fn create_bmc_user( }; tracing::info!( - "Creating BMC User {} ({role}) on {endpoint_address}", - req.create_username, + username = %req.create_username, + role = %role, + endpoint_address = %endpoint_address, + "Creating BMC user", ); do_create_bmc_user( @@ -941,8 +949,10 @@ pub(crate) async fn create_bmc_user( .await?; tracing::info!( - "Successfully created BMC User {} ({role}) on {endpoint_address}", - req.create_username + username = %req.create_username, + role = %role, + endpoint_address = %endpoint_address, + "Successfully created BMC user", ); Ok(Response::new(rpc::CreateBmcUserResponse {})) @@ -972,15 +982,17 @@ pub(crate) async fn delete_bmc_user( let endpoint_address = &bmc_endpoint_request.ip_address; tracing::info!( - "Deleting BMC User {} on {endpoint_address}", - req.delete_username, + username = %req.delete_username, + endpoint_address = %endpoint_address, + "Deleting BMC user", ); do_delete_bmc_user(api, &bmc_endpoint_request, &req.delete_username).await?; tracing::info!( - "Successfully deleted BMC User {} on {endpoint_address}", - req.delete_username + username = %req.delete_username, + endpoint_address = %endpoint_address, + "Successfully deleted BMC user", ); Ok(Response::new(rpc::DeleteBmcUserResponse {})) diff --git a/crates/api-core/src/handlers/component_manager.rs b/crates/api-core/src/handlers/component_manager.rs index 2ce59fbadc..4472468f81 100644 --- a/crates/api-core/src/handlers/component_manager.rs +++ b/crates/api-core/src/handlers/component_manager.rs @@ -1745,7 +1745,8 @@ async fn power_control_health_override( tracing::warn!( %machine_id, error = %e, - "failed to {action} health report override for power control" + action, + "Failed to apply health report override for power control", ); } diff --git a/crates/api-core/src/handlers/dns.rs b/crates/api-core/src/handlers/dns.rs index 6b920d658e..74b8046293 100644 --- a/crates/api-core/src/handlers/dns.rs +++ b/crates/api-core/src/handlers/dns.rs @@ -40,7 +40,7 @@ async fn lookup_soa_record( db: impl DbReader<'_>, query_name: &str, ) -> Result { - tracing::debug!("Looking up SOA record for {}", query_name); + tracing::debug!(query_name, "Looking up SOA record",); let record = resource_record::get_soa_record(db, query_name) .await .map_err(CarbideError::from)? @@ -64,7 +64,7 @@ async fn lookup_records_by_qname( txn: impl DbReader<'_>, query_name: &str, ) -> Result, tonic::Status> { - tracing::debug!("Looking up records for {}", query_name); + tracing::debug!(query_name, "Looking up DNS records",); // dns_records view expects trailing dots (FQDN format) let qname_with_dot = if !query_name.ends_with('.') { diff --git a/crates/api-core/src/handlers/dpu_remediation.rs b/crates/api-core/src/handlers/dpu_remediation.rs index 7c7a00f686..cf3d647ecd 100644 --- a/crates/api-core/src/handlers/dpu_remediation.rs +++ b/crates/api-core/src/handlers/dpu_remediation.rs @@ -37,7 +37,10 @@ pub fn external_user_name(request: &Request) -> Result { tracing::error!( - "Found {} MachineInterface entries for MAC address {mac}. Should be impossible", - interfaces.len() + interface_count = interfaces.len(), + mac_address = %mac, + "Found machine interface entries; should be impossible", ); } Err(err) => { @@ -545,11 +546,16 @@ async fn by_mac( [iface] => return Ok(Some((iface.id.to_string(), rpc::MacOwner::DpaInterface))), [] => {} // expected, continue to search other object types _ => tracing::error!( - "Found {} DpaInterfaces entries for MAC address {mac}. Should be impossible", - ifs.len() + dpa_interface_count = ifs.len(), + mac_address = %mac, + "Found DPA interface entries; should be impossible", ), }, - Err(e) => tracing::error!("by_mac - Error from find_by_mac_addr for DPA: {e}"), + Err(e) => tracing::error!( + mac_address = %mac, + error = %e, + "Failed to find DPA interface", + ), }; // Any other MAC addresses to search? diff --git a/crates/api-core/src/handlers/firmware.rs b/crates/api-core/src/handlers/firmware.rs index 5e40e23592..cf6bf027f8 100644 --- a/crates/api-core/src/handlers/firmware.rs +++ b/crates/api-core/src/handlers/firmware.rs @@ -59,10 +59,10 @@ pub(crate) async fn set_firmware_update_time_window( let mut txn = api.txn_begin().await?; tracing::info!( - "set_firmware_update_time_window: Setting update start/end ({:?} {:?}) for {:?}", - chrono::Utc.timestamp_opt(start, 0), - chrono::Utc.timestamp_opt(end, 0), - request.machine_ids + start_time = ?chrono::Utc.timestamp_opt(start, 0), + end_time = ?chrono::Utc.timestamp_opt(end, 0), + machine_ids = ?request.machine_ids, + "Setting firmware update time window", ); db::machine::update_firmware_update_time_window_start_end( diff --git a/crates/api-core/src/handlers/instance.rs b/crates/api-core/src/handlers/instance.rs index ee0a7a01c6..ff808e329b 100644 --- a/crates/api-core/src/handlers/instance.rs +++ b/crates/api-core/src/handlers/instance.rs @@ -1027,7 +1027,11 @@ pub(crate) async fn invoke_power( .await .map_err(|err| { // print actual error for debugging, but don't leak internal info to user. - tracing::error!(machine=%machine_id, "{:?}", err); + tracing::error!( + machine_id = %machine_id, + error = ?err, + "failed to approve DPU reprovision request", + ); // TODO: What does this error actually mean CarbideError::internal( @@ -1042,7 +1046,11 @@ pub(crate) async fn invoke_power( .await .map_err(|err| { // print actual error for debugging, but don't leak internal info to user. - tracing::error!(machine=%machine_id, "{:?}", err); + tracing::error!( + machine_id = %machine_id, + error = ?err, + "failed to approve host reprovision request", + ); CarbideError::internal( "Internal Failure. Try again after some time.".to_string(), @@ -1184,7 +1192,10 @@ pub(crate) async fn update_instance_config( Some(config) => config.try_into().map_err(CarbideError::from)?, }; - tracing::info!("SPX update_instance_config config: {:?}", config.spxconfig); + tracing::info!( + spx_config = ?config.spxconfig, + "Updating instance SPX config", + ); // Network validation is done only if network update is requested. config @@ -1370,16 +1381,16 @@ pub(crate) async fn update_instance_config( .await?; tracing::debug!( - "Updating instance {} with NVLink config {:?}", - instance.id, - config.nvlink + instance_id = %instance.id, + nvlink = ?config.nvlink, + "Updating instance NVLink configuration", ); update_instance_nvlink_config(&mh_snapshot, &instance, &config.nvlink, &mut txn).await?; tracing::debug!( - "Updating instance {} with Spx config {:?}", - instance.id, - config.spxconfig + instance_id = %instance.id, + spx_config = ?config.spxconfig, + "Updating instance SPX configuration", ); update_instance_spx_config(&mh_snapshot, &instance, &mut config.spxconfig, &mut txn).await?; diff --git a/crates/api-core/src/handlers/instance_type.rs b/crates/api-core/src/handlers/instance_type.rs index c608ae68f6..53a71a8f62 100644 --- a/crates/api-core/src/handlers/instance_type.rs +++ b/crates/api-core/src/handlers/instance_type.rs @@ -596,7 +596,9 @@ pub(crate) async fn associate_machines( if ids.len() != machine_versions.len() { tracing::error!( - "Not all machine's instances updated. This should never happen because we take row-lock. Something is terribly wrong. ids: {ids:?}; versions: {machine_versions:?}" + ids = ?ids, + machine_versions = ?machine_versions, + "Not all machine's instances updated. This should never happen because we take row-lock. Something is terribly wrong.", ) } diff --git a/crates/api-core/src/handlers/machine.rs b/crates/api-core/src/handlers/machine.rs index d717eb51d5..cda4c900ec 100644 --- a/crates/api-core/src/handlers/machine.rs +++ b/crates/api-core/src/handlers/machine.rs @@ -23,7 +23,6 @@ use ::rpc::model::machine::ManagedHostStateSnapshotRpc; use carbide_redfish::libredfish::RedfishAuth; use carbide_secrets::credentials::{BmcCredentialType, CredentialKey}; use carbide_uuid::machine::MachineId; -use itertools::Itertools; use libredfish::SystemPowerControl; use model::hardware_info::MachineNvLinkInfo; use model::machine::machine_search_config::MachineSearchConfig; @@ -345,8 +344,11 @@ pub(crate) async fn admin_force_delete_machine( .unwrap_or("unknown"); tracing::info!( - "admin_force_delete_machine query='{query}' machine_id={} serial='{serial}' issued_by={issued_by:?}", - machine.id + query = %query, + machine_id = %machine.id, + serial, + issued_by = ?issued_by, + "Admin force-delete machine request", ); if machine.instance_type_id.is_some() { @@ -363,7 +365,11 @@ pub(crate) async fn admin_force_delete_machine( let dpu_machines; if machine.is_dpu() { if let Some(host) = db::machine::find_host_by_dpu_machine_id(&mut txn, &machine.id).await? { - tracing::info!("Found host Machine {:?}", machine.id.to_string()); + tracing::info!( + host_machine_id = %host.id, + dpu_machine_id = %machine.id, + "Found host machine", + ); // Get all DPUs attached to this host, in case there are more than one. dpu_machines = db::machine::find_dpus_by_host_machine_id(&mut txn, &host.id).await?; host_machine = Some(host); @@ -374,8 +380,8 @@ pub(crate) async fn admin_force_delete_machine( } else { dpu_machines = db::machine::find_dpus_by_host_machine_id(&mut txn, &machine.id).await?; tracing::info!( - "Found dpu Machines {:?}", - dpu_machines.iter().map(|m| m.id.to_string()).join(", ") + dpu_machine_ids = ?dpu_machines.iter().map(|m| &m.id).collect::>(), + "Found DPU machines", ); host_machine = Some(machine); } @@ -584,14 +590,14 @@ pub(crate) async fn admin_force_delete_machine( } } else { tracing::warn!( - "Failed to unlock this host because Forge could not retrieve the BMC MAC address for machine {}", - machine.id + machine_id = %machine.id, + "Failed to unlock host because the BMC MAC address is missing", ); } } else { tracing::warn!( - "Failed to unlock this host because Forge could not retrieve the BMC IP address for machine {}", - machine.id + machine_id = %machine.id, + "Failed to unlock host because the BMC IP address is missing", ); } @@ -637,7 +643,11 @@ pub(crate) async fn admin_force_delete_machine( if let Some(machine) = &host_machine && let Some(addr) = machine.bmc_info.ip { - tracing::info!("Cleaning up explored endpoint at {addr} {}", machine.id); + tracing::info!( + endpoint_address = %addr, + machine_id = %machine.id, + "Cleaning up explored endpoint", + ); // If this delete waited out a concurrent exploration rewrite, its // statement snapshot can miss the row that rewrite re-inserted; the @@ -649,7 +659,11 @@ pub(crate) async fn admin_force_delete_machine( } for dpu_machine in dpu_machines.iter() { if let Some(addr) = dpu_machine.bmc_info.ip { - tracing::info!("Cleaning up explored endpoint at {addr} {}", dpu_machine.id); + tracing::info!( + endpoint_address = %addr, + machine_id = %dpu_machine.id, + "Cleaning up explored endpoint", + ); db::explored_endpoints::delete(&mut txn, addr).await?; } @@ -692,9 +706,9 @@ pub(crate) async fn admin_force_delete_machine( { // just log the error and carry on tracing::error!( - "Could not remove EK cert status for machine with id {}: {}", - machine.id, - e + machine_id = %machine.id, + error = %e, + "Could not remove EK certificate status", ); } } @@ -818,9 +832,9 @@ fn snapshot_map_to_rpc_machines( async fn clear_bmc_credentials(api: &Api, machine: &Machine) -> Result<(), CarbideError> { if let Some(mac_address) = machine.bmc_info.mac { tracing::info!( - "Cleaning up BMC credentials in vault at {} for machine {}", - mac_address, - machine.id + mac_address = %mac_address, + machine_id = %machine.id, + "Cleaning up BMC credentials in vault", ); crate::handlers::credential::delete_bmc_root_credentials_by_mac(api, mac_address).await?; } @@ -875,16 +889,16 @@ pub async fn get_machine_position_info( .filter_map(|(machine_id, ip_opt)| match ip_opt { Some(ip_str) => ip_str.parse().ok().or_else(|| { tracing::warn!( - "Failed to parse BMC IP '{}' for machine {}", - ip_str, - machine_id + bmc_ip_address = %ip_str, + machine_id = %machine_id, + "Failed to parse BMC IP", ); None }), None => { tracing::warn!( - "Machine {} has topology but no BMC IP configured", - machine_id + machine_id = %machine_id, + "Machine has topology but no BMC IP configured", ); None } diff --git a/crates/api-core/src/handlers/machine_discovery.rs b/crates/api-core/src/handlers/machine_discovery.rs index e0cff00ef2..6b68a04810 100644 --- a/crates/api-core/src/handlers/machine_discovery.rs +++ b/crates/api-core/src/handlers/machine_discovery.rs @@ -417,10 +417,10 @@ pub(crate) async fn discover_machine( ) } else { tracing::info!( - "Attestation enabled is {}. Is_DPU is {}. Vending certs to machine with id {}", - api.runtime_config.attestation_enabled, - hardware_info.is_dpu(), - stable_machine_id, + attestation_enabled = api.runtime_config.attestation_enabled, + is_dpu = hardware_info.is_dpu(), + %stable_machine_id, + "Vending attestation certificates", ); None diff --git a/crates/api-core/src/handlers/machine_scout.rs b/crates/api-core/src/handlers/machine_scout.rs index 423f682b47..eda69f792f 100644 --- a/crates/api-core/src/handlers/machine_scout.rs +++ b/crates/api-core/src/handlers/machine_scout.rs @@ -208,12 +208,12 @@ pub(crate) async fn forge_agent_control( }, } => { tracing::info!( - " context : {} id: {} is_enabled: {}, completed {}, total {}", - context, - id, - is_enabled, - completed, - total, + context = %context, + machine_validation_id = %id, + is_enabled = *is_enabled, + completed = *completed, + total = *total, + "Machine validation progress reported by scout", ); if *is_enabled { db::machine_validation::update_status( @@ -289,9 +289,9 @@ pub(crate) async fn forge_agent_control( let last_cleanup_time = host_machine.last_cleanup_time; let state_version = host_machine.state.version; tracing::info!( - "last_cleanup_time: {:?}, state_version: {:?}", - last_cleanup_time, - state_version + last_cleanup_time = ?last_cleanup_time, + state_version = ?state_version, + "Checking whether machine cleanup is current", ); // Check scout has already cleaned up the machine if last_cleanup_time.unwrap_or_default() > state_version.timestamp() { @@ -305,9 +305,9 @@ pub(crate) async fn forge_agent_control( bom_validating_state: BomValidating::UpdatingInventory(_), } => { tracing::info!( - "Request Discovery {} < {}", - machine.last_discovery_time.unwrap_or_default(), - machine.current_version().timestamp() + last_discovery_time = %machine.last_discovery_time.unwrap_or_default(), + current_version_time = %machine.current_version().timestamp(), + "Checking whether machine discovery is stale", ); if machine.last_discovery_time.unwrap_or_default() < machine.current_version().timestamp() @@ -325,7 +325,11 @@ pub(crate) async fn forge_agent_control( match crate::handlers::svpc::process_scout_req(api, machine_id).await { Ok(action) => (action, None), Err(e) => { - tracing::error!("Error returned from process_scout_req: {e}"); + tracing::error!( + machine_id = %machine_id, + error = %e, + "Failed to process Scout request", + ); (Action::noop(), None) } } @@ -349,7 +353,8 @@ pub(crate) async fn forge_agent_control( Ok(task) => Action::FirmwareUpgrade(fac::FirmwareUpgrade { task: Some(task) }), Err(e) => { tracing::warn!( - "Could not deserialize firmware upgrade task, sending no-op action to scout: {e}" + error = %e, + "Could not deserialize firmware upgrade task, sending no-op action to scout", ); Action::noop() } diff --git a/crates/api-core/src/handlers/machine_validation.rs b/crates/api-core/src/handlers/machine_validation.rs index a79d223dc5..2ef5b5a61b 100644 --- a/crates/api-core/src/handlers/machine_validation.rs +++ b/crates/api-core/src/handlers/machine_validation.rs @@ -284,13 +284,19 @@ pub(crate) async fn persist_validation_result( ManagedHostState::Validation { validation_state } => { match validation_state { ValidationState::MachineValidation { .. } => { - tracing::info!("machine state is {}", machine.current_state()); + tracing::info!( + machine_state = %machine.current_state(), + "Machine is in validation state", + ); //Continue to persist data } } } _ => { - tracing::error!("invalid host machine state {}", machine.current_state()); + tracing::error!( + machine_state = %machine.current_state(), + "invalid host machine state", + ); return Err( CarbideError::InvalidArgument("wrong host machine state".to_string()).into(), ); @@ -1030,9 +1036,9 @@ pub async fn apply_config_on_startup( for test_config in &config.tests { if let Some(test) = tests.iter().find(|t| t.test_id == test_config.id) { tracing::info!( - "Updating test '{}' to state {} from config", - test.test_id, - test_config.enable + test_id = %test.test_id, + enable = test_config.enable, + "Updating test to state from config", ); machine_validation_suites::enable_disable( @@ -1065,9 +1071,9 @@ pub async fn apply_config_on_startup( }; tracing::info!( - "Setting test '{}' to state {} (EnableAll mode)", - test.test_id, - enable_state + test_id = %test.test_id, + enable_state, + "Setting test to state (EnableAll mode)", ); machine_validation_suites::enable_disable( @@ -1099,9 +1105,9 @@ pub async fn apply_config_on_startup( }; tracing::info!( - "Setting test '{}' to state {} (DisableAll mode)", - test.test_id, - enable_state + test_id = %test.test_id, + enable_state, + "Setting test to state (DisableAll mode)", ); machine_validation_suites::enable_disable( diff --git a/crates/api-core/src/handlers/rack.rs b/crates/api-core/src/handlers/rack.rs index da3bacbe1d..8e9194dbae 100644 --- a/crates/api-core/src/handlers/rack.rs +++ b/crates/api-core/src/handlers/rack.rs @@ -821,7 +821,10 @@ pub(crate) async fn on_demand_rack_maintenance( return Err(status); } - tracing::info!("On-demand maintenance scheduled for rack {}", rack_id,); + tracing::info!( + rack_id = %rack_id, + "On-demand maintenance scheduled", + ); Ok(Response::new(rpc::RackMaintenanceOnDemandResponse {})) } diff --git a/crates/api-core/src/handlers/redfish.rs b/crates/api-core/src/handlers/redfish.rs index 78b0dfd256..64f991b2b0 100644 --- a/crates/api-core/src/handlers/redfish.rs +++ b/crates/api-core/src/handlers/redfish.rs @@ -271,7 +271,12 @@ pub async fn redfish_apply_action( // Enclosing function may have returned. Nowhere to return error to. update_response_in_tx(&pool, request, index, response) .await - .inspect_err(|e| tracing::error!("Error applying redfish action: {e}")) + .inspect_err(|e| { + tracing::error!( + error = %e, + "Error applying redfish action", + ) + }) .ok(); } }); @@ -313,7 +318,11 @@ async fn handle_request( (Err(error), _) | (_, Some(error)) => { // Make a UUID for easy log correlation let failure_uuid = Uuid::new_v4(); - tracing::error!("Redfish client creation failure {failure_uuid}: {error}"); + tracing::error!( + failure_uuid = %failure_uuid, + error = %error, + "Redfish client creation failure", + ); // Set the "response" to indicate we couldn't get a redfish client. Don't // leak error string in case of credentials/etc. diff --git a/crates/api-core/src/handlers/scout_stream.rs b/crates/api-core/src/handlers/scout_stream.rs index 2adb4b61bb..1184a6af5e 100644 --- a/crates/api-core/src/handlers/scout_stream.rs +++ b/crates/api-core/src/handlers/scout_stream.rs @@ -57,7 +57,10 @@ pub(crate) async fn scout_stream( } }; - tracing::info!("scout agent connected for machine: {machine_id}"); + tracing::info!( + machine_id = %machine_id, + "Scout agent connected", + ); // Now we create channels for bidirectional communication. The API // will receive on one side, process whatever is packed into the oneof field @@ -84,7 +87,10 @@ pub(crate) async fn scout_stream( // If/when the connection breaks, unregister the scout // agent connection from the connection registry. - tracing::info!("scout agent disconnected for machine: {machine_id}"); + tracing::info!( + machine_id = %machine_id, + "Scout agent disconnected", + ); registry_clone.unregister(machine_id).await; }); diff --git a/crates/api-core/src/handlers/site_explorer.rs b/crates/api-core/src/handlers/site_explorer.rs index 3839401549..214f04cfe8 100644 --- a/crates/api-core/src/handlers/site_explorer.rs +++ b/crates/api-core/src/handlers/site_explorer.rs @@ -257,7 +257,10 @@ pub(crate) async fn clear_site_exploration_error( // actually retries preingestion instead of requiring a force-delete of the // endpoint. Non-failed states are left untouched. if db::explored_endpoints::reset_failed_preingestion(bmc_ip, &mut txn).await? { - tracing::info!("Reset failed preingestion to initial for {bmc_ip} on error clear"); + tracing::info!( + %bmc_ip, + "Reset failed preingestion to initial after clearing the site exploration error", + ); } txn.commit().await?; diff --git a/crates/api-core/src/handlers/svpc.rs b/crates/api-core/src/handlers/svpc.rs index 2e3495c9d3..a8a16e7b83 100644 --- a/crates/api-core/src/handlers/svpc.rs +++ b/crates/api-core/src/handlers/svpc.rs @@ -67,8 +67,8 @@ pub(crate) async fn process_scout_req( if dpa_snapshots.is_empty() { tracing::error!( - "process_scout_req no dpa_snapshots for machine: {:#?}", - machine_id + %machine_id, + "No DPA snapshots found", ); return Ok(fac::Action::noop()); } @@ -113,7 +113,10 @@ pub(crate) async fn process_scout_req( Ok(action) => device_actions.push(action), Err(e) => { // Would only happen if the op is an ApplyProfile command with invalid YAML - tracing::error!("process_scout_req Error encoding DpaCommand for dpa: {e}"); + tracing::error!( + error = %e, + "Failed to encode DPA command", + ); } } } @@ -355,7 +358,7 @@ async fn process_mlx_observation( let req = request.into_inner(); let Some(rep) = req.report else { - tracing::error!("process_mlx_observation without report req: {:#?}", req); + tracing::error!("MLX observation request is missing its report"); return Err(CarbideError::GenericErrorFromReport(eyre!( "process_mlx_observation without report req: {:#?}", req @@ -364,8 +367,8 @@ async fn process_mlx_observation( let Some(machine_id) = rep.machine_id else { tracing::error!( - "process_mlx_observation without machine_id report: {:#?}", - rep + observation_count = rep.observations.len(), + "MLX device report is missing its machine ID", ); return Err(CarbideError::GenericErrorFromReport(eyre!( "process_mlx_observation without machine_id report: {:#?}", @@ -383,8 +386,8 @@ async fn process_mlx_observation( if dpa_snapshots.is_empty() { tracing::error!( - "process_mlx_observation no dpa snapshots for machine: {:#?}", - machine_id + %machine_id, + "No DPA snapshots found", ); return Err(CarbideError::GenericErrorFromReport(eyre!( "process_mlx_observation no dpa snapshots for machine: {:#?}", @@ -394,8 +397,9 @@ async fn process_mlx_observation( if rep.observations.is_empty() { tracing::error!( - "process_mlx_observation no observations in report: {:#?}", - rep + %machine_id, + observation_count = rep.observations.len(), + "MLX device report contains no observations", ); return Err(CarbideError::GenericErrorFromReport(eyre!( "process_mlx_observation no observations in report: {:#?}", @@ -406,8 +410,8 @@ async fn process_mlx_observation( for obs in rep.observations { let Some(devinfo) = obs.device_info else { tracing::error!( - "process_mlx_observation no device_info observation: {:#?}", - obs + %machine_id, + "MLX device observation contains no device info", ); continue; }; @@ -416,9 +420,10 @@ async fn process_mlx_observation( Ok(dpa) => dpa, Err(e) => { tracing::error!( - "process_mlx_observation dpa not found for device {:#?} error: {:#?}", - devinfo, - e + pci_name = %devinfo.pci_name, + mac_address = %devinfo.base_mac, + error = %e, + "DPA interface not found", ); continue; } @@ -426,8 +431,11 @@ async fn process_mlx_observation( if dpa.interface_type != DpaInterfaceType::Svpc { tracing::error!( - "process_mlx_observation dpa interface type is not Svpc, skipping: {:#?}", - dpa + dpa_interface_id = %dpa.id, + %machine_id, + pci_name = %dpa.pci_name, + interface_type = ?dpa.interface_type, + "DPA interface is not an SVPC interface; skipping", ); continue; } @@ -446,7 +454,10 @@ async fn process_mlx_observation( let ls = match DpaLockMode::try_from(lock_status) { Ok(ls) => ls, Err(e) => { - tracing::error!("process_mlx_observation Error from LockStatus::try_from {e}"); + tracing::error!( + error = %e, + "Failed to convert DPA lock status", + ); continue; } }; @@ -474,7 +485,10 @@ async fn process_mlx_observation( match dpa_interface::update_card_state(&mut txn, dpa).await { Ok(_id) => (), Err(e) => { - tracing::error!("process_mlx_observation update_card_state error: {e}"); + tracing::error!( + error = %e, + "Failed to update DPA card state", + ); } } } @@ -503,9 +517,9 @@ pub(crate) async fn publish_mlx_device_report( .try_into() .map_err(|e: String| CarbideError::Internal { message: e })?; tracing::info!( - "received MlxDeviceReport hostname={} device_count={}", - report.hostname, - report.devices.len(), + hostname = %report.hostname, + device_count = report.devices.len(), + "received MlxDeviceReport", ); // Without a machine_id, we can't create dpa interfaces @@ -620,11 +634,16 @@ pub(crate) async fn publish_mlx_device_report( } tracing::info!( - "spx nics count: {spx_nics} machine_id: {:#?}", - report.machine_id + spx_nic_count = spx_nics, + %machine_id, + "counted SPX NICs", ); } else { - tracing::warn!("MlxDeviceReport without machine_id: {:#?}", report); + tracing::warn!( + hostname = %report.hostname, + device_count = report.devices.len(), + "MLX device report is missing its machine ID", + ); } } else { tracing::warn!("no embedded MlxDeviceReport published"); diff --git a/crates/api-core/src/handlers/uefi.rs b/crates/api-core/src/handlers/uefi.rs index 788c3bdec6..86c3d379b6 100644 --- a/crates/api-core/src/handlers/uefi.rs +++ b/crates/api-core/src/handlers/uefi.rs @@ -241,7 +241,10 @@ pub(crate) async fn clear_host_uefi_password( .client_by_info(&bmc_access_info) .await .map_err(|e| { - tracing::error!("unable to create redfish client: {}", e); + tracing::error!( + error = %e, + "unable to create redfish client", + ); CarbideError::Internal { message: format!( "Could not create connection to Redfish API to {machine_id}, check logs" @@ -350,7 +353,10 @@ pub(crate) async fn set_host_uefi_password( .client_by_info(&bmc_access_info) .await .map_err(|e| { - tracing::error!("unable to create redfish client: {}", e); + tracing::error!( + error = %e, + "unable to create redfish client", + ); CarbideError::RedfishClientCreation { inner: e.into(), machine_id, @@ -392,7 +398,10 @@ pub(crate) async fn set_host_uefi_password( }) .await? .map_err(|e| { - tracing::error!("Failed to update bios_password_set_time: {}", e); + tracing::error!( + error = %e, + "Failed to update bios_password_set_time", + ); CarbideError::Internal { message: format!("Failed to update BIOS password timestamp: {e}"), } diff --git a/crates/api-core/src/instance/mod.rs b/crates/api-core/src/instance/mod.rs index f9000d29a3..1daae87e26 100644 --- a/crates/api-core/src/instance/mod.rs +++ b/crates/api-core/src/instance/mod.rs @@ -443,9 +443,9 @@ pub fn allocate_ib_port_guid( let mut guids: Vec = Vec::new(); for request in &mut updated_ib_config.ib_interfaces { tracing::debug!( - "request IB device:{}, device_instance:{}", - request.device.clone(), - request.device_instance + device = %request.device, + device_instance = request.device_instance, + "Requested InfiniBand device", ); // TOTO: will support VF in the future. Currently, it will return err when the function_id is not PF. @@ -461,7 +461,10 @@ pub fn allocate_ib_port_guid( request.pf_guid = Some(ib.guid.clone()); request.guid = Some(ib.guid.clone()); guids.push(ib.guid.clone()); - tracing::debug!("select IB device GUID {}", ib.guid.clone()); + tracing::debug!( + ib_guid = %ib.guid, + "select IB device GUID", + ); } else { return Err(CarbideError::InvalidArgument(format!( "not enough ib device {} (machine {})", @@ -732,7 +735,11 @@ pub async fn batch_allocate_instances( })?; if let Err(e) = mh_snapshot.is_usable_as_instance(request.allow_unhealthy_machine) { - tracing::error!(%machine_id, "Host can not be used as instance due to reason: {}", e); + tracing::error!( + %machine_id, + error = %e, + "Host can not be used as instance due to reason", + ); return Err(match e { NotAllocatableReason::InvalidState(s) => CarbideError::InvalidArgument(format!( "Could not create instance on machine {machine_id} given machine state {s:?}" @@ -1278,7 +1285,8 @@ pub async fn batch_validate_spx_partition_ownership( for (partition_id, expected_tenant) in validations { let partition = partition_map.get(partition_id).ok_or_else(|| { tracing::error!( - "batch_validate_spx_partition_ownership partition not found: {partition_id}" + partition_id = %partition_id, + "SPX partition not found while validating ownership", ); ConfigValidationError::invalid_value(format!( "SPX partition {partition_id} is not created" @@ -1287,7 +1295,8 @@ pub async fn batch_validate_spx_partition_ownership( if &partition.tenant_organization_id != *expected_tenant { tracing::error!( - "batch_validate_spx_partition_ownership partition not owned by the tenant: {partition_id}" + partition_id = %partition_id, + "SPX partition is not owned by the tenant", ); return Err(CarbideError::InvalidArgument(format!( "SPX Partition {partition_id} is not owned by the tenant {expected_tenant}", @@ -1394,8 +1403,8 @@ pub fn sort_spx_by_slot(spx_hw_info_vec: &[DpaInterface]) -> HashMap String { tracing::info!( - "machine_type: {machine_type}; machine interface ID: {machine_interface_id}; mac address: {mac_address}" + machine_type = %machine_type, + machine_interface_id = %machine_interface_id, + mac_address = %mac_address, + "machine network boot parameters", ); match arch { rpc::MachineArchitecture::Arm => { @@ -342,9 +345,9 @@ exit || )); } else { tracing::warn!( - "Unsupported DPU type. Product is '{}', but architecture is {:?}", - product, - target.arch, + product = %product, + arch = ?target.arch, + "Unsupported DPU type", ) } }; diff --git a/crates/api-core/src/listener.rs b/crates/api-core/src/listener.rs index 53c40b8a7c..4d92555027 100644 --- a/crates/api-core/src/listener.rs +++ b/crates/api-core/src/listener.rs @@ -143,9 +143,9 @@ fn get_tls_acceptor(tls_config: &ApiTlsConfig) -> Option { .build() .inspect_err(|error| { tracing::error!( - "Could not build client cert verifier. Does root CA file at {} contain no root trust anchors? {}", - tls_config.root_cafile_path, - error + root_cafile_path = %tls_config.root_cafile_path, + error = %error, + "Could not build client certificate verifier; the root CA file may contain no trust anchors", ); }) .ok()?; @@ -284,9 +284,7 @@ pub async fn start( .as_ref() .and_then(|c| c.trust.as_ref()) .cloned() - .inspect(|trust_config| { - tracing::info!("TrustConfig rendered from config: {trust_config:?}") - }) + .inspect(|trust_config| tracing::info!(?trust_config, "TrustConfig rendered from config",)) .map(SpiffeContext::try_from) .transpose()? .ok_or(CarbideError::InvalidConfiguration( @@ -427,7 +425,11 @@ pub async fn start( .service(app); if let Err(error) = http.serve_connection(conn, TowerToHyperService::new(app_with_ext)).await { - tracing::debug!(%error, "error servicing tls http request: {error:?}"); + tracing::debug!( + %error, + error_debug = ?error, + "error servicing tls http request", + ); } } Err(error) => { @@ -464,7 +466,11 @@ pub async fn start( }; if let Err(error) = result { - tracing::debug!(%error, "error servicing plain http connection: {error:?}"); + tracing::debug!( + error = %error, + error_debug = ?error, + "error servicing plain http connection", + ); } } }) diff --git a/crates/api-core/src/logging/setup.rs b/crates/api-core/src/logging/setup.rs index 8c8e5bb38b..fb8888bce7 100644 --- a/crates/api-core/src/logging/setup.rs +++ b/crates/api-core/src/logging/setup.rs @@ -208,7 +208,10 @@ pub fn setup_logging( LevelFilter::current() )) } else { - tracing::info!("current log level: {}", LevelFilter::current()); + tracing::info!( + configured_log_level = %LevelFilter::current(), + "current log level", + ); Ok(Logging { filter: ActiveLevel::new( initial_log_filter, diff --git a/crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs b/crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs index 0003e84655..f6fc0d268f 100644 --- a/crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs +++ b/crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs @@ -70,7 +70,10 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate { match dpu_machine_update::get_reprovisioning_machines(txn).await { Ok(current_updating_machines) => current_updating_machines, Err(e) => { - tracing::warn!("Error getting outstanding reprovisioning count: {}", e); + tracing::warn!( + error = %e, + "Error getting outstanding reprovisioning count", + ); vec![] } }; @@ -166,7 +169,10 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate { async fn clear_completed_updates(&self, txn: &mut PgConnection) -> CarbideResult<()> { let updated_machines = dpu_machine_update::get_updated_machines(txn, self.config.host_health).await?; - tracing::debug!("found {} updated machines", updated_machines.len()); + tracing::debug!( + updated_machine_count = updated_machines.len(), + "found updated machines", + ); for updated_machine in updated_machines { if self .config @@ -179,7 +185,8 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate { { tracing::warn!( machine_id = %updated_machine.dpu_machine_id, - "Failed to remove machine update markers: {}", e + error = %e, + "Failed to remove machine update markers", ); } else if let Ok(mut reported) = self.reported_wrong_versions.lock() { reported.retain(|(dpu, _)| *dpu != updated_machine.dpu_machine_id); @@ -295,7 +302,10 @@ impl DpuNicFirmwareUpdate { ) { Ok(machine_updates) => machine_updates, Err(e) => { - tracing::warn!("Failed to find machines needing updates: {}", e); + tracing::warn!( + error = %e, + "Failed to find machines needing updates", + ); vec![] } } diff --git a/crates/api-core/src/machine_update_manager/host_firmware.rs b/crates/api-core/src/machine_update_manager/host_firmware.rs index 74e64ecbc8..e2d7747588 100644 --- a/crates/api-core/src/machine_update_manager/host_firmware.rs +++ b/crates/api-core/src/machine_update_manager/host_firmware.rs @@ -73,7 +73,7 @@ impl MachineUpdateModule for HostFirmwareUpdate { if firmware_catalog_last_read.as_ref() != Some(&catalog_marker) { // Save the firmware config in an SQL table so that we can filter for hosts with non-matching firmware there. let fw_config_snapshot = self.effective_firmware_config_snapshot(&mut txn).await?; - tracing::info!("Firmware config now: {:?}", fw_config_snapshot); + tracing::info!(?fw_config_snapshot, "Firmware config now",); let models = fw_config_snapshot.into_values().collect::>(); desired_firmware::snapshot_desired_firmware(&mut txn, &models).await?; *firmware_catalog_last_read = Some(catalog_marker); @@ -183,7 +183,7 @@ impl HostFirmwareUpdate { meter: opentelemetry::metrics::Meter, firmware_config: FirmwareConfig, ) -> Option { - tracing::info!("Using firmware configuration: {firmware_config:?}"); + tracing::info!(?firmware_config, "Using firmware configuration",); let metrics = HostFirmwareUpdateMetrics::new(); metrics.register_callbacks(&meter); diff --git a/crates/api-core/src/machine_update_manager/mod.rs b/crates/api-core/src/machine_update_manager/mod.rs index 1875641f95..90dc3772ee 100644 --- a/crates/api-core/src/machine_update_manager/mod.rs +++ b/crates/api-core/src/machine_update_manager/mod.rs @@ -245,7 +245,10 @@ impl MachineUpdateManager { if (current_updating_machines.len() as i32) >= max_concurrent_updates { break; } - tracing::debug!("in progress: {:?}", current_updating_machines); + tracing::debug!( + current_updating_machines = ?current_updating_machines, + "Machine updates in progress", + ); let available_updates = max_concurrent_updates - current_updating_machines.len() as i32; let updates_started = update_module @@ -256,7 +259,10 @@ impl MachineUpdateManager { &snapshots, ) .await?; - tracing::debug!("started: {:?}", updates_started); + tracing::debug!( + updates_started = ?updates_started, + "Machine updates started", + ); updates_started_count += updates_started.len(); diff --git a/crates/api-core/src/machine_validation/mod.rs b/crates/api-core/src/machine_validation/mod.rs index 95804c84e0..e412f68c09 100644 --- a/crates/api-core/src/machine_validation/mod.rs +++ b/crates/api-core/src/machine_validation/mod.rs @@ -281,10 +281,10 @@ impl MachineValidationManager { ) .await?; tracing::debug!( - "MachineValidation metrics: completed {} failed {} in_progress {}", - metrics.completed_validation, - metrics.failed_validation, - metrics.in_progress_validation, + completed_validation = metrics.completed_validation, + failed_validation = metrics.failed_validation, + in_progress_validation = metrics.in_progress_validation, + "Machine validation metrics computed", ); self.metric_holder.update_metrics(metrics); diff --git a/crates/api-core/src/measured_boot/metrics_collector/mod.rs b/crates/api-core/src/measured_boot/metrics_collector/mod.rs index 4fefdc5338..f1095acf4a 100644 --- a/crates/api-core/src/measured_boot/metrics_collector/mod.rs +++ b/crates/api-core/src/measured_boot/metrics_collector/mod.rs @@ -100,7 +100,10 @@ impl MeasuredBootMetricsCollector { async fn run(&self, cancel_token: CancellationToken) { loop { if let Err(e) = self.run_single_iteration().await { - tracing::warn!("MeasuredBootMetricsCollector error: {}", e); + tracing::warn!( + error = %e, + "MeasuredBootMetricsCollector error", + ); } tokio::select! { diff --git a/crates/api-core/src/mqtt_state_change_hook/hook.rs b/crates/api-core/src/mqtt_state_change_hook/hook.rs index 214258fe4a..a0500757e8 100644 --- a/crates/api-core/src/mqtt_state_change_hook/hook.rs +++ b/crates/api-core/src/mqtt_state_change_hook/hook.rs @@ -100,7 +100,10 @@ impl StateChangeHook for MqttStateChangeHook { deadline, }; if let Err(e) = self.sender.try_send(queued) { - tracing::warn!("MQTT state change event dropped (queue full): {e}"); + tracing::warn!( + error = %e, + "MQTT state change event dropped (queue full)", + ); self.metrics.record_overflow(); } } diff --git a/crates/api-core/src/run.rs b/crates/api-core/src/run.rs index 4036f747e9..eebe140a8b 100644 --- a/crates/api-core/src/run.rs +++ b/crates/api-core/src/run.rs @@ -132,12 +132,15 @@ pub async fn run( // Redact credentials before printing the config let print_config = carbide_config.redacted(); - tracing::info!("Using configuration: {:#?}", print_config); tracing::info!( - "Tokio worker thread count: {} (num_cpus::get()={}, TOKIO_WORKER_THREADS={})", - tokio::runtime::Handle::current().metrics().num_workers(), - num_cpus::get(), - std::env::var("TOKIO_WORKER_THREADS").unwrap_or_else(|_| "UNSET".to_string()) + print_config = ?print_config, + "Using configuration", + ); + tracing::info!( + worker_count = tokio::runtime::Handle::current().metrics().num_workers(), + cpu_count = num_cpus::get(), + tokio_worker_threads = %std::env::var("TOKIO_WORKER_THREADS").unwrap_or_else(|_| "UNSET".to_string()), + "Tokio worker thread configuration", ); let metrics = create_metrics()?; @@ -172,7 +175,11 @@ pub async fn run( ) .await { - tracing::error!("Metrics endpoint failed with error: {}", e); + tracing::error!( + metrics_address = %metrics_address, + error = %e, + "Metrics endpoint failed", + ); } } })?; diff --git a/crates/api-core/src/scout_stream.rs b/crates/api-core/src/scout_stream.rs index 5a2ca5b9ec..c9f5c68e80 100644 --- a/crates/api-core/src/scout_stream.rs +++ b/crates/api-core/src/scout_stream.rs @@ -100,7 +100,10 @@ impl ConnectionRegistry { }; let Some(response) = response else { - tracing::info!("scout agent connection closed (machine_id={machine_id})"); + tracing::info!( + %machine_id, + "Scout agent connection closed", + ); break; }; @@ -115,12 +118,18 @@ impl ConnectionRegistry { if let Some(sender) = flows.remove(&flow_uuid) { if let Err(send_err) = sender.send(response) { tracing::warn!( - "error relaying flow response (machine_id={machine_id}, flow_uuid={flow_uuid}): {send_err:?}" + %machine_id, + %flow_uuid, + error = ?send_err, + "Failed to relay Scout flow response", ); } } else { tracing::warn!( - "dropping flow response for unknown flow_uuid (machine_id={machine_id}, flow_uuid={flow_uuid}): {response:?}" + %machine_id, + %flow_uuid, + ?response, + "Dropping response for unknown Scout flow", ); } } @@ -128,18 +137,25 @@ impl ConnectionRegistry { let mut connections = self.connections.write().await; connections.insert(machine_id, connection); - tracing::info!("registered scout agent connection for machine: {machine_id}"); + tracing::info!( + %machine_id, + "Scout agent connection registered", + ); } // unregister removes a scout agent connection from the registry. pub async fn unregister(&self, machine_id: MachineId) -> bool { let mut connections = self.connections.write().await; if connections.remove(&machine_id).is_some() { - tracing::info!("unregistered scout agent connection for machine: {machine_id}"); + tracing::info!( + %machine_id, + "Scout agent connection unregistered", + ); true } else { tracing::info!( - "could not unregister scout agent connection for machine (not found): {machine_id}" + %machine_id, + "Scout agent connection was not registered", ); false } @@ -199,7 +215,9 @@ impl ConnectionRegistry { // And now the request to the scout agent. tracing::info!( - "sending request to scout agent (machine_id={machine_id}, flow_uuid={flow_uuid})" + %machine_id, + %flow_uuid, + "Sending request to Scout agent", ); connection_tx.send(Ok(request)).await.map_err(|e| CarbideError::Internal { @@ -232,7 +250,10 @@ impl ConnectionRegistry { connections .iter() .map(|(machine_id, conn)| { - tracing::debug!("active scout stream connection: {}", conn.machine_id); + tracing::debug!( + machine_id = %conn.machine_id, + "active scout stream connection", + ); (*machine_id, conn.connected_at) }) .collect() @@ -247,13 +268,18 @@ fn extract_flow_uuid( ) -> Result { let flow_uuid_pb = response.flow_uuid.as_ref().ok_or_else(|| { tracing::warn!( - "dropping flow response with empty flow_uuid (machine_id={machine_id}): {response:?}" + %machine_id, + ?response, + "Dropping Scout flow response with empty flow UUID", ); })?; flow_uuid_pb.clone().try_into().map_err(|e| { tracing::warn!( - "failed to decode flow_uuid (machine_id={machine_id}): {flow_uuid_pb:?}: {e:?}" + %machine_id, + ?flow_uuid_pb, + error = ?e, + "Failed to decode Scout flow UUID", ); }) } diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 8f880bbb82..3086cf103f 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -157,7 +157,10 @@ pub fn parse_carbide_config( .iter() .filter(|(_, host)| host.vendor == bmc_vendor::BMCVendor::Unknown) { - tracing::error!("Host firmware configuration has invalid vendor for {label}") + tracing::error!( + label = %label, + "Host firmware configuration has invalid vendor", + ) } // If the carbide config does not say whether to allow dynamically changing the bmc_proxy or @@ -244,7 +247,10 @@ pub fn parse_carbide_config( .wrap_err("Invalid configuration")); } - tracing::trace!("Carbide config: {:#?}", config.redacted()); + tracing::trace!( + config = ?config.redacted(), + "Carbide config", + ); Ok(Arc::new(config)) } @@ -552,10 +558,10 @@ pub async fn start_api( { Ok(cm) => { tracing::info!( - "Component manager configured (nv_switch={}, power_shelf={}, compute_tray={})", - cm.nv_switch.name(), - cm.power_shelf.name(), - cm.compute_tray.name() + nv_switch_backend = cm.nv_switch.name(), + power_shelf_backend = cm.power_shelf.name(), + compute_tray_backend = cm.compute_tray.name(), + "Component manager configured", ); Some(cm) } @@ -571,8 +577,8 @@ pub async fn start_api( // TODO: make the three backends individually optional so a bad // config for one backend does not disable the others. tracing::error!( - "Component manager NOT initialized; failed to build one of the \ - nv-switch / power-shelf / compute-tray backends: {e}" + error = %e, + "Component manager NOT initialized; failed to build one of the nv-switch / power-shelf / compute-tray backends", ); None } @@ -919,8 +925,7 @@ impl<'a> SeedData<'a> { object_kind = %kind, object_name = %name, source = %source, - "{kind} `{name}` is defined in {source}. Defining initial objects in {source} \ - is deprecated; move the definitions into `initial_objects_file`.", + "Initial object is defined in a deprecated configuration source; move the definition into `initial_objects_file`.", ); } Ok(Cow::Borrowed(cc)) @@ -973,8 +978,7 @@ impl<'a> SeedData<'a> { object_kind = %kind, object_name = %name, source = %source, - "{kind} `{name}` is still defined in {source}. \ - Move it into initial_objects_file to silence this warning.", + "Initial object is still defined in a deprecated configuration source; move it into `initial_objects_file`.", ); } Ok(merged) @@ -1027,7 +1031,10 @@ async fn initialize_and_start_controllers<'a>( if let Some(domain_name) = &carbide_config.initial_domain_name && db_init::create_initial_domain(db_pool.clone(), domain_name).await? { - tracing::info!("Created initial domain {domain_name}"); + tracing::info!( + domain_name = %domain_name, + "Created initial domain", + ); } // Probe the helm-chart layout first, then the forged-kustomize layout. @@ -1047,14 +1054,18 @@ async fn initialize_and_start_controllers<'a>( .await .wrap_err_with(|| format!("Failed to read {path_used}"))?; let expected_machines = serde_json::from_str::>(file_str.as_str()).inspect_err(|err| { - tracing::error!("expected_machines.json file exists, but unable to parse expected_machines file, nothing was written to db, bailing: {err}."); + tracing::error!( + error = %err, + "expected_machines.json file exists, but unable to parse expected_machines file, nothing was written to db, bailing.", + ); })?; let mut txn = Transaction::begin(db_pool).await?; crate::handlers::expected_machine::create_missing_from(&mut txn, &expected_machines) .await .inspect_err(|err| { tracing::error!( - "Unable to update database from expected_machines list, bailing: {err}" + error = %err, + "Unable to update database from expected_machines list, bailing", ); })?; txn.commit().await?; @@ -1142,7 +1153,10 @@ async fn initialize_and_start_controllers<'a>( .await?; if let Err(e) = update_dpu_asns(db_pool, common_pools).await { - tracing::warn!("Failed to update ASN for DPUs: {e}"); + tracing::warn!( + error = %e, + "Failed to update ASN for DPUs", + ); } let downloader = FirmwareDownloader::new(); @@ -1195,9 +1209,9 @@ async fn initialize_and_start_controllers<'a>( client.register_metrics(&meter, "dsx_event_bus"); tracing::info!( - "DSX Exchange Event Bus enabled, publishing to {}:{}", - config.mqtt_endpoint, - config.mqtt_broker_port + mqtt_endpoint = %config.mqtt_endpoint, + mqtt_broker_port = config.mqtt_broker_port, + "DSX Exchange Event Bus enabled", ); let bms_client = BmsDsxExchangeHandle::new( diff --git a/crates/api-core/src/tests/common/api_fixtures/dpu.rs b/crates/api-core/src/tests/common/api_fixtures/dpu.rs index ed23e3f362..83dc97661e 100644 --- a/crates/api-core/src/tests/common/api_fixtures/dpu.rs +++ b/crates/api-core/src/tests/common/api_fixtures/dpu.rs @@ -62,7 +62,10 @@ pub async fn create_dpu_machine_in_waiting_for_network_install( } pub async fn create_machine_inventory(env: &TestEnv, machine_id: MachineId) { - tracing::debug!("Creating machine inventory for {}", machine_id); + tracing::debug!( + machine_id = %machine_id, + "Creating machine inventory", + ); env.api .update_agent_reported_inventory(Request::new(rpc::forge::DpuAgentInventoryReport { machine_id: Some(machine_id), diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 5a04a00f5f..7d5a8d34f6 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -2311,10 +2311,10 @@ pub async fn network_configured_with_health_and_ext_services( astra_config_status: None, }; tracing::trace!( - "network_configured machine={} instance_network={} instance={}", - status.network_config_version.as_ref().unwrap(), - instance_network_config_version.clone().unwrap_or_default(), - instance_config_version.clone().unwrap_or_default(), + network_config_version = %status.network_config_version.as_ref().unwrap(), + instance_network_config_version = ?instance_network_config_version, + instance_config_version = ?instance_config_version, + "machine network configured", ); let _ = env .api @@ -2641,7 +2641,10 @@ pub async fn reboot_completed( env: &TestEnv, machine_id: carbide_uuid::machine::MachineId, ) -> rpc::forge::MachineRebootCompletedResponse { - tracing::info!("Machine ={} rebooted", machine_id); + tracing::info!( + machine_id = %machine_id, + "Machine rebooted", + ); env.api .reboot_completed(Request::new(rpc::forge::MachineRebootCompletedRequest { machine_id: Some(machine_id), diff --git a/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs b/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs index 65b36d5ab3..30a1862bff 100644 --- a/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs +++ b/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs @@ -1340,7 +1340,10 @@ impl<'a> MockExploredHost<'a> { let sku = db::sku::generate_sku_from_machine(txn.as_mut(), host_machine_id) .await .unwrap(); - tracing::info!("creating sku: {}", sku.id); + tracing::info!( + sku_id = %sku.id, + "creating sku", + ); db::sku::create(&mut txn, &sku).await.unwrap(); tracing::info!("assigning sku"); diff --git a/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs b/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs index 1b1b756800..a490e55739 100644 --- a/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs @@ -90,14 +90,19 @@ impl TestMachine { } pub async fn reboot_completed(&self) -> rpc::forge::MachineRebootCompletedResponse { - tracing::info!("Machine ={} rebooted", self.id); - self.api + let response = self + .api .reboot_completed(Request::new(rpc::forge::MachineRebootCompletedRequest { machine_id: self.id.into(), })) .await .unwrap() - .into_inner() + .into_inner(); + tracing::info!( + machine_id = %self.id, + "Machine rebooted", + ); + response } pub async fn forge_agent_control(&self) -> rpc::forge::ForgeAgentControlResponse { diff --git a/crates/api-core/src/tests/dns.rs b/crates/api-core/src/tests/dns.rs index c850dfba20..d9af33340f 100644 --- a/crates/api-core/src/tests/dns.rs +++ b/crates/api-core/src/tests/dns.rs @@ -59,7 +59,10 @@ async fn test_dns(pool: sqlx::PgPool) { let fqdn2 = interface2.fqdn; let ip2 = interface2.address; - tracing::info!("FQDN1: {}", fqdn1); + tracing::info!( + fqdn1 = %fqdn1, + "FQDN1", + ); let dns_record = api .lookup_record(tonic::Request::new( rpc::protos::dns::DnsResourceRecordLookupRequest { @@ -74,8 +77,14 @@ async fn test_dns(pool: sqlx::PgPool) { .await .unwrap() .into_inner(); - tracing::info!("DNS Record: {:?}", dns_record); - tracing::info!("IP: {}", ip1); + tracing::info!( + dns_record = ?dns_record, + "DNS Record", + ); + tracing::info!( + ip1 = %ip1, + "IP", + ); assert_eq!( ip1.split('/').collect::>()[0], &*dns_record.records[0].content @@ -227,7 +236,10 @@ async fn test_dns(pool: sqlx::PgPool) { .unwrap() .into_inner(); - tracing::info!("Status: {:?}", status); + tracing::info!( + status = ?status, + "Status", + ); assert_eq!(status.records.len(), 0); } } diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 6871251981..1d7ebb904b 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -1356,7 +1356,7 @@ async fn test_reboot_no_retry_during_firmware_update(pool: sqlx::PgPool) { let dpu = mh.dpu().db_machine(&mut txn).await; let last_reboot_requested = host.last_reboot_requested.as_ref().unwrap(); - tracing::info!("power request: {:?}", last_reboot_requested); + tracing::info!(?last_reboot_requested, "power request",); assert!(matches!( host.last_reboot_requested.as_ref().unwrap().mode, MachineLastRebootRequestedMode::Reboot @@ -1955,7 +1955,11 @@ async fn test_instance_reprov_restart_failed_impl(pool: sqlx::PgPool) { let dpu = mh.dpu().db_machine(&mut txn).await; - tracing::info!(machine_id = %dpu.id, "{} {}", dpu.current_state(), "curr state:"); + tracing::info!( + machine_id = %dpu.id, + dpu_state = %dpu.current_state(), + "current DPU state", + ); assert!(matches!( dpu.current_state(), diff --git a/crates/api-core/src/tests/instance.rs b/crates/api-core/src/tests/instance.rs index 20b707efda..0a74d690f3 100644 --- a/crates/api-core/src/tests/instance.rs +++ b/crates/api-core/src/tests/instance.rs @@ -1072,7 +1072,10 @@ async fn test_instance_dns_resolution(_: PgPoolOptions, options: PgConnectOption .unwrap() .into_inner(); - tracing::info!("dns_record is {:?}: ", dns_record); + tracing::info!( + dns_record = ?dns_record, + "Fetched DNS record", + ); assert_eq!(dns_record.records.first().unwrap().content, "192.0.2.3"); //DHCP response uses hostname set during allocation diff --git a/crates/api-core/src/tests/machine_network.rs b/crates/api-core/src/tests/machine_network.rs index c0263de99b..8b45b02a47 100644 --- a/crates/api-core/src/tests/machine_network.rs +++ b/crates/api-core/src/tests/machine_network.rs @@ -1390,8 +1390,8 @@ async fn test_managed_host_network_status(pool: sqlx::PgPool) { assert_eq!(response.instances.len(), 1); let instance = &response.instances[0]; tracing::info!( - "instance_network_config_version: {}", - instance.network_config_version + network_config_version = %instance.network_config_version, + "Instance network config version", ); assert_eq!( instance.status.as_ref().unwrap().configs_synced, diff --git a/crates/api-core/src/tests/sku.rs b/crates/api-core/src/tests/sku.rs index a9188e83d7..00a0e7db4e 100644 --- a/crates/api-core/src/tests/sku.rs +++ b/crates/api-core/src/tests/sku.rs @@ -236,7 +236,10 @@ pub mod tests { &mh.host().id, 3, |machine| { - tracing::info!("waiting for inventory update: {}", machine.current_state()); + tracing::info!( + machine_state = %machine.current_state(), + "waiting for inventory update", + ); matches!( machine.current_state(), ManagedHostState::BomValidating { @@ -484,9 +487,15 @@ pub mod tests { actual_sku.created = expected_sku.created; let actual_sku_json: String = serde_json::ser::to_string_pretty(&actual_sku)?; - tracing::info!("actual_sku_json: {}", actual_sku_json); + tracing::info!( + actual_sku_json = %actual_sku_json, + "Serialized actual SKU", + ); let expected_sku_json = serde_json::ser::to_string_pretty(&expected_sku)?; - tracing::info!("expected_sku_json: {}", expected_sku_json); + tracing::info!( + expected_sku_json = %expected_sku_json, + "Serialized expected SKU", + ); assert_eq!(actual_sku_json, expected_sku_json); @@ -1288,7 +1297,10 @@ pub mod tests { .pop() .unwrap(); - tracing::info!("SKU1: {:?}", original_sku); + tracing::info!( + original_sku = ?original_sku, + "SKU1", + ); let mut broken_sku = original_sku.clone(); broken_sku.id = "Broken SKU".to_string(); @@ -1296,7 +1308,10 @@ pub mod tests { db::sku::create(&mut txn, &broken_sku).await?; - tracing::info!("SKU2: {:?}", broken_sku); + tracing::info!( + broken_sku = ?broken_sku, + "SKU2", + ); db::machine::unassign_sku(&mut txn, &machine_id).await?; db::machine::assign_sku(&mut txn, &machine_id, &broken_sku.id).await?; diff --git a/crates/api-core/tests/integration/machine_bmc_metadata.rs b/crates/api-core/tests/integration/machine_bmc_metadata.rs index 77bb24feb7..1eda4f79cf 100644 --- a/crates/api-core/tests/integration/machine_bmc_metadata.rs +++ b/crates/api-core/tests/integration/machine_bmc_metadata.rs @@ -100,7 +100,12 @@ async fn fetch_bmc_credentials(pool: PgPool) { ] .into_iter() { - tracing::info!("Looking up credentials for {:?}", request); + let lookup_selector = if request.machine_id.is_some() { + "machine_id" + } else { + "bmc_endpoint" + }; + tracing::info!(lookup_selector, "Looking up BMC credentials"); let metadata = env .api() .get_bmc_meta_data(tonic::Request::new(request)) diff --git a/crates/api-db/src/bmc_metadata.rs b/crates/api-db/src/bmc_metadata.rs index e458ca7398..8c4eacd3fd 100644 --- a/crates/api-db/src/bmc_metadata.rs +++ b/crates/api-db/src/bmc_metadata.rs @@ -32,7 +32,10 @@ async fn update_bmc_network_into_topologies( "BMC Info in machine_topologies does not have a MAC address for machine {machine_id}" ))); } - tracing::info!("put bmc_info: {:?}", bmc_info); + tracing::info!( + bmc_info = ?bmc_info, + "Updating BMC info", + ); // A entry with same machine id is already created by discover_machine call. // Just update json by adding a ipmi_ip entry. @@ -123,10 +126,10 @@ pub async fn enrich_mac_address( let bmc_mac_address = bmc_machine_interface.mac_address; tracing::info!( - "{} is enriching BMC Info for machine {} with a BMC mac address of {:#?}", - caller, - machine_id, - bmc_machine_interface.mac_address, + caller = %caller, + machine_id = %machine_id, + mac_address = ?bmc_machine_interface.mac_address, + "Enriching BMC information", ); bmc_info.mac = Some(bmc_mac_address); bmc_info.machine_interface_id = Some(bmc_machine_interface.id); @@ -136,9 +139,10 @@ pub async fn enrich_mac_address( } else { // This should never happen. Should we return an error here? tracing::info!( - "{} failed to enrich the BMC Info for machine {} with a MAC: cannot cannot find a machine interface with IP address {bmc_ip_address}", - caller, - machine_id + caller = %caller, + machine_id = %machine_id, + bmc_ip_address = %bmc_ip_address, + "Failed to enrich BMC information: machine interface not found", ); } } diff --git a/crates/api-db/src/carbide_version.rs b/crates/api-db/src/carbide_version.rs index 69cec5dda0..483602c087 100644 --- a/crates/api-db/src/carbide_version.rs +++ b/crates/api-db/src/carbide_version.rs @@ -61,7 +61,8 @@ pub async fn observe_as_latest_version( } else if superseded_count > 1 { tracing::warn!( version, - "observed a new forge version, superseded {superseded_count} versions" + superseded_count, + "observed a new forge version, superseded versions", ); } diff --git a/crates/api-db/src/dns/resource_record.rs b/crates/api-db/src/dns/resource_record.rs index ba7e920c76..bba71ec25d 100644 --- a/crates/api-db/src/dns/resource_record.rs +++ b/crates/api-db/src/dns/resource_record.rs @@ -102,7 +102,7 @@ pub async fn find_record( COALESCE(q_type, CASE WHEN family(resource_record) = 6 THEN 'AAAA' ELSE 'A' END) as q_type from dns_records WHERE q_name=$1"#; - tracing::info!("Looking up record using query_name: {}", query_name); + tracing::info!(query_name, "Looking up DNS record",); let result = sqlx::query_as::<_, DbResourceRecord>(query) .bind(query_name) .fetch_all(txn) diff --git a/crates/api-db/src/expected_power_shelf.rs b/crates/api-db/src/expected_power_shelf.rs index 048ced4293..2874fd224c 100644 --- a/crates/api-db/src/expected_power_shelf.rs +++ b/crates/api-db/src/expected_power_shelf.rs @@ -315,8 +315,8 @@ pub async fn create_missing_from( for expected_power_shelf in expected_power_shelves { if existing_map.contains_key(&expected_power_shelf.bmc_mac_address.to_string()) { tracing::debug!( - "Not overwriting expected-power-shelf with mac_addr: {}", - expected_power_shelf.bmc_mac_address.to_string() + bmc_mac_address = %expected_power_shelf.bmc_mac_address, + "Expected power shelf already exists; not overwriting", ); continue; } diff --git a/crates/api-db/src/expected_switch.rs b/crates/api-db/src/expected_switch.rs index 3815533e08..58199c05e4 100644 --- a/crates/api-db/src/expected_switch.rs +++ b/crates/api-db/src/expected_switch.rs @@ -511,8 +511,8 @@ pub async fn create_missing_from( for expected_switch in expected_switches { if existing_map.contains_key(&expected_switch.bmc_mac_address.to_string()) { tracing::debug!( - "Not overwriting expected-switch with mac_addr: {}", - expected_switch.bmc_mac_address.to_string() + bmc_mac_address = %expected_switch.bmc_mac_address, + "Expected switch already exists; not overwriting", ); continue; } diff --git a/crates/api-db/src/instance.rs b/crates/api-db/src/instance.rs index cb14cc380b..922be45c8e 100644 --- a/crates/api-db/src/instance.rs +++ b/crates/api-db/src/instance.rs @@ -501,8 +501,9 @@ pub async fn update_phone_home_last_contact( .map_err(|e| DatabaseError::query(query, e))?; tracing::info!( - "Phone home last contact updated for instance {}", - query_result.0 + instance_id = %instance_id, + phone_home_last_contact = %query_result.0, + "Phone home last contact updated", ); Ok(query_result.0) } @@ -519,7 +520,10 @@ pub async fn clear_phone_home_last_contact( .await .map_err(|e| DatabaseError::query(query, e))?; - tracing::info!("Phone home last contact cleared for instance {instance_id}"); + tracing::info!( + instance_id = %instance_id, + "Phone home last contact cleared", + ); Ok(()) } @@ -1186,8 +1190,9 @@ pub async fn batch_update_spx_config( // Verify all rows were updated (version check passed) if result.rows_affected() != expected_count { tracing::error!( - "batch_update_spx_config affected != expected: {:#?} != {expected_count}", - result.rows_affected() + rows_affected = result.rows_affected(), + expected_rows = expected_count, + "SPX config batch update affected an unexpected number of rows", ); return Err(DatabaseError::FailedPrecondition( "Spx config version mismatch during batch update".to_string(), diff --git a/crates/api-db/src/lib.rs b/crates/api-db/src/lib.rs index d55ae36ae6..f68dc968ff 100644 --- a/crates/api-db/src/lib.rs +++ b/crates/api-db/src/lib.rs @@ -534,7 +534,12 @@ impl From for tonic::Status { if f.len() == 2 { let handler = f[0].trim(); let location = f[1].trim().replace("at ", ""); - tracing::error!("{from} location={location} handler='{handler}'"); + tracing::error!( + error = %from, + error_location = %location, + handler, + "database error conversion", + ); true } else { false @@ -546,7 +551,7 @@ impl From for tonic::Status { if !printed { match from { DatabaseError::NotImplemented => {} - _ => tracing::error!("{from}"), + _ => tracing::error!(error = %from, "database error conversion"), } } diff --git a/crates/api-db/src/machine.rs b/crates/api-db/src/machine.rs index 2dcd277cfe..8102f55a87 100644 --- a/crates/api-db/src/machine.rs +++ b/crates/api-db/src/machine.rs @@ -1033,8 +1033,8 @@ pub async fn update_spx_status_observation( observation: &MachineSpxStatusObservation, ) -> Result<(), DatabaseError> { tracing::debug!( - "update_spx_status_observation: observation {:#?}", - observation + observation = ?observation, + "updating SPX status observation", ); let query = "UPDATE machines SET spx_status_observation = $1::json WHERE id = $2 AND (spx_status_observation->>'observed_at' IS NULL OR spx_status_observation->>'observed_at' <= $3) RETURNING id"; @@ -1058,7 +1058,12 @@ async fn debug_failed_machine_status_update( ) { let serialized_data = serde_json::to_string_pretty(column_data).unwrap_or_else(|_| "Invalid".to_string()); - tracing::error!(machine_id=%machine_id, column_name, "Failed to update column. New column data: {serialized_data}"); + tracing::error!( + machine_id=%machine_id, + column_name, + serialized_data = %serialized_data, + "Failed to update column. New column data", + ); // Dump the raw Machine state for debugging purposes let query = "SELECT * from machines WHERE id = $1"; match sqlx::query(query) @@ -1067,13 +1072,19 @@ async fn debug_failed_machine_status_update( .await { Ok(Some(row)) => { - tracing::error!("Machine Data: {:?}", row); + tracing::error!( + row = ?row, + "Machine Data", + ); } Ok(None) => { tracing::error!("No Machine Data"); } Err(e) => { - tracing::error!("Failed to load Machine Data. Error: {e}"); + tracing::error!( + error = %e, + "Failed to load machine data", + ); } } } @@ -1564,7 +1575,11 @@ pub async fn create( { Ok(asn) => Some(asn as i64), Err(e) => { - tracing::info!("Failed to allocate asn for dpu {stable_machine_id}: {e}"); + tracing::info!( + stable_machine_id = %stable_machine_id, + error = %e, + "Failed to allocate asn for dpu", + ); None } } diff --git a/crates/api-db/src/machine_interface.rs b/crates/api-db/src/machine_interface.rs index 4cd3e718d8..6e45e3100b 100644 --- a/crates/api-db/src/machine_interface.rs +++ b/crates/api-db/src/machine_interface.rs @@ -609,7 +609,8 @@ async fn find_or_create_machine_interface_inner( None => { tracing::info!( %mac_address, - "Found no existing machine with mac address {mac_address} using networks with relays {relaystr}", + relays = ?relays, + "No existing machine found", ); let mut interface = validate_existing_mac_and_create_inner( &mut *txn, @@ -667,7 +668,8 @@ pub async fn find_or_create_observed_machine_interface( None => { tracing::info!( %mac_address, - "Found no existing machine with mac address {mac_address} using networks with relays {relaystr}", + relays = ?relays, + "No existing machine found", ); // Return an existing row when the MAC is already known on this segment. diff --git a/crates/api-db/src/machine_topology.rs b/crates/api-db/src/machine_topology.rs index 3c961c38d3..dd772a5556 100644 --- a/crates/api-db/src/machine_topology.rs +++ b/crates/api-db/src/machine_topology.rs @@ -128,8 +128,9 @@ pub async fn create_or_update_with_bom_validation( let age = Utc::now() - topology.updated; if bom_validation_enabled && age > TimeDelta::days(1) { tracing::debug!( - "Received inventory update from {}, bom_validation is enabled, existing data is old, updating", - machine_id + machine_id = %machine_id, + inventory_age_days = age.num_days(), + "Received stale inventory update while BOM validation is enabled", ); set_topology_update_needed(txn, machine_id, true).await?; } diff --git a/crates/api-db/src/measured_boot/bundle.rs b/crates/api-db/src/measured_boot/bundle.rs index 4f68e08f06..6f07217088 100644 --- a/crates/api-db/src/measured_boot/bundle.rs +++ b/crates/api-db/src/measured_boot/bundle.rs @@ -563,8 +563,8 @@ async fn find_closest_bundle( if active_bundles.is_empty() { tracing::info!( - "No Active or Obsolete bundles were found for profile_id {0}", - profile_id + profile_id = %profile_id, + "No active or obsolete measured boot bundles found", ); return Ok(None); } diff --git a/crates/api-db/src/vpc_dpu_loopback.rs b/crates/api-db/src/vpc_dpu_loopback.rs index 5f56a5a505..7c09d61f3c 100644 --- a/crates/api-db/src/vpc_dpu_loopback.rs +++ b/crates/api-db/src/vpc_dpu_loopback.rs @@ -58,8 +58,8 @@ pub async fn delete_and_deallocate( if admin_vpcs.is_empty() { tracing::warn!( - "No VPC attached to one or more admin segments: {:?}.", - admin_segments + ?admin_segments, + "No VPC attached to one or more admin segments.", ); } else { // We only allow a single admin VPC, so it seems this could easily be diff --git a/crates/api-db/src/work_lock_manager.rs b/crates/api-db/src/work_lock_manager.rs index 95c3d4d5bc..9daf436a66 100644 --- a/crates/api-db/src/work_lock_manager.rs +++ b/crates/api-db/src/work_lock_manager.rs @@ -151,7 +151,10 @@ async fn run_loop( match try_acquire_lock(db, &work_key, keepalive_timeout).await { Ok(Some(worker_id)) => { reply_tx.send(Ok(worker_id)).ok(); - tracing::debug!("Acquired work lock {work_key}"); + tracing::debug!( + work_key = %work_key, + "Acquired work lock", + ); } Ok(None) => { reply_tx @@ -171,7 +174,11 @@ async fn run_loop( release_lock(db, &work_key, worker_id) .await .inspect_err(|e| { - tracing::error!(%work_key, "Could not release work lock: {e}"); + tracing::error!( + %work_key, + error = %e, + "Could not release work lock", + ); }) .ok(); tracing::debug!(%work_key, "Released work lock"); @@ -277,9 +284,9 @@ pub struct WorkLock { impl Drop for WorkLock { fn drop(&mut self) { tracing::debug!( - "Releasing lock for {} worker {}", - self.work_key, - self.worker_id + work_key = %self.work_key, + worker_id = %self.worker_id, + "Releasing work lock", ); // Let the keepalive loop stop @@ -293,7 +300,12 @@ impl Drop for WorkLock { worker_id: self.worker_id, }) .inspect_err(|e| { - tracing::error!("ERROR! Could not release lock for {} worker {}: WorkLockManager queue is full! Database is likely overloaded: {}", self.work_key, self.worker_id, e); + tracing::error!( + work_key = %self.work_key, + worker_id = %self.worker_id, + error = %e, + "Could not release work lock: WorkLockManager queue is full; database is likely overloaded", + ); }) .ok(); } @@ -307,36 +319,49 @@ impl WorkLock { keepalive_interval: Duration, ) -> Self { let (keepalive_stop_tx, mut keepalive_stop_rx) = oneshot::channel(); - let join_handle = tokio::task::Builder::new().name(&format!("keepalive loop for {work_key} worker {worker_id}")).spawn({ - let manager = manager.clone(); - let work_key = work_key.clone(); - let mut keepalive_timer = tokio::time::interval(keepalive_interval); - keepalive_timer.set_missed_tick_behavior(MissedTickBehavior::Skip); - let fut = async move { - loop { - tokio::select! { - _ = keepalive_timer.tick() => { - match manager.keep_lock_alive(work_key.clone(), worker_id).await { - Ok(_) => {} - Err(KeepAliveError::LockLost(msg)) => { - tracing::error!("{work_key} worker {worker_id} lost lock: {msg}"); - return; - } - Err(e) => { - tracing::error!("Error sending keepalive for {work_key} worker {worker_id} (will retry): {e}"); + let join_handle = tokio::task::Builder::new() + .name(&format!("keepalive loop for {work_key} worker {worker_id}")) + .spawn({ + let manager = manager.clone(); + let work_key = work_key.clone(); + let mut keepalive_timer = tokio::time::interval(keepalive_interval); + keepalive_timer.set_missed_tick_behavior(MissedTickBehavior::Skip); + let fut = async move { + loop { + tokio::select! { + _ = keepalive_timer.tick() => { + match manager.keep_lock_alive(work_key.clone(), worker_id).await { + Ok(_) => {} + Err(KeepAliveError::LockLost(msg)) => { + tracing::error!( + work_key = %work_key, + worker_id = %worker_id, + detail = %msg, + "worker lost lock", + ); + return; + } + Err(e) => { + tracing::error!( + work_key = %work_key, + worker_id = %worker_id, + error = %e, + "Failed to send work-lock keepalive; retrying", + ); + } } } - } - _ = &mut keepalive_stop_rx => { - break; + _ = &mut keepalive_stop_rx => { + break; + } } } - } - }; - // Note: don't inherit the callers span, since child spans can't outlive their parent. - // This prevents a crash in tracing-subscriber. - fut.instrument(tracing::debug_span!(parent: None, "WorkLock keepalive loop")) - }).expect("could not spawn tokio task"); + }; + // Note: don't inherit the callers span, since child spans can't outlive their parent. + // This prevents a crash in tracing-subscriber. + fut.instrument(tracing::debug_span!(parent: None, "WorkLock keepalive loop")) + }) + .expect("could not spawn tokio task"); if !cfg!(test) { _ = join_handle; diff --git a/crates/api-integration-tests/tests/lib.rs b/crates/api-integration-tests/tests/lib.rs index 8755583942..c5c2bd5bc0 100644 --- a/crates/api-integration-tests/tests/lib.rs +++ b/crates/api-integration-tests/tests/lib.rs @@ -659,7 +659,10 @@ async fn test_machine_a_tron_multidpu( let machine_id = machine_handle .observed_machine_id() .expect("Machine ID should be set if host is ready"); - tracing::info!("Machine {machine_id} has made it to Ready, allocating instance"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready, allocating instance", + ); let instance_id = instance::create( carbide_api_addrs, &machine_id, @@ -702,14 +705,18 @@ async fn test_machine_a_tron_multidpu( assert_eq!(gateways.len(), 1); tracing::info!( - "Machine {machine_id} has made it to Assigned/Ready, releasing instance" + machine_id = %machine_id, + "Machine has made it to Assigned/Ready, releasing instance", ); instance::release(carbide_api_addrs, &machine_id, &instance_id, false).await?; machine_handle .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(90)) .await?; - tracing::info!("Machine {machine_id} has made it to Ready again, all done"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready again, all done", + ); Ok::<(), eyre::Report>(()) } }, @@ -806,7 +813,10 @@ async fn test_machine_a_tron_zerodpu( let machine_id = machine_handle .observed_machine_id() .expect("Machine ID should be set if host is ready"); - tracing::info!("Machine {machine_id} has made it to Ready, allocating instance"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready, allocating instance", + ); // Zero-DPU tenants pass `auto: true` with empty interfaces; the // allocator resolves the host's HostInband segment(s) from the @@ -827,7 +837,8 @@ async fn test_machine_a_tron_zerodpu( .wait_until_machine_up_with_api_state("Assigned/Ready", Duration::from_secs(90)) .await?; tracing::info!( - "Machine {machine_id} has made it to Assigned/Ready, releasing instance" + machine_id = %machine_id, + "Machine has made it to Assigned/Ready, releasing instance", ); instance::release(carbide_api_addrs, &machine_id, &instance_id, false).await?; @@ -835,7 +846,10 @@ async fn test_machine_a_tron_zerodpu( machine_handle .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(90)) .await?; - tracing::info!("Machine {machine_id} has made it to Ready again, all done"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready again, all done", + ); Ok::<(), eyre::Report>(()) } }, @@ -867,7 +881,10 @@ async fn test_machine_a_tron_singledpu_nic_mode( let machine_id = machine_handle .observed_machine_id() .expect("Machine ID should be set if host is ready"); - tracing::info!("Machine {machine_id} has made it to Ready, allocating instance"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready, allocating instance", + ); // For a DPU in NIC-mode, the DPU is treated as a plain NIC, meaning // allocation goes through HostInband the same way the zero-DPU path @@ -889,7 +906,8 @@ async fn test_machine_a_tron_singledpu_nic_mode( .wait_until_machine_up_with_api_state("Assigned/Ready", Duration::from_secs(90)) .await?; tracing::info!( - "Machine {machine_id} has made it to Assigned/Ready, releasing instance" + machine_id = %machine_id, + "Machine has made it to Assigned/Ready, releasing instance", ); instance::release(carbide_api_addrs, &machine_id, &instance_id, false).await?; @@ -897,7 +915,10 @@ async fn test_machine_a_tron_singledpu_nic_mode( machine_handle .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(90)) .await?; - tracing::info!("Machine {machine_id} has made it to Ready again, all done"); + tracing::info!( + machine_id = %machine_id, + "Machine has made it to Ready again, all done", + ); Ok::<(), eyre::Report>(()) } }, @@ -945,7 +966,9 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( .observed_machine_id() .expect("Machine ID should be set if host is ready"); tracing::info!( - "Machine {initial_machine_id} (bmc_mac {bmc_mac}) is Ready in DPU mode; flipping to NIC mode" + initial_machine_id = %initial_machine_id, + bmc_mac_address = %bmc_mac, + "Machine is Ready in DPU mode; flipping to NIC mode", ); // 2. Flip the ExpectedMachine to NIC mode. Get the current record, @@ -965,7 +988,10 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( Some(&expected_machine), ) .await?; - tracing::info!("ExpectedMachine for {bmc_mac} now declares NIC mode"); + tracing::info!( + bmc_mac = %bmc_mac, + "ExpectedMachine for now declares NIC mode", + ); // 3. Force-delete the managed-DPU machine so site-explorer // re-ingests the host under the new declared mode. This is the @@ -987,7 +1013,10 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( Some(&force_delete_req), ) .await?; - tracing::info!("Force-deleted machine {initial_machine_id}; awaiting re-ingestion as NicMode"); + tracing::info!( + initial_machine_id = %initial_machine_id, + "Force-deleted machine; awaiting re-ingestion as NicMode", + ); // 4. Wait for the host to re-ingest as a zero-managed-DPU machine, // asserted directly against the database. The host's TPM-derived @@ -1034,8 +1063,9 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( if host_exists && nic_count >= 1 && managed_dpu_count == 0 { tracing::info!( - "Host re-ingested as zero-managed-DPU machine {host_id} \ - (nic_count={nic_count}); DPU->NIC flip applied" + host_machine_id = %host_id, + nic_interface_count = nic_count, + "Host re-ingested as a zero-managed-DPU machine; DPU-to-NIC flip applied", ); break; } @@ -1053,7 +1083,10 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( // the host BMC now serves an event log so the controller's // restart verification can confirm reboots, and the zero-DPU // lockdown short-circuit lets it skip the DPU-down wait. - tracing::info!("Waiting for re-ingested NicMode host {host_id} to reach Ready"); + tracing::info!( + host_id = %host_id, + "Waiting for re-ingested NicMode host to reach Ready", + ); let ready_deadline = time::Instant::now() + Duration::from_secs(240); loop { let resp = api_test_helper::grpcurl::grpcurl( @@ -1065,7 +1098,11 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( let resp: serde_json::Value = serde_json::from_str(&resp)?; let state = resp["machines"][0]["state"].as_str().unwrap_or(""); if state == "Ready" { - tracing::info!("Re-ingested NicMode host {host_id} reached Ready ({state})"); + tracing::info!( + host_id = %host_id, + state, + "Re-ingested NicMode host reached Ready", + ); break; } if time::Instant::now() >= ready_deadline { @@ -1111,7 +1148,8 @@ async fn test_machine_a_tron_dual_stack( .observed_machine_id() .expect("Machine ID should be set if host is ready"); tracing::info!( - "Machine {machine_id} is Ready, allocating dual-stack instance via ipv6 config" + machine_id = %machine_id, + "Machine is Ready, allocating dual-stack instance via ipv6 config", ); let instance_id = instance::create_with_vpc_prefixes( carbide_api_addrs, @@ -1169,7 +1207,9 @@ async fn test_machine_a_tron_dual_stack( ); tracing::info!( - "Machine {machine_id} dual-stack allocation verified: addresses = {addr_strings:?}" + machine_id = %machine_id, + addresses = ?addr_strings, + "Machine dual-stack allocation verified", ); instance::release(carbide_api_addrs, &machine_id, &instance_id, false) @@ -1179,7 +1219,8 @@ async fn test_machine_a_tron_dual_stack( .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(90)) .await?; tracing::info!( - "Machine {machine_id} back to Ready after dual-stack release" + machine_id = %machine_id, + "Machine back to Ready after dual-stack release", ); Ok::<(), eyre::Report>(()) } @@ -1218,7 +1259,8 @@ async fn test_machine_a_tron_dual_stack_l2( .observed_machine_id() .expect("Machine ID should be set if host is ready"); tracing::info!( - "Machine {machine_id} is Ready, allocating dual-stack L2 instance" + machine_id = %machine_id, + "Machine is Ready, allocating dual-stack L2 instance", ); let instance_id = instance::create( carbide_api_addrs, @@ -1239,17 +1281,18 @@ async fn test_machine_a_tron_dual_stack_l2( .await?; tracing::info!( - "Machine {machine_id} dual-stack L2 instance allocated and reached Assigned/Ready" + machine_id = %machine_id, + "Machine dual-stack L2 instance allocated and reached Assigned/Ready", ); - instance::release(carbide_api_addrs, &machine_id, &instance_id, false) - .await?; + instance::release(carbide_api_addrs, &machine_id, &instance_id, false).await?; machine_handle .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(90)) .await?; tracing::info!( - "Machine {machine_id} back to Ready after dual-stack L2 release" + machine_id = %machine_id, + "Machine back to Ready after dual-stack L2 release", ); Ok::<(), eyre::Report>(()) } diff --git a/crates/api-model/src/dpa_interface/mod.rs b/crates/api-model/src/dpa_interface/mod.rs index 0f606cdb25..46ee735c2f 100644 --- a/crates/api-model/src/dpa_interface/mod.rs +++ b/crates/api-model/src/dpa_interface/mod.rs @@ -300,8 +300,8 @@ impl DpaInterface { // If we haven't yet seen any observations, we are not synced let Some(spx_status_observation) = spx_status_observation else { tracing::info!( - "DPA interface {dpa_id} is not synced because no SPX status observation is available", - dpa_id = self.id + dpa_id = %self.id, + "DPA interface is not synced because no SPX status observation is available", ); return false; }; @@ -330,10 +330,10 @@ impl DpaInterface { { if config_version != dpa_expected_version { tracing::info!( - "DPA interface {dpa_id} is not synced version mismatch: {config_version} != {dpa_expected_version}", - dpa_id = self.id, - config_version = config_version, - dpa_expected_version = dpa_expected_version + dpa_id = %self.id, + config_version = %config_version, + dpa_expected_version = %dpa_expected_version, + "DPA interface is not synced version mismatch!", ); return false; } @@ -342,9 +342,9 @@ impl DpaInterface { } tracing::info!( - "DPA interface {dpa_id} is not synced verrsion mismatch: {dpa_expected_version}", - dpa_id = self.id, - dpa_expected_version = dpa_expected_version + dpa_id = %self.id, + dpa_expected_version = %dpa_expected_version, + "DPA interface is not synced because no matching SPX status attachment was found", ); false diff --git a/crates/api-model/src/firmware.rs b/crates/api-model/src/firmware.rs index 20bdce2c70..ec9eb0c353 100644 --- a/crates/api-model/src/firmware.rs +++ b/crates/api-model/src/firmware.rs @@ -142,9 +142,10 @@ impl Firmware { .find(|&x| self.matching_version_id(&x.id, firmware_type)) { tracing::debug!( - "find_version {:?}: For {firmware_type:?} found {:?}", - report.machine_id, - matching_inventory.version + machine_id = ?report.machine_id, + ?firmware_type, + version = ?matching_inventory.version, + "Found matching firmware version", ); return matching_inventory.version.clone(); }; diff --git a/crates/api-model/src/instance/status/network.rs b/crates/api-model/src/instance/status/network.rs index 945667d887..a5bdf992d3 100644 --- a/crates/api-model/src/instance/status/network.rs +++ b/crates/api-model/src/instance/status/network.rs @@ -24,7 +24,6 @@ use carbide_uuid::vpc::VpcId; use chrono::{DateTime, Utc}; use config_version::{ConfigVersion, Versioned}; use ipnetwork::IpNetwork; -use itertools::Itertools; use mac_address::MacAddress; use serde::{Deserialize, Serialize}; @@ -222,8 +221,8 @@ impl InstanceNetworkStatus { > 1 { tracing::error!( - "Found multiple physical interfaces when no device specified: {:?}", - config + ?config, + "Found multiple physical interfaces when no device specified", ); return Self::unsynchronized_for_config(&config); } @@ -292,8 +291,8 @@ impl InstanceNetworkStatus { if !missing_dpus.is_empty() { tracing::info!( - "Missing observations for DPUs: {}", - missing_dpus.into_iter().join(",") + missing_dpu_ids = ?missing_dpus, + "Missing observations for DPUs", ); } @@ -393,11 +392,16 @@ impl InstanceInterfaceStatus { // if any of them were not found. let gateways = prefix_ids .iter() - .map(|id| if let Some(gw) = value.network_segment_gateways.remove(id) { - Some(gw) - } else { - tracing::warn!("Missing gateway in InstanceInterfaceConfig for network prefix {id}, gateways field will be empty."); - None + .map(|id| { + if let Some(gw) = value.network_segment_gateways.remove(id) { + Some(gw) + } else { + tracing::warn!( + network_prefix_id = %id, + "Missing gateway in InstanceInterfaceConfig; gateways field will be empty", + ); + None + } }) .collect::>>() .unwrap_or_default(); diff --git a/crates/api-model/src/instance/status/nvlink.rs b/crates/api-model/src/instance/status/nvlink.rs index bac1a08332..8e88a7f8dd 100644 --- a/crates/api-model/src/instance/status/nvlink.rs +++ b/crates/api-model/src/instance/status/nvlink.rs @@ -72,8 +72,8 @@ impl InstanceNvLinkStatus { } None => { tracing::error!( - "could not find matching status gpu {:?}", - cfg.device_instance + device_instance = cfg.device_instance, + "could not find matching status gpu", ); configs_synced = SyncState::Pending; InstanceNvLinkGpuStatus { diff --git a/crates/api-model/src/instance/status/spx.rs b/crates/api-model/src/instance/status/spx.rs index 9073b1889a..03e4c079e2 100644 --- a/crates/api-model/src/instance/status/spx.rs +++ b/crates/api-model/src/instance/status/spx.rs @@ -74,8 +74,8 @@ impl InstanceSpxStatus { } None => { tracing::error!( - "could not find matching status spx attachment {:?}", - cfg.device_instance + device_instance = cfg.device_instance, + "could not find matching status spx attachment", ); configs_synced = SyncState::Pending; InstanceSpxAttachmentStatus { diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 3eb8e8dbc5..1493254f0f 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -2542,7 +2542,8 @@ pub fn state_sla( { tracing::debug!( machine_id = %machine_id, - "Skipping state machine SLA for machine due to {exclude} classification" + health_alert_classification = %exclude, + "Skipping state machine SLA due to classification", ); return StateSla::no_sla(); } @@ -2923,8 +2924,8 @@ pub fn dpf_based_dpu_provisioning_possible( // DPF should be enabled for host. if !state.host_snapshot.dpf.enabled { tracing::info!( - "DPF based DPU provisioning is not possible because DPF is not enabled for the host {}.", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "DPF based DPU provisioning is not possible because DPF is not enabled for the host.", ); tracing::warn!( machine_id = %state.host_snapshot.id, @@ -2947,9 +2948,8 @@ pub fn dpf_based_dpu_provisioning_possible( .all(|dpu| dpu.reprovision_requested.is_some()) { tracing::info!( - "DPF based DPU reprovisioning is not possible for host {} because initial ingestion is not done with DPF \ - and not all DPUs are being reprovisioned.", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "DPF based DPU reprovisioning is not possible for host because initial ingestion is not done with DPF and not all DPUs are being reprovisioned.", ); tracing::warn!( machine_id = %state.host_snapshot.id, @@ -2969,8 +2969,8 @@ pub fn dpf_based_dpu_provisioning_possible( .unwrap_or(false) }) { tracing::info!( - "DPF based DPU provisioning is not possible because some DPUs are Bluefield 2 in {}.", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "DPF-based DPU provisioning is not possible because some DPUs are BlueField-2", ); tracing::warn!( machine_id = %state.host_snapshot.id, diff --git a/crates/api-model/src/machine/nvlink.rs b/crates/api-model/src/machine/nvlink.rs index 254ad7662a..5a3c78dc7e 100644 --- a/crates/api-model/src/machine/nvlink.rs +++ b/crates/api-model/src/machine/nvlink.rs @@ -73,8 +73,8 @@ pub fn nvlink_config_synced( .find(|gpu| gpu.device_instance == gpu_config.device_instance) else { tracing::error!( - "could not find matching status gpu {:?}", - gpu_config.device_instance + device_instance = gpu_config.device_instance, + "could not find matching status gpu", ); return Err(NvLinkConfigNotSyncedReason( "No matching NvLink status observation found for GPU in config".to_string(), diff --git a/crates/api-model/src/machine/spx.rs b/crates/api-model/src/machine/spx.rs index c05eb8e79a..3e755ad0ac 100644 --- a/crates/api-model/src/machine/spx.rs +++ b/crates/api-model/src/machine/spx.rs @@ -80,8 +80,8 @@ pub fn spx_config_synced( conf_att.mac_address.as_deref().unwrap_or_default() == obs_att.mac_address.to_string() }) else { tracing::error!( - "could not find matching status instance {:?}", - conf_att.device_instance + device_instance = conf_att.device_instance, + "could not find matching status instance", ); return Err(SpxConfigNotSyncedReason( "No matching SPX status observation found for attachment in config".to_string(), diff --git a/crates/api-model/src/site_explorer/mod.rs b/crates/api-model/src/site_explorer/mod.rs index 6f5b81fdc7..a1a0ab0e7c 100644 --- a/crates/api-model/src/site_explorer/mod.rs +++ b/crates/api-model/src/site_explorer/mod.rs @@ -254,9 +254,10 @@ impl ExploredEndpoint { .find(|&x| fw_info.matching_version_id(&x.id, firmware_type)) { tracing::debug!( - "find_version {}: For {firmware_type:?} found {:?}", - self.address, - matching_inventory.version + address = %self.address, + firmware_type = ?firmware_type, + version = ?matching_inventory.version, + "Found matching firmware version", ); return matching_inventory.version.as_ref(); }; @@ -283,10 +284,11 @@ impl ExploredEndpoint { } tracing::debug!( - "find_all_versions {}: Found {} versions for {firmware_type:?}: {:?}", - self.address, - versions.len(), - versions + address = %self.address, + version_count = versions.len(), + firmware_type = ?firmware_type, + versions = ?versions, + "Found firmware versions", ); versions diff --git a/crates/api-test-helper/src/domain.rs b/crates/api-test-helper/src/domain.rs index 79c458b2f6..d846a4aace 100644 --- a/crates/api-test-helper/src/domain.rs +++ b/crates/api-test-helper/src/domain.rs @@ -26,6 +26,9 @@ pub async fn create(carbide_api_addrs: &[SocketAddr], name: &str) -> eyre::Resul "name": name, }); let domain_id = grpcurl_id(carbide_api_addrs, "CreateDomain", &data.to_string()).await?; - tracing::info!("Domain created with ID {domain_id}"); + tracing::info!( + domain_id = %domain_id, + "Domain created", + ); Ok(domain_id) } diff --git a/crates/api-test-helper/src/instance.rs b/crates/api-test-helper/src/instance.rs index fa29649cea..28b74f6485 100644 --- a/crates/api-test-helper/src/instance.rs +++ b/crates/api-test-helper/src/instance.rs @@ -32,8 +32,9 @@ pub async fn create( keyset_ids: &[&str], ) -> eyre::Result { tracing::info!( - "Creating instance with machine: {host_machine_id}, with network segment: {}", - segment_id.unwrap_or("") + host_machine_id = %host_machine_id, + network_segment_id = segment_id.unwrap_or(""), + "Creating instance", ); let mut tenant = serde_json::json!({ @@ -91,7 +92,10 @@ pub async fn create( }, }); let instance_id = grpcurl_id(addrs, "AllocateInstance", &data.to_string()).await?; - tracing::info!("Instance created with ID {instance_id}"); + tracing::info!( + instance_id = %instance_id, + "Instance created", + ); if !wait_until_ready { return Ok(instance_id); @@ -114,7 +118,10 @@ pub async fn create( wait_for_instance_state(addrs, &instance_id, "READY").await?; wait_for_state(addrs, host_machine_id, "Assigned/Ready").await?; - tracing::info!("Instance with ID {instance_id} is ready"); + tracing::info!( + instance_id = %instance_id, + "Instance is ready", + ); Ok(instance_id) } @@ -129,7 +136,7 @@ pub async fn create_with_vpc_prefixes( tracing::info!( %host_machine_id, ?vpc_prefix_ids, - "Creating instance with VPC prefix allocation", + "Creating instance", ); let v4_id = vpc_prefix_ids @@ -169,7 +176,11 @@ pub async fn create_with_vpc_prefixes( }); let instance_id = grpcurl_id(addrs, "AllocateInstance", &data.to_string()).await?; - tracing::info!("Dual-stack instance created with ID {instance_id}"); + tracing::info!( + instance_id = %instance_id, + ?vpc_prefix_ids, + "Instance created", + ); Ok(instance_id) } @@ -179,13 +190,20 @@ pub async fn release( instance_id: &str, wait_until_ready: bool, ) -> eyre::Result<()> { - tracing::info!("Releasing instance {instance_id} on machine: {host_machine_id}"); + tracing::info!( + instance_id, + host_machine_id = %host_machine_id, + "Releasing instance", + ); let data = serde_json::json!({ "id": {"value": instance_id} }); let resp = grpcurl(addrs, "ReleaseInstance", Some(data)).await?; - tracing::info!("ReleaseInstance response: {}", resp); + tracing::info!( + release_instance_response = %resp, + "ReleaseInstance response", + ); if !wait_until_ready { return Ok(()); @@ -211,9 +229,9 @@ pub async fn release( }) }); if let Some(ip_address) = ip_address { - tracing::info!("Instance with ID {instance_id} at {ip_address} is terminating"); + tracing::info!(instance_id, ip_address, "Instance is terminating",); } else { - tracing::info!("Instance with ID {instance_id} is terminating"); + tracing::info!(instance_id, "Instance is terminating",); } wait_for_state(addrs, host_machine_id, "WaitingForCleanup/HostCleanup").await?; @@ -222,10 +240,13 @@ pub async fn release( }); let response = grpcurl(addrs, "FindInstancesByIds", Some(&data)).await?; let resp: serde_json::Value = serde_json::from_str(&response)?; - tracing::info!("FindInstancesByIds Response: {}", resp); + tracing::info!( + find_instances_response = %resp, + "FindInstancesByIds Response", + ); assert!(resp["instances"].as_array().unwrap().is_empty()); - tracing::info!("Instance with ID {instance_id} is released"); + tracing::info!(instance_id, "Instance is released",); Ok(()) } @@ -239,7 +260,11 @@ pub async fn phone_home( "instance_id": {"value": instance_id}, }); - tracing::info!(%host_machine_id, "Phoning home for instance {instance_id}"); + tracing::info!( + %host_machine_id, + instance_id, + "Phoning home", + ); grpcurl(addrs, "UpdateInstancePhoneHomeLastContact", Some(&data)).await?; @@ -257,7 +282,10 @@ pub async fn get_instance_state(addrs: &[SocketAddr], instance_id: &str) -> eyre .as_str() .unwrap() .to_string(); - tracing::info!("\tCurrent instance state: {state}"); + tracing::info!( + state = %state, + "Current instance state", + ); Ok(state) } @@ -282,14 +310,17 @@ pub async fn wait_for_instance_state( let mut latest_state = String::new(); - tracing::info!("Waiting for Instance {instance_id} state {target_state}"); + tracing::info!(instance_id, target_state, "Waiting for Instance state",); while start.elapsed() < MAX_WAIT { latest_state = get_instance_state(addrs, instance_id).await?; if latest_state.contains(target_state) { return Ok(()); } - tracing::info!("\tCurrent instance state: {latest_state}"); + tracing::info!( + latest_state = %latest_state, + "Current instance state", + ); std::thread::sleep(std::time::Duration::from_secs(1)); } diff --git a/crates/api-test-helper/src/machine.rs b/crates/api-test-helper/src/machine.rs index 8233c4a133..52414883f2 100644 --- a/crates/api-test-helper/src/machine.rs +++ b/crates/api-test-helper/src/machine.rs @@ -34,7 +34,11 @@ pub async fn wait_for_state( let data = serde_json::json!({ "machine_ids": [{"id": machine_id}], }); - tracing::info!("Waiting for Machine {machine_id} state {target_state}"); + tracing::info!( + machine_id = %machine_id, + target_state, + "Waiting for Machine state", + ); let mut i = 0; while i < MAX_RETRY { let response = grpcurl(addrs, "FindMachinesByIds", Some(&data)).await?; @@ -43,7 +47,7 @@ pub async fn wait_for_state( if state.contains(target_state) { break; } - tracing::info!("\tCurrent: {state}"); + tracing::info!(state, "\tCurrent",); thread::sleep(time::Duration::from_millis(500)); i += 1; } diff --git a/crates/api-test-helper/src/machine_a_tron.rs b/crates/api-test-helper/src/machine_a_tron.rs index 1a1be33a6c..6682b9e456 100644 --- a/crates/api-test-helper/src/machine_a_tron.rs +++ b/crates/api-test-helper/src/machine_a_tron.rs @@ -75,8 +75,8 @@ pub async fn run_local( .entries; tracing::info!( - "Got desired firmware versions from the server: {:?}", - desired_firmware + ?desired_firmware, + "Got desired firmware versions from the server", ); let app_context = Arc::new(MachineATronContext { diff --git a/crates/api-test-helper/src/subnet.rs b/crates/api-test-helper/src/subnet.rs index 1e24556f13..c28bc4f7da 100644 --- a/crates/api-test-helper/src/subnet.rs +++ b/crates/api-test-helper/src/subnet.rs @@ -37,11 +37,17 @@ pub async fn create( }); let segment_id = grpcurl_id(carbide_api_addrs, "CreateNetworkSegment", &data.to_string()).await?; - tracing::info!("Network Segment created with ID {segment_id}"); + tracing::info!( + segment_id = %segment_id, + "Network segment created", + ); wait_for_network_segment_state(carbide_api_addrs, &segment_id, "READY").await?; - tracing::info!("Network Segment with ID {segment_id} is ready"); + tracing::info!( + segment_id = %segment_id, + "Network segment is ready", + ); Ok(segment_id) } @@ -59,7 +65,11 @@ pub async fn wait_for_network_segment_state( }); let mut latest_state: String; - tracing::info!("Waiting for Network Segment {segment_id} state {target_state}"); + tracing::info!( + segment_id, + target_state, + "Waiting for Network Segment state", + ); while start.elapsed() < MAX_WAIT { let response = grpcurl(addrs, "FindNetworkSegmentsByIds", Some(&data)).await?; let resp: serde_json::Value = serde_json::from_str(&response)?; @@ -70,7 +80,10 @@ pub async fn wait_for_network_segment_state( if latest_state.contains(target_state) { return Ok(()); } - tracing::info!("\tCurrent network segment state: {latest_state}"); + tracing::info!( + latest_state = %latest_state, + "\tCurrent network segment state", + ); std::thread::sleep(std::time::Duration::from_secs(1)); } @@ -104,11 +117,17 @@ pub async fn create_dual_stack( }); let segment_id = grpcurl_id(carbide_api_addrs, "CreateNetworkSegment", &data.to_string()).await?; - tracing::info!("Dual-stack network segment created with ID {segment_id}"); + tracing::info!( + segment_id = %segment_id, + "Dual-stack network segment created", + ); wait_for_network_segment_state(carbide_api_addrs, &segment_id, "READY").await?; - tracing::info!("Dual-stack network segment with ID {segment_id} is ready"); + tracing::info!( + segment_id = %segment_id, + "Dual-stack network segment is ready", + ); Ok(segment_id) } diff --git a/crates/api-test-helper/src/tenant.rs b/crates/api-test-helper/src/tenant.rs index 87a3729cb1..3ade393d4a 100644 --- a/crates/api-test-helper/src/tenant.rs +++ b/crates/api-test-helper/src/tenant.rs @@ -34,7 +34,7 @@ pub async fn create( } }); grpcurl(carbide_api_addrs, "CreateTenant", Some(&data.to_string())).await?; - tracing::info!("Tenant created with name {name}"); + tracing::info!(tenant_name = name, "Tenant created",); Ok(()) } @@ -69,7 +69,10 @@ pub mod keyset { Some(&data.to_string()), ) .await?; - tracing::info!("Tenant keyset created with id {id}"); + tracing::info!( + keyset_id = %id, + "Tenant keyset created", + ); Ok(()) } } diff --git a/crates/api-test-helper/src/vpc.rs b/crates/api-test-helper/src/vpc.rs index d2efc95481..75ab39d0e0 100644 --- a/crates/api-test-helper/src/vpc.rs +++ b/crates/api-test-helper/src/vpc.rs @@ -31,7 +31,10 @@ pub async fn create(carbide_api_addrs: &[SocketAddr], tenant_org_id: &str) -> ey "tenantOrganizationId": tenant_org_id, }); let vpc_id = grpcurl_id(carbide_api_addrs, "CreateVpc", &data.to_string()).await?; - tracing::info!("VPC created with ID {vpc_id}"); + tracing::info!( + vpc_id = %vpc_id, + "VPC created", + ); Ok(vpc_id) } @@ -48,7 +51,10 @@ pub async fn create_fnn( "network_virtualization_type": 5, // FNN }); let vpc_id = grpcurl_id(carbide_api_addrs, "CreateVpc", &data.to_string()).await?; - tracing::info!("FNN VPC created with ID {vpc_id}"); + tracing::info!( + vpc_id = %vpc_id, + "FNN VPC created", + ); Ok(vpc_id) } @@ -66,6 +72,9 @@ pub async fn create_flat( "network_virtualization_type": 6, // FLAT }); let vpc_id = grpcurl_id(carbide_api_addrs, "CreateVpc", &data.to_string()).await?; - tracing::info!("Flat VPC created with ID {vpc_id}"); + tracing::info!( + vpc_id = %vpc_id, + "Flat VPC created", + ); Ok(vpc_id) } diff --git a/crates/api-test-helper/src/vpc_prefix.rs b/crates/api-test-helper/src/vpc_prefix.rs index 97ce414b0a..c6c5b8471a 100644 --- a/crates/api-test-helper/src/vpc_prefix.rs +++ b/crates/api-test-helper/src/vpc_prefix.rs @@ -25,7 +25,7 @@ pub async fn create( prefix: &str, name: &str, ) -> eyre::Result { - tracing::info!("Creating VPC prefix {prefix} ({name})"); + tracing::info!(prefix, vpc_prefix_name = name, "Creating VPC prefix",); let data = serde_json::json!({ "vpc_id": { "value": vpc_id }, @@ -38,6 +38,9 @@ pub async fn create( }, }); let prefix_id = grpcurl_id(carbide_api_addrs, "CreateVpcPrefix", &data.to_string()).await?; - tracing::info!("VPC prefix created with ID {prefix_id}"); + tracing::info!( + prefix_id = %prefix_id, + "VPC prefix created", + ); Ok(prefix_id) } diff --git a/crates/api-web/src/auth.rs b/crates/api-web/src/auth.rs index 446814af6d..74947a92b7 100644 --- a/crates/api-web/src/auth.rs +++ b/crates/api-web/src/auth.rs @@ -423,7 +423,10 @@ impl IntoResponse for AuthCallbackResponse { match self { AuthCallbackResponse::Response(response) => response, AuthCallbackResponse::Error(error) => { - tracing::error!("internal server error running auth_callback: {error}"); + tracing::error!( + error = %error, + "internal server error running auth_callback", + ); (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response() } } @@ -439,9 +442,9 @@ where fn from(value: (StatusCode, S)) -> Self { if !value.0.is_success() { tracing::info!( - "auth_callback was unsuccessful with status code {}: {}", - value.0.as_u16(), - value.1.to_string() + status_code = value.0.as_u16(), + response_detail = %value.1.to_string(), + "auth_callback returned an unsuccessful response", ); } AuthCallbackResponse::Response((value.0, value.1.to_string()).into_response()) diff --git a/crates/api-web/src/interface.rs b/crates/api-web/src/interface.rs index 97901c85d0..8e0d934c72 100644 --- a/crates/api-web/src/interface.rs +++ b/crates/api-web/src/interface.rs @@ -351,7 +351,12 @@ pub async fn detail( }; if machine_interfaces.interfaces.len() != 1 { - tracing::error!(%interface_id, "Expected exactly 1 match, found {}", machine_interfaces.interfaces.len()); + tracing::error!( + %interface_id, + expected_interface_count = 1, + matching_interface_count = machine_interfaces.interfaces.len(), + "Unexpected number of matching interfaces", + ); return ( StatusCode::INTERNAL_SERVER_ERROR, "Expected exactly one interface to match", diff --git a/crates/api-web/src/lib.rs b/crates/api-web/src/lib.rs index a902f69b6c..743fc4a013 100644 --- a/crates/api-web/src/lib.rs +++ b/crates/api-web/src/lib.rs @@ -408,8 +408,8 @@ pub fn routes(api: Arc) -> eyre::Result> { } "none" | "" => { tracing::warn!( - "{}: admin web UI has no in-process authentication; restrict access with network policy, a private network, or an authenticating reverse proxy (for example OAuth2 Proxy)", - AUTH_TYPE_ENV + auth_type_env_var = AUTH_TYPE_ENV, + "admin web UI has no in-process authentication; restrict access with network policy, a private network, or an authenticating reverse proxy (for example OAuth2 Proxy)", ); None } diff --git a/crates/api-web/src/machine.rs b/crates/api-web/src/machine.rs index e1de2ce593..67e9eb1c88 100644 --- a/crates/api-web/src/machine.rs +++ b/crates/api-web/src/machine.rs @@ -1008,7 +1008,7 @@ pub async fn quarantine( .await .map(|_| ()), unknown => { - tracing::error!("Expected action to be 'enable' or 'disable' but got {unknown}"); + tracing::error!(action = unknown, "Unexpected quarantine action",); return Redirect::to(&view_url).into_response(); } }; @@ -1072,7 +1072,7 @@ pub async fn sku( .await .map(|_| "SKU association removed successfully".to_string()), unknown => { - tracing::error!("Expected SKU action to be 'assign' or 'remove' but got {unknown}"); + tracing::error!(action = unknown, "Unexpected SKU action",); return Redirect::to(&view_url).into_response(); } }; diff --git a/crates/api-web/src/managed_host.rs b/crates/api-web/src/managed_host.rs index 0dba7c859c..ca57113488 100644 --- a/crates/api-web/src/managed_host.rs +++ b/crates/api-web/src/managed_host.rs @@ -502,7 +502,10 @@ pub async fn show_html( } if active_gpu_filter != "all" { let Ok(gf) = active_gpu_filter.parse::() else { - tracing::warn!("Invalid GPU filter: '{active_gpu_filter}'"); + tracing::warn!( + active_gpu_filter = %active_gpu_filter, + "Invalid GPU filter", + ); continue; }; if gf != m.num_gpus { @@ -511,7 +514,10 @@ pub async fn show_html( } if active_ib_filter != "all" { let Ok(ibf) = active_ib_filter.parse::() else { - tracing::warn!("Invalid IB IFs filter: '{active_ib_filter}'"); + tracing::warn!( + active_ib_filter = %active_ib_filter, + "Invalid IB IFs filter", + ); continue; }; if ibf != m.num_ib_ifs { @@ -817,7 +823,7 @@ fn mem_to_size(mem: &str) -> isize { .collect_tuple() .map(|(size, unit)| (size.parse::(), unit)) else { - tracing::warn!("Invalid memory format: '{mem}'"); + tracing::warn!(mem, "Invalid memory format",); return 0; }; @@ -825,7 +831,7 @@ fn mem_to_size(mem: &str) -> isize { "GiB" => size, "TiB" => size * 1024.0, _ => { - tracing::warn!("Invalid unit '{}' in mem string '{mem}'", unit); + tracing::warn!(unit, mem, "Invalid unit in mem string",); return 0; } }) as isize diff --git a/crates/api-web/src/search.rs b/crates/api-web/src/search.rs index c842cb9844..10d24002c5 100644 --- a/crates/api-web/src/search.rs +++ b/crates/api-web/src/search.rs @@ -117,7 +117,10 @@ async fn find_by_uuid(state: Arc, u: uuid::Uuid) -> Response { } }; let Ok(object_type) = forgerpc::UuidType::try_from(out.object_type) else { - tracing::error!("Invalid UuidType from carbide api: {}", out.object_type); + tracing::error!( + object_type = out.object_type, + "Invalid UuidType from carbide api", + ); return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; use forgerpc::UuidType::*; @@ -157,7 +160,10 @@ async fn find_by_mac(state: Arc, mac: mac_address::MacAddress) -> impl Into } }; let Ok(object_type) = forgerpc::MacOwner::try_from(out.object_type) else { - tracing::error!("Invalid MacOwner from carbide api: {}", out.object_type); + tracing::error!( + object_type = out.object_type, + "Invalid MacOwner from carbide api", + ); return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; use forgerpc::MacOwner::*; diff --git a/crates/bmc-explorer/src/chassis.rs b/crates/bmc-explorer/src/chassis.rs index e67244cc9d..37cd659569 100644 --- a/crates/bmc-explorer/src/chassis.rs +++ b/crates/bmc-explorer/src/chassis.rs @@ -148,7 +148,10 @@ impl ExploredChassisCollection { .iter() .map(|ps| format!("{}:{:?}", ps.id, ps.power_state)) .join(", "); - tracing::warn!("Delta power shelf power state is unknown: {detail}"); + tracing::warn!( + power_supply_states = %detail, + "Delta power shelf power state is unknown" + ); } state } @@ -470,7 +473,7 @@ fn delta_psu_power_on(ps: &NvPowerSupply) -> Option { match ps.oem_delta() { Ok(oem) => oem.and_then(|d| d.power()), Err(e) => { - tracing::warn!("failed to parse Delta OEM power supply data: {e:?}"); + tracing::warn!(error = ?e, "Failed to parse Delta OEM power supply data"); None } } @@ -529,7 +532,10 @@ impl LiteOnSuppliesState<'_> { } else if off { ModelPowerState::Off } else { - tracing::warn!("powershelf power state is unknown: {self}"); + tracing::warn!( + power_supply_states = %self, + "powershelf power state is unknown" + ); ModelPowerState::Unknown } } diff --git a/crates/bmc-explorer/src/computer_system.rs b/crates/bmc-explorer/src/computer_system.rs index ecc5f4fbcb..669ac4bf08 100644 --- a/crates/bmc-explorer/src/computer_system.rs +++ b/crates/bmc-explorer/src/computer_system.rs @@ -158,7 +158,8 @@ impl ExploredComputerSystem { == Some(ErrorClass::NotFound) { tracing::warn!( - "received 404 on system's ethernet collection fetch. Retrying. {retries_remaining} tries left" + retries_remaining, + "received 404 while fetching the system ethernet collection; retrying" ); retries_remaining -= 1; tokio::time::sleep(config.explore.retry_timeout).await; @@ -196,7 +197,11 @@ impl ExploredComputerSystem { v.inner() .parse() .inspect_err(|err| { - tracing::warn!("Failed to parse BaseMAC: {err} (mac: {v})"); + tracing::warn!( + error = %err, + mac_address = %v, + "failed to parse BaseMAC" + ); }) .ok() }); @@ -354,25 +359,31 @@ impl ExploredComputerSystem { &self, hw_type: Option, ) -> Result, Error> { - let mut result = self.ethernet_interfaces.iter() + let mut result = self + .ethernet_interfaces + .iter() .map(|iface| { let mac_address = iface .mac_address() .map(|addr| { - deserialize_input_mac_to_address(addr.as_str()) - .map_err(|e| Error::InvalidValue(format!("MAC address not valid: {addr} (err: {e})"))) + deserialize_input_mac_to_address(addr.as_str()).map_err(|e| { + Error::InvalidValue(format!("MAC address not valid: {addr} (err: {e})")) + }) }) .transpose() .or_else(|err| { if iface - .interface_enabled().is_some_and(|is_enabled| !is_enabled) + .interface_enabled() + .is_some_and(|is_enabled| !is_enabled) { // disabled interfaces sometimes populate the MAC address with junk, // ignore this error and create the interface with an empty mac address // in the exploration report tracing::debug!( - "could not parse MAC address for a disabled interface {} (link_status: {:#?}): {err}", - iface.id(), iface.link_status() + interface_id = %iface.id(), + link_status = ?iface.link_status(), + error = %err, + "could not parse MAC address for a disabled interface" ); Ok(None) } else { @@ -395,7 +406,8 @@ impl ExploredComputerSystem { link_status: iface.link_status().map(|s| format!("{s:?}")), uefi_device_path, }) - }).collect::, _>>()?; + }) + .collect::, _>>()?; if hw_type.is_some_and(|v| v == hw::HwType::Bluefield) && !result.iter().any(|iface| { diff --git a/crates/bmc-explorer/src/manager.rs b/crates/bmc-explorer/src/manager.rs index e60f53003f..9eb479f218 100644 --- a/crates/bmc-explorer/src/manager.rs +++ b/crates/bmc-explorer/src/manager.rs @@ -149,61 +149,69 @@ impl ExploredManager { } pub fn to_model(&self) -> Result> { - let ethernet_interfaces = self.eth_interfaces.iter().map(|iface| { - let mac_address = iface - .mac_address() - .map(|addr| { - deserialize_input_mac_to_address(addr.as_str()) - .map_err(|e| Error::InvalidValue(format!("MAC address not valid: {addr} (err: {e})"))) + let ethernet_interfaces = self + .eth_interfaces + .iter() + .map(|iface| { + let mac_address = iface + .mac_address() + .map(|addr| { + deserialize_input_mac_to_address(addr.as_str()).map_err(|e| { + Error::InvalidValue(format!("MAC address not valid: {addr} (err: {e})")) + }) + }) + .transpose() + .or_else(|err| { + if iface + .interface_enabled() + .is_some_and(|is_enabled| !is_enabled) + { + // disabled interfaces sometimes populate the MAC address with junk, + // ignore this error and create the interface with an empty mac address + // in the exploration report + tracing::debug!( + interface_id = %iface.id(), + link_status = ?iface.link_status(), + error = %err, + "could not parse MAC address for a disabled interface" + ); + Ok(None) + } else { + Err(err) + } + })?; + + // Warn if the manager eth0 MAC is locally-administered: a real BMC MAC is + // globally unique, so this signals transient pre-sync data (seen briefly + // after a BMC reboot) that would poison anything keyed on the BMC MAC. + if iface.id().inner().eq_ignore_ascii_case("eth0") + && let Some(mac) = mac_address + && is_locally_administered_mac(mac) + { + tracing::warn!( + manager_id = %self.manager.id().inner(), + eth0_mac = %mac, + "manager eth0 MAC is locally-administered (transient pre-sync data?)", + ); + } + + let uefi_device_path = iface + .uefi_device_path() + .map(|v| v.into_inner()) + .map(UefiDevicePath::from_str) + .transpose() + .map_err(|err| Error::InvalidValue(format!("UefiDevicePath: {err}")))?; + + Ok(ModelEthernetInterface { + description: iface.description().map(|v| v.to_string()), + id: Some(iface.id().to_string()), + interface_enabled: iface.interface_enabled(), + mac_address, + link_status: iface.link_status().map(|s| format!("{s:?}")), + uefi_device_path, }) - .transpose() - .or_else(|err| { - if iface - .interface_enabled().is_some_and(|is_enabled| !is_enabled) - { - // disabled interfaces sometimes populate the MAC address with junk, - // ignore this error and create the interface with an empty mac address - // in the exploration report - tracing::debug!( - "could not parse MAC address for a disabled interface {} (link_status: {:#?}): {err}", - iface.id(), iface.link_status() - ); - Ok(None) - } else { - Err(err) - } - })?; - - // Warn if the manager eth0 MAC is locally-administered: a real BMC MAC is - // globally unique, so this signals transient pre-sync data (seen briefly - // after a BMC reboot) that would poison anything keyed on the BMC MAC. - if iface.id().inner().eq_ignore_ascii_case("eth0") - && let Some(mac) = mac_address - && is_locally_administered_mac(mac) - { - tracing::warn!( - manager_id = %self.manager.id().inner(), - eth0_mac = %mac, - "manager eth0 MAC is locally-administered (transient pre-sync data?)", - ); - } - - let uefi_device_path = iface - .uefi_device_path() - .map(|v| v.into_inner()) - .map(UefiDevicePath::from_str) - .transpose() - .map_err(|err| Error::InvalidValue(format!("UefiDevicePath: {err}")))?; - - Ok(ModelEthernetInterface { - description: iface.description().map(|v| v.to_string()), - id: Some(iface.id().to_string()), - interface_enabled: iface.interface_enabled(), - mac_address, - link_status: iface.link_status().map(|s| format!("{s:?}")), - uefi_device_path, }) - }).collect::, _>>()?; + .collect::, _>>()?; Ok(ModelManager { id: self.manager.id().inner().to_string(), diff --git a/crates/bmc-mock/src/combined_server.rs b/crates/bmc-mock/src/combined_server.rs index e821561d4d..dc95602c7b 100644 --- a/crates/bmc-mock/src/combined_server.rs +++ b/crates/bmc-mock/src/combined_server.rs @@ -60,7 +60,10 @@ impl Drop for CombinedServer { if let Some(join_handle) = self.join_handle.take() && !join_handle.is_finished() { - tracing::info!("Stopping BMC Mock at {}", self.address); + tracing::info!( + address = %self.address, + "Stopping BMC mock", + ); self.axum_handle.shutdown(); join_handle.abort() } @@ -114,7 +117,10 @@ impl CombinedServer { ) } }; - tracing::info!("Listening on {}", addr); + tracing::info!( + listen_address = %addr, + "BMC mock listening", + ); // Inject middleware to normalize request URIs by dropping the trailing slash let router = router.layer(NormalizePathLayer::trim_trailing_slash()); @@ -125,7 +131,11 @@ impl CombinedServer { .serve(router.into_make_service()) .await .inspect_err(|e| { - tracing::error!("BMC mock could not listen on address {}: {}", addr, e) + tracing::error!( + listen_address = %addr, + error = %e, + "BMC mock could not listen", + ) })?; Ok(()) }) diff --git a/crates/bmc-mock/src/combined_service.rs b/crates/bmc-mock/src/combined_service.rs index 9762756030..d26ecf37bd 100644 --- a/crates/bmc-mock/src/combined_service.rs +++ b/crates/bmc-mock/src/combined_service.rs @@ -111,7 +111,12 @@ fn no_router_response( let err = format!( "no router configured for forwarded_host/host/authority: {forwarded_host:?}/{host:?}/{authority:?}" ); - tracing::info!("{err}"); + tracing::info!( + forwarded_host = ?forwarded_host, + host = ?host, + authority = ?authority, + "No BMC mock router is configured for request", + ); Response::builder() .status(StatusCode::NOT_FOUND) .body(err.into()) diff --git a/crates/bmc-mock/src/main.rs b/crates/bmc-mock/src/main.rs index 4a6cd6fdef..60675aee5f 100644 --- a/crates/bmc-mock/src/main.rs +++ b/crates/bmc-mock/src/main.rs @@ -73,9 +73,9 @@ async fn main() -> Result<(), Box> { if let Some(ip_routers) = args.ip_router { for ip_router in ip_routers { info!( - "Using archive {} for {}", - ip_router.targz.to_string_lossy(), - ip_router.ip_address + archive_path = %ip_router.targz.to_string_lossy(), + ip_address = %ip_router.ip_address, + "Using BMC mock archive", ); let r = tar_router::tar_router( TarGzOption::Disk(&ip_router.targz), @@ -87,9 +87,15 @@ async fn main() -> Result<(), Box> { } let listen_addr = args.port.map(|p| SocketAddr::from(([0, 0, 0, 0], p))); - info!("Using cert_path: {:?}", args.cert_path); + info!( + cert_path = ?args.cert_path, + "Using BMC mock certificate path", + ); let router = if let Some(tar_path) = args.targz { - info!("Using archive {} as default", tar_path.to_string_lossy()); + info!( + archive_path = %tar_path.to_string_lossy(), + "Using default BMC mock archive", + ); tar_router::tar_router(TarGzOption::Disk(&tar_path), Some(&mut tar_router_entries)).unwrap() } else { info!("Using default BMC mock"); @@ -132,7 +138,10 @@ fn spawn_qemu_reboot_handler() -> mpsc::UnboundedSender { continue; } Err(err) => { - tracing::error!("Error trying to run 'virsh reboot ManagedHost'. {}", err); + tracing::error!( + error = %err, + "Failed to run virsh reboot for managed host", + ); continue; } }; @@ -142,11 +151,15 @@ fn spawn_qemu_reboot_handler() -> mpsc::UnboundedSender { tracing::debug!("Rebooted qemu managed host..."); } Some(exit_code) => { - tracing::error!( - "Reboot command 'virsh reboot ManagedHost' failed with exit code {exit_code}." + tracing::error!(exit_code, "virsh reboot failed for managed host",); + tracing::info!( + stdout = %String::from_utf8_lossy(&reboot_output.stdout), + "virsh reboot standard output", + ); + tracing::info!( + stderr = %String::from_utf8_lossy(&reboot_output.stderr), + "virsh reboot standard error", ); - tracing::info!("STDOUT: {}", String::from_utf8_lossy(&reboot_output.stdout)); - tracing::info!("STDERR: {}", String::from_utf8_lossy(&reboot_output.stderr)); } None => { tracing::error!("Reboot command killed by signal"); diff --git a/crates/bmc-mock/src/redfish/expander_router.rs b/crates/bmc-mock/src/redfish/expander_router.rs index 07576cdbad..9dae723cd8 100644 --- a/crates/bmc-mock/src/redfish/expander_router.rs +++ b/crates/bmc-mock/src/redfish/expander_router.rs @@ -93,7 +93,10 @@ async fn process(State(mut state): State, request: Request) -> R .filter_map(|member| match member { Value::Object(object) => Some(object), _ => { - tracing::warn!("Invalid member JSON, expected Object: {:?}", member); + tracing::warn!( + member = ?member, + "Invalid member JSON, expected Object", + ); None } }) @@ -101,8 +104,8 @@ async fn process(State(mut state): State, request: Request) -> R Some(Value::String(id)) => Some(id.to_owned()), _ => { tracing::warn!( - "Invalid member JSON, expected @odata.id string: {:?}", - object + object = ?object, + "Invalid member JSON, expected @odata.id string", ); None } diff --git a/crates/bmc-mock/src/tar_router.rs b/crates/bmc-mock/src/tar_router.rs index d6cba26c0a..7105ee296e 100644 --- a/crates/bmc-mock/src/tar_router.rs +++ b/crates/bmc-mock/src/tar_router.rs @@ -140,7 +140,10 @@ async fn get_from_tar( match cache.entries.lock().unwrap().get(&path) { None => { - tracing::trace!("Not found: {path}"); + tracing::trace!( + path = %path, + "BMC mock archive path not found", + ); (StatusCode::NOT_FOUND, path).into_response() } Some(s) => (StatusCode::OK, s.clone()).into_response(), @@ -149,7 +152,11 @@ async fn get_from_tar( // We should never get here, but axum's matchit bug means we sometimes do: https://github.com/tokio-rs/axum/issues/1986 async fn not_found_handler(req: Request) -> (StatusCode, String) { - tracing::warn!("fallback: No route for {} {}", req.method(), req.uri()); + tracing::warn!( + method = %req.method(), + uri = %req.uri(), + "No route for BMC mock request", + ); ( StatusCode::NOT_FOUND, format!("No route for {} {}", req.method(), req.uri()), diff --git a/crates/bmc-mock/src/tls.rs b/crates/bmc-mock/src/tls.rs index 95d950e462..37a604d66a 100644 --- a/crates/bmc-mock/src/tls.rs +++ b/crates/bmc-mock/src/tls.rs @@ -95,7 +95,11 @@ pub fn server_config(cert_path: Option>) -> Result acceptor, Err(e) => { - tracing::error!("Error reloading TLS certificate, will retry: {e}"); + tracing::error!( + error = %e, + "Error reloading TLS certificate, will retry", + ); emit(TlsConnectionFailed { reason: ConnectionFailReason::TlsCertificateInvalid, }); @@ -352,7 +355,11 @@ impl BmcProxy { .serve_connection(conn, TowerToHyperService::new(app_with_ext)) .await { - tracing::debug!(%error, "error servicing tls http request: {error:?}"); + tracing::debug!( + %error, + error_debug = ?error, + "error servicing tls http request", + ); } } Err(error) => { @@ -488,7 +495,7 @@ fn get_tls_acceptor(tls_config: &TlsConfig) -> Result( auth_config: &AuthConfig, ) -> Result, BmcProxyError> { - tracing::info!("TrustConfig rendered from config: {:?}", auth_config.trust); + tracing::info!(trust_config = ?auth_config.trust, "TrustConfig rendered from config"); let spiffe_context = SpiffeContext::try_from(auth_config.trust.clone()).map_err(|e| { BmcProxyError::InvalidConfiguration(format!( "Invalid trust config in bmc-proxy config toml file: {e}" @@ -651,7 +658,7 @@ async fn ip_for_forwarded_target( .iter() .filter_map(|s| { IpAddr::from_str(s) - .inspect_err(|e| tracing::error!("Invalid IP address returned by API: {e}")) + .inspect_err(|e| tracing::error!(error = %e, "Invalid IP address returned by API")) .ok() }) .collect::>(); @@ -666,13 +673,9 @@ async fn ip_for_forwarded_target( (0, 1..) => { if v6_ips.len() > 1 { tracing::warn!( - "Multiple IPv6 BMC IP's found for {} ({}), using first one", - lookup_by_str, - v6_ips - .iter() - .map(|ip| ip.to_string()) - .collect::>() - .join(", "), + lookup_by = %lookup_by_str, + ip_addresses = ?v6_ips, + "Multiple IPv6 BMC IP's found, using first one", ); } v6_ips.into_iter().next() @@ -682,13 +685,9 @@ async fn ip_for_forwarded_target( // first, in case of broken dual-stack setups. if v4_ips.len() > 1 { tracing::warn!( - "Multiple IPv4 BMC IP's found for {} ({}), using first one", - lookup_by_str, - v4_ips - .iter() - .map(|ip| ip.to_string()) - .collect::>() - .join(", "), + lookup_by = %lookup_by_str, + ip_addresses = ?v4_ips, + "Multiple IPv4 BMC IP's found, using first one", ); } v4_ips.into_iter().next() diff --git a/crates/bmc-proxy/src/setup.rs b/crates/bmc-proxy/src/setup.rs index 84d269719a..5febd1c236 100644 --- a/crates/bmc-proxy/src/setup.rs +++ b/crates/bmc-proxy/src/setup.rs @@ -61,7 +61,7 @@ pub fn setup_logging(debug: bool) -> SetupResult<()> { ) .try_init()?; - tracing::info!("current log level: {}", LevelFilter::current()); + tracing::info!(log_level = %LevelFilter::current(), "current log level"); Ok(()) } diff --git a/crates/dhcp-server/src/cache.rs b/crates/dhcp-server/src/cache.rs index 4d7cfc5fe2..0f54259091 100644 --- a/crates/dhcp-server/src/cache.rs +++ b/crates/dhcp-server/src/cache.rs @@ -57,14 +57,14 @@ pub fn get( ) -> Option { let key = &key(mac_address, link_address, circuit_id, remote_id, vendor_id); if key.len() < MIN_KEY_LEN { - tracing::debug!("Unexpected cache key, skipping: '{key}'"); + tracing::debug!(key = key.as_str(), "Unexpected cache key, skipping"); return None; } if let Some(entry) = cache.get(key) { if !entry.has_expired() { return Some(entry.clone()); } else { - tracing::debug!("removed expired cached response for {mac_address:?}"); + tracing::debug!(?mac_address, "removed expired cached response"); let _removed = cache.pop_entry(key); } } diff --git a/crates/dhcp-server/src/grpc_server.rs b/crates/dhcp-server/src/grpc_server.rs index c9bd0400b4..45b38ce9a9 100644 --- a/crates/dhcp-server/src/grpc_server.rs +++ b/crates/dhcp-server/src/grpc_server.rs @@ -215,7 +215,7 @@ impl DhcpServerControl for DhcpServerControlService { ) -> Result, Status> { let mut ts = DhcpTimestamps::new(DhcpTimestampsFilePath::Hbn); if let Err(e) = ts.read() { - tracing::warn!("Failed to read DHCP timestamps file: {e}"); + tracing::warn!(error = %e, "Failed to read DHCP timestamps file"); } let entries = ts .into_iter() @@ -233,13 +233,13 @@ impl DhcpServerControl for DhcpServerControlService { /// Start the plain (no-TLS) gRPC control server and block until it exits. pub async fn run_grpc_server(addr: SocketAddr, ctrl_tx: mpsc::Sender) { let service = DhcpServerControlService { ctrl_tx }; - tracing::info!("gRPC config-reload server listening on {}", addr); + tracing::info!(listen_address = %addr, "gRPC config-reload server listening"); if let Err(e) = tonic::transport::Server::builder() .add_service(DhcpServerControlServer::new(service)) .serve(addr) .await { - tracing::error!("gRPC server exited with error: {}", e); + tracing::error!(listen_address = %addr, error = %e, "gRPC server exited"); } } diff --git a/crates/dhcp-server/src/main.rs b/crates/dhcp-server/src/main.rs index 22b6e20491..62045c9243 100644 --- a/crates/dhcp-server/src/main.rs +++ b/crates/dhcp-server/src/main.rs @@ -69,7 +69,7 @@ async fn run_dhcp_server(args: Args, cancel_token: CancellationToken) { let config__ = match init(args.clone()).await { Ok(c) => c, Err(e) => { - tracing::error!("Failed to initialise DHCP server config: {}", e); + tracing::error!(error = %e, "Failed to initialise DHCP server config"); return; } }; @@ -88,7 +88,7 @@ async fn run_dhcp_server(args: Args, cancel_token: CancellationToken) { // and pollute the logs. We could have read() skip NotFound errors, but that // could be misleading in other scenarios. Let's just "init" the file. if let Err(e) = d.write() { - tracing::error!("Failed to init DHCP timestamps file: {}", e); + tracing::error!(error = %e, "Failed to init DHCP timestamps file"); return; } d @@ -116,10 +116,10 @@ async fn run_dhcp_server(args: Args, cancel_token: CancellationToken) { let socket = get_socket(listen_address, interface.clone()).await; tracing::info!( - "Listening on {:?} on interface: {}, mode: {:?}", - listen_address, - interface, - handler + %listen_address, + interface = interface.as_str(), + mode = ?handler, + "DHCP server listening" ); let mut server = Server { @@ -140,8 +140,8 @@ async fn run_dhcp_server(args: Args, cancel_token: CancellationToken) { tokio::select! { _ = cancel.cancelled() => { tracing::info!( - "DHCP server on interface {} received cancellation, shutting down", - interface + interface = interface.as_str(), + "DHCP server received cancellation, shutting down" ); break; } @@ -152,10 +152,19 @@ async fn run_dhcp_server(args: Args, cancel_token: CancellationToken) { // We don't know after this read is failed, will we be able to read again // from this socket? Mostly no. In this case, recreate the socket. // We observed this fluctuation during admin to tenant network switch. - tracing::error!("Socket recv failed with error: {err}"); + tracing::error!( + %listen_address, + interface = interface.as_str(), + error = %err, + "Socket receive failed" + ); // Try to close the existing socket. drop(server.socket); - tracing::info!("Recreating the socket on {listen_address}, {interface}"); + tracing::info!( + %listen_address, + interface = interface.as_str(), + "Recreating the socket" + ); server.socket = Arc::new(get_socket(listen_address, interface.clone()).await); continue; @@ -266,7 +275,7 @@ async fn handle_update_config( tokio::fs::write(&new_dhcp, &dhcp_yaml) .await .map_err(|e| -> Box { format!("write {new_dhcp}: {e}").into() })?; - tracing::info!("dhcp_config changed – staged at {new_dhcp}"); + tracing::info!(path = new_dhcp.as_str(), "dhcp_config changed – staged"); } if let (Some(yaml), Some(path)) = (host_yaml, &args.host_config) { @@ -276,7 +285,7 @@ async fn handle_update_config( tokio::fs::write(&new_host, &yaml) .await .map_err(|e| -> Box { format!("write {new_host}: {e}").into() })?; - tracing::info!("host_config changed – staged at {new_host}"); + tracing::info!(path = new_host.as_str(), "host_config changed – staged"); } } Ok(()) @@ -382,7 +391,7 @@ async fn run_with_grpc_control( .map_err(|e| -> Box { format!("create_dir_all {}: {e}", dir.display()).into() })?; - tracing::info!("Created config directory {}", dir.display()); + tracing::info!(path = %dir.display(), "Created config directory"); } // Channel through which the gRPC handlers deliver control requests. @@ -424,7 +433,7 @@ async fn run_with_grpc_control( None => std::future::pending().await, } } => { - tracing::error!("DHCP server exited unexpectedly: {:?}", result); + tracing::error!(?result, "DHCP server exited unexpectedly"); return Ok(()); } @@ -508,9 +517,9 @@ async fn main() -> Result<(), Box> { // default HealthController state), not packet-serving readiness -- // don't point a DHCP-serving probe at them. tokio::spawn(async move { - tracing::info!("Spawning metrics endpoint on {}", metrics_config.address); + tracing::info!(address = %metrics_config.address, "Spawning metrics endpoint"); if let Err(e) = run_metrics_endpoint(&metrics_config).await { - tracing::error!("Metrics endpoint error: {}", e); + tracing::error!(error = %e, "Metrics endpoint error"); } }); } @@ -645,7 +654,7 @@ async fn process( return; } - tracing::debug!("Received packet [{}] from {}", buf[0], addr); + tracing::debug!(bootp_op = buf[0], source_address = %addr, "Received DHCP packet"); let packet = match packet_handler::process_packet( buf, @@ -687,8 +696,9 @@ async fn process( dhcp_timestamps.add_timestamp(host_config.host_interface_id, Utc::now().to_rfc3339()); if let Err(e) = dhcp_timestamps.write() { tracing::error!( - "Failed writing to {}: {e}", - DhcpTimestampsFilePath::HbnTmp.path_str() + dhcp_timestamps_path = DhcpTimestampsFilePath::HbnTmp.path_str(), + error = %e, + "Failed to write DHCP timestamps file" ); } } diff --git a/crates/dhcp-server/src/modes/controller.rs b/crates/dhcp-server/src/modes/controller.rs index 66c9556b5e..465fa06c7b 100644 --- a/crates/dhcp-server/src/modes/controller.rs +++ b/crates/dhcp-server/src/modes/controller.rs @@ -73,12 +73,12 @@ impl DhcpMode for Controller { &mut machine_cache, ) { tracing::info!( - "returning cached response for (mac: {}, link(or relay)_address: {}, circuit_id: {:?}, remote: {:?}, vendor: {})", - discovery_request.mac_address, - link_address, - &discovery_request.circuit_id, - &discovery_request.remote_id, - &vendor_id, + mac_address = %discovery_request.mac_address, + %link_address, + circuit_id = ?discovery_request.circuit_id, + remote_id = ?discovery_request.remote_id, + %vendor_id, + "returning cached response" ); return Ok(cache_entry.dhcp_record); diff --git a/crates/dhcp-server/src/packet_handler.rs b/crates/dhcp-server/src/packet_handler.rs index f484ad33f9..787fa811f3 100644 --- a/crates/dhcp-server/src/packet_handler.rs +++ b/crates/dhcp-server/src/packet_handler.rs @@ -229,7 +229,7 @@ impl Packet { dst_address: SocketAddrV4, socket: Arc, ) -> Result<(), String> { - tracing::debug!("Sending packet to {:?}", dst_address); + tracing::debug!(?dst_address, "Sending packet"); socket .send_to(&self.encoded_packet, dst_address) .await diff --git a/crates/dhcp-server/src/util.rs b/crates/dhcp-server/src/util.rs index 1709f26e84..29fcbe4b58 100644 --- a/crates/dhcp-server/src/util.rs +++ b/crates/dhcp-server/src/util.rs @@ -25,7 +25,11 @@ macro_rules! socket_opr { ($socket:expr, $statement:expr, $retry:expr) => { if let Err(e) = $statement { drop($socket); - tracing::info!("Socket set option failed. Retry: {}, error: {e}", $retry); + tracing::info!( + retry = $retry, + error = %e, + "Socket set option failed" + ); tokio::time::sleep(std::time::Duration::from_secs(2)).await; continue; } @@ -94,7 +98,7 @@ pub async fn get_socket(listen_address: core::net::SocketAddr, interface: String ) { Ok(socket) => socket, Err(e) => { - tracing::info!("Socket creation failed. Retry: {retry}, error: {e}"); + tracing::info!(retry, error = %e, "Socket creation failed"); tokio::time::sleep(std::time::Duration::from_secs(2)).await; continue; } @@ -109,7 +113,11 @@ pub async fn get_socket(listen_address: core::net::SocketAddr, interface: String let mut retries_left = 10; while retries_left > 0 && socket.bind_device(Some(interface.as_bytes())).is_err() { retries_left -= 1; - tracing::info!("Interface {interface} not ready, retrying {retries_left} more times"); + tracing::info!( + interface = interface.as_str(), + retries_left, + "Interface not ready, retrying" + ); tokio::time::sleep(std::time::Duration::from_secs(2)).await; } if retries_left == 0 { diff --git a/crates/dns-record/src/lib.rs b/crates/dns-record/src/lib.rs index 0dcfb32bc4..60789e00c2 100644 --- a/crates/dns-record/src/lib.rs +++ b/crates/dns-record/src/lib.rs @@ -172,17 +172,18 @@ impl SoaRecord { } else { // Increment the last two digits if the date hasn't changed let incremented_serial = self.serial + 1; - debug!("DNS serial number incremented: {}", incremented_serial); + debug!(serial = incremented_serial, "DNS serial number incremented"); self.serial = incremented_serial; } } pub fn generate_new_serial() -> u32 { let now = Utc::now(); let formatted_data = now.format("%Y%m%d").to_string() + "01"; - debug!("Serial generated for zone {}", formatted_data); - formatted_data + let serial = formatted_data .parse::() - .expect("Unable to generate new serial for zone") + .expect("Unable to generate new serial for zone"); + debug!(serial, "generated serial for DNS zone"); + serial } pub fn new(domain_name: &str) -> SoaRecord { diff --git a/crates/dns/src/lib.rs b/crates/dns/src/lib.rs index e02b9e97ff..588f349176 100644 --- a/crates/dns/src/lib.rs +++ b/crates/dns/src/lib.rs @@ -544,13 +544,13 @@ impl DnsServer { pub async fn run(config: Config) -> Result<(), Report> { let listen = config.listen_address; - info!("Starting DNS server on {}", listen); + info!(%listen, "Starting DNS server"); let forge_client_config = config.forge_client_config(); let api_uri = config.api_uri.to_string(); let api_config = ApiConfig::new(api_uri.as_str(), &forge_client_config); - info!("Connecting to carbide-api at {}", api_uri); + info!(%api_uri, "Connecting to carbide-api"); let client = Mutex::new(ForgeTlsClient::retry_build(&api_config).await?); @@ -578,9 +578,9 @@ impl DnsServer { }; tokio::spawn(async move { - tracing::info!("Spawning metrics endpoint on {}", metrics_config.address); + tracing::info!(address = %metrics_config.address, "Spawning metrics endpoint"); if let Err(e) = run_metrics_endpoint(&metrics_config).await { - tracing::error!("Metrics endpoint error: {}", e); + tracing::error!(error = %e, "Metrics endpoint error"); } }); @@ -611,9 +611,9 @@ impl DnsServer { srv.register_listener(tcp_socket, Duration::new(5, 0), 32); info!( - "Started DNS server on {} version {}", - listen, - carbide_version::version!() + %listen, + version = carbide_version::version!(), + "Started DNS server", ); match srv.block_until_done().await { @@ -622,7 +622,7 @@ impl DnsServer { } Err(e) => { let error_msg = format!("Carbide-dns has encountered an error: {e}"); - error!("{}", error_msg); + error!(error = %e, "Carbide-dns has encountered an error"); return Err(eyre::eyre!("{}", error_msg)); } } diff --git a/crates/dns/src/main.rs b/crates/dns/src/main.rs index bfe030a421..ec4a681376 100644 --- a/crates/dns/src/main.rs +++ b/crates/dns/src/main.rs @@ -63,11 +63,6 @@ async fn main() -> Result<(), eyre::Report> { Command::Run(run_command) => { let config: Config = run_command.try_into()?; - tracing::info!( - "OpenTelemetry tracing enabled, exporting to endpoint: {}", - &config.otlp_endpoint.to_string() - ); - let otlp_exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .with_endpoint(config.otlp_endpoint.to_string()) @@ -96,6 +91,11 @@ async fn main() -> Result<(), eyre::Report> { .with(otel_layer) .try_init()?; + tracing::info!( + endpoint = %config.otlp_endpoint, + "OpenTelemetry tracing enabled", + ); + DnsServer::run(config) .await .wrap_err("Failed to start DNS service")?; diff --git a/crates/dpa-manager/src/card_handler/astra.rs b/crates/dpa-manager/src/card_handler/astra.rs index 4bf02096ad..3218a3b9ca 100644 --- a/crates/dpa-manager/src/card_handler/astra.rs +++ b/crates/dpa-manager/src/card_handler/astra.rs @@ -37,8 +37,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_provisioning idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_provisioning index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -73,8 +74,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_ready idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_ready index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -110,8 +112,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_unlocking idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_unlocking index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -122,8 +125,8 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { let dpa_interface = &mh.dpa_interface_snapshots[idx]; tracing::warn!( - "Astra DPA interface state unexpected state: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "Astra DPA interface is in an unexpected state", ); return Ok(HandlerResult { @@ -141,8 +144,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_apply_firmware idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_apply_firmware index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -153,8 +157,8 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { let dpa_interface = &mh.dpa_interface_snapshots[idx]; tracing::warn!( - "Astra DPA interface state unexpected state: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "Astra DPA interface is in an unexpected state", ); return Ok(HandlerResult { @@ -172,8 +176,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_apply_profile idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_apply_profile index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -183,8 +188,8 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { let dpa_interface = &mh.dpa_interface_snapshots[idx]; tracing::warn!( - "Astra DPA interface state unexpected state: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "Astra DPA interface is in an unexpected state", ); return Ok(HandlerResult { new_state: None, @@ -201,8 +206,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_locking idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_locking index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -212,8 +218,8 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { let dpa_interface = &mh.dpa_interface_snapshots[idx]; tracing::warn!( - "Astra DPA interface state unexpected state: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "Astra DPA interface is in an unexpected state", ); return Ok(HandlerResult { new_state: None, @@ -230,8 +236,9 @@ impl DpaInterfaceStateHandler for AstraInterfaceHandler { ) -> DpaManagerResult { if idx >= mh.dpa_interface_snapshots.len() { tracing::error!( - "handle_assigned idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_assigned index out of bounds", ); return Ok(HandlerResult { new_state: None, diff --git a/crates/dpa-manager/src/card_handler/svpc.rs b/crates/dpa-manager/src/card_handler/svpc.rs index 2fe75aef10..2cef280406 100644 --- a/crates/dpa-manager/src/card_handler/svpc.rs +++ b/crates/dpa-manager/src/card_handler/svpc.rs @@ -50,7 +50,10 @@ impl SvpcInterfaceHandler { fn at_most_one(mut iter: I, ctx: &str) -> DpaManagerResult> { let first = iter.next(); if first.is_some() && iter.next().is_some() { - tracing::error!("{ctx}: more than one match"); + tracing::error!( + context = %ctx, + "Multiple matches found", + ); return Err(DpaManagerError::InvalidArgument(format!( "{ctx}: more than one match" ))); @@ -232,9 +235,10 @@ impl SvpcInterfaceHandler { || (observed_attachment.config_version != Some(nic_version)); tracing::debug!( - "[{}] reconcile_ready_state: need_deletion {need_deletion}, need_heartbeat {}", - chrono::Utc::now(), - !need_deletion + timestamp = %chrono::Utc::now(), + need_deletion = need_deletion, + heartbeat_needed = !need_deletion, + "Reconciled ready-state action", ); if need_deletion { @@ -271,8 +275,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_provisioning idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_provisioning index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -305,8 +310,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_ready idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_ready index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -359,8 +365,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_unlocking idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_unlocking index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -370,8 +377,8 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { let Some(ref cs) = dpa_interface.card_state else { tracing::error!( - "Unexpected - card_state none for dpa: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "DPA interface has no card state", ); return Ok(HandlerResult { new_state: None, @@ -404,8 +411,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_apply_firmware idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_apply_firmware index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -415,8 +423,8 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { let Some(ref card_state) = dpa_interface.card_state else { tracing::info!( - "no firmware report, because card_state none for dpa: {:#?}, waiting for retry", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "DPA interface has no card state; waiting to retry firmware report", ); return Ok(HandlerResult { new_state: None, @@ -462,8 +470,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_apply_profile idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_apply_profile index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -483,8 +492,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_locking idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_locking index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -494,8 +504,8 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { let Some(ref cs) = dpa_interface.card_state else { tracing::error!( - "Unexpected - card_state none for dpa: {:#?}", - dpa_interface.id + dpa_interface_id = %dpa_interface.id, + "DPA interface has no card state", ); return Ok(HandlerResult { new_state: None, @@ -551,8 +561,9 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { ) -> DpaManagerResult { let Some(dpa_interface) = mh.dpa_interface_snapshots.get(idx) else { tracing::error!( - "handle_assigned idx out of bounds: {idx}, len: {}", - mh.dpa_interface_snapshots.len() + index = idx, + dpa_interface_snapshot_count = mh.dpa_interface_snapshots.len(), + "handle_assigned index out of bounds", ); return Ok(HandlerResult { new_state: None, @@ -602,8 +613,8 @@ impl DpaInterfaceStateHandler for SvpcInterfaceHandler { fn apply_profile(state: &DpaInterface) -> DpaManagerResult { let Some(ref cs) = state.card_state else { tracing::info!( - "no profile report, because card_state none for dpa: {:#?}, waiting for retry", - state.id + dpa_interface_id = %state.id, + "DPA interface has no card state; waiting to retry profile report", ); return Ok(HandlerResult { new_state: None, diff --git a/crates/dpa-manager/src/lib.rs b/crates/dpa-manager/src/lib.rs index 07f56d272e..0974949980 100644 --- a/crates/dpa-manager/src/lib.rs +++ b/crates/dpa-manager/src/lib.rs @@ -121,7 +121,10 @@ impl DpaMonitor { } } Err(e) => { - tracing::warn!("DpaMonitor error: {}", e); + tracing::warn!( + error = %e, + "DPA monitor error", + ); } } @@ -165,7 +168,8 @@ impl DpaMonitor { Ok(lock) => lock, Err(e) => { tracing::warn!( - "DpaMonitor failed to acquire work lock: Another instance of carbide running? {e}" + error = %e, + "DpaMonitor failed to acquire work lock: Another instance of carbide running?", ); return Ok(0); } @@ -426,11 +430,11 @@ impl DpaMonitor { db::AnnotatedSqlxError::new("dpa_monitor hb begin txn", e) })?; let res = db::dpa_interface::update_last_hb_time(state, &mut txn).await; - if res.is_err() { + if let Err(error) = res { tracing::error!( - "Error updating last_hb_time for dpa id: {} res: {:#?}", - state.id, - res + dpa_interface_id = %state.id, + error = %error, + "Error updating DPA interface heartbeat time", ); } Ok(Some(txn)) diff --git a/crates/dpa/src/lib.rs b/crates/dpa/src/lib.rs index 93dc5b04fe..33c2da3c88 100644 --- a/crates/dpa/src/lib.rs +++ b/crates/dpa/src/lib.rs @@ -71,10 +71,10 @@ pub async fn send_dpa_command( } Err(e) => { tracing::error!( - "send_dpa_command - error: {:#?} sending message: {:#?} to topic: {}", - e, - svni, - topic + error = ?e, + payload = ?svni, + %topic, + "failed to send DPA command" ); return Err(eyre::eyre!("send_message error: {e}")); } diff --git a/crates/dpf/src/sdk.rs b/crates/dpf/src/sdk.rs index 715eb1e710..4bde3edc60 100644 --- a/crates/dpf/src/sdk.rs +++ b/crates/dpf/src/sdk.rs @@ -320,14 +320,14 @@ async fn refresh_bmc_secret_if_changed( match provider.get_bmc_password().await { Ok(new_pw) if new_pw != last_password => { if let Err(e) = write_bmc_secret::(repo, namespace, &new_pw).await { - tracing::error!("Failed to refresh BMC secret: {e}"); + tracing::error!(error = %e, "Failed to refresh BMC secret"); last_password } else { new_pw } } Err(e) => { - tracing::error!("Failed to read BMC password: {e}"); + tracing::error!(error = %e, "Failed to read BMC password"); last_password } _ => last_password, @@ -1546,7 +1546,7 @@ impl DpfSdk { if let Err(e) = DpuNodeRepository::patch(&*self.repo, node_name, &self.namespace, patch).await { - tracing::warn!("Failed to remove label from DPU node {}: {}", node_name, e); + tracing::warn!(node_name, error = %e, "Failed to remove label from DPU node"); } DpuNodeRepository::delete(&*self.repo, node_name, &self.namespace).await @@ -1657,7 +1657,8 @@ impl DpfSdk { let Some(owner_label) = owner_label else { tracing::debug!( dpu = %cr_name, - "DPU is missing {DPU_OWNED_BY_DEPLOYMENT_LABEL} label; skipping" + label = DPU_OWNED_BY_DEPLOYMENT_LABEL, + "DPU is missing label; skipping" ); return None; }; @@ -1795,12 +1796,12 @@ impl eyre::Result<()> { config_path.display().to_string(), ), }; - tracing::info!("Using configuration from {path}: {agent:?}"); + tracing::info!( + config_path = path.as_str(), + ?agent, + "Using agent configuration" + ); let forge_client_config = Arc::new( ForgeClientConfig::new( diff --git a/crates/dpu-remediation/src/remediation.rs b/crates/dpu-remediation/src/remediation.rs index f2bfc8ff41..31d166cd49 100644 --- a/crates/dpu-remediation/src/remediation.rs +++ b/crates/dpu-remediation/src/remediation.rs @@ -128,7 +128,10 @@ impl RemediationExecutor { let status_file_str = tokio::fs::read_to_string(&status_path).await?; let results = if !status_file_str.is_empty() { serde_json::from_str::>(&status_file_str).map_err(|err| { - tracing::error!("Unable to deserialize json into hashmap from status output file, error was {:#?}", err); + tracing::error!( + error = ?err, + "Unable to deserialize json into hashmap from status output file", + ); }).unwrap_or_default() } else { HashMap::new() @@ -224,10 +227,7 @@ impl RemediationExecutor { tracing::debug!("Remediation successfully applied."); } Err(error) => { - tracing::error!( - "Remediation failed with error: {:#?}.", - error - ); + tracing::error!(?error, "Remediation failed",); } } } @@ -236,24 +236,26 @@ impl RemediationExecutor { } _ => { tracing::error!( - "received a response with one of id or script but not both, skipping, will retry next loop, response: {:#?}", - next_remediation + has_script = next_remediation.remediation_script.is_some(), + has_remediation_id = + next_remediation.remediation_id.is_some(), + "received a response with one of id or script but not both, skipping, will retry next loop", ); } } } Err(remediation_fetch_error) => { tracing::error!( - "Remediation executor unable to fetch next remediation this loop, will retry next loop, error was: {:#?}", - remediation_fetch_error + error = ?remediation_fetch_error, + "Remediation executor unable to fetch next remediation this loop, will retry next loop", ); } } } Err(client_creation_error) => { tracing::error!( - "Remediation executor unable to create forge client this loop, will retry next loop, error was: {:#?}", - client_creation_error + error = ?client_creation_error, + "Remediation executor unable to create forge client this loop, will retry next loop", ); } } diff --git a/crates/dsx-exchange-consumer/src/api_client.rs b/crates/dsx-exchange-consumer/src/api_client.rs index ea67553c2f..edd34e2a24 100644 --- a/crates/dsx-exchange-consumer/src/api_client.rs +++ b/crates/dsx-exchange-consumer/src/api_client.rs @@ -117,9 +117,9 @@ impl RackHealthReportSink for ConsoleRackHealthSink { ) -> Result<(), DsxConsumerError> { tracing::info!( rack_id = %rack_id, - "Inserting rack health override: {} successes and {} alerts", - report.successes.len(), - report.alerts.len() + success_count = report.successes.len(), + alert_count = report.alerts.len(), + "Inserting rack health override" ); for alert in &report.alerts { tracing::warn!(rack_id = %rack_id, alert = ?alert, "Rack health alert"); diff --git a/crates/firmware/src/config.rs b/crates/firmware/src/config.rs index a17ced186e..79b5d0cdcb 100644 --- a/crates/firmware/src/config.rs +++ b/crates/firmware/src/config.rs @@ -65,7 +65,11 @@ impl FirmwareConfigSnapshot { } }) .cloned(); - tracing::debug!("FirmwareConfig::find: key {key} found {ret:?}"); + tracing::debug!( + %key, + firmware = ?ret, + "Firmware config lookup completed", + ); ret } @@ -137,7 +141,11 @@ impl FirmwareConfig { // Fake configs to merge for unit tests for ovrd in &self.test_overrides { if let Err(err) = self.merge_from_string(&mut data, ovrd.clone()) { - tracing::error!("Bad override {ovrd}: {err}"); + tracing::error!( + override_config = %ovrd, + error = %err, + "Failed to merge test firmware override", + ); } } } @@ -174,7 +182,7 @@ impl FirmwareConfig { firmware_directory: &PathBuf, ) { if !firmware_directory.is_dir() { - tracing::error!("Missing firmware directory {:?}", firmware_directory); + tracing::error!(?firmware_directory, "Firmware directory does not exist",); return; } @@ -192,12 +200,20 @@ impl FirmwareConfig { let metadata = match fs::read_to_string(metadata_path.clone()) { Ok(str) => str, Err(e) => { - tracing::error!("Could not read {metadata_path:?}: {e}"); + tracing::error!( + ?metadata_path, + error = %e, + "Could not read firmware metadata", + ); continue; } }; if let Err(e) = self.merge_from_string(map, metadata) { - tracing::error!("Failed to merge in metadata from {:?}: {e}", dir.path()); + tracing::error!( + metadata_directory = ?dir.path(), + error = %e, + "Failed to merge firmware metadata", + ); } } } @@ -375,7 +391,10 @@ fn vendor_model_to_key(vendor: bmc_vendor::BMCVendor, model: &str) -> String { fn subdirectories_sorted_by_modification_date(topdir: &PathBuf) -> Vec { let Ok(dirs) = topdir.read_dir() else { - tracing::error!("Unreadable firmware directory {:?}", topdir); + tracing::error!( + firmware_directory = ?topdir, + "Unreadable firmware directory", + ); return vec![]; }; diff --git a/crates/firmware/src/downloader.rs b/crates/firmware/src/downloader.rs index 4b29e1f956..277bd3c73d 100644 --- a/crates/firmware/src/downloader.rs +++ b/crates/firmware/src/downloader.rs @@ -159,7 +159,10 @@ impl FirmwareDownloader { } if url.is_empty() { - tracing::error!("Firmware with file not present has no URL: {filename:?}"); + tracing::error!( + firmware_path = ?filename, + "Firmware artifact is missing and has no URL", + ); return false; } @@ -273,14 +276,16 @@ fn cached_file_status(filename: &Path, sha256: &str) -> CachedFileStatus { Ok(()) => CachedFileStatus::Available, Err(err) => { tracing::warn!( - "Cached firmware artifact {} failed checksum verification: {err}", - filename.display() + filename = %filename.display(), + error = %err, + "Cached firmware artifact failed checksum verification", ); if let Err(err) = std::fs::remove_file(filename) { tracing::error!( - "Failed to remove stale cached firmware artifact {}: {err}", - filename.display() + filename = %filename.display(), + error = %err, + "Failed to remove stale cached firmware artifact", ); return CachedFileStatus::Unusable; } diff --git a/crates/fmds/src/http_request_metrics.rs b/crates/fmds/src/http_request_metrics.rs index a8ab566f07..6fa6bf4ae6 100644 --- a/crates/fmds/src/http_request_metrics.rs +++ b/crates/fmds/src/http_request_metrics.rs @@ -130,14 +130,18 @@ pub fn with_http_request_trace_layer(router: Router, metrics: Arc, _span: &Span| { metrics_request.http_counter.add(1, &[]); - tracing::info!("started {} {}", request.method(), request.uri().path()); + tracing::info!( + method = %request.method(), + request_path = %request.uri().path(), + "started request", + ); }) .on_response( move |_response: &Response, latency: Duration, _span: &Span| { metrics_response .http_req_latency_histogram .record(latency.as_secs_f64() * 1000.0, &[]); - tracing::info!("response generated in {:?}", latency); + tracing::info!(?latency, "response generated"); }, ); diff --git a/crates/fmds/src/main.rs b/crates/fmds/src/main.rs index ef71d52f9e..c098b4da86 100644 --- a/crates/fmds/src/main.rs +++ b/crates/fmds/src/main.rs @@ -118,7 +118,7 @@ async fn main() -> eyre::Result<()> { let server = axum_server::Server::bind(metrics_address); tracing::info!(%metrics_address, "Prometheus /metrics listening"); if let Err(err) = server.serve(router.into_make_service()).await { - tracing::error!("Prometheus metrics server error: {err}"); + tracing::error!(error = %err, "Prometheus metrics server error"); } }); @@ -139,7 +139,7 @@ async fn main() -> eyre::Result<()> { tracing::info!(%rest_address, "REST server listening"); if let Err(err) = server.serve(router.into_make_service()).await { - tracing::error!("REST server error: {err}"); + tracing::error!(error = %err, "REST server error"); } }); diff --git a/crates/fmds/src/nic_init.rs b/crates/fmds/src/nic_init.rs index 9e7506cbcf..3d54055e01 100644 --- a/crates/fmds/src/nic_init.rs +++ b/crates/fmds/src/nic_init.rs @@ -77,7 +77,7 @@ pub async fn setup_metadata_routing(interface_name: &str, cidr: IpNetwork) -> ey eyre::bail!("ip rule add failed: {stderr}"); } } - tracing::info!(src = %src_ip, "policy rule: from {src_ip} lookup 100"); + tracing::info!(source_ip = %src_ip, table = 100, "policy rule configured"); let route = TokioCommand::new("ip") .args([ @@ -99,7 +99,12 @@ pub async fn setup_metadata_routing(interface_name: &str, cidr: IpNetwork) -> ey eyre::bail!("ip route add failed: {stderr}"); } } - tracing::info!(gateway = %gateway, interface = interface_name, "policy route: default via {gateway} dev {interface_name} table 100"); + tracing::info!( + gateway = %gateway, + interface = interface_name, + table = 100, + "default policy route configured", + ); Ok(()) } diff --git a/crates/fmds/src/phone_home.rs b/crates/fmds/src/phone_home.rs index d39530c6aa..e6c111bc78 100644 --- a/crates/fmds/src/phone_home.rs +++ b/crates/fmds/src/phone_home.rs @@ -64,9 +64,9 @@ pub async fn phone_home(state: &Arc) -> Result<(), eyre::Error> { .ok_or_else(|| eyre!("timestamp is empty in response"))?; tracing::info!( - "Successfully phoned home for Machine {} at {}", - machine_id, - timestamp, + %machine_id, + %timestamp, + "Successfully phoned home", ); Ok(()) diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index de83283648..085af41b9c 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -278,7 +278,7 @@ impl ApiEndpointSource { self.prune_bmc_client_cache(&endpoints); - tracing::info!("Prepared total {} endpoints", endpoints.len()); + tracing::info!(endpoint_count = endpoints.len(), "Prepared endpoints"); Ok(endpoints) } @@ -309,7 +309,10 @@ impl ApiEndpointSource { .await .map_err(HealthError::ApiInvocationError)?; - tracing::info!("Found {} machines", machine_ids.machine_ids.len(),); + tracing::info!( + machine_count = machine_ids.machine_ids.len(), + "Found machines" + ); let mut endpoints = Vec::new(); @@ -325,8 +328,9 @@ impl ApiEndpointSource { .await .map_err(HealthError::ApiInvocationError)?; tracing::debug!( - "Fetched details for {} machines with chunk size of 100", - machines.machines.len(), + machine_count = machines.machines.len(), + chunk_size = ids_chunk.len(), + "Fetched machine details" ); for machine in machines.machines { diff --git a/crates/health/src/collectors/logs/periodic.rs b/crates/health/src/collectors/logs/periodic.rs index 2f184592f0..8cd9709112 100644 --- a/crates/health/src/collectors/logs/periodic.rs +++ b/crates/health/src/collectors/logs/periodic.rs @@ -206,8 +206,8 @@ impl LogsCollector { match self.discover_log_services().await { Ok(services) => { tracing::info!( - "Service discovery complete. Found {} log services", - services.len() + service_count = services.len(), + "Log service discovery complete" ); let persistent_state = self.load_persistent_state().await; diff --git a/crates/health/src/discovery/cleanup.rs b/crates/health/src/discovery/cleanup.rs index a7347259b1..8c4ddbc8c5 100644 --- a/crates/health/src/discovery/cleanup.rs +++ b/crates/health/src/discovery/cleanup.rs @@ -20,20 +20,36 @@ use std::collections::HashSet; use super::context::{CollectorKind, DiscoveryLoopContext}; +#[derive(Clone, Copy)] +enum CollectorStopReason { + EndpointRemoved, + SwitchEndpointNoLongerEligible, +} + +impl std::fmt::Display for CollectorStopReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::EndpointRemoved => "endpoint removed", + Self::SwitchEndpointNoLongerEligible => "switch endpoint is no longer eligible", + }) + } +} + fn stop_collectors_for_keys( ctx: &mut DiscoveryLoopContext, kind: CollectorKind, removed_keys: &HashSet>, - stop_message: &'static str, + stop_reason: CollectorStopReason, ) { let collectors = ctx.collectors.map_mut(kind); for key in removed_keys { if let Some(collector) = collectors.remove(key) { tracing::info!( endpoint_key = %key, + collector_kind = ?kind, + %stop_reason, remaining_collectors = collectors.len(), - "{}", - stop_message + "Stopping collector" ); tokio::spawn(async move { collector.stop().await; @@ -49,7 +65,12 @@ pub(super) fn stop_removed_bmc_collectors( let removed_keys = ctx.collectors.removed_keys(active_endpoints); for kind in CollectorKind::ALL { - stop_collectors_for_keys(ctx, kind, &removed_keys, kind.stop_message()); + stop_collectors_for_keys( + ctx, + kind, + &removed_keys, + CollectorStopReason::EndpointRemoved, + ); } for key in &removed_keys { @@ -92,7 +113,7 @@ pub(super) fn stop_ineligible_nmxc_collectors( ctx, CollectorKind::Nmxc, &ineligible_keys, - "Stopping NMX-C streaming collector for ineligible switch endpoint", + CollectorStopReason::SwitchEndpointNoLongerEligible, ); if !ineligible_keys.is_empty() { diff --git a/crates/health/src/discovery/context.rs b/crates/health/src/discovery/context.rs index 0ea6fa1c21..736042b9b1 100644 --- a/crates/health/src/discovery/context.rs +++ b/crates/health/src/discovery/context.rs @@ -65,27 +65,6 @@ impl CollectorKind { CollectorKind::NvueRest, CollectorKind::NvueGnmi, ]; - - pub(super) fn stop_message(self) -> &'static str { - match self { - CollectorKind::Discovery => { - "Stopping entity discovery collector for removed BMC endpoint" - } - CollectorKind::Sensor => "Stopping sensor collector for removed BMC endpoint", - CollectorKind::Metrics => "Stopping entity metrics collector for removed BMC endpoint", - CollectorKind::Logs => "Stopping logs collector for removed BMC endpoint", - CollectorKind::Firmware => "Stopping firmware collector for removed BMC endpoint", - CollectorKind::LeakDetector => { - "Stopping leak detector collector for removed BMC endpoint" - } - CollectorKind::Nmxt => "Stopping NMX-T collector for removed BMC endpoint", - CollectorKind::Nmxc => "Stopping NMX-C streaming collector for removed switch endpoint", - CollectorKind::NvueRest => "Stopping NVUE REST collector for removed BMC endpoint", - CollectorKind::NvueGnmi => { - "Stopping NVUE gNMI streaming collector for removed switch endpoint" - } - } - } } pub(super) struct CollectorState { diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 57b2abc527..385ca80567 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -115,8 +115,8 @@ fn spawn_generic_redfish_collectors( Err(error) => { tracing::error!( ?error, - "Could not start entity discovery collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start entity discovery collector" ); } } @@ -158,8 +158,8 @@ fn spawn_generic_redfish_collectors( Err(error) => { tracing::error!( ?error, - "Could not start sensor collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start sensor collector" ); } } @@ -200,8 +200,8 @@ fn spawn_generic_redfish_collectors( Err(error) => { tracing::error!( ?error, - "Could not start entity metrics collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start entity metrics collector" ); } } @@ -336,8 +336,8 @@ fn spawn_generic_redfish_collectors( tracing::error!( ?error, mode = ?logs_cfg.mode, - "Could not start logs collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start logs collector" ); } None => {} @@ -376,8 +376,8 @@ fn spawn_generic_redfish_collectors( Err(error) => { tracing::error!( ?error, - "Could not start firmware collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start firmware collector" ) } } @@ -418,8 +418,8 @@ fn spawn_generic_redfish_collectors( Err(error) => { tracing::error!( ?error, - "Could not start leak detector collector for: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start leak detector collector" ) } } @@ -475,8 +475,8 @@ fn spawn_switch_host_collectors( Err(error) => { tracing::error!( ?error, - "Could not start NMX-T collector for switch host: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start NMX-T collector for switch host" ) } } @@ -578,8 +578,8 @@ fn spawn_switch_host_collectors( Err(error) => { tracing::error!( ?error, - "Could not start NVUE REST collector for switch host: {:?}", - endpoint.addr + endpoint = ?endpoint.addr, + "Could not start NVUE REST collector for switch host" ) } } diff --git a/crates/health/src/metrics.rs b/crates/health/src/metrics.rs index 1fb612d3f1..616fdd995a 100644 --- a/crates/health/src/metrics.rs +++ b/crates/health/src/metrics.rs @@ -273,7 +273,11 @@ impl Collector for SubRegistry { impl Drop for CollectorRegistry { fn drop(&mut self) { if let Err(e) = self.parent.unregister(self.registry.clone()) { - tracing::error!(e=?e, "Could not properly drop registry for collector {}", self.prefix()) + tracing::error!( + error = ?e, + collector_prefix = self.prefix().as_str(), + "Could not properly drop registry for collector" + ) } } } @@ -465,8 +469,8 @@ pub async fn run_metrics_server( .map_err(|e| Box::new(e) as BoxedErr)?; tracing::info!( - "Metrics server listening on {} (paths: /metrics, /telemetry, /livez)", - metrics_endpoint + %metrics_endpoint, + "Metrics server listening (paths: /metrics, /telemetry, /livez)" ); loop { diff --git a/crates/host-support/src/hardware_enumeration.rs b/crates/host-support/src/hardware_enumeration.rs index 4aef6052b4..3bf60a7c9d 100644 --- a/crates/host-support/src/hardware_enumeration.rs +++ b/crates/host-support/src/hardware_enumeration.rs @@ -228,12 +228,20 @@ fn get_numa_node_from_syspath(syspath: Option<&Path>) -> Result HardwareEnumerationResult> { let device_debug_log = |device: &Device| { - tracing::debug!("SysPath - {:?}", device.syspath()); + tracing::debug!(syspath = ?device.syspath(), "SysPath"); for p in device.properties() { - tracing::trace!("Property - {:?} - {:?}", p.name(), p.value()); + tracing::trace!( + property_name = ?p.name(), + property_value = ?p.value(), + "Property" + ); } for a in device.attributes() { - tracing::trace! {"attribute - {:?} - {:?}", a.name(), a.value()} + tracing::trace! { + attribute_name = ?a.name(), + attribute_value = ?a.value(), + "attribute" + } } }; @@ -249,9 +257,9 @@ pub fn discovery_ibs() -> HardwareEnumerationResult properties_ext, Err(e) => { tracing::error!( - "Failed to enumerate properties of device {:?}: {}", - device.devpath(), - e + device_path = ?device.devpath(), + error = %e, + "Failed to enumerate properties of device" ); continue; } @@ -408,9 +416,13 @@ fn enumerate_hardware_inner( for device in devices { let sys_path = device.syspath(); - tracing::debug!("SysPath - {:?}", sys_path); + tracing::debug!(?sys_path, "SysPath"); for p in device.properties() { - tracing::trace!("net device property - {:?} - {:?}", p.name(), p.value()); + tracing::trace!( + property_name = ?p.name(), + property_value = ?p.value(), + "net device property" + ); } //for a in device.attributes() { // tracing::trace!("attribute - {:?} - {:?}", a.name(), a.value()); @@ -423,15 +435,15 @@ fn enumerate_hardware_inner( Ok(properties_ext) => properties_ext, Err(e) => { tracing::error!( - "Failed to enumerate properties of device {:?}: {}", - device.devpath(), - e + device_path = ?device.devpath(), + error = %e, + "Failed to enumerate properties of device" ); continue; } }; - tracing::trace!("properties: {:?}", properties_ext); + tracing::trace!(?properties_ext, "properties"); // discovery DPU and non ib capable device // Note: @@ -598,11 +610,11 @@ fn enumerate_hardware_inner( // skip the device if its hidden if convert_sysattr_to_string("hidden", &device).is_ok_and(|v| v == "1") { tracing::info!( - "Ignoring hidden device {}", - device + syspath = device .syspath() .and_then(|v| v.to_str()) - .unwrap_or_default() + .unwrap_or_default(), + "Ignoring hidden device" ); continue; } @@ -610,11 +622,11 @@ fn enumerate_hardware_inner( // skip the device if its removable if convert_sysattr_to_string("removable", &device).is_ok_and(|v| v != "0") { tracing::info!( - "Ignoring removable device {}", - device + syspath = device .syspath() .and_then(|v| v.to_str()) - .unwrap_or_default() + .unwrap_or_default(), + "Ignoring removable device" ); continue; } @@ -623,11 +635,11 @@ fn enumerate_hardware_inner( .is_ok_and(|v| v.contains("virtual")) { tracing::info!( - "Ignoring virtual device {}", - device + syspath = device .syspath() .and_then(|v| v.to_str()) - .unwrap_or_default() + .unwrap_or_default(), + "Ignoring virtual device" ); continue; } @@ -687,7 +699,7 @@ fn enumerate_hardware_inner( // There is only expected to be a single set, and we don't want to // accidentally overwrite it with other data if let Some(device) = devices.next() { - tracing::debug!("DMI device syspath: {:?}", device.syspath()); + tracing::debug!(syspath = ?device.syspath(), "DMI device syspath"); // e.g. 'DRAM'. We will use this later if smbios fails. backup_ram_type = device @@ -714,9 +726,9 @@ fn enumerate_hardware_inner( dmi.product_name = convert_sysattr_to_string("product_name", &device)?.to_string(); if cpu_part == BF3_CPU_PART && dmi.product_name == BF2_PRODUCT_NAME { tracing::info!( - "Overriding product name {} with {}", - dmi.product_name, - BF3_PRODUCT_NAME + product_name = dmi.product_name.as_str(), + new_product_name = BF3_PRODUCT_NAME, + "Overriding product name" ); dmi.product_name = BF3_PRODUCT_NAME.to_owned(); } @@ -747,7 +759,7 @@ fn enumerate_hardware_inner( ))); } Err(e) => { - tracing::error!("Could not read TPM EK certificate: {:?}", e); + tracing::error!(error = ?e, "Could not read TPM EK certificate"); None } }; @@ -756,7 +768,7 @@ fn enumerate_hardware_inner( "https://www.mellanox.com" | "Nvidia" => match dpu::get_dpu_info() { Ok(dpu_data) => Some(dpu_data), Err(e) => { - tracing::error!("Could not get DPU data: {:?}", e); + tracing::error!(error = ?e, "Could not get DPU data"); None } }, @@ -818,7 +830,9 @@ fn enumerate_hardware_inner( } Err(err) => { warn!( - "Could not discover host memory using smbios device, using {mem_info_path}: {err}" + mem_info_path, + error = %err, + "Could not discover host memory using smbios device" ); let meminfo = std::fs::read_to_string(mem_info_path).map_err(|e| { HardwareEnumerationError::GenericError(format!("Err reading {mem_info_path}: {e}")) @@ -832,19 +846,22 @@ fn enumerate_hardware_inner( } } - tracing::debug!("Discovered Disks: {:?}", disks); + tracing::debug!(?disks, "Discovered Disks"); if !cpus.is_empty() { - tracing::debug!("Discovered CPUs[0]: {:?}", cpus[0]); + tracing::debug!(cpu = ?cpus[0], "Discovered CPUs[0]"); } - tracing::debug!("Discovered NICS: {:?}", nics); - tracing::debug!("Discovered IBS: {:?}", ibs); - tracing::debug!("Discovered NVMES: {:?}", nvmes); - tracing::debug!("Discovered DMI: {:?}", dmi); - tracing::debug!("Discovered GPUs: {:?}", gpus); - tracing::debug!("Discovered Machine Architecture: {}", info.machine.as_str()); - tracing::debug!("Discovered DPU: {:?}", dpu_vpd); + tracing::debug!(?nics, "Discovered NICS"); + tracing::debug!(?ibs, "Discovered IBS"); + tracing::debug!(?nvmes, "Discovered NVMES"); + tracing::debug!(?dmi, "Discovered DMI"); + tracing::debug!(?gpus, "Discovered GPUs"); + tracing::debug!( + architecture = info.machine.as_str(), + "Discovered Machine Architecture" + ); + tracing::debug!(dpu = ?dpu_vpd, "Discovered DPU"); if let Some(cert) = tpm_ek_certificate.as_ref() { - tracing::debug!("TPM EK certificate (base64): {}", cert); + tracing::debug!(certificate = cert.as_str(), "TPM EK certificate (base64)"); } Ok(rpc_discovery::DiscoveryInfo { diff --git a/crates/host-support/src/hardware_enumeration/dpu.rs b/crates/host-support/src/hardware_enumeration/dpu.rs index aa72c21c48..726ccc7c72 100644 --- a/crates/host-support/src/hardware_enumeration/dpu.rs +++ b/crates/host-support/src/hardware_enumeration/dpu.rs @@ -98,7 +98,7 @@ pub fn get_lldp_port_info(port: &str) -> Result { if cfg!(test) { const TEST_DATA: &str = "test/lldp_query.json"; std::fs::read_to_string(TEST_DATA).map_err(|e| { - warn!("Could not read LLDP json: {e}"); + warn!(error = %e, "Could not read LLDP json"); DpuEnumerationError::Read(TEST_DATA, e.to_string()) }) } else { @@ -107,7 +107,7 @@ pub fn get_lldp_port_info(port: &str) -> Result { .args(vec!["-c", lldp_cmd.as_str()]) .output() .map_err(|e| { - warn!("Could not discover LLDP peer for {port}, {e}"); + warn!(port, error = %e, "Could not discover LLDP peer"); DpuEnumerationError::Lldp(e.to_string()) }) } @@ -134,7 +134,7 @@ pub fn wait_until_all_ports_available() { } } - debug!("lldp: Ports {:?} are read successfully.", ports_read); + debug!(ports = ?ports_read, "lldp: Ports are read successfully."); } // LLDP was broken in multiple forge versions. It was fixed in HBN 2.1/ doca 2.6, as per @@ -164,7 +164,11 @@ pub fn get_port_lldp_info(port: &str) -> Result x, Err(e) => { - warn!("Could not deserialize LLDP response {lldp_json}, {e}"); + warn!( + lldp_json = lldp_json.as_str(), + error = %e, + "Could not deserialize LLDP response" + ); return Err(DpuEnumerationError::Lldp(e.to_string())); } }; diff --git a/crates/host-support/src/hardware_enumeration/gpu.rs b/crates/host-support/src/hardware_enumeration/gpu.rs index 4348d32020..020c0a41a9 100644 --- a/crates/host-support/src/hardware_enumeration/gpu.rs +++ b/crates/host-support/src/hardware_enumeration/gpu.rs @@ -45,7 +45,7 @@ pub fn get_nvidia_smi_data() -> HardwareEnumerationResult> { for result in csv_reader.deserialize() { match result { Ok(gpu) => gpus.push(gpu), - Err(error) => tracing::error!("Could not parse nvidia-smi output: {}", error), + Err(error) => tracing::error!(%error, "Could not parse nvidia-smi output"), } } diff --git a/crates/host-support/src/hardware_enumeration/tpm.rs b/crates/host-support/src/hardware_enumeration/tpm.rs index 0f367ffb65..ee244a6b2b 100644 --- a/crates/host-support/src/hardware_enumeration/tpm.rs +++ b/crates/host-support/src/hardware_enumeration/tpm.rs @@ -97,14 +97,16 @@ fn get_ek_certificate_with_runner(runner: &impl CommandRunner) -> Result Ok(cert) => Ok(cert), Err(primary_error) => { tracing::warn!( - "Could not read TPM EK certificate using {TPM2_GET_EK_CERTIFICATE}: {primary_error:?}; probing known NV indices" + command = TPM2_GET_EK_CERTIFICATE, + error = ?primary_error, + "Could not read TPM EK certificate; probing known NV indices" ); let mut certs = vec![]; let mut nv_errors = vec![]; for index in TPM_EK_CERT_NV_INDICES { match get_ek_certificate_from_nv_index(runner, index) { Ok(cert) => { - tracing::info!("Read TPM EK certificate from NV index {index}"); + tracing::info!(index, "Read TPM EK certificate from NV index"); certs.extend_from_slice(&cert); } Err(e) => nv_errors.push(format!("{index}: {e}")), diff --git a/crates/host-support/src/registration.rs b/crates/host-support/src/registration.rs index ad09c1f4a8..195eed209b 100644 --- a/crates/host-support/src/registration.rs +++ b/crates/host-support/src/registration.rs @@ -101,18 +101,15 @@ impl<'a, 'c> RegistrationClient<'a, 'c> { // let response = connection.your_api_endpoint(request).await?; // async fn connect(&self, purpose: &str) -> Result { - tracing::debug!("creating tls client connection for {purpose}"); + tracing::debug!(purpose, "creating tls client connection"); let client = ForgeTlsClient::new(self.config); match client.build(self.api_url.to_string()).await { Ok(connection) => { - tracing::debug!( - "created tls client connection for {purpose}: {:?}", - connection - ); + tracing::debug!(purpose, ?connection, "created tls client connection"); Ok(connection) } Err(e) => { - tracing::error!("could not create tls client for {purpose}: {:?}", e); + tracing::error!(purpose, error = ?e, "could not create tls client"); Err(RegistrationError::TransportError(e.to_string())) } } @@ -127,14 +124,14 @@ impl<'a, 'c> RegistrationClient<'a, 'c> { info: MachineDiscoveryInfo, attempt: u32, ) -> Result { - tracing::info!("Attempting to discover_machine (attempt: {})", attempt); + tracing::info!(attempt, "Attempting to discover_machine"); // Create a new connection off of the ForgeTlsClient. let mut connection = self.connect("discover_machine_once").await?; // Create a new request with the provided MachineDiscoveryInfo. let request = tonic::Request::new(info); - tracing::debug!("register_machine request {:?}", request); + tracing::debug!(?request, "discover_machine request"); // And now attempt to send the request. Ok(connection @@ -142,9 +139,9 @@ impl<'a, 'c> RegistrationClient<'a, 'c> { .await .inspect_err(|err| { tracing::error!( - "Error attempting to discover_machine (attempt: {}): {}", attempt, - err.to_string() + error = %err, + "Error attempting to discover_machine" ); })? .into_inner()) @@ -180,15 +177,13 @@ impl<'a, 'c> RegistrationClient<'a, 'c> { // Create a new request with the provided AttestQuoteRequest. let request = tonic::Request::new(quote.clone()); - tracing::debug!("attest_quote request {:?}", request); + tracing::debug!(?request, "attest_quote request"); // And now attempt to send the request. Ok(connection .attest_quote(request) .await - .inspect_err(|err| { - tracing::error!("Error attempting to attest_quote: {}", err.to_string()) - })? + .inspect_err(|err| tracing::error!(error = %err, "Error attempting to attest_quote"))? .into_inner()) } } @@ -207,7 +202,7 @@ fn create_client_config( .map_err(|e| RegistrationError::TransportError(e.to_string()))?, false => ForgeClientConfig::new(root_ca, None), }; - tracing::debug!("{purpose} client_config {:?}", forge_client_config); + tracing::debug!(purpose, ?forge_client_config, "client_config"); Ok(forge_client_config) } @@ -243,7 +238,7 @@ pub async fn register_machine( discovery_reporter: discovery_reporter as i32, discovery_reporter_version: reporter_version, }; - tracing::info!("register_machine discovery_info {:?}", info); + tracing::info!(?info, "register_machine discovery_info"); let forge_client_config = create_client_config("register_machine", use_mgmt_vrf, root_ca)?; let response = RegistrationClient::new(forge_api, &forge_client_config, retry) @@ -289,14 +284,7 @@ pub async fn attest_quote( let _ = write_certs(response.machine_certificate, None).await; - tracing::info!( - "Attestation result is {}", - if response.success { - "SUCCESS" - } else { - "FAILURE" - } - ); + tracing::info!(success = response.success, "Attestation result"); Ok(response.success) } @@ -322,7 +310,7 @@ pub async fn write_certs( .wrap_err(format!( "Failed to write new machine certificate PEM to {client_cert}" ))?; - tracing::info!("Wrote new machine certificate PEM to: {:?}", client_cert); + tracing::info!(%client_cert, "Wrote new machine certificate PEM"); tokio::fs::write(client_key, machine_certificate.private_key.as_slice()) .await diff --git a/crates/http-connector/src/connector.rs b/crates/http-connector/src/connector.rs index a944923a4e..5ab5275946 100644 --- a/crates/http-connector/src/connector.rs +++ b/crates/http-connector/src/connector.rs @@ -205,7 +205,7 @@ impl ConnectorMetricsInner { self.total_successes.fetch_add(1, Ordering::SeqCst); GLOBAL_TOTALS.successes.fetch_add(1, Ordering::Relaxed); self.addr_maps.lock().unwrap().add_success_by_addr(addr); - trace!("connected to {addr}"); + trace!(address = %addr, "connected"); } // connect_error is the "inner" function for handling a @@ -216,7 +216,7 @@ impl ConnectorMetricsInner { self.total_errors.fetch_add(1, Ordering::SeqCst); GLOBAL_TOTALS.errors.fetch_add(1, Ordering::Relaxed); self.addr_maps.lock().unwrap().add_error_by_addr(addr); - info!("connect error for {}: {:?}", addr, e); + info!(address = %addr, error = ?e, "connect error"); } } @@ -412,7 +412,7 @@ fn connect( conf = conf.with_retries(retries) } if let Err(e) = socket.set_tcp_keepalive(&conf) { - warn!("tcp set_keepalive error: {}", e); + warn!(error = %e, "tcp set_keepalive error"); } } #[cfg(target_os = "linux")] @@ -449,19 +449,19 @@ fn connect( if config.reuse_address && let Err(e) = socket.set_reuseaddr(true) { - warn!("tcp set_reuse_address error: {}", e); + warn!(error = %e, "tcp set_reuse_address error"); } if let Some(size) = config.send_buffer_size && let Err(e) = socket.set_send_buffer_size(size.try_into().unwrap_or(u32::MAX)) { - warn!("tcp set_buffer_size error: {}", e); + warn!(error = %e, "tcp set_buffer_size error"); } if let Some(size) = config.recv_buffer_size && let Err(e) = socket.set_recv_buffer_size(size.try_into().unwrap_or(u32::MAX)) { - warn!("tcp set_recv_buffer_size error: {}", e); + warn!(error = %e, "tcp set_recv_buffer_size error"); } let connect = socket.connect(*addr); @@ -520,7 +520,7 @@ impl<'a> ConnectingTcp<'a> { config: &'a Config, metrics: ConnectorMetrics, ) -> Self { - trace!("ConnectingTcp config: {:?}", config); + trace!(?config, "ConnectingTcp config"); if let Some(fallback_timeout) = config.happy_eyeballs_timeout { let (preferred_addrs, fallback_addrs) = remote_addrs @@ -927,10 +927,10 @@ impl tower_service::Service for ForgeHttpConnector { fn get_host_port<'u>(config: &Config, dst: &'u Uri) -> Result<(&'u str, u16), ConnectError> { trace!( - "Http::connect; scheme={:?}, host={:?}, port={:?}", - dst.scheme(), - dst.host(), - dst.port(), + scheme = ?dst.scheme(), + host = ?dst.host(), + port = ?dst.port(), + "Http::connect" ); if config.enforce_http { @@ -1006,7 +1006,7 @@ impl ForgeHttpConnector { ); tryhard::retry_fn(|| { - trace!("establishing new tcp connection for {dst}"); + trace!(%dst, "establishing new tcp connection"); let c = ConnectingTcp::new(addrs.clone(), config, self.metrics.clone()); c.connect() }) diff --git a/crates/http-connector/src/resolver.rs b/crates/http-connector/src/resolver.rs index eb50ced785..5b1abd0671 100644 --- a/crates/http-connector/src/resolver.rs +++ b/crates/http-connector/src/resolver.rs @@ -299,7 +299,7 @@ impl Service for HickoryResolver { Box::pin(async move { let response = resolver.lookup_ip(name.to_string()).await?; - trace!("response from DNS Server{:?}", response); + trace!(?response, "response from DNS Server"); let addresses = response.iter(); Ok(SocketAddrs { diff --git a/crates/ib-fabric/src/lib.rs b/crates/ib-fabric/src/lib.rs index 914eacf7c9..b5bf02dfc6 100755 --- a/crates/ib-fabric/src/lib.rs +++ b/crates/ib-fabric/src/lib.rs @@ -21,10 +21,9 @@ pub mod ib; mod metrics; use std::collections::{HashMap, HashSet}; -use std::fmt::Write; -use std::io; use std::sync::Arc; use std::time::Duration; +use std::{fmt, io}; use carbide_utils::periodic_timer::PeriodicTimer; use carbide_uuid::infiniband::IBPartitionId; @@ -138,7 +137,7 @@ impl IbFabricMonitor { } } Err(e) => { - tracing::warn!("IbFabricMonitor error: {}", e); + tracing::warn!(error = %e, "IB fabric monitor iteration failed"); } } @@ -194,7 +193,7 @@ impl IbFabricMonitor { *num_changes } Err(e) => { - tracing::error!("IbFabricMonitor run failed due to: {:?}", e); + tracing::error!(error = ?e, "IB fabric monitor run failed"); check_ib_fabrics_span.record("otel.status_code", "error"); // Writing this field will set the span status to error // Therefore we only write it on errors @@ -389,7 +388,7 @@ async fn load_single_fabric_data( fabric, &fabric_definition.endpoints, &e, - "failed to build the IB fabric client", + FabricFailureStage::BuildClient, ); return None; } @@ -401,7 +400,7 @@ async fn load_single_fabric_data( fabric, &fabric_definition.endpoints, &e, - "IB fabric health check failed", + FabricFailureStage::HealthCheck, ); // There's no point in loading other information case the fabric is down return Some(conn); @@ -417,7 +416,7 @@ async fn load_single_fabric_data( fabric, &fabric_definition.endpoints, &e, - "Loading port information failed", + FabricFailureStage::LoadPorts, ); // There's no point in loading other information case the fabric is down return Some(conn); @@ -434,7 +433,7 @@ async fn load_single_fabric_data( fabric, &fabric_definition.endpoints, &e, - "Loading partition information failed", + FabricFailureStage::LoadPartitions, ); // There's no point in loading other information case the fabric is down return Some(conn); @@ -447,8 +446,27 @@ async fn load_single_fabric_data( Some(conn) } -/// Records a failed stage of a fabric's data load: logs the failure under the -/// stage's `message` and stores the error as the fabric's error metric. +#[derive(Clone, Copy)] +enum FabricFailureStage { + BuildClient, + HealthCheck, + LoadPorts, + LoadPartitions, +} + +impl fmt::Display for FabricFailureStage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::BuildClient => "build_client", + Self::HealthCheck => "health_check", + Self::LoadPorts => "load_ports", + Self::LoadPartitions => "load_partitions", + }) + } +} + +/// Records a failed stage of a fabric's data load and stores the error as the +/// fabric's error metric. /// /// TODO: Storing the raw error string isn't efficient because we will get a /// lot of different dimensions. We need to have better defined errors from the @@ -458,9 +476,15 @@ fn note_fabric_error( fabric: &str, endpoints: &[String], error: &IbError, - message: &'static str, + failure_stage: FabricFailureStage, ) { - tracing::error!(fabric, endpoints = endpoints.join(","), error = %error, "{message}"); + tracing::error!( + fabric, + endpoints = endpoints.join(","), + %failure_stage, + error = %error, + "IB fabric operation failed", + ); fabric_metrics.fabric_error = error.to_string(); } @@ -1056,37 +1080,27 @@ async fn record_machine_infiniband_status_observation( if !result.missing_guid_pkeys.is_empty() { metrics.num_machines_with_missing_pkeys += 1; - let mut msg = "Machine is missing pkeys on UFM: ".to_string(); - for (idx, (_fabric, guid, pkey)) in result.missing_guid_pkeys.iter().enumerate() { - if idx != 0 { - msg.push(','); - } - write!(&mut msg, "(guid: {guid}, pkey: {pkey})").unwrap(); - } - tracing::warn!(machine_id = %machine_id, msg); + tracing::warn!( + machine_id = %machine_id, + missing_guid_pkeys = ?result.missing_guid_pkeys, + "Machine is missing pkeys on UFM", + ); } if !result.unexpected_guid_pkeys.is_empty() { metrics.num_machines_with_unexpected_pkeys += 1; - let mut msg = "Machine has unexpected registered pkeys on UFM: ".to_string(); - for (idx, (_fabric, guid, pkey)) in result.unexpected_guid_pkeys.iter().enumerate() { - if idx != 0 { - msg.push(','); - } - write!(&mut msg, "(guid: {guid}, pkey: {pkey})").unwrap(); - } - tracing::warn!(machine_id = %machine_id, msg); + tracing::warn!( + machine_id = %machine_id, + unexpected_guid_pkeys = ?result.unexpected_guid_pkeys, + "Machine has unexpected registered pkeys on UFM", + ); } if !result.unknown_guid_pkeys.is_empty() { metrics.num_machines_with_unknown_pkeys += 1; - let mut msg = - "Machine has registered pkeys on UFM that do not map to IB PartitionIDs: ".to_string(); - for (idx, (_fabric, guid, pkey)) in result.unknown_guid_pkeys.iter().enumerate() { - if idx != 0 { - msg.push(','); - } - write!(&mut msg, "(guid: {guid}, pkey: {pkey})").unwrap(); - } - tracing::warn!(machine_id = %machine_id, msg); + tracing::warn!( + machine_id = %machine_id, + unknown_guid_pkeys = ?result.unknown_guid_pkeys, + "Machine has registered pkeys on UFM that do not map to IB PartitionIDs", + ); } let has_existing_ib_port_down_alert = mh_snapshot diff --git a/crates/ib-partition-controller/src/handler.rs b/crates/ib-partition-controller/src/handler.rs index 89e1fcf9e5..b249d36bd2 100644 --- a/crates/ib-partition-controller/src/handler.rs +++ b/crates/ib-partition-controller/src/handler.rs @@ -101,8 +101,7 @@ impl StateHandler for IBPartitionStateHandler { tracing::info!( %partition_id, instance_count, - "Postponing IB partition deletion: \ - {instance_count} instance(s) still reference this partition", + "Postponing IB partition deletion because instances still reference it", ); return Ok(StateHandlerOutcome::wait(format!( "Waiting for {instance_count} instance(s) to release IB partition" diff --git a/crates/ipmi/src/tool.rs b/crates/ipmi/src/tool.rs index 3d2e5f8f03..1c7fef838d 100644 --- a/crates/ipmi/src/tool.rs +++ b/crates/ipmi/src/tool.rs @@ -151,7 +151,7 @@ impl IPMIToolImpl { .args(&args) .attempts(self.attempts); - tracing::info!("Running command: {:?}", cmd); + tracing::info!(command = ?cmd, "Running IPMI command"); let result = cmd.env("IPMITOOL_PASSWORD", password).output().await; count_ipmi_command(command, &result); result diff --git a/crates/kms-provider/src/providers/transit.rs b/crates/kms-provider/src/providers/transit.rs index 8dfd159b47..b52235b6ac 100644 --- a/crates/kms-provider/src/providers/transit.rs +++ b/crates/kms-provider/src/providers/transit.rs @@ -85,7 +85,10 @@ impl TransitKmsProvider { match vaultrs::token::lookup_self(client.as_ref()).await { Ok(info) => break info, Err(e) => { - tracing::warn!("failed to look up Transit KMS token, retrying: {e}"); + tracing::warn!( + error = %e, + "failed to look up Transit KMS token; retrying" + ); tokio::select! { _ = cancel.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(30)) => {} @@ -121,7 +124,7 @@ impl TransitKmsProvider { ); } Err(e) => { - tracing::warn!("failed to renew Transit KMS vault token: {e}"); + tracing::warn!(error = %e, "failed to renew Transit KMS vault token"); next_renewal = Duration::from_secs(30); } } diff --git a/crates/libmlx/src/device/discovery.rs b/crates/libmlx/src/device/discovery.rs index 206e2cc2d8..cdaa042fbb 100644 --- a/crates/libmlx/src/device/discovery.rs +++ b/crates/libmlx/src/device/discovery.rs @@ -116,7 +116,7 @@ pub fn discover_devices() -> Result, String> { { let xml_content = String::from_utf8_lossy(&output.stdout); - warn!("mlxfwmanager XML output: {}", xml_content); + warn!(xml = %xml_content, "mlxfwmanager XML output"); parse_mlxfwmanager_xml(&xml_content) } } @@ -125,7 +125,7 @@ pub fn discover_devices() -> Result, String> { // The actual XML returned is still "devices", but will only // contain the target device. pub fn discover_device(device: &str) -> Result { - debug!("Running mlxfwmanager to discover device: {device}"); + debug!(device, "Running mlxfwmanager to discover device"); let output = Command::new("mlxfwmanager") .args(["--dev", device, "--query-format", "xml"]) @@ -148,7 +148,7 @@ pub fn discover_device(device: &str) -> Result { } let xml_content = String::from_utf8_lossy(&output.stdout); - debug!("mlxfwmanager XML output: {}", xml_content); + debug!(xml = %xml_content, "mlxfwmanager XML output"); let devices = parse_mlxfwmanager_xml(&xml_content)?; if devices.len() > 1 { @@ -172,20 +172,21 @@ pub fn discover_devices_with_filters(filter: DeviceFilter) -> Result Result, S devices.push(device_info); } - debug!("Discovered {} MLX devices", devices.len()); + debug!(device_count = devices.len(), "Discovered MLX devices"); Ok(devices) } @@ -273,8 +274,9 @@ pub fn convert_pci_name_to_address(pci_name: &str) -> Result { }; debug!( - "Converted PCI name '{}' to address '{}'", - pci_name, cleaned_address + pci_name, + address = cleaned_address.as_str(), + "Converted PCI name to address" ); Ok(cleaned_address) } diff --git a/crates/libnmxc/src/lib.rs b/crates/libnmxc/src/lib.rs index 8b0242cccb..1b23382bc3 100644 --- a/crates/libnmxc/src/lib.rs +++ b/crates/libnmxc/src/lib.rs @@ -390,7 +390,7 @@ impl ChannelConnector for TlsChannelConnector { .await? }; - debug!("Connected to NMX-C at {}", endpoint.uri); + debug!(endpoint = %endpoint.uri, "Connected to NMX-C"); Ok(channel) } diff --git a/crates/libnmxm/src/lib.rs b/crates/libnmxm/src/lib.rs index afd47d016d..5f093f0e95 100644 --- a/crates/libnmxm/src/lib.rs +++ b/crates/libnmxm/src/lib.rs @@ -356,7 +356,11 @@ impl NmxmApiClient { source: e, })?; let response_body = String::from_utf8_lossy(&response_buffer).to_string(); - debug!("RX {status_code} {}", truncate(&response_body, 1500)); + debug!( + status_code = status_code.as_u16(), + response_body = truncate(&response_body, 1500), + "Received NMX-M response" + ); if !status_code.is_success() { return Err(NmxmApiError::HTTPErrorCode { diff --git a/crates/machine-a-tron/src/api_client.rs b/crates/machine-a-tron/src/api_client.rs index 5eb32ddc13..e8ba1e56ee 100644 --- a/crates/machine-a-tron/src/api_client.rs +++ b/crates/machine-a-tron/src/api_client.rs @@ -354,14 +354,14 @@ impl ApiClient { Ok(vpc_id_list) => { match vpc_id_list.vpc_ids.len() { 0 => tracing::error!( - "There are no VPC ids associated with {}. Should not have happened.", - *vpc_name + vpc_name = %*vpc_name, + "No VPC IDs are associated with VPC name; this should not happen", ), 1 => {} _ => tracing::warn!( - "There are {} VPC ids associated with {}. Should not have happened. Clean up DB and start over.", - vpc_id_list.vpc_ids.len(), - vpc_name + vpc_id_count = vpc_id_list.vpc_ids.len(), + vpc_name = %vpc_name, + "Multiple VPC IDs are associated with VPC name; clean up the database and restart", ), } diff --git a/crates/machine-a-tron/src/api_throttler.rs b/crates/machine-a-tron/src/api_throttler.rs index 25f3d16b76..8f9e107b2c 100644 --- a/crates/machine-a-tron/src/api_throttler.rs +++ b/crates/machine-a-tron/src/api_throttler.rs @@ -56,7 +56,10 @@ pub fn run(mut interval: Interval, api_client: ApiClient) -> ApiThrottler { chunk.iter().map(|id| **id).collect(), ) .await - .inspect_err(|e| tracing::error!("API failure getting machines: {e}")).unwrap_or_default(); + .inspect_err(|e| tracing::error!( + error = %e, + "API failure getting machines", + )).unwrap_or_default(); // Index the result by ID for m in machines { diff --git a/crates/machine-a-tron/src/bmc_mock_wrapper.rs b/crates/machine-a-tron/src/bmc_mock_wrapper.rs index b3b51b407a..f737d82f7c 100644 --- a/crates/machine-a-tron/src/bmc_mock_wrapper.rs +++ b/crates/machine-a-tron/src/bmc_mock_wrapper.rs @@ -92,7 +92,12 @@ impl BmcMockWrapper { &self.app_context.app_config.interface, ) .await - .inspect_err(|e| tracing::warn!("{}", e)) + .inspect_err(|e| { + tracing::warn!( + error = %e, + "failed to add BMC mock address to interface", + ) + }) .map_err(MachineStateError::ListenAddressConfigError)?; } @@ -138,7 +143,10 @@ impl BmcMockWrapper { None }; - tracing::info!("Starting bmc mock on {:?}", address); + tracing::info!( + listen_address = ?address, + "Starting BMC mock", + ); let tls_server_config = bmc_mock::tls::server_config(Some(certs_dir))?; let bmc_mock_router = self.bmc_mock_router.clone(); diff --git a/crates/machine-a-tron/src/dhcp_wrapper.rs b/crates/machine-a-tron/src/dhcp_wrapper.rs index a24823430b..4823b66493 100644 --- a/crates/machine-a-tron/src/dhcp_wrapper.rs +++ b/crates/machine-a-tron/src/dhcp_wrapper.rs @@ -44,7 +44,10 @@ pub async fn request_ip( api_client: ApiClient, request_info: DhcpRequestInfo, ) -> DhcpRelayResult { - tracing::debug!("requesting ip for mac: {}", request_info.mac_address); + tracing::debug!( + mac_address = %request_info.mac_address, + "Requesting IP address", + ); let dhcp_record = api_client .discover_dhcp( @@ -55,15 +58,18 @@ pub async fn request_ip( ) .await .inspect_err(|e| { - tracing::warn!("discover_dhcp failed: {e}"); + tracing::warn!( + error = %e, + "discover_dhcp failed", + ); })?; tracing::info!( - "dhcp request for {} through relay {} got address {} (machine id {:?})", - request_info.mac_address, - request_info.relay_address, - dhcp_record.address, - dhcp_record.machine_id, + mac_address = %request_info.mac_address, + relay_address = %request_info.relay_address, + assigned_address = %dhcp_record.address, + machine_id = ?dhcp_record.machine_id, + "DHCP request received an address", ); let interface_uuid = dhcp_record.machine_interface_id.ok_or_else(|| { diff --git a/crates/machine-a-tron/src/host_machine.rs b/crates/machine-a-tron/src/host_machine.rs index 37080e1e38..ac5204a8c4 100644 --- a/crates/machine-a-tron/src/host_machine.rs +++ b/crates/machine-a-tron/src/host_machine.rs @@ -314,7 +314,10 @@ impl HostMachine { } } Some(cmd) = self.bmc_control_rx.recv() => { - tracing::debug!("HOST set_system_power request: {cmd:?}"); + tracing::debug!( + command = ?cmd, + "Received host power command", + ); match cmd { BmcCommand::SetSystemPower { request, reply } => { let response = self.set_system_power(request); @@ -402,7 +405,7 @@ impl HostMachine { } fn set_system_power(&mut self, request: SystemPowerControl) -> SetSystemPowerResult { - tracing::debug!("Host set_system_power request: {request:?}"); + tracing::debug!(?request, "Received host system-power request",); match request { // Force-restart does not restart DPUs @@ -421,7 +424,8 @@ impl HostMachine { _ = dpu.set_system_power(request).inspect_err(|e| { tracing::error!( error = %e, - "Could not send power request to DPU {dpu_index}", + dpu_index, + "Could not send power request to DPU", ) }); } @@ -663,8 +667,8 @@ impl HostMachineHandle { { Some(machine_id) => { tracing::info!( - "Attempting to delete machine with id: {} from db.", - machine_id + %machine_id, + "Attempting to delete machine from database", ); machine_id.to_string() } @@ -672,7 +676,10 @@ impl HostMachineHandle { // force_delete_machine also supports sending MAC address (which could break if there is 0 DPUs on this host) match self.0.host_info.system_mac_address() { Some(mac) => { - tracing::info!("Attempting to delete machine with mac: {} from db.", mac); + tracing::info!( + mac_address = %mac, + "Attempting to delete machine from database", + ); mac.to_string() } None => { diff --git a/crates/machine-a-tron/src/machine_a_tron.rs b/crates/machine-a-tron/src/machine_a_tron.rs index 36150ed67a..0d393acf06 100644 --- a/crates/machine-a-tron/src/machine_a_tron.rs +++ b/crates/machine-a-tron/src/machine_a_tron.rs @@ -94,7 +94,10 @@ impl MachineATron { .as_mut() .and_then(|m| m.remove(config_name.as_str())) { - tracing::info!("Recovering persisted machines for config {}", config_name); + tracing::info!( + config_name = %config_name, + "Recovering persisted machines", + ); persisted_machines .into_iter() .map(|persisted| -> eyre::Result { @@ -117,7 +120,10 @@ impl MachineATron { }) .collect::>() } else { - tracing::info!("Constructing machines for config {}", config_name); + tracing::info!( + config_name = %config_name, + "Constructing machines", + ); (0..config.host_count) .map(|_| { let mac_range = mac_address_pool.allocate_range_config()?; @@ -221,8 +227,8 @@ impl MachineATron { } } else { tracing::info!( - "register_expected_machines=false; skipping auto-registration of {} mock host(s)", - machines.len() + machine_count = machines.len(), + "register_expected_machines=false; skipping auto-registration of mock host(s)", ); } @@ -248,7 +254,10 @@ impl MachineATron { { let host_port_str = format!("{}:{}", host_str, self.app_context.app_config.bmc_mock_port); - tracing::info!("Configuring carbide API to use {host_port_str} as bmc_proxy",); + tracing::info!( + bmc_proxy_address = %host_port_str, + "Configuring carbide API to use as bmc_proxy", + ); _ = self .app_context .api_client() @@ -279,7 +288,10 @@ impl MachineATron { subnet_handles.push(subnet); } Err(e) => { - tracing::error!("Error creating network segment: {}", e); + tracing::error!( + error = %e, + "Error creating network segment", + ); } } } @@ -354,7 +366,10 @@ impl MachineATron { tracing::info!("allocate_instance was successful. "); } Err(e) => { - tracing::info!("allocate_instance failed with {} ", e); + tracing::info!( + error = %e, + "allocate_instance failed", + ); } }; } @@ -365,21 +380,27 @@ impl MachineATron { // It rather soft deletes the VPCs by updating the deleted column of a vpc. if self.app_context.app_config.cleanup_on_quit { for vpc in vpc_handles { - tracing::info!("Attempting to delete VPC with id: {} from db.", vpc.vpc_id); + tracing::info!( + vpc_id = %vpc.vpc_id, + "Attempting to delete VPC from database", + ); if let Err(e) = self .app_context .forge_api_client .delete_vpc(vpc.vpc_id) .await { - tracing::error!("Delete VPC Api call failed with {}", e) + tracing::error!( + error = %e, + "Delete VPC API call failed", + ) } } for subnet in subnet_handles { tracing::info!( - "Attempting to delete network segment with id: {} from db.", - subnet.segment_id + segment_id = %subnet.segment_id, + "Attempting to delete network segment from database", ); if let Err(e) = self .app_context @@ -387,7 +408,10 @@ impl MachineATron { .delete_network_segment(subnet.segment_id) .await { - tracing::error!("Delete network segment Api call failed with {}", e) + tracing::error!( + error = %e, + "Delete network segment API call failed", + ) } } } diff --git a/crates/machine-a-tron/src/machine_state_machine.rs b/crates/machine-a-tron/src/machine_state_machine.rs index 35eb714767..e519735f24 100644 --- a/crates/machine-a-tron/src/machine_state_machine.rs +++ b/crates/machine-a-tron/src/machine_state_machine.rs @@ -513,15 +513,16 @@ impl MachineStateMachine { .await .inspect(|_| { tracing::debug!( - "BMC DHCP Request for {} took {}ms", - self.machine_info.bmc_mac_address(), - start.elapsed().as_millis() + bmc_mac_address = %self.machine_info.bmc_mac_address(), + elapsed_ms = start.elapsed().as_millis(), + "BMC DHCP request completed", ); }) .inspect_err(|err| { tracing::warn!( - "BMC DHCP Request failed after {}ms: {err}", - start.elapsed().as_millis() + elapsed_ms = start.elapsed().as_millis(), + error = %err, + "BMC DHCP request failed", ); }) } @@ -531,72 +532,76 @@ impl MachineStateMachine { return Err(MachineStateError::NoMachineMacAddress); }; - tracing::debug!("Sending Admin DHCP Request for {}", primary_mac); + tracing::debug!( + primary_mac = %primary_mac, + "Sending admin DHCP request", + ); let start = Instant::now(); // Bound the relay wait so a vanished DPU (one mid-flip to NIC mode, its // relay server gone) can't block the host actor indefinitely. const DPU_DHCP_RELAY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - let machine_dhcp_info_result = if let Some(DpuDhcpRelay::HostEnd(relay_tx)) = - &self.dpu_dhcp_relay - { - // Relay this host's data-plane DHCP through its managed DPU, but fall - // back to a direct request on the host's own MAC if the relay does not - // answer promptly -- e.g. the DPU just flipped to NIC mode, so its relay - // server is gone. The same MAC is used either way, so a retained boot - // interface still matches on re-ingestion. - let (reply_tx, reply_rx) = oneshot::channel(); - let relayed = match relay_tx.send(reply_tx) { - Ok(()) => match tokio::time::timeout(DPU_DHCP_RELAY_TIMEOUT, reply_rx).await { - // The relay answered -- use its result (success OR a real - // DHCP/API error). Only fall back to a direct request when the - // relay is genuinely gone: send failed, channel canceled, or - // timed out. - Ok(Ok(result)) => Some(result), - Ok(Err(_)) | Err(_) => None, - }, - Err(_) => None, - }; - match relayed { - Some(result) => result, - None => { - tracing::warn!( - "DPU DHCP relay did not answer for {primary_mac}; falling back to a direct request (the DPU likely flipped to NIC mode)" - ); - dhcp_wrapper::request_ip( - self.app_context.api_client(), - DhcpRequestInfo { - mac_address: primary_mac, - relay_address: self.config.admin_dhcp_relay_address, - template_dir: self.config.template_dir.clone(), - }, - ) - .await + let machine_dhcp_info_result = + if let Some(DpuDhcpRelay::HostEnd(relay_tx)) = &self.dpu_dhcp_relay { + // Relay this host's data-plane DHCP through its managed DPU, but fall + // back to a direct request on the host's own MAC if the relay does not + // answer promptly -- e.g. the DPU just flipped to NIC mode, so its relay + // server is gone. The same MAC is used either way, so a retained boot + // interface still matches on re-ingestion. + let (reply_tx, reply_rx) = oneshot::channel(); + let relayed = match relay_tx.send(reply_tx) { + Ok(()) => match tokio::time::timeout(DPU_DHCP_RELAY_TIMEOUT, reply_rx).await { + // The relay answered -- use its result (success OR a real + // DHCP/API error). Only fall back to a direct request when the + // relay is genuinely gone: send failed, channel canceled, or + // timed out. + Ok(Ok(result)) => Some(result), + Ok(Err(_)) | Err(_) => None, + }, + Err(_) => None, + }; + match relayed { + Some(result) => result, + None => { + tracing::warn!( + primary_mac = %primary_mac, + "DPU DHCP relay unavailable; falling back to a direct request", + ); + dhcp_wrapper::request_ip( + self.app_context.api_client(), + DhcpRequestInfo { + mac_address: primary_mac, + relay_address: self.config.admin_dhcp_relay_address, + template_dir: self.config.template_dir.clone(), + }, + ) + .await + } } - } - } else { - dhcp_wrapper::request_ip( - self.app_context.api_client(), - DhcpRequestInfo { - mac_address: primary_mac, - relay_address: self.config.admin_dhcp_relay_address, - template_dir: self.config.template_dir.clone(), - }, - ) - .await - }; + } else { + dhcp_wrapper::request_ip( + self.app_context.api_client(), + DhcpRequestInfo { + mac_address: primary_mac, + relay_address: self.config.admin_dhcp_relay_address, + template_dir: self.config.template_dir.clone(), + }, + ) + .await + }; machine_dhcp_info_result .inspect(|_| { tracing::debug!( - "Admin DHCP Request for {} took {}ms", - primary_mac, - start.elapsed().as_millis() + primary_mac = %primary_mac, + elapsed_ms = start.elapsed().as_millis(), + "Admin DHCP request completed", ); }) .map_err(|err| { tracing::debug!( - "Admin DHCP Request for {} failed after {}ms: {err}", - primary_mac, - start.elapsed().as_millis() + primary_mac = %primary_mac, + elapsed_ms = start.elapsed().as_millis(), + error = %err, + "Admin DHCP request failed", ); err.into() }) @@ -713,9 +718,9 @@ impl MachineStateMachine { return Err(MachineStateError::MachineNotFound(machine_id)); }; tracing::trace!( - "get action took {}ms; action={:?}", - start.elapsed().as_millis(), - control_response.action, + elapsed_ms = start.elapsed().as_millis(), + action = ?control_response.action, + "forge_agent_control action received", ); match &control_response.action { @@ -741,9 +746,9 @@ impl MachineStateMachine { Some(Action::Noop(_)) => {} _ => { tracing::warn!( - "Unknown action from forge_agent_control: {:?} for OS image {}", - control_response.action, - os_image, + action = ?control_response.action, + os_image = %os_image, + "Unknown forge_agent_control action for OS image", ); } } @@ -884,7 +889,10 @@ impl MachineStateMachine { ) .await?; - tracing::trace!("discover_machine took {}ms", start.elapsed().as_millis()); + tracing::trace!( + elapsed_ms = start.elapsed().as_millis(), + "discover_machine completed", + ); Ok(machine_discovery_result) } @@ -981,7 +989,7 @@ impl MachineStateMachine { GracefulShutdown | ForceOff => self.fsm_event(Event::PowerOff), PushPowerButton | Nmi | Suspend | Pause | Resume => { let msg = format!("Machine-a-tron mock: unsupported power request {request:?}",); - tracing::warn!("{msg}"); + tracing::warn!(?request, "unsupported machine-a-tron mock power request",); return Err(SetSystemPowerError::BadRequest(msg)); } }; @@ -1063,7 +1071,10 @@ impl MachineStateMachine { .discovery_completed(*machine_id) .await .map_err(ClientApiError::InvocationError)?; - tracing::trace!("discovery_complete took {}ms", start.elapsed().as_millis()); + tracing::trace!( + elapsed_ms = start.elapsed().as_millis(), + "discovery_complete completed", + ); Ok(()) } diff --git a/crates/machine-a-tron/src/machine_utils.rs b/crates/machine-a-tron/src/machine_utils.rs index 271e6ddc37..fbabeff119 100644 --- a/crates/machine-a-tron/src/machine_utils.rs +++ b/crates/machine-a-tron/src/machine_utils.rs @@ -67,7 +67,10 @@ pub async fn forge_agent_control( if e.code() == tonic::Code::NotFound { return None; } - tracing::warn!("Error getting control action: {e}"); + tracing::warn!( + error = %e, + "Error getting control action", + ); Some(ForgeAgentControlResponse::noop()) } } @@ -116,15 +119,15 @@ pub async fn send_pxe_boot_request( PxeResponse::Scout } else { tracing::error!( - "Could not determine what to do with kernel URL returned by PXE script, will treat as 'exit': {}", - pxe_script + pxe_script = %pxe_script, + "Could not determine what to do with kernel URL returned by PXE script, will treat as 'exit'", ); PxeResponse::Exit } } else { tracing::error!( - "Could not determine what to do with PXE script (no kernel line, no exit line), will treat as 'exit': {}", - pxe_script + pxe_script = %pxe_script, + "Could not determine what to do with PXE script (no kernel line, no exit line), will treat as 'exit'", ); PxeResponse::Exit }; @@ -154,14 +157,18 @@ pub async fn add_address_to_interface( ) -> Result<(), AddressConfigError> { if interface_has_address(interface, address).await? { tracing::info!( - "Skipping adding address {} to interface {}, as it is already configured.", - address, - interface + address = %address, + interface = %interface, + "Skipping address addition; already configured", ); return Ok(()); } - tracing::info!("Adding address {} to interface {}", address, interface); + tracing::info!( + address = %address, + interface = %interface, + "Adding address", + ); let wrapper_cmd = find_sudo_command(); let mut cmd = tokio::process::Command::new(wrapper_cmd); #[cfg(not(target_os = "macos"))] diff --git a/crates/machine-a-tron/src/main.rs b/crates/machine-a-tron/src/main.rs index 2605fb6b6a..2843d86d0b 100644 --- a/crates/machine-a-tron/src/main.rs +++ b/crates/machine-a-tron/src/main.rs @@ -104,8 +104,12 @@ async fn main() -> Result<(), Box> { args.client_key_path, file_config.as_ref(), ); - let proxy = - get_proxy_info().inspect_err(|e| tracing::error!("Failed to get proxy info: {}", e))?; + let proxy = get_proxy_info().inspect_err(|e| { + tracing::error!( + error = %e, + "Failed to get proxy info", + ) + })?; let mut forge_client_config = ForgeClientConfig::new(forge_root_ca_path.clone(), Some(forge_client_cert)); @@ -136,8 +140,8 @@ async fn main() -> Result<(), Box> { .entries; tracing::info!( - "Got desired firmware versions from the server: {:?}", - desired_firmware_versions + desired_firmware_versions = ?desired_firmware_versions, + "Got desired firmware versions from the server", ); let bmc_mock_port = app_config.bmc_mock_port; @@ -184,7 +188,10 @@ async fn main() -> Result<(), Box> { }); let info = app_context.forge_api_client.version(false).await?; - tracing::info!("version: {}", info.build_version); + tracing::info!( + build_version = %info.build_version, + "machine-a-tron version", + ); let mut mat = MachineATron::new(app_context.clone()); @@ -301,13 +308,21 @@ async fn main() -> Result<(), Box> { if let Some(tui_handle) = tui_handle { if let Some(tui_quit_tx) = tui_quit_tx.as_ref() { - _ = tui_quit_tx - .try_send(()) - .inspect_err(|e| tracing::warn!("Could not send quit signal to TUI: {e}")); + _ = tui_quit_tx.try_send(()).inspect_err(|e| { + tracing::warn!( + error = %e, + "Could not send quit signal to TUI", + ) + }); } tui_handle .await - .inspect_err(|e| tracing::warn!("Error running TUI: {e}")) + .inspect_err(|e| { + tracing::warn!( + error = %e, + "Error running TUI", + ) + }) .ok(); } diff --git a/crates/machine-a-tron/src/mock_ssh_server.rs b/crates/machine-a-tron/src/mock_ssh_server.rs index e33c9b5f72..7ce10b0244 100644 --- a/crates/machine-a-tron/src/mock_ssh_server.rs +++ b/crates/machine-a-tron/src/mock_ssh_server.rs @@ -119,7 +119,10 @@ impl Server { tokio::spawn(async move { if config.nodelay && let Err(e) = socket.set_nodelay(true) { - tracing::warn!("set_nodelay() failed: {e:?}"); + tracing::warn!( + error = ?e, + "set_nodelay failed", + ); } let session = match run_stream(config, socket, handler).await { @@ -283,7 +286,8 @@ impl server::Handler for MockSshHandler { Ok(server::Auth::Accept) } else { tracing::info!( - "got incorrect auth_password, rejecting. user={user}, password={password}" + user = %user, + "Incorrect SSH password; rejecting authentication", ); Ok(server::Auth::Reject { proceed_with_methods: None, @@ -292,7 +296,8 @@ impl server::Handler for MockSshHandler { } } else { tracing::info!( - "configured to accept any credentials, accepting user={user}, password={password}" + user = %user, + "Accepting SSH credentials because any credentials are allowed", ); Ok(server::Auth::Accept) } @@ -367,8 +372,8 @@ impl server::Handler for MockSshHandler { (b"\x1c", PromptBehavior::Dell) => { // ssh-console should have prevented this, make it a warning. tracing::warn!( - "Got ctrl+\\ in system console, dropping to BMC prompt {:?}", - self.console_state + console_state = ?self.console_state, + "Got ctrl+\\ in system console, dropping to BMC prompt", ); // ctrl+\ self.console_state = ConsoleState::Bmc; diff --git a/crates/machine-a-tron/src/subnet.rs b/crates/machine-a-tron/src/subnet.rs index 98944144c8..4eb59f2a77 100644 --- a/crates/machine-a-tron/src/subnet.rs +++ b/crates/machine-a-tron/src/subnet.rs @@ -48,7 +48,10 @@ impl Subnet { .create_network_segment(&vpc.metadata.name, vpc.network_virtualization_type) .await .map_err(|e| { - tracing::error!("Error creating network segment: {}", e); + tracing::error!( + error = %e, + "Error creating network segment", + ); Status::internal("Failed to create network segment.") })?; @@ -68,7 +71,12 @@ impl Subnet { _ = ui_event_tx .send(UiUpdate::Subnet(details)) .await - .inspect_err(|e| tracing::warn!("Error sending TUI event: {}", e)); + .inspect_err(|e| { + tracing::warn!( + error = %e, + "Error sending TUI event", + ) + }); } Ok(new_subnet) diff --git a/crates/machine-a-tron/src/tui.rs b/crates/machine-a-tron/src/tui.rs index a948e01eb0..cae21a3258 100644 --- a/crates/machine-a-tron/src/tui.rs +++ b/crates/machine-a-tron/src/tui.rs @@ -267,7 +267,10 @@ impl Tui { .1 } _ => { - tracing::warn!("Unexpected event: {:?}", event); + tracing::warn!( + event = ?event, + "Unexpected event", + ); false } } @@ -459,7 +462,10 @@ impl Tui { Some(Ok(event)) => { list_updated = self.handle_event(event).await; } - Some(Err(e)) => tracing::warn!("Error: {:?}", e), + Some(Err(e)) => tracing::warn!( + error = ?e, + "TUI error", + ), None => break, } } diff --git a/crates/machine-a-tron/src/vpc.rs b/crates/machine-a-tron/src/vpc.rs index e09170d2c9..53d350265a 100644 --- a/crates/machine-a-tron/src/vpc.rs +++ b/crates/machine-a-tron/src/vpc.rs @@ -63,7 +63,12 @@ impl Vpc { _ = ui_event_tx .send(UiUpdate::Vpc(details)) .await - .inspect_err(|e| tracing::warn!("Error sending TUI event: {}", e)); + .inspect_err(|e| { + tracing::warn!( + error = %e, + "Error sending TUI event", + ) + }); } new_vpc diff --git a/crates/machine-controller/src/dpf.rs b/crates/machine-controller/src/dpf.rs index cac8cc972f..1234f6ab65 100644 --- a/crates/machine-controller/src/dpf.rs +++ b/crates/machine-controller/src/dpf.rs @@ -591,7 +591,8 @@ impl DpfOperations for DpfSdkOps { let Some(machine_id_str) = m.dpu_labels.get(DPU_MACHINE_ID_LABEL) else { tracing::warn!( dpu = %m.dpu_cr_name, - "Outdated DPU missing {DPU_MACHINE_ID_LABEL} label; skipping" + label = DPU_MACHINE_ID_LABEL, + "Outdated DPU missing label; skipping" ); continue; }; @@ -601,8 +602,9 @@ impl DpfOperations for DpfSdkOps { tracing::warn!( dpu = %m.dpu_cr_name, label_value = %machine_id_str, + label = DPU_MACHINE_ID_LABEL, error = %e, - "Outdated DPU has invalid {DPU_MACHINE_ID_LABEL} label; skipping" + "Outdated DPU has invalid label; skipping" ); continue; } diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 3f01c4a6ce..d2160f0990 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -738,12 +738,11 @@ impl MachineStateHandler { && let Some((machine_id, details)) = get_failed_state(mh_snapshot) { tracing::error!( - %machine_id, - "ManagedHost {}/{} (failed machine: {}) is moved to Failed state with cause: {:?}", - mh_snapshot.host_snapshot.id, - get_display_ids(&mh_snapshot.dpu_snapshots), - machine_id, - details + machine_id = %mh_snapshot.host_snapshot.id, + dpu_machine_ids = %get_display_ids(&mh_snapshot.dpu_snapshots), + failed_machine_id = %machine_id, + failure_details = ?details, + "ManagedHost is moved to Failed state" ); let next_state = match mh_state { ManagedHostState::Assigned { .. } => ManagedHostState::Assigned { @@ -979,9 +978,9 @@ impl MachineStateHandler { && mh_snapshot.host_snapshot.bios_password_set_time.is_none() { tracing::info!( - "transitioning legacy {} host {} to UefiSetupState::UnlockHost while it is in ManagedHostState::Ready so that the BIOS password can be configured", - mh_snapshot.host_snapshot.bmc_vendor(), - mh_snapshot.host_snapshot.id + vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + machine_id = %mh_snapshot.host_snapshot.id, + "transitioning legacy host to UefiSetupState::UnlockHost while it is in ManagedHostState::Ready so that the BIOS password can be configured" ); return Ok(StateHandlerOutcome::transition( ManagedHostState::HostInit { @@ -1485,12 +1484,11 @@ impl MachineStateHandler { // Do nothing. // Handle error cause and decide how to recover if possible. tracing::error!( - %machine_id, - "ManagedHost {} is in Failed state with machine/cause {}/{}. Failed at: {}, Ignoring.", - host_machine_id, - machine_id, - details.cause, - details.failed_at, + machine_id = %host_machine_id, + failed_machine_id = %machine_id, + failure_cause = %details.cause, + failed_at = %details.failed_at, + "ManagedHost is in Failed state. Ignoring.", ); // TODO: Should this be StateHandlerError::ManualInterventionRequired ? Ok(StateHandlerOutcome::do_nothing()) @@ -1781,7 +1779,11 @@ impl MachineStateHandler { if let Err(err) = handler_host_power_control(state, ctx, SystemPowerControl::ForceRestart).await { - tracing::error!(%host_machine_id, "Host reboot failed with error: {err}"); + tracing::error!( + machine_id = %host_machine_id, + error = %err, + "Host reboot failed" + ); } set_managed_host_topology_update_needed( ctx.pending_db_writes, @@ -2045,7 +2047,10 @@ async fn handle_restart_verification( verified: Some(true), attempts: 0, }); - tracing::info!("Restart verified for host {}", mh_snapshot.host_snapshot.id); + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Restart verified for host" + ); return Ok(None); } @@ -2064,9 +2069,9 @@ async fn handle_restart_verification( }); tracing::info!( - "Issued force-restart for host {} after {} failed verifications", - mh_snapshot.host_snapshot.id, - verification_attempts + machine_id = %mh_snapshot.host_snapshot.id, + verification_attempts, + "Issued force-restart for host after failed verifications" ); return Ok(None); } @@ -2150,7 +2155,7 @@ async fn handle_restart_verification( verified: Some(true), attempts: 0, }); - tracing::info!("Restart verified for DPU {}", dpu.id); + tracing::info!(machine_id = %dpu.id, "Restart verified for DPU"); } else if verification_attempts >= MAX_VERIFICATION_ATTEMPTS { dpu_redfish_client .power(SystemPowerControl::ForceRestart) @@ -2166,9 +2171,9 @@ async fn handle_restart_verification( }); tracing::info!( - "Issued force-restart for DPU {} after {} failed verifications", - dpu.id, - verification_attempts + machine_id = %dpu.id, + verification_attempts, + "Issued force-restart for DPU after failed verifications" ); } else { ctx.pending_db_writes @@ -2218,7 +2223,7 @@ pub async fn check_restart_in_logs( let logs = redfish_client.get_bmc_event_log(Some(restart_time)).await?; for log in &logs { - tracing::debug!("BMC log message: {}", log.message); + tracing::debug!(bmc_message = %log.message, "BMC log message"); } let restart_found = logs.iter().any(|log| { @@ -2282,7 +2287,11 @@ async fn are_dpus_up_trigger_reboot_if_needed( .await { Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), + Err(e) => tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "could not reboot dpu" + ), } return false; } @@ -2315,7 +2324,7 @@ impl StateHandler for MachineStateHandler { .is_empty() && mh_snapshot.dpu_snapshots.is_empty() { - tracing::error!("No DPU snapshot found for host {}", host_machine_id); + tracing::error!(machine_id = %host_machine_id, "No DPU snapshot found for host"); return Err(StateHandlerError::GenericError(eyre!( "No DPU snapshot found." ))); @@ -2524,9 +2533,9 @@ async fn handle_bfb_install_state( .await .map_err(|e| redfish_error("update_firmware_simple_update", e))?; tracing::info!( - "DPU {} OS install task {} submitted.", - dpu_snapshot.id, - task.id + machine_id = %dpu_snapshot.id, + task_id = %task.id, + "DPU OS install task submitted." ); Ok(StateHandlerOutcome::transition( next_state_resolver.next_bfb_install_state( @@ -2547,15 +2556,18 @@ async fn handle_bfb_install_state( .map_err(|e| redfish_error("get_task", e))?; tracing::info!( - "DPU {} OS install task {}: {:#?}", - dpu_snapshot.id, - task.id, - task.task_state + machine_id = %dpu_snapshot.id, + task_id = %task.id, + task_state = ?task.task_state, + "DPU OS install task" ); match task.task_state { Some(TaskState::Completed) => { - tracing::info!("Install BFB on {:#?} completed", dpu_snapshot.bmc_addr()); + tracing::info!( + bmc_address = ?dpu_snapshot.bmc_addr(), + "Install BFB completed" + ); let next_state = next_state_resolver.next_bfb_install_state( &state.managed_state, &InstallDpuOsState::Completed, @@ -2570,7 +2582,12 @@ async fn handle_bfb_install_state( dpu_snapshot.bmc_addr(), task.messages.iter().map(|t| t.message.clone()).join("\n") ); - tracing::error!(msg); + tracing::error!( + machine_id = %dpu_snapshot.id, + %task_id, + reason = %msg, + "BFB install task failed", + ); let next_state = next_state_resolver.next_bfb_install_state( &state.managed_state, &InstallDpuOsState::InstallationError { msg }, @@ -2595,7 +2612,13 @@ async fn handle_bfb_install_state( task_state, task.messages.iter().map(|t| t.message.clone()).join("\n") ); - tracing::error!(msg); + tracing::error!( + machine_id = %dpu_snapshot.id, + %task_id, + task_state = ?task_state, + reason = %msg, + "BFB install task failed", + ); let next_state = next_state_resolver.next_bfb_install_state( &state.managed_state, &InstallDpuOsState::InstallationError { msg }, @@ -2883,8 +2906,8 @@ async fn handle_dpu_reprovision( // Host is not powered-off yet. Try again. if power_state != libredfish::PowerState::Off { tracing::error!( - "Machine {} is still not power-off state. Turning off for host again.", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "Machine is still not power-off state. Turning off for host again." ); handler_host_power_control(state, ctx, SystemPowerControl::ForceOff).await?; @@ -2960,7 +2983,7 @@ async fn handle_dpu_reprovision( for dsnapshot in &state.dpu_snapshots { if !is_dpu_up(state, dsnapshot) { let msg = format!("Waiting for DPU {} to come up", dsnapshot.id); - tracing::warn!("{msg}"); + tracing::warn!(machine_id = %dsnapshot.id, "Waiting for DPU to come up"); let mut reboot_status = None; // Only the DPU handled by this invocation should trigger its @@ -2987,7 +3010,10 @@ async fn handle_dpu_reprovision( dsnapshot, state.host_snapshot.network_config.version, ) { - tracing::warn!("Waiting for network to be ready for DPU {}", dsnapshot.id); + tracing::warn!( + machine_id = %dsnapshot.id, + "Waiting for network to be ready for DPU" + ); // The install path already requested a DPU reboot. If this // specific DPU remains unsynced, let trigger_reboot_if_needed @@ -3415,8 +3441,8 @@ async fn handle_dpu_reprovision( // At this point, all of the host's DPU have finished the NIC FW Update, been power cycled, and the ARM has come up on the DPU. if state.host_snapshot.bmc_vendor().is_lenovo() { tracing::info!( - "Initiating BMC reset of lenovo machine {}", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "Initiating BMC reset of lenovo machine" ); let redfish_client = ctx @@ -3426,8 +3452,9 @@ async fn handle_dpu_reprovision( if let Err(redfish_error) = redfish_client.bmc_reset().await { tracing::warn!( - "Failed to reboot BMC for {} through redfish, will try ipmitool: {redfish_error}", - &state.host_snapshot.id + machine_id = %state.host_snapshot.id, + error = %redfish_error, + "Failed to reboot BMC through redfish, will try ipmitool" ); let bmc_mac_address = state.host_snapshot.bmc_info.mac.ok_or_else(|| { @@ -3456,8 +3483,9 @@ async fn handle_dpu_reprovision( .await { tracing::warn!( - "Failed to reset BMC for {} through IPMI tool: {ipmitool_error}", - &state.host_snapshot.id + machine_id = %state.host_snapshot.id, + error = %ipmitool_error, + "Failed to reset BMC through IPMI tool" ); return Err(StateHandlerError::GenericError(eyre!( @@ -3690,7 +3718,11 @@ async fn should_wait_for_dpus_before_host_boot_config( .await { Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), + Err(e) => tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "could not reboot dpu" + ), } return true; } @@ -3793,14 +3825,22 @@ async fn check_fw_component_version( let inventory = match redfish_client.get_firmware(inventory_id).await { Ok(inventory) => inventory, Err(e) => { - tracing::error!(machine_id=%dpu_snapshot.id, "redfish command get_firmware error {}", e.to_string()); + tracing::error!( + machine_id = %dpu_snapshot.id, + error = %e, + "redfish command get_firmware error" + ); return Err(redfish_error("get_firmware", e)); } }; if inventory.version.is_none() { let msg = format!("Unknown {component_name:?} version"); - tracing::error!(machine_id=%dpu_snapshot.id, msg); + tracing::error!( + machine_id = %dpu_snapshot.id, + component = ?component_name, + "Unknown firmware version", + ); return Err(StateHandlerError::FirmwareUpdateError(eyre!(msg))); }; @@ -3831,20 +3871,20 @@ async fn check_fw_component_version( { // For this case need to run host power cycle tracing::info!( - machine_id=%dpu_snapshot.id, - "Need to launch host power cycle to update CEC FW from {} to {}", - cur_version, - expected_version + machine_id = %dpu_snapshot.id, + current_version = %cur_version, + expected_version = %expected_version, + "Need to launch host power cycle to update CEC FW" ); return Ok(None); } tracing::debug!( - machine_id=%dpu_snapshot.id, - "{:#?} FW not yet at the expected version. Expected: {}, Current: {}", - component, - expected_version, - cur_version, + machine_id = %dpu_snapshot.id, + component = ?component, + expected_version = %expected_version, + current_version = %cur_version, + "FW not yet at the expected version", ); // Don't return Error. In case of the error, reboot time won't be updated in db. @@ -3857,10 +3897,10 @@ async fn check_fw_component_version( } tracing::info!( - machine_id=%dpu_snapshot.id, - "{:#?} FW updated successfully to {}", - component, - expected_version, + machine_id = %dpu_snapshot.id, + component = ?component, + expected_version = %expected_version, + "FW updated successfully", ); // BMC FW version need to update in machine_topology->bmc_info @@ -3875,8 +3915,11 @@ async fn check_fw_component_version( .get_firmware("DPU_UEFI") .await .inspect_err(|e| { - tracing::error!("redfish command get_firmware error {}", e.to_string()); - tracing::error!(machine_id=%dpu_snapshot.id, "redfish command get_firmware error {}", e.to_string()); + tracing::error!( + machine_id = %dpu_snapshot.id, + error = %e, + "redfish command get_firmware error" + ); }) .ok() .and_then(|uefi| uefi.version) @@ -4051,10 +4094,11 @@ impl DpuMachineStateHandler { Ok(StateHandlerOutcome::transition(next_state)) } DpuDiscoveringState::EnableRshim => { - let _ = dpu_redfish_client + dpu_redfish_client .enable_rshim_bmc() .await - .map_err(|e| tracing::info!("failed to enable rshim on DPU {e}")); + .inspect_err(|e| tracing::info!(error = %e, "failed to enable rshim on DPU")) + .ok(); let next_dpu_discovering_state = DpuDiscoveringState::next_substate_based_on_bfb_support( @@ -4064,13 +4108,14 @@ impl DpuMachineStateHandler { ); tracing::info!( - "DPU {dpu_machine_id} (BMC FW version: {}); next_state: {}.", - dpu_snapshot + machine_id = %dpu_machine_id, + bmc_firmware_version = %dpu_snapshot .bmc_info .firmware_version .clone() .unwrap_or("unknown".to_string()), - next_dpu_discovering_state + next_state = %next_dpu_discovering_state, + "DPU discovery state selected." ); let next_state = @@ -4214,8 +4259,8 @@ impl DpuMachineStateHandler { } tracing::debug!( - "ManagedHostState::DPUNotReady::Init: firmware update enabled = {}", - self.dpu_nic_firmware_initial_update_enabled + firmware_update_enabled = self.dpu_nic_firmware_initial_update_enabled, + "ManagedHostState::DPUNotReady::Init" ); // All DPUs are discovered. Reboot them to proceed. @@ -4311,7 +4356,11 @@ impl DpuMachineStateHandler { "failed to create redfish client for DPU {}, potentially because we turned the host off as part of error handling in this state. err: {}", dpu_snapshot.id, e ); - tracing::warn!(msg); + tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "failed to create redfish client for DPU, potentially because we turned the host off as part of error handling in this state", + ); // If we cannot create a redfish client for the DPU, this function call will never result in an actual DPU reboot. // The only side effect is turning the DPU's host back on if we turned it off earlier. let reboot_status = trigger_reboot_if_needed( @@ -4367,7 +4416,11 @@ impl DpuMachineStateHandler { "redfish machine_setup failed for DPU {}, potentially due to known race condition between UEFI POST and BMC. issuing a force-restart. err: {}", dpu_snapshot.id, e ); - tracing::warn!(msg); + tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "redfish machine_setup failed for DPU, potentially due to known race condition between UEFI POST and BMC; issuing a force-restart", + ); let reboot_status = trigger_reboot_if_needed( dpu_snapshot, state, @@ -4399,7 +4452,11 @@ impl DpuMachineStateHandler { "Failed to run uefi_setup call failed for DPU {}: {}", dpu_snapshot.id, e ); - tracing::warn!(msg); + tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "Failed to run uefi_setup for DPU", + ); let reboot_status = trigger_reboot_if_needed( dpu_snapshot, state, @@ -4489,7 +4546,11 @@ impl DpuMachineStateHandler { "DPU {} BIOS attributes not ready ({e}); issuing a force-restart to mitigate the known UEFI POST/BMC race", dpu_snapshot.id ); - tracing::warn!("{msg}"); + tracing::warn!( + machine_id = %dpu_snapshot.id, + error = %e, + "DPU BIOS attributes not ready; issuing a force-restart to mitigate the known UEFI POST/BMC race", + ); let reboot_status = trigger_reboot_if_needed( dpu_snapshot, state, @@ -4559,8 +4620,8 @@ impl DpuMachineStateHandler { } DpuInitState::WaitingForNetworkInstall => { tracing::warn!( - "Invalid State WaitingForNetworkInstall for dpu Machine {}", - dpu_machine_id + machine_id = %dpu_machine_id, + "Invalid State WaitingForNetworkInstall for dpu Machine" ); Err(StateHandlerError::InvalidHostState( *dpu_machine_id, @@ -4598,8 +4659,10 @@ impl DpuMachineStateHandler { if count > 0 && !has_dpu_finished_booting { tracing::info!( - "Waiting for DPU {} to finish booting; boot progress: {dpu_boot_progress:#?}; SetSecureBoot cycle: {count}", - dpu_snapshot.id + machine_id = %dpu_snapshot.id, + boot_progress = ?dpu_boot_progress, + cycle = count, + "Waiting for DPU to finish booting; SetSecureBoot cycle" ) } @@ -4721,9 +4784,9 @@ impl DpuMachineStateHandler { } Err(StateHandlerError::MissingData { object_id, missing }) => { tracing::info!( - "Missing data in secure boot status response for DPU {}: {}; rebooting DPU as a work-around", - object_id, - missing + machine_id = %object_id, + missing = %missing, + "Missing data in secure boot status response for DPU; rebooting DPU as a work-around" ); /*** @@ -5112,9 +5175,9 @@ pub async fn trigger_reboot_if_needed_with_location( if let MachineLastRebootRequestedMode::PowerOff = last_reboot_requested.mode { // PowerOn the host. tracing::info!( - "Machine {} is in power-off state. Turning on for host: {}", - target.id, - host.id, + machine_id = %target.id, + host_machine_id = %host.id, + "Machine is in power-off state. Turning on for host", ); if wait( @@ -5143,14 +5206,18 @@ pub async fn trigger_reboot_if_needed_with_location( SystemPowerControl::On } else { tracing::error!( - "Machine {} is still not power-off state. Turning off again for host: {}", - target.id, - host.id, + machine_id = %target.id, + host_machine_id = %host.id, + "Machine is still not power-off state. Turning off again for host", ); SystemPowerControl::ForceOff }; - tracing::trace!(machine_id=%target.id, "Redfish setting host power state to {action}"); + tracing::trace!( + machine_id = %target.id, + %action, + "Redfish setting host power state" + ); handler_host_power_control_with_location(state, ctx, action, trigger_location).await?; return Ok(RebootStatus { increase_retry_count: false, @@ -5161,8 +5228,8 @@ pub async fn trigger_reboot_if_needed_with_location( // Check if reboot is prevented by health override. if state.aggregate_health.is_reboot_blocked_in_state_machine() { tracing::info!( - "Not trying to reboot {} since health override is set to prevent reboot.", - target.id, + machine_id = %target.id, + "Not trying to reboot since health override is set to prevent reboot.", ); return Ok(RebootStatus { increase_retry_count: false, @@ -5265,10 +5332,11 @@ pub async fn trigger_reboot_if_needed_with_location( ) }; - tracing::info!(machine_id=%target.id, - "triggered reboot for machine in managed-host state {}: {}", - state.managed_state, - status, + tracing::info!( + machine_id = %target.id, + managed_host_state = %state.managed_state, + reboot_status = %status, + "Triggered reboot in managed-host state", ); Ok(RebootStatus { @@ -5416,8 +5484,9 @@ async fn handle_host_boot_order_setup( set_boot_order_info: Option, ) -> Result, StateHandlerError> { tracing::info!( - "Starting Boot Order Configuration for {}: {set_boot_order_info:#?}", - mh_snapshot.host_snapshot.id + machine_id = %mh_snapshot.host_snapshot.id, + set_boot_order_info = ?set_boot_order_info, + "Starting Boot Order Configuration" ); let redfish_client = ctx @@ -5629,7 +5698,12 @@ async fn handle_host_uefi_setup( // For all other vendors, allow ingestion even though we couldnt set the bios password // An operator will have to set the bios password manually - tracing::info!(msg); + tracing::info!( + machine_id = %state.host_snapshot.id, + bmc_vendor = %state.host_snapshot.bmc_vendor(), + error = %e, + "failed to set the BIOS password", + ); Ok(StateHandlerOutcome::transition( ManagedHostState::HostInit { @@ -5818,12 +5892,15 @@ impl StateHandler for HostMachineStateHandler { match redfish_client.lockdown_status().await { Err(libredfish::RedfishError::NotSupported(_)) => { tracing::info!( - "BMC vendor does not support checking lockdown status for {host_machine_id}." + machine_id = %host_machine_id, + "BMC vendor does not support checking lockdown status." ); } Err(e) => { tracing::warn!( - "Error fetching lockdown status for {host_machine_id} during machine_setup check: {e}" + machine_id = %host_machine_id, + error = %e, + "Error fetching lockdown status during machine_setup check" ); return Ok(StateHandlerOutcome::wait(format!( "Failed to fetch lockdown status: {}", @@ -5832,7 +5909,8 @@ impl StateHandler for HostMachineStateHandler { } Ok(lockdown_status) if !lockdown_status.is_fully_disabled() => { tracing::info!( - "Lockdown is enabled for {host_machine_id} during machine_setup, disabling now." + machine_id = %host_machine_id, + "Lockdown is enabled during machine_setup, disabling now." ); let next_state = ManagedHostState::HostInit { machine_state: MachineState::WaitingForLockdown { @@ -6080,10 +6158,9 @@ impl StateHandler for HostMachineStateHandler { ) { tracing::trace!( machine_id = %host_machine_id, - "Waiting for forge-scout to report host online. \ - Host last seen {:?}, must come after DPU's {}", - mh_snapshot.host_snapshot.last_discovery_time, - mh_snapshot.host_snapshot.state.version.timestamp() + host_last_seen = ?mh_snapshot.host_snapshot.last_discovery_time, + minimum_discovery_time = %mh_snapshot.host_snapshot.state.version.timestamp(), + "Waiting for forge-scout to report host online; host discovery predates the current state transition" ); let status = trigger_reboot_if_needed( &mh_snapshot.host_snapshot, @@ -6315,7 +6392,8 @@ impl StateHandler for HostMachineStateHandler { } Err(libredfish::RedfishError::NotSupported(_)) => { tracing::info!( - "BMC vendor does not support checking lockdown status for {host_machine_id}." + machine_id = %host_machine_id, + "BMC vendor does not support checking lockdown status." ); Ok(StateHandlerOutcome::transition(next_state)) } @@ -7290,11 +7368,11 @@ impl StateHandler for InstanceStateHandler { // 2. If failed during reprovision, fix the config/hw issue and // retrigger DPU reprovision. tracing::warn!( - "Instance id {}/machine: {} stuck in failed state. details: {:?}, failed machine: {}", - instance.id, - host_machine_id, - details, - machine_id + instance_id = %instance.id, + machine_id = %host_machine_id, + failure_details = ?details, + failed_machine_id = %machine_id, + "Instance machine stuck in failed state" ); Ok(StateHandlerOutcome::do_nothing()) } @@ -7428,8 +7506,8 @@ async fn process_dpu_use_admin_network_state_change( mh_snapshot: &ManagedHostStateSnapshot, ) -> Result<(), StateHandlerError> { tracing::info!( - "Set use_admin_network_changed flag as host {} has changed use_admin_network state and site-restart-ovs is set", - &mh_snapshot.host_snapshot.id + machine_id = %mh_snapshot.host_snapshot.id, + "Set use_admin_network_changed flag as host has changed use_admin_network state and site-restart-ovs is set" ); // Determine which DPUs have tenant interface configs. A DPU matches if: @@ -7468,7 +7546,10 @@ async fn process_dpu_use_admin_network_state_change( }); if dpu_has_tenant_interface_config { - tracing::info!("Set DPU use_admin_network_changed flag for dpu {}", &dpu.id); + tracing::info!( + machine_id = %dpu.id, + "Set DPU use_admin_network_changed flag for dpu" + ); db::machine::set_use_admin_network_changed(txn, &dpu.id, true).await?; } } @@ -7672,9 +7753,9 @@ async fn handle_instance_network_config_update_request( .collect_vec(); tracing::info!( - "Releasing network resources for instance {}: addresses: {:?}", - instance.id, - addresses, + instance_id = %instance.id, + addresses = ?addresses, + "Releasing network resources", ); db::instance_address::delete_addresses(&mut txn, &addresses).await?; release_network_segments_with_vpc_prefix(&resources_to_be_released, &mut txn) @@ -7784,15 +7865,19 @@ fn check_instance_network_synced_and_dpu_healthy( .iter() .filter(|dpu_machine_id| { if let Some(device) = id_to_device_map.get(dpu_machine_id) { - tracing::info!("Found device {} for dpu {}", device, dpu_machine_id); + tracing::info!( + %device, + machine_id = %dpu_machine_id, + "Found device for dpu" + ); if let Some(id_vec) = device_to_id_map.get(device) && let Some(device_instance) = id_vec.iter().position(|id| id == *dpu_machine_id) { tracing::info!( - "Found device_instance {} for dpu {}", device_instance, - dpu_machine_id + machine_id = %dpu_machine_id, + "Found device_instance for dpu" ); let device_locator = DeviceLocator { device: device.clone(), @@ -7813,14 +7898,14 @@ fn check_instance_network_synced_and_dpu_healthy( if instance.observations.network.len() != dpu_machine_ids.len() { tracing::info!( - "obs: {} dpus: {}", - instance.observations.network.len(), - dpu_machine_ids.len() + network_observation_count = instance.observations.network.len(), + dpu_count = dpu_machine_ids.len(), + "network observation count does not match DPU count" ); let mut missing_dpus = Vec::default(); for dpu_id in dpu_machine_ids { - tracing::info!("checking dpu: {}", dpu_id); + tracing::info!(machine_id = %dpu_id, "checking dpu"); if !instance.observations.network.contains_key(&dpu_id) { tracing::info!("missing"); missing_dpus.push(dpu_id); @@ -8256,8 +8341,8 @@ impl HostUpgradeState { if let Some(result) = result { if result.success { tracing::info!( - "Scout firmware upgrade succeeded for {}", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + "Scout firmware upgrade succeeded" ); let next_reprov_state = HostReprovisionState::ResetForNewFirmware { final_version: final_version.to_string(), @@ -8279,9 +8364,9 @@ impl HostUpgradeState { result.error.clone() }; tracing::warn!( - "Scout firmware upgrade failed for {}: {}", - state.host_snapshot.id, - reason + machine_id = %state.host_snapshot.id, + reason = %reason, + "Scout firmware upgrade failed" ); Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::FailedFirmwareUpgrade { @@ -8294,9 +8379,9 @@ impl HostUpgradeState { } } else if Utc::now() > *deadline { tracing::warn!( - "Scout firmware upgrade timed out for {} (deadline {})", - state.host_snapshot.id, - deadline, + machine_id = %state.host_snapshot.id, + %deadline, + "Scout firmware upgrade timed out", ); Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::FailedFirmwareUpgrade { @@ -8525,8 +8610,8 @@ impl HostUpgradeState { // For now, only gb200s need manual upgrades. if requires_manual_firmware_upgrade(state, &ctx.services.site_config.firmware_global) { tracing::info!( - "Machine {} (GB200) requires manual firmware upgrade, transitioning to WaitingForManualUpgrade", - machine_id + %machine_id, + "Machine (GB200) requires manual firmware upgrade, transitioning to WaitingForManualUpgrade" ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::WaitingForManualUpgrade { @@ -8558,7 +8643,7 @@ impl HostUpgradeState { else { // find_explored_refreshed_endpoint's behavior is to return None to indicate we're waiting for an update, not to indicate there isn't anything. - tracing::debug!("Managed host {machine_id} waiting for site explorer to revisit"); + tracing::debug!(%machine_id, "Managed host waiting for site explorer to revisit"); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { firmware_type: current_firmware_type, @@ -8672,11 +8757,12 @@ impl HostUpgradeState { .await; } - tracing::info!(%machine_id, - "Installing {:?} (number #{}) on {}", - to_install, + tracing::info!( + %machine_id, + firmware = ?to_install, firmware_number, - explored_endpoint.address + address = %explored_endpoint.address, + "Installing firmware" ); if !repeat && to_install.pre_update_resets { @@ -8717,7 +8803,11 @@ impl HostUpgradeState { // Note that this is different from the place where we do something similar false } else { - tracing::warn!("Could not get lockdown status for {machine_id}: {e}",); + tracing::warn!( + %machine_id, + error = %e, + "Could not get lockdown status" + ); return Ok(StateHandlerOutcome::do_nothing()); } } @@ -8729,7 +8819,7 @@ impl HostUpgradeState { .lockdown(libredfish::EnabledDisabled::Enabled) .await { - tracing::error!("Could not set lockdown for {machine_id}: {e}"); + tracing::error!(%machine_id, error = %e, "Could not set lockdown"); return Ok(StateHandlerOutcome::do_nothing()); } match scenario { @@ -8827,7 +8917,10 @@ impl HostUpgradeState { Ok(cmd) => cmd, Err(e) => { tracing::error!( - "Upgrade script {machine_id} {address} command creation failed: {e}" + %machine_id, + %address, + error = %e, + "Upgrade script command creation failed" ); upgrade_script_state.completed(machine_id.to_string(), false); return; @@ -8835,7 +8928,11 @@ impl HostUpgradeState { }; let Some(stdout) = cmd.stdout.take() else { - tracing::error!("Upgrade script {machine_id} {address} STDOUT creation failed"); + tracing::error!( + %machine_id, + %address, + "Upgrade script STDOUT creation failed" + ); let _ = cmd.kill().await; let _ = cmd.wait().await; upgrade_script_state.completed(machine_id.to_string(), false); @@ -8844,7 +8941,11 @@ impl HostUpgradeState { let stdout = tokio::io::BufReader::new(stdout); let Some(stderr) = cmd.stderr.take() else { - tracing::error!("Upgrade script {machine_id} {address} STDERR creation failed"); + tracing::error!( + %machine_id, + %address, + "Upgrade script STDERR creation failed" + ); let _ = cmd.kill().await; let _ = cmd.wait().await; upgrade_script_state.completed(machine_id.to_string(), false); @@ -8870,19 +8971,27 @@ impl HostUpgradeState { match cmd.wait().await { Err(e) => { tracing::info!( - "Upgrade script {machine_id} {address} FAILED: Wait failure {e}" + %machine_id, + %address, + error = %e, + "Upgrade script FAILED: Wait failure" ); upgrade_script_state.completed(machine_id.to_string(), false); } Ok(errorcode) => { if errorcode.success() { tracing::info!( - "Upgrade script {machine_id} {address} completed successfully" + %machine_id, + %address, + "Upgrade script completed successfully" ); upgrade_script_state.completed(machine_id.to_string(), true); } else { tracing::warn!( - "Upgrade script {machine_id} {address} FAILED: Exited with {errorcode}" + %machine_id, + %address, + exit_status = %errorcode, + "Upgrade script FAILED: Exited" ); upgrade_script_state.completed(machine_id.to_string(), false); } @@ -8905,9 +9014,9 @@ impl HostUpgradeState { if let Some(completed_at) = state.host_snapshot.manual_firmware_upgrade_completed { tracing::info!( - "Manual firmware upgrade completed for {} at {}, proceeding to automatic upgrades", - machine_id, - completed_at + %machine_id, + %completed_at, + "Manual firmware upgrade completed, proceeding to automatic upgrades" ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( @@ -8920,8 +9029,8 @@ impl HostUpgradeState { } tracing::debug!( - "Machine {} still waiting for manual firmware upgrade to be marked complete", - machine_id + %machine_id, + "Machine still waiting for manual firmware upgrade to be marked complete" ); Ok(StateHandlerOutcome::do_nothing()) } @@ -9068,9 +9177,9 @@ impl HostUpgradeState { ResolvedFirmwareArtifactSource::Remote { url, sha256 } => { if !self.downloader.available(&artifact.local_path, url, sha256) { tracing::debug!( - "{} is being downloaded from {}, update deferred", - artifact.local_path.display(), - url + artifact_path = %artifact.local_path.display(), + %url, + "firmware artifact is being downloaded, update deferred" ); return Ok(StateHandlerOutcome::do_nothing()); @@ -9083,7 +9192,11 @@ impl HostUpgradeState { artifact.local_path.display(), snapshot.id ); - tracing::warn!("{reason}"); + tracing::warn!( + artifact_path = %artifact.local_path.display(), + machine_id = %snapshot.id, + "firmware artifact is not present and no firmware artifact URL is configured", + ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::FailedFirmwareUpgrade { firmware_type: *component_type, @@ -9098,9 +9211,9 @@ impl HostUpgradeState { let Ok(_active) = self.upload_limiter.try_acquire() else { tracing::debug!( - "Deferring installation of {:?} on {}, too many uploads already active", - to_install, - snapshot.id, + firmware = ?to_install, + machine_id = %snapshot.id, + "Deferring installation, too many uploads already active", ); return Ok(StateHandlerOutcome::do_nothing()); }; @@ -9120,8 +9233,9 @@ impl HostUpgradeState { true } else { tracing::warn!( - "Could not get lockdown status for {}: {e}", - state.host_snapshot.id + machine_id = %state.host_snapshot.id, + error = %e, + "Could not get lockdown status" ); return Ok(StateHandlerOutcome::do_nothing()); } @@ -9136,7 +9250,7 @@ impl HostUpgradeState { .lockdown(libredfish::EnabledDisabled::Disabled) .await { - tracing::warn!("Could not set lockdown for {}: {e}", address.to_string()); + tracing::warn!(%address, error = %e, "Could not set lockdown"); return Ok(StateHandlerOutcome::do_nothing()); } if fw_info.model == "Dell" { @@ -9216,7 +9330,9 @@ impl HostUpgradeState { match self.async_firmware_uploader.upload_status(&machine_id) { None => { tracing::info!( - "Apparent restart before upload to {machine_id} {address} completion, returning to CheckingFirmware" + %machine_id, + %address, + "Apparent restart before upload completion, returning to CheckingFirmware" ); Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { @@ -9229,7 +9345,11 @@ impl HostUpgradeState { Some(upload_status) => { match upload_status { None => { - tracing::debug!("Upload to {machine_id} {address} not yet complete"); + tracing::debug!( + %machine_id, + %address, + "Upload not yet complete" + ); Ok(StateHandlerOutcome::do_nothing()) } Some(result) => { @@ -9237,7 +9357,10 @@ impl HostUpgradeState { UploadResult::Success { task_id } => { // We want to remove the machine ID from the hashmap, but do not do it here, because we may fail the commit. Run it in the next state handling. Failure case doesn't matter, it would have identical behavior. tracing::info!( - "Upload to {machine_id} {address} completed with task ID {task_id}" + %machine_id, + %address, + %task_id, + "Upload completed" ); // Upload complete and updated started, will monitor task in future iterations let reprovision_state = @@ -9333,10 +9456,10 @@ impl HostUpgradeState { | Some(TaskState::Running) | Some(TaskState::Pending) => { tracing::debug!( - "Upgrade task for {} not yet complete, current state {:?} message {:?}", - machine_id, - task_info.task_state, - task_info.messages, + %machine_id, + task_state = ?task_info.task_state, + messages = ?task_info.messages, + "Upgrade task not yet complete", ); Ok(StateHandlerOutcome::do_nothing()) } @@ -9378,10 +9501,11 @@ impl HostUpgradeState { .unwrap_or(false); if has_more_artifacts { tracing::debug!( - "Moving {:?} chain step {} on {} to CheckingFirmware", - selected_firmware, + %machine_id, + firmware = ?selected_firmware, firmware_number, - endpoint.address + address = %endpoint.address, + "Moving firmware chain step to CheckingFirmware" ); // There are more files to install. @@ -9403,8 +9527,8 @@ impl HostUpgradeState { } tracing::debug!( - "Saw completion of host firmware upgrade task for {}", - machine_id + %machine_id, + "Saw completion of host firmware upgrade task" ); let reprovision_state = HostReprovisionState::ResetForNewFirmware { @@ -9434,7 +9558,11 @@ impl HostUpgradeState { .last() .map_or("".to_string(), |m| m.message.clone()) ); - tracing::warn!(msg); + tracing::warn!( + %machine_id, + reason = %msg, + "Firmware upgrade task failed", + ); // We need site explorer to requery the version, just in case it actually did get done let mut txn = ctx.services.db_pool.begin().await?; @@ -9458,7 +9586,11 @@ impl HostUpgradeState { "Unrecognized task state for {}: {:?}", machine_id, task_info.task_state ); - tracing::warn!(msg); + tracing::warn!( + %machine_id, + reason = %msg, + "Unrecognized firmware upgrade task state", + ); let reprovision_state = HostReprovisionState::FailedFirmwareUpgrade { firmware_type: *firmware_type, @@ -9489,8 +9621,8 @@ impl HostUpgradeState { && current_version == final_version { tracing::info!( - "Marking completion of Redfish task of firmware upgrade for {} with missing task", - &endpoint.address + address = %endpoint.address, + "Marking completion of Redfish task of firmware upgrade with missing task" ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( @@ -9507,9 +9639,10 @@ impl HostUpgradeState { && Utc::now().signed_duration_since(started_waiting) > chrono::TimeDelta::minutes(15) { - tracing::info!(%machine_id, - "Timed out with missing Redfish task for firmware upgrade for {}, returning to CheckingFirmware", - &endpoint.address + tracing::info!( + %machine_id, + address = %endpoint.address, + "Timed out with missing Redfish task for firmware upgrade, returning to CheckingFirmware" ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { @@ -9569,7 +9702,7 @@ impl HostUpgradeState { }; let Some(endpoint) = find_explored_refreshed_endpoint(state, machine_id, ctx).await? else { - tracing::debug!("Waiting for site explorer to revisit {machine_id}"); + tracing::debug!(%machine_id, "Waiting for site explorer to revisit"); return Ok(StateHandlerOutcome::do_nothing()); }; @@ -9578,8 +9711,9 @@ impl HostUpgradeState { && *delay_until > chrono::Utc::now().timestamp() { tracing::info!( - "Waiting after {last_power_drain_operation:?} of {}", - &endpoint.address + last_power_drain_operation = ?last_power_drain_operation, + address = %endpoint.address, + "Waiting after power drain operation" ); return Ok(StateHandlerOutcome::do_nothing()); } @@ -9588,13 +9722,13 @@ impl HostUpgradeState { None | Some(PowerDrainState::On) => { // The 1000 is for unit tests; values above this will skip delays. if *power_drains_needed == 0 || *power_drains_needed == 1000 { - tracing::info!("Power drains for {} done", &endpoint.address); + tracing::info!(address = %endpoint.address, "Power drains done"); // This path, and only this path of the match, exits the match and lets us proceed. All others should return after updating state. } else { tracing::info!( - "Upgrade task has completed for {} but needs {} power drain(s), initiating one", - &endpoint.address, - power_drains_needed + address = %endpoint.address, + power_drains_needed = *power_drains_needed, + "Upgrade task has completed but needs power drain(s), initiating one" ); handler_host_power_control(state, ctx, SystemPowerControl::ForceOff) .await?; @@ -9617,7 +9751,7 @@ impl HostUpgradeState { } } Some(PowerDrainState::Off) => { - tracing::info!("Doing powercycle now for {}", &endpoint.address); + tracing::info!(address = %endpoint.address, "Doing powercycle now"); handler_host_power_control(state, ctx, SystemPowerControl::ACPowercycle) .await?; @@ -9637,7 +9771,7 @@ impl HostUpgradeState { ))); } Some(PowerDrainState::Powercycle) => { - tracing::info!("Turning back on {}", &endpoint.address); + tracing::info!(address = %endpoint.address, "Turning back on"); handler_host_power_control(state, ctx, SystemPowerControl::On).await?; let delay = if *power_drains_needed < 1000 { 5 } else { 0 }; @@ -9658,8 +9792,8 @@ impl HostUpgradeState { }; } else if firmware_type.is_uefi() { tracing::debug!( - "Upgrade task has completed for {} but needs reboot, initiating one", - &endpoint.address + address = %endpoint.address, + "Upgrade task has completed but needs reboot, initiating one" ); handler_host_power_control(state, ctx, SystemPowerControl::ForceRestart).await?; @@ -9674,8 +9808,8 @@ impl HostUpgradeState { .is_dell() { tracing::debug!( - "Upgrade task has completed for {} but needs BMC reboot, initiating one", - &endpoint.address + address = %endpoint.address, + "Upgrade task has completed but needs BMC reboot, initiating one" ); let redfish_client = ctx .services @@ -9683,7 +9817,7 @@ impl HostUpgradeState { .await?; if let Err(e) = redfish_client.bmc_reset().await { - tracing::warn!("Failed to reboot {}: {e}", &endpoint.address); + tracing::warn!(address = %endpoint.address, error = %e, "Failed to reboot"); return Ok(StateHandlerOutcome::do_nothing()); } } @@ -9701,12 +9835,20 @@ impl HostUpgradeState { // We previously possibly tried to use ACPowerycle here, however that requires enough time for the BMC to come back. We use // the power_drains_needed setting instead for that which is already aware of how to keep track of that sort of thing. if let Err(e) = redfish_client.power(SystemPowerControl::ForceOff).await { - tracing::error!("Failed to power off {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power off" + ); return Ok(StateHandlerOutcome::do_nothing()); } tokio::time::sleep(self.hgx_bmc_gpu_reboot_delay).await; if let Err(e) = redfish_client.power(SystemPowerControl::On).await { - tracing::error!("Failed to power on {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power on" + ); return Ok(StateHandlerOutcome::do_nothing()); } // Okay to proceed @@ -9757,13 +9899,13 @@ impl HostUpgradeState { }; let Some(endpoint) = find_explored_refreshed_endpoint(state, machine_id, ctx).await? else { - tracing::debug!("Waiting for site explorer to revisit {machine_id}"); + tracing::debug!(%machine_id, "Waiting for site explorer to revisit"); return Ok(StateHandlerOutcome::do_nothing()); }; let fw_config_snapshot = self.host_firmware_config_snapshot(ctx).await?; let Some(fw_info) = fw_config_snapshot.find_fw_info_for_host(&endpoint) else { - tracing::error!("Could no longer find firmware info for {machine_id}"); + tracing::error!(%machine_id, "Could no longer find firmware info"); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { firmware_type: Some(*firmware_type), @@ -9775,7 +9917,7 @@ impl HostUpgradeState { let current_versions = endpoint.find_all_versions(&fw_info, *firmware_type); if current_versions.is_empty() { - tracing::error!("Could no longer find current versions for {machine_id}"); + tracing::error!(%machine_id, "Could no longer find current versions"); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { firmware_type: Some(*firmware_type), @@ -9788,15 +9930,16 @@ impl HostUpgradeState { let versions_match_final_version = current_versions.iter().all(|v| *v == final_version); if !versions_match_final_version { tracing::warn!( - "{}: Not all firmware versions match. Expected: {final_version}, Found: {:?}", - endpoint.address, - current_versions + address = %endpoint.address, + %final_version, + current_versions = ?current_versions, + "Not all firmware versions match" ); } if versions_match_final_version { // Done waiting, go back to overall checking of version`2s - tracing::debug!("Done waiting for {machine_id} to reach version"); + tracing::debug!(%machine_id, "Done waiting to reach version"); Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::CheckingFirmwareRepeatV2 { firmware_type: Some(*firmware_type), @@ -9813,7 +9956,14 @@ impl HostUpgradeState { let reason = format!( "Firmware version did not converge after completed update for {firmware_type}: expected {final_version}, found {current_versions:?} after {reset_retry_count} reset retries" ); - tracing::warn!(%machine_id, "{reason}"); + tracing::warn!( + %machine_id, + firmware_type = %firmware_type, + expected_version = %final_version, + current_versions = ?current_versions, + reset_retry_count = *reset_retry_count, + "Firmware version did not converge after completed update", + ); return Ok(StateHandlerOutcome::transition(scenario.actual_new_state( HostReprovisionState::FailedFirmwareUpgrade { firmware_type: *firmware_type, @@ -9825,9 +9975,9 @@ impl HostUpgradeState { } tracing::info!( - "Upgrade for {} {:?} has taken more than 30 minutes to report new version; resetting again.", - &endpoint.address, - firmware_type + address = %endpoint.address, + firmware_type = ?firmware_type, + "Upgrade has taken more than 30 minutes to report new version; resetting again." ); let details = &HostReprovisionState::ResetForNewFirmware { final_version: final_version.to_string(), @@ -9843,7 +9993,11 @@ impl HostUpgradeState { .await; } tracing::info!( - "Waiting for {machine_id} {firmware_type:?} to reach version {final_version} currently {current_versions:?}" + %machine_id, + firmware_type = ?firmware_type, + %final_version, + current_versions = ?current_versions, + "Waiting to reach firmware version" ); let mut txn = ctx.services.db_pool.begin().await?; @@ -9922,7 +10076,9 @@ impl AsyncFirmwareUploader { // // Log it so we can see what's going on in case there's problems. tracing::info!( - "Uploading conflict for {id} {address}; our upload should still be in progress." + machine_id = %id, + %address, + "Uploading conflict; our upload should still be in progress." ); return; } @@ -9948,7 +10104,12 @@ impl AsyncFirmwareUploader { hashmap.insert(id, Some(UploadResult::Success { task_id })); } Err(e) => { - tracing::warn!("Failed uploading firmware to {id} {address}: {e}"); + tracing::warn!( + machine_id = %id, + %address, + error = %e, + "Failed uploading firmware" + ); let mut hashmap = active_uploads.lock().expect("lock poisoned"); hashmap.insert(id, Some(UploadResult::Failure)); } @@ -10285,9 +10446,9 @@ async fn wait_for_boss_controller_job_to_scheduled( ), libredfish::JobState::Completed => { tracing::warn!( - "CreateBossVolume: job {} for {} completed before being scheduled, skipping reboot", - job_id, - mh_snapshot.host_snapshot.id, + %job_id, + machine_id = %mh_snapshot.host_snapshot.id, + "CreateBossVolume: job completed before being scheduled, skipping reboot", ); waiting_for_cleanup_state( @@ -10630,7 +10791,11 @@ async fn restart_dpu( .map_err(|e| { // We use a Dell to mock our BMC responses in the integration tests. UefiHttp boot is not implemented // for Dells, so this call is failing in our tests. Regardless, it is ok to not make this call blocking. - tracing::error!(%e, "Failed to configure DPU {} to boot once", machine.id); + tracing::error!( + machine_id = %machine.id, + error = %e, + "Failed to configure DPU to boot once" + ); }); } @@ -10851,7 +11016,8 @@ fn handle_no_dpu_error( match (result, expected_dpu_count) { (Err(RedfishError::NoDpu), 0) => { tracing::info!( - "redfish {operation} failed with NoDpu on a zero-DPU host; treating as Ok" + operation, + "redfish operation failed with NoDpu on a zero-DPU host; treating as Ok" ); Ok(None) } @@ -11066,7 +11232,11 @@ async fn handle_instance_host_platform_config( } Err(e) => { // TODO: Dell's return a generic error if in lockdown which needs to be changed in Redfish SDK - tracing::warn!("Failed to AC Powercycle host, skipping to power on: {e}"); + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "Failed to AC Powercycle host, skipping to power on" + ); } }; } @@ -11523,9 +11693,9 @@ async fn set_host_boot_order( // it hasn't been fixed/addressed yet, and it appears to be logged all over // the place now. tracing::warn!( - "redfish set_boot_order_dpu_first failed for {}, potentially due to known race condition between UEFI POST and BMC. triggering force-restart if needed. err: {}", - mh_snapshot.host_snapshot.id, - e + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "redfish set_boot_order_dpu_first failed, potentially due to known race condition between UEFI POST and BMC. triggering force-restart if needed" ); let reboot_status = if mh_snapshot.host_snapshot.last_reboot_requested.is_none() @@ -11703,11 +11873,11 @@ async fn set_host_boot_order( } tracing::warn!( - "SetBootOrder: job {} lookup failed for {} after {} minutes, transitioning to HandleJobFailure: {}", - job_id, - mh_snapshot.host_snapshot.id, + %job_id, + machine_id = %mh_snapshot.host_snapshot.id, minutes_since_state_change, - e + error = %e, + "SetBootOrder: job lookup failed, transitioning to HandleJobFailure" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11727,9 +11897,10 @@ async fn set_host_boot_order( } _ if job_state.is_error_state() => { tracing::warn!( - "SetBootOrder: job {} failed for {} with state {job_state:#?}, transitioning to HandleJobFailure", - job_id, - mh_snapshot.host_snapshot.id, + %job_id, + machine_id = %mh_snapshot.host_snapshot.id, + job_state = ?job_state, + "SetBootOrder: job failed, transitioning to HandleJobFailure", ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11785,9 +11956,9 @@ async fn set_host_boot_order( // Host is powered off, reset the BMC tracing::info!( - "HandleJobFailure: Resetting BMC for {} after failure: {}", - mh_snapshot.host_snapshot.id, - failure + machine_id = %mh_snapshot.host_snapshot.id, + failure = %failure, + "HandleJobFailure: Resetting BMC after failure" ); redfish_client @@ -11840,8 +12011,8 @@ async fn set_host_boot_order( // Host is powered on, transition to CheckBootOrder to verify and retry tracing::info!( - "HandleJobFailure: BMC reset complete and host powered on for {}, transitioning to CheckBootOrder", - mh_snapshot.host_snapshot.id, + machine_id = %mh_snapshot.host_snapshot.id, + "HandleJobFailure: BMC reset complete and host powered on, transitioning to CheckBootOrder", ); Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11879,8 +12050,8 @@ async fn set_host_boot_order( if boot_order_configured { tracing::info!( - "Boot order verified for {} - the host has its boot order configured properly", - mh_snapshot.host_snapshot.id, + machine_id = %mh_snapshot.host_snapshot.id, + "Boot order verified - the host has its boot order configured properly", ); return Ok(SetBootOrderOutcome::Done); } @@ -11890,10 +12061,10 @@ async fn set_host_boot_order( mh_snapshot.host_snapshot.state.version.since_state_change(); tracing::warn!( - "Boot order check failed for {} - the host does not have its boot order configured properly after SetBootOrder (retry_count: {}, time_in_state: {} minutes)", - mh_snapshot.host_snapshot.id, + machine_id = %mh_snapshot.host_snapshot.id, retry_count, - time_since_state_change.num_minutes() + time_in_state_minutes = time_since_state_change.num_minutes(), + "Boot order check failed - the host does not have its boot order configured properly after SetBootOrder" ); // If we've been stuck for 30+ minutes and haven't exhausted retries, retry SetBootOrder @@ -11901,11 +12072,11 @@ async fn set_host_boot_order( && retry_count < MAX_BOOT_ORDER_RETRIES { tracing::info!( - "Boot order check timed out for {} after {} minutes, retrying SetBootOrder (retry {} of {})", - mh_snapshot.host_snapshot.id, - time_since_state_change.num_minutes(), - retry_count + 1, - MAX_BOOT_ORDER_RETRIES + machine_id = %mh_snapshot.host_snapshot.id, + time_in_state_minutes = time_since_state_change.num_minutes(), + retry = retry_count + 1, + max_retries = MAX_BOOT_ORDER_RETRIES, + "Boot order check timed out, retrying SetBootOrder" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { diff --git a/crates/machine-controller/src/handler/attestation.rs b/crates/machine-controller/src/handler/attestation.rs index 08dcebcfb1..ac93052272 100644 --- a/crates/machine-controller/src/handler/attestation.rs +++ b/crates/machine-controller/src/handler/attestation.rs @@ -81,8 +81,8 @@ pub async fn trigger_attestation( Some(product_name) => product_name, None => { tracing::info!( - "ServiceRoot.Product is None, not scheduling SPDM attestation for machine: {}", - machine_id + %machine_id, + "ServiceRoot.Product is None; not scheduling SPDM attestation" ); return Ok(0); } @@ -90,9 +90,9 @@ pub async fn trigger_attestation( if !is_supported_product(&product) { tracing::info!( - "ServiceRoot.Product - {} - is not supported, not scheduling SPDM attestation for machine: {}", - product, - machine_id + %machine_id, + %product, + "ServiceRoot.Product is not supported; not scheduling SPDM attestation" ); return Ok(0); } @@ -143,9 +143,9 @@ pub async fn trigger_attestation( txn.commit().await?; tracing::info!( - "SPDM attestation commenced for machine {}, scheduled {} SPDM device attestations", - machine_id, - records_inserted + %machine_id, + records_inserted, + "SPDM attestation commenced; scheduled SPDM device attestations" ); Ok(records_inserted) diff --git a/crates/machine-controller/src/handler/bios_config.rs b/crates/machine-controller/src/handler/bios_config.rs index b94aaa5992..abd856d317 100644 --- a/crates/machine-controller/src/handler/bios_config.rs +++ b/crates/machine-controller/src/handler/bios_config.rs @@ -104,9 +104,9 @@ pub(super) async fn configure_host_bios( { Err(e) => { tracing::warn!( - "redfish machine_setup failed for {}, potentially due to known race condition between UEFI POST and BMC. triggering force-restart if needed. err: {}", - mh_snapshot.host_snapshot.id, - e + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "redfish machine_setup failed, potentially due to known race condition between UEFI POST and BMC. triggering force-restart if needed" ); // if machine_setup failed, reboot to potentially work around diff --git a/crates/machine-controller/src/handler/dpf.rs b/crates/machine-controller/src/handler/dpf.rs index a26e8f7cde..17fa8cb3d3 100644 --- a/crates/machine-controller/src/handler/dpf.rs +++ b/crates/machine-controller/src/handler/dpf.rs @@ -465,7 +465,7 @@ async fn handle_dpf_reprovisioning( return Ok(outcome.with_txn(txn)); } - tracing::info!("DPF initiate reprovision of DPU {}", dpu_snapshot.id); + tracing::info!(machine_id = %dpu_snapshot.id, "DPF initiate reprovision of DPU"); dpf_sdk .reprovision_dpu(&dpf_id(dpu_snapshot)?, &node_name) .await diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 7950f844cd..d45859a147 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -137,12 +137,12 @@ pub(crate) async fn handle_machine_validation_state( is_enabled, } => { tracing::trace!( - "context = {} id = {} completed = {} total = {}, is_enabled = {} ", - context, - id, - completed, - total, - is_enabled, + machine_validation_context = %context, + machine_validation_id = %id, + completed_count = completed, + total_count = total, + validation_enabled = is_enabled, + "machine validation progress", ); if !rebooted(&mh_snapshot.host_snapshot) { // Ensure the boot config is still what it should be while @@ -202,8 +202,8 @@ pub(crate) async fn handle_machine_validation_state( if machine_validation_completed(&mh_snapshot.host_snapshot) { if mh_snapshot.host_snapshot.failure_details.cause == FailureCause::NoError { tracing::info!( - "{} machine validation completed", - mh_snapshot.host_snapshot.id + machine_id = %mh_snapshot.host_snapshot.id, + "machine validation completed" ); let machine_validation = db::machine_validation::find_by_id(&mut ctx.services.db_reader, id) @@ -227,7 +227,10 @@ pub(crate) async fn handle_machine_validation_state( }, )); } else { - tracing::info!("{} machine validation failed", mh_snapshot.host_snapshot.id); + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "machine validation failed" + ); return Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { details: mh_snapshot.host_snapshot.failure_details.clone(), machine_id: mh_snapshot.host_snapshot.id, diff --git a/crates/machine-controller/src/handler/power.rs b/crates/machine-controller/src/handler/power.rs index d66493f240..4fd290dde2 100644 --- a/crates/machine-controller/src/handler/power.rs +++ b/crates/machine-controller/src/handler/power.rs @@ -62,8 +62,8 @@ pub async fn handle_power( } } else { tracing::warn!( - "Power options are not available for host: {}", - mh_snapshot.host_snapshot.id + machine_id = %mh_snapshot.host_snapshot.id, + "Power options are not available for host" ); Ok(PowerHandlingOutcome::new(None, true, None)) } @@ -115,7 +115,8 @@ pub async fn handle_power_desired_on( } UsablePowerState::NotUsable(s) => { tracing::warn!( - "Not usable power state {s}. Since desired state is On, continuing state machine. Will check in next cycle." + power_state = %s, + "Not usable power state. Since desired state is On, continuing state machine. Will check in next cycle." ); return Ok(PowerHandlingOutcome::new( Some(updated_power_options), diff --git a/crates/machine-controller/src/handler/sku.rs b/crates/machine-controller/src/handler/sku.rs index 64aa02ef6f..a21873985b 100644 --- a/crates/machine-controller/src/handler/sku.rs +++ b/crates/machine-controller/src/handler/sku.rs @@ -104,15 +104,18 @@ async fn generate_missing_sku_for_machine( } let Some(sku_id) = mh_snapshot.host_snapshot.hw_sku.as_ref() else { tracing::debug!( - "No SKU assigned to machine {}", - mh_snapshot.host_snapshot.id + machine_id = %mh_snapshot.host_snapshot.id, + "No SKU assigned" ); return false; }; // its unlikely we got here without a bmc mac let Some(bmc_mac_address) = mh_snapshot.host_snapshot.bmc_info.mac else { - tracing::debug!("No bmc mac for machine {}", mh_snapshot.host_snapshot.id); + tracing::debug!( + machine_id = %mh_snapshot.host_snapshot.id, + "No BMC MAC address configured" + ); return false; }; @@ -123,7 +126,7 @@ async fn generate_missing_sku_for_machine( .flatten() .is_none_or(|em| em.data.sku_id.as_ref().is_none_or(|id| id != sku_id)) { - tracing::debug!("No expected machine for bmc {}", bmc_mac_address); + tracing::debug!(%bmc_mac_address, "No expected machine for bmc"); return false; } @@ -552,7 +555,11 @@ pub(crate) async fn handle_bom_validation_state( let diffs = diff_skus(&actual_sku, &expected_sku); for diff in &diffs { - tracing::error!(machine_id=%mh_snapshot.host_snapshot.id, "{}", diff); + tracing::error!( + machine_id = %mh_snapshot.host_snapshot.id, + difference = %diff, + "SKU validation mismatch", + ); } if diffs.is_empty() { diff --git a/crates/machine-validation/src/machine_validation.rs b/crates/machine-validation/src/machine_validation.rs index 67fa8784de..13cf69989c 100644 --- a/crates/machine-validation/src/machine_validation.rs +++ b/crates/machine-validation/src/machine_validation.rs @@ -107,8 +107,14 @@ impl MachineValidation { .get_external_config(file_name.clone(), Some("container_auth".to_string())) .await { - Ok(()) => trace!("Fetched {} config", file_name), - Err(e) => trace!("Error - {}", e.to_string()), + Ok(()) => trace!( + external_config_file = %file_name, + "Fetched external machine validation config", + ), + Err(e) => trace!( + error = %e, + "Failed to fetch container authentication config", + ), } Ok(()) } @@ -117,7 +123,10 @@ impl MachineValidation { external_config_file: String, external_config_name: Option, ) -> Result<(), MachineValidationError> { - tracing::info!("{}", external_config_file); + tracing::info!( + external_config_file = %external_config_file, + "Fetching external machine validation config", + ); let name = if let Some(name) = external_config_name { name @@ -172,7 +181,10 @@ impl MachineValidation { self, data: Option, ) -> Result<(), MachineValidationError> { - tracing::info!("{}", data.clone().unwrap().name); + tracing::info!( + validation_name = %data.as_ref().expect("validation result").name, + "Persisting machine validation result", + ); let mut client = self.create_forge_client().await?; let request = tonic::Request::new(rpc::forge::MachineValidationResultPostRequest { result: data }); @@ -250,7 +262,10 @@ impl MachineValidation { self, test_request: rpc::forge::MachineValidationTestsGetRequest, ) -> Result, MachineValidationError> { - tracing::info!("{:?}", test_request); + tracing::info!( + request = ?test_request, + "Fetching machine validation tests", + ); let mut client = self.create_forge_client().await?; let request = tonic::Request::new(test_request); let response = client @@ -296,9 +311,15 @@ impl MachineValidation { for image_name in list.images { match Self::import_container(&image_name, MACHINE_VALIDATION_RUNNER_TAG).await { Ok(data) => { - trace!("Import successful '{}'", data) + trace!( + image_reference = %data, + "Imported machine validation container image", + ) } - Err(e) => error!("Failed to import '{}'", e.to_string()), + Err(e) => error!( + error = %e, + "Failed to import machine validation container image", + ), }; } Ok(()) @@ -316,7 +337,10 @@ impl MachineValidation { MachineValidationManager::download_file(&url, MACHINE_VALIDATION_IMAGE_FILE).await?; let command_string = format!(" ctr images import {MACHINE_VALIDATION_IMAGE_FILE}"); - info!("Executing command '{}'", command_string); + info!( + command = %command_string, + "Executing machine validation command", + ); TokioCmd::new("sh") .args(vec!["-c".to_string(), command_string]) .timeout(DEFAULT_TIMEOUT) @@ -338,8 +362,15 @@ impl MachineValidation { .output_with_timeout() .await { - Ok(result) => info!("pulled: {}", result.stdout), - Err(e) => error!("Failed to image pull{} '{}'", image_name, e), + Ok(result) => info!( + stdout = %result.stdout, + "Pulled machine validation container image", + ), + Err(e) => error!( + image_name = %image_name, + error = %e, + "Failed to pull machine validation container image", + ), } } async fn execute_machinevalidation_command( @@ -401,7 +432,10 @@ impl MachineValidation { .get_external_config(file_name.clone(), None) .await { - Ok(()) => trace!("Fetched {} config", file_name), + Ok(()) => trace!( + external_config_file = %file_name, + "Fetched external machine validation config", + ), Err(e) => { mc_result.start_time = Some(Utc::now().into()); mc_result.end_time = Some(Utc::now().into()); @@ -466,7 +500,10 @@ impl MachineValidation { command_string ); }; - info!("Executing command '{}'", command_string); + info!( + command = %command_string, + "Executing machine validation command", + ); let _ = std::fs::remove_file("/tmp/forge_env_variables"); match File::create("/tmp/forge_env_variables") { @@ -559,7 +596,10 @@ impl MachineValidation { self, data: rpc::forge::MachineValidationRunRequest, ) -> Result<(), MachineValidationError> { - tracing::info!("{:?}", data.clone()); + tracing::info!( + request = ?data, + "Updating machine validation run", + ); let mut client = self.create_forge_client().await?; let request = tonic::Request::new(data); let _response = client @@ -585,7 +625,7 @@ impl MachineValidation { self.clone().get_container_auth_config().await?; match Self::get_container_images().await { Ok(_) => info!("Successfully fetched container images"), - Err(e) => error!("{}", e.to_string()), + Err(e) => error!(error = %e, "Failed to fetch container images"), } if execute_tests_sequentially { for test in tests { @@ -612,8 +652,14 @@ impl MachineValidation { heartbeat.stop().await; } match persist_result { - Ok(_) => info!("Successfully sent to api server - {}", test.name), - Err(e) => error!("{}", e.to_string()), + Ok(_) => info!( + test_name = %test.name, + "Sent machine validation result to API server", + ), + Err(e) => error!( + error = %e, + "Failed to send machine validation result to API server", + ), } } } else { diff --git a/crates/metrics-endpoint/src/lib.rs b/crates/metrics-endpoint/src/lib.rs index 45d31f57ba..858da3cd83 100644 --- a/crates/metrics-endpoint/src/lib.rs +++ b/crates/metrics-endpoint/src/lib.rs @@ -184,7 +184,7 @@ pub async fn run_metrics_endpoint_with_listener( let stream = match result { Ok((stream, _addr)) => stream, Err(e) => { - tracing::error!("error accepting TCP connection: {e}"); + tracing::error!(error = %e, "error accepting TCP connection"); continue; } }; diff --git a/crates/metrics-utils/src/lib.rs b/crates/metrics-utils/src/lib.rs index 7582f681bc..ee9386f5af 100644 --- a/crates/metrics-utils/src/lib.rs +++ b/crates/metrics-utils/src/lib.rs @@ -69,7 +69,11 @@ pub fn new_view( .with_unit(i.unit().to_owned()) .build() .inspect_err(|e| { - tracing::error!("BUG: Could not build stream from view {}: {}", i.name(), e) + tracing::error!( + view = i.name(), + error = %e, + "BUG: Could not build stream from view" + ) }) .ok() })) diff --git a/crates/mqttea-example/src/main.rs b/crates/mqttea-example/src/main.rs index fe78f7989d..8937a53dc8 100644 --- a/crates/mqttea-example/src/main.rs +++ b/crates/mqttea-example/src/main.rs @@ -164,8 +164,11 @@ async fn main() -> Result<(), Box> { client .on_message(|client, message: HelloWorld, topic| async move { info!( - "HelloWorld on {}: '{}' from device {} (timestamp: {})", - topic, message.message, message.device_id, message.timestamp + topic = %topic, + payload = %message.message, + device_id = %message.device_id, + timestamp = message.timestamp, + "received HelloWorld message" ); let response_message: StringMessage = format!("i got your message that said: {}", message.message).into(); @@ -179,7 +182,10 @@ async fn main() -> Result<(), Box> { ) .await { - warn!("failed to publish response message: {:?}", publish_err); + warn!( + error = ?publish_err, + "failed to publish response message" + ); } }) .await; @@ -188,8 +194,10 @@ async fn main() -> Result<(), Box> { client .on_message(|_client, message: CatStatus, topic| async move { info!( - "CatStatus on {}: {} is feeling {}", - topic, message.name, message.mood, + topic = %topic, + cat_name = %message.name, + mood = %message.mood, + "received CatStatus message" ); }) .await; @@ -199,11 +207,15 @@ async fn main() -> Result<(), Box> { .on_message(|_client, message: RawMessage, topic| async move { println!("FYI: Received message on unmapped topic: '{topic}'"); match String::from_utf8(message.payload.clone()) { - Ok(text) => info!("RawMessage on {}: '{}'", topic, text), + Ok(text) => info!( + topic = %topic, + payload = %text, + "received raw text message" + ), Err(_) => info!( - "RawMessage on {}: {} bytes of binary data", - topic, - message.payload.len() + topic = %topic, + payload_bytes = message.payload.len(), + "received raw binary message" ), } }) diff --git a/crates/mqttea/src/client/core.rs b/crates/mqttea/src/client/core.rs index e204c07c87..f93799bec0 100644 --- a/crates/mqttea/src/client/core.rs +++ b/crates/mqttea/src/client/core.rs @@ -139,7 +139,7 @@ impl MqtteaClient { .as_ref() .and_then(|opts| opts.credentials_provider.clone()); - info!("Created MQTT client for {}:{}", broker_host, broker_port); + info!(%broker_host, broker_port, "Created MQTT client"); Ok(Arc::new(Self { client: Arc::new(client), @@ -178,8 +178,9 @@ impl MqtteaClient { for (topic, qos) in subs_snapshot { if let Err(e) = self.client.subscribe(topic.as_str(), qos).await { error!( - "Failed to re-subscribe to {} after fresh session: {:?}", - topic, e + %topic, + error = %e, + "Failed to re-subscribe after fresh session", ); } } @@ -273,8 +274,8 @@ impl MqtteaClient { } Err(mpsc::error::TrySendError::Full(_)) => { warn!( - "Message queue full, dropping message from topic: {}", - publish.topic + topic = %publish.topic, + "Message queue full, dropping message from topic", ); queue_stats_producer.increment_dropped(payload_size); tokio::time::sleep(backoff_strategy.next_delay()).await; @@ -292,13 +293,16 @@ impl MqtteaClient { } else { queue_stats_producer.increment_unmatched_topics(); if warn_on_unmatched_topic { - warn!("No registered pattern matched topic: {}", publish.topic); + warn!( + topic = %publish.topic, + "No registered pattern matched topic", + ); } } } } Err(e) => { - error!("MQTT event loop connection error: {:?}", e); + error!(error = %e, "MQTT event loop connection error"); connection_state.set_connected(false); queue_stats_producer.increment_event_loop_errors(); @@ -319,8 +323,8 @@ impl MqtteaClient { } Err(cred_err) => { error!( - "Failed to refresh credentials for reconnection: {:?}", - cred_err + error = %cred_err, + "Failed to refresh credentials for reconnection", ); } } @@ -352,14 +356,19 @@ impl MqtteaClient { .decrement_pending_increment_processed(payload_size); } Err(e) => { - error!("Handler error for {}: {}", msg.type_name, e); + error!( + message_type = %msg.type_name, + error = %e, + "Handler error", + ); queue_stats_processor.decrement_pending_increment_failed(payload_size); } } } else { warn!( - "No handler registered for message type '{}' on topic '{}'", - msg.type_name, msg.topic + message_type = %msg.type_name, + topic = %msg.topic, + "No handler registered for message type on topic", ); queue_stats_processor.decrement_pending_increment_failed(payload_size); } @@ -450,8 +459,8 @@ impl MqtteaClient { let mut handlers_guard = self.handlers.write().await; handlers_guard.insert(std::any::type_name::().to_string(), type_erased_handler); info!( - "Registered handler for message type: {}", - std::any::type_name::() + message_type = std::any::type_name::(), + "Registered handler for message type", ); } @@ -471,7 +480,7 @@ impl MqtteaClient { .await .map_err(MqtteaClientError::ConnectionError)?; - info!("Subscribed to topic: {} (QoS: {:?})", topic, qos); + info!(%topic, ?qos, "Subscribed to topic"); Ok(()) } @@ -520,7 +529,7 @@ impl MqtteaClient { match self.client.publish(topic, qos, retain, payload).await { Ok(_) => { self.publish_stats.increment_published(payload_size); - debug!("Published message to topic: {}", topic); + debug!(%topic, "Published message to topic"); Ok(()) } Err(e) => { @@ -709,8 +718,8 @@ impl SuperBasicBackoff { let delay = self.current; self.current = std::cmp::min(self.current * 2, self.max); warn!( - "Message event loop backoff updated: {}ms", - delay.as_millis() + delay_ms = delay.as_millis(), + "Message event loop backoff updated" ); delay } diff --git a/crates/mqttea/src/client/messages.rs b/crates/mqttea/src/client/messages.rs index 9078837921..9d2e5bd78c 100644 --- a/crates/mqttea/src/client/messages.rs +++ b/crates/mqttea/src/client/messages.rs @@ -58,7 +58,7 @@ impl ReceivedMessage { let payload = publish.payload.to_vec(); let payload_size = payload.len(); - debug!("Looking for pattern match for topic: {}", topic); + debug!(%topic, "Looking for pattern match for topic"); let registry_guard = registry.read().await; registry_guard .find_matching_type_for_topic(&topic) diff --git a/crates/mqttea/src/registry/core.rs b/crates/mqttea/src/registry/core.rs index 07c8ec7ca5..ecf52222e6 100644 --- a/crates/mqttea/src/registry/core.rs +++ b/crates/mqttea/src/registry/core.rs @@ -99,8 +99,9 @@ impl MqttRegistry { self.topic_patterns.push((regex, type_name.to_string())); debug!( - "Registered pattern '{}' for type '{}'", - regex_pattern, type_name + pattern = %regex_pattern, + message_type = %type_name, + "Registered MQTT topic pattern", ); } @@ -133,10 +134,10 @@ impl MqttRegistry { for (regex, type_name) in &self.topic_patterns { if regex.is_match(topic) { debug!( - "Topic '{}' matched pattern '{}' -> type '{}'", - topic, - regex.as_str(), - type_name + %topic, + pattern = %regex.as_str(), + message_type = %type_name, + "Topic matched registered pattern", ); return self .entries @@ -144,7 +145,7 @@ impl MqttRegistry { .map(|entry| &entry.message_type_info); } } - debug!("Topic '{}' did not match any registered patterns", topic); + debug!(%topic, "Topic did not match any registered patterns"); None } diff --git a/crates/nvlink-manager/src/lib.rs b/crates/nvlink-manager/src/lib.rs index 5e677c6bcd..adccf66664 100644 --- a/crates/nvlink-manager/src/lib.rs +++ b/crates/nvlink-manager/src/lib.rs @@ -478,7 +478,10 @@ impl PartitionProcessingContext { } true } else { - tracing::error!("logical partition {} not found!!", logical_partition_id); + tracing::error!( + logical_partition_id = %logical_partition_id, + "Logical partition not found", + ); false } } @@ -541,9 +544,10 @@ impl PartitionProcessingContext { .collect(), None => { tracing::error!( - "NMX-C partition not found for machine {}, GPU index {}", - machine_id, - device_instance + machine_id = %machine_id, + device_instance, + partition_id = partition_nmx_c_id.partition_id, + "NMX-C partition not found", ); return None; } @@ -561,9 +565,10 @@ impl PartitionProcessingContext { .collect(), None => { tracing::error!( - "NMX-C partition not found for machine {}, GPU index {}", - machine_id, - device_instance + machine_id = %machine_id, + device_instance, + partition_id = partition_nmx_c_id.partition_id, + "NMX-C partition not found", ); return None; } @@ -602,9 +607,10 @@ impl PartitionProcessingContext { .collect(), None => { tracing::error!( - "NMX-C partition not found for machine {}, GPU index {}", - machine_id, - device_instance + machine_id = %machine_id, + device_instance, + partition_id = partition_nmx_c_id.partition_id, + "NMX-C partition not found", ); return None; } @@ -622,9 +628,10 @@ impl PartitionProcessingContext { .collect(), None => { tracing::error!( - "NMX-C partition not found for machine {}, GPU index {}", - machine_id, - device_instance + machine_id = %machine_id, + device_instance, + partition_id = partition_nmx_c_id.partition_id, + "NMX-C partition not found", ); return None; } @@ -1040,7 +1047,10 @@ impl NvlPartitionMonitor { } } Err(e) => { - tracing::warn!("NvlPartitionMonitor error: {}", e); + tracing::warn!( + error = %e, + "NVLink partition monitor error", + ); } } @@ -1094,7 +1104,8 @@ impl NvlPartitionMonitor { Ok(lock) => lock, Err(e) => { tracing::warn!( - "NvlPartitionMonitor failed to acquire work lock: Another instance of carbide running? {e}" + error = %e, + "NvlPartitionMonitor failed to acquire work lock: Another instance of carbide running?", ); return Ok(0); } @@ -1467,7 +1478,10 @@ impl NvlPartitionMonitor { let nmx_c_operations = partition_processing_context.nmx_c_operations; if !nmx_c_operations.is_empty() { - tracing::debug!("NMX-C operations: {:?}", nmx_c_operations); + tracing::debug!( + nmx_c_operations = ?nmx_c_operations, + "Starting NMX-C operations", + ); } // Execute any NMX-C operations and collect successful completions. @@ -1477,8 +1491,8 @@ impl NvlPartitionMonitor { if !completed_nmx_c_operations.is_empty() { tracing::debug!( - "Completed NMX-C operations: {:?}", - completed_nmx_c_operations + completed_nmx_c_operations = ?completed_nmx_c_operations, + "Completed NMX-C operations", ); } @@ -1558,7 +1572,10 @@ impl NvlPartitionMonitor { .get(&instance.machine_id) .cloned() else { - tracing::warn!("No nvlink_info found for machine {}", instance.machine_id); + tracing::warn!( + machine_id = %instance.machine_id, + "No NVLink info found", + ); machine_gpu_statuses.insert( instance.machine_id, MachineNvLinkStatusObservation { @@ -1617,8 +1634,8 @@ impl NvlPartitionMonitor { if db_logical_partition_id.is_none() { // How can this happen? tracing::error!( - "No logical partition ID associated with physical partition {:?}", - partition_id.to_string() + partition_id, + "No logical partition ID associated with physical partition", ); continue; } else if gpu_config.logical_partition_id @@ -1651,11 +1668,11 @@ impl NvlPartitionMonitor { if is_nmx_c_default_partition(&nmxc_partition) { if instance_gpu_config.is_some() { tracing::info!( - "Removing GPU {} in machine {} and instance {} from default partition {}", - nvlink_gpu.guid, - instance.machine_id, - instance.id, - partition_id + gpu_guid = nvlink_gpu.guid, + machine_id = %instance.machine_id, + instance_id = %instance.id, + partition_id, + "Removing GPU in machine and instance from default partition", ); gpu_action = GpuAction::RemoveFromUnknownPartition; gpu_ctx.partition_nmx_c_id = @@ -1668,9 +1685,9 @@ impl NvlPartitionMonitor { // Monitor does not know about this partition, so just remove the GPU. On the next iteration // the monitor will put the GPU in the correct partition (or leave it if the config says no partition) tracing::warn!( - "Removing GPU {} from unknown partition with NMX-C ID {}", - nvlink_gpu.guid, - partition_id + gpu_guid = nvlink_gpu.guid, + partition_id, + "Removing GPU from unknown partition with NMX-C ID", ); gpu_action = GpuAction::RemoveFromUnknownPartition; gpu_ctx.partition_nmx_c_id = @@ -1735,7 +1752,8 @@ impl NvlPartitionMonitor { tracing::error!( gpu_guid = %gpu_ctx.gpu_guid, machine_id = %instance.machine_id, - "Failed to handle GPU addition to existing partition: {e}" + error = %e, + "Failed to handle GPU addition to existing partition", ); } } else { @@ -1746,7 +1764,8 @@ impl NvlPartitionMonitor { tracing::error!( gpu_guid = %gpu_ctx.gpu_guid, machine_id = %instance.machine_id, - "Failed to handle GPU addition to new partition: {e}" + error = %e, + "Failed to handle GPU addition to new partition", ); } } @@ -1770,7 +1789,8 @@ impl NvlPartitionMonitor { tracing::error!( gpu_guid = %gpu_ctx.gpu_guid, machine_id = %instance.machine_id, - "Failed to handle GPU removal from partition: {e}" + error = %e, + "Failed to handle GPU removal from partition", ); } } @@ -1793,15 +1813,16 @@ impl NvlPartitionMonitor { tracing::error!( gpu_guid = %gpu_ctx.gpu_guid, machine_id = %instance.machine_id, - "Failed to handle GPU removal from unknown partition: {e}" + error = %e, + "Failed to handle GPU removal from unknown partition", ); } } else { tracing::error!( gpu_guid = %gpu_ctx.gpu_guid, machine_id = %instance.machine_id, - "No default partition found with NMX-C ID = {}", - gpu_ctx.partition_nmx_c_id.partition_id + partition_id = gpu_ctx.partition_nmx_c_id.partition_id, + "NMX-C partition not found for GPU removal", ); continue; } @@ -1811,7 +1832,10 @@ impl NvlPartitionMonitor { } } None => { - tracing::warn!("No nvlink_info found for machine {}", instance.machine_id); + tracing::warn!( + machine_id = %instance.machine_id, + "No NVLink info found", + ); } } // Now we've generated the operations, record an observation. @@ -2152,7 +2176,8 @@ impl NvlPartitionMonitor { Err(e) => { tracing::warn!( %logical_partition_id, - "Failed to retry create partition on NMX-C with multicast_groups_limit=0: {e}" + error = %e, + "Failed to retry create partition on NMX-C with multicast_groups_limit=0", ); false } @@ -2163,7 +2188,8 @@ impl NvlPartitionMonitor { tracing::warn!( %logical_partition_id, create_partition_request = ?request, - "Failed to issue create partition to NMX-C, continuing with other operations: {e}" + error = %e, + "Failed to issue create partition to NMX-C, continuing with other operations", ); false } @@ -2184,7 +2210,8 @@ impl NvlPartitionMonitor { tracing::warn!( %logical_partition_id, %nmx_c_partition_id, - "Failed to issue delete partition to NMX-C, continuing with other operations: {e}" + error = %e, + "Failed to issue delete partition to NMX-C, continuing with other operations", ); false } @@ -2223,7 +2250,8 @@ impl NvlPartitionMonitor { tracing::warn!( %logical_partition_id, %nmx_c_partition_id, - "Failed to get partition info from NMX-C before update: {e}" + error = %e, + "Failed to get partition info from NMX-C before update", ); false } @@ -2265,7 +2293,8 @@ impl NvlPartitionMonitor { tracing::warn!( %logical_partition_id, %nmx_c_partition_id, - "Failed to remove GPUs from partition on NMX-C: {e}" + error = %e, + "Failed to remove GPUs from partition on NMX-C", ); ok = false; } @@ -2287,7 +2316,8 @@ impl NvlPartitionMonitor { tracing::warn!( %logical_partition_id, %nmx_c_partition_id, - "Failed to add GPUs to partition on NMX-C: {e}" + error = %e, + "Failed to add GPUs to partition on NMX-C", ); ok = false; } @@ -2346,8 +2376,8 @@ impl NvlPartitionMonitor { Some(p) => p, None => { tracing::error!( - "NMX-C partition not found for name {}", - operation.name + operation_name = %operation.name, + "NMX-C partition not found", ); continue; } @@ -2358,15 +2388,15 @@ impl NvlPartitionMonitor { .map(|id| id.partition_id) else { tracing::error!( - "NMX-C partition ID not found for name {}", - operation.name + operation_name = %operation.name, + "NMX-C partition ID not found", ); continue; }; let Ok(nmx_c_partition_id) = i32::try_from(nmx_c_partition_id) else { tracing::error!( - "NMX-C partition ID does not fit in database column for name {}", - operation.name + operation_name = %operation.name, + "NMX-C partition ID does not fit in database column", ); continue; }; @@ -2374,7 +2404,7 @@ impl NvlPartitionMonitor { if operation.name.starts_with("tray_partition_") { tracing::debug!( logical_partition_id = %logical_partition_id, - name = %operation.name, + operation_name = %operation.name, "Skipping nvl_partition DB insert for tray partition" ); continue; diff --git a/crates/nvlink-manager/src/switch_cert_monitor.rs b/crates/nvlink-manager/src/switch_cert_monitor.rs index 02f6a4c8b9..9d14fb4df7 100644 --- a/crates/nvlink-manager/src/switch_cert_monitor.rs +++ b/crates/nvlink-manager/src/switch_cert_monitor.rs @@ -589,7 +589,10 @@ impl SwitchCertificateMonitor { loop { let tick = timer.tick(); if let Err(e) = self.run_single_iteration(&cancel_token).await { - tracing::warn!("SwitchCertificateMonitor error: {}", e); + tracing::warn!( + error = %e, + "Switch certificate monitor error", + ); } tokio::select! { @@ -647,7 +650,8 @@ impl SwitchCertificateMonitor { Ok(lock) => lock, Err(e) => { tracing::warn!( - "SwitchCertificateMonitor failed to acquire work lock: Another instance of carbide running? {e}" + error = %e, + "SwitchCertificateMonitor failed to acquire work lock: Another instance of carbide running?", ); return Ok(()); } diff --git a/crates/power-shelf-controller/src/configuring.rs b/crates/power-shelf-controller/src/configuring.rs index 51f6b14f66..3384f3ed2d 100644 --- a/crates/power-shelf-controller/src/configuring.rs +++ b/crates/power-shelf-controller/src/configuring.rs @@ -36,8 +36,8 @@ pub async fn handle_configuring( _ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, ) -> Result, StateHandlerError> { tracing::info!( - "Configuring PowerShelf {}, transitioning to Ready", - power_shelf_id + %power_shelf_id, + "Configuring PowerShelf; transitioning to Ready" ); Ok(StateHandlerOutcome::transition( PowerShelfControllerState::Ready, diff --git a/crates/power-shelf-controller/src/deleting.rs b/crates/power-shelf-controller/src/deleting.rs index fb7e0bd708..64fef1a2c1 100644 --- a/crates/power-shelf-controller/src/deleting.rs +++ b/crates/power-shelf-controller/src/deleting.rs @@ -36,7 +36,7 @@ pub async fn handle_deleting( _state: &mut PowerShelf, ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, ) -> Result, StateHandlerError> { - tracing::info!("Deleting PowerShelf {}", power_shelf_id); + tracing::info!(%power_shelf_id, "Deleting PowerShelf"); let mut txn = ctx.services.db_pool.begin().await?; db_power_shelf::final_delete(*power_shelf_id, &mut txn).await?; Ok(StateHandlerOutcome::deleted().with_txn(txn)) diff --git a/crates/power-shelf-controller/src/fetching_data.rs b/crates/power-shelf-controller/src/fetching_data.rs index 48339e2c1f..ccdfa8e889 100644 --- a/crates/power-shelf-controller/src/fetching_data.rs +++ b/crates/power-shelf-controller/src/fetching_data.rs @@ -36,8 +36,8 @@ pub async fn handle_fetching_data( _ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, ) -> Result, StateHandlerError> { tracing::info!( - "Fetching PowerShelf {} data, transitioning to Configuring", - power_shelf_id + %power_shelf_id, + "Fetching PowerShelf data; transitioning to Configuring" ); Ok(StateHandlerOutcome::transition( PowerShelfControllerState::Configuring, diff --git a/crates/power-shelf-controller/src/initializing.rs b/crates/power-shelf-controller/src/initializing.rs index 08dc0cc41f..81e25d2776 100644 --- a/crates/power-shelf-controller/src/initializing.rs +++ b/crates/power-shelf-controller/src/initializing.rs @@ -37,8 +37,8 @@ pub async fn handle_initializing( _ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, ) -> Result, StateHandlerError> { tracing::info!( - "PowerShelf {} initializing, transitioning to FetchingData", - power_shelf_id + %power_shelf_id, + "PowerShelf initializing; transitioning to FetchingData" ); Ok(StateHandlerOutcome::transition( PowerShelfControllerState::FetchingData, diff --git a/crates/preingestion-manager/src/bfb_rshim_copier.rs b/crates/preingestion-manager/src/bfb_rshim_copier.rs index 3bce6af314..c9f880a816 100644 --- a/crates/preingestion-manager/src/bfb_rshim_copier.rs +++ b/crates/preingestion-manager/src/bfb_rshim_copier.rs @@ -135,7 +135,7 @@ impl BfbRshimCopier { .is_rshim_enabled(bmc_ip_address, credentials.clone()) .await? { - tracing::warn!("RSHIM is not enabled on {bmc_ip_address}"); + tracing::warn!(%bmc_ip_address, "RSHIM is not enabled"); self.enable_rshim(bmc_ip_address, credentials.clone()) .await?; @@ -160,7 +160,10 @@ impl BfbRshimCopier { let _lock = self.bfb_file_lock.lock().await; if fs::metadata(UNIFIED_PREINGESTION_BFB_PATH).await.is_err() { - tracing::info!("Writing {UNIFIED_PREINGESTION_BFB_PATH}"); + tracing::info!( + path = UNIFIED_PREINGESTION_BFB_PATH, + "Writing unified preingestion BFB" + ); let bf_cfg_contents = format!( "BMC_USER=\"{username}\"\nBMC_PASSWORD=\"{password}\"\nBMC_REBOOT=\"yes\"\nCEC_REBOOT=\"yes\"\n" ); @@ -180,7 +183,7 @@ impl BfbRshimCopier { let mut buffer = vec![0; 1024 * 1024].into_boxed_slice(); // 1 MB buffer - tracing::info!("Writing BFB to {UNIFIED_PREINGESTION_BFB_PATH}"); + tracing::info!(path = UNIFIED_PREINGESTION_BFB_PATH, "Writing BFB payload"); loop { let n = preingestion_bfb.read(&mut buffer).await.map_err(|err| { BfbRshimCopyError::Other { @@ -201,7 +204,10 @@ impl BfbRshimCopier { })?; } - tracing::info!("Writing bf.cfg to {UNIFIED_PREINGESTION_BFB_PATH}"); + tracing::info!( + path = UNIFIED_PREINGESTION_BFB_PATH, + "Writing bf.cfg payload" + ); unified_bfb .write_all(bf_cfg_contents.as_bytes()) diff --git a/crates/preingestion-manager/src/lib.rs b/crates/preingestion-manager/src/lib.rs index a101c33f34..a206360be7 100644 --- a/crates/preingestion-manager/src/lib.rs +++ b/crates/preingestion-manager/src/lib.rs @@ -177,7 +177,7 @@ impl PreingestionManager { let res = self.run_single_iteration().await; if let Err(e) = &res { - tracing::warn!("Preingestion manager error: {}", e); + tracing::warn!(error = %e, "Preingestion manager error"); } // If we were able to go through everything (few or no uploads), or if we ran into a database error, @@ -208,9 +208,9 @@ impl PreingestionManager { Err(e) => { // Unable to obtain the lock, we'll sleep and try again later. There must be another instance of carbide-api running. tracing::warn!( - "Unable to acquire lock for {}. Will try again on next iteration: {}", - Self::ITERATION_WORK_KEY, - e + work_key = Self::ITERATION_WORK_KEY, + error = %e, + "Unable to acquire lock; will try again on next iteration" ); return Ok(()); } @@ -223,15 +223,18 @@ impl PreingestionManager { if !items.is_empty() && items.len() < 3 { // Show states if a modest amount, just count otherwise tracing::debug!( - "PreingestionManager: Working on {} items {:?}", - items.len(), - items + item_count = items.len(), + items = ?items .iter() .map(|x| format!("({}: {:?})", x.address, x.preingestion_state)) - .collect::>() + .collect::>(), + "Preingestion manager working on items" ); } else { - tracing::debug!("PreingestionManager: Working on {} items", items.len()) + tracing::debug!( + item_count = items.len(), + "Preingestion manager working on items" + ) } // Limit the number of concurrent preingestion tasks. @@ -262,11 +265,11 @@ impl PreingestionManager { } } Err(e) => { - tracing::warn!("Error handling preingestion update: {e}"); + tracing::warn!(error = %e, "Error handling preingestion update"); } }, Err(e) => { - tracing::warn!("Error handling preingestion update: {e}"); + tracing::warn!(error = %e, "Error handling preingestion update"); } } } @@ -280,10 +283,10 @@ impl PreingestionManager { .max(0) as usize; tracing::debug!( - "Preingestion metrics: in_preingestion {} waiting {} delayed {}", - metrics.machines_in_preingestion, - metrics.waiting_for_installation, - metrics.delayed_uploading, + machines_in_preingestion = metrics.machines_in_preingestion, + waiting_for_installation = metrics.waiting_for_installation, + delayed_uploading = metrics.delayed_uploading, + "Preingestion metrics updated" ); self.metric_holder.update_metrics(metrics); @@ -300,7 +303,7 @@ async fn one_endpoint( endpoint: &ExploredEndpoint, static_info: Arc, ) -> PreingestionManagerResult { - tracing::debug!("Preingestion on endpoint {:?}", endpoint); + tracing::debug!(?endpoint, "Running preingestion"); // BFB-related preingestion doesn't work for a DPU running in NIC // mode -- the Arm OS doesn't boot, so the `in_bfb_installation_wait` @@ -494,7 +497,8 @@ async fn one_endpoint( PreingestionState::Complete => { // This should have been filtered out by the query that got us this list. tracing::warn!( - "Endpoint showed complete preingestion and should not have been here: {endpoint:?}" + ?endpoint, + "Endpoint showed complete preingestion and should not have been here" ); false } @@ -542,8 +546,8 @@ impl PreingestionManagerStatic { let fw_info = match self.find_fw_info_for_host(db, endpoint).await? { None => { tracing::debug!( - "check_firmware_versions_below_preingestion {}: No matching firmware info found", - endpoint.address + address = %endpoint.address, + "No matching firmware info found during preingestion check" ); // No desired firmware description found for this host, nothing to do. // This is the expected path for DPUs. @@ -560,16 +564,20 @@ impl PreingestionManagerStatic { && let Some(current) = endpoint.find_version(&fw_info, *fwtype) { tracing::info!( - "check_firmware_versions_below_preingestion {}: {fwtype:?} min preingestion {min_preingestion:?} current {current:?}", - endpoint.address + address = %endpoint.address, + ?fwtype, + ?min_preingestion, + ?current, + "Checking firmware version against preingestion minimum" ); if version_compare::compare(current, min_preingestion) .is_ok_and(|c| c == version_compare::Cmp::Lt) { tracing::info!( - "check_firmware_versions_below_preingestion {}: Start upload of {fwtype:?}", - endpoint.address + address = %endpoint.address, + ?fwtype, + "Starting firmware upload during preingestion check" ); // One or both of the versions are low enough to absolutely need upgrades first - do them both while we're at it. let delayed_upgrade = self @@ -578,16 +586,17 @@ impl PreingestionManagerStatic { return Ok(delayed_upgrade); } else { tracing::info!( - "check_firmware_versions_below_preingestion {}: {fwtype:?} is good", - endpoint.address + address = %endpoint.address, + ?fwtype, + "Firmware version satisfies preingestion minimum" ); } } } tracing::debug!( - "check_firmware_versions_below_preingestion {}: Satisfied and marking complete", - endpoint.address + address = %endpoint.address, + "Firmware versions satisfy preingestion requirements; marking complete" ); // Good enough for now at least, proceed with ingestion. db.with_txn(|txn| { @@ -609,8 +618,8 @@ impl PreingestionManagerStatic { ) -> PreingestionManagerResult { if endpoint.waiting_for_explorer_refresh { tracing::debug!( - "start_firmware_uploads_or_continue {}: Waiting for explorer refresh", - endpoint.address + address = %endpoint.address, + "Waiting for explorer refresh before continuing firmware uploads" ); // We've updated something and are waiting for site explorer to get back around to it return Ok(false); @@ -620,8 +629,8 @@ impl PreingestionManagerStatic { // We can't check machine IDs here as they may not be available yet, so use the global value only. if !self.config.autoupdate { tracing::debug!( - "start_firmware_uploads_or_continue {}: Auto updates disabled", - endpoint.address + address = %endpoint.address, + "Automatic firmware updates disabled" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_complete(endpoint.address, txn).boxed() @@ -634,8 +643,8 @@ impl PreingestionManagerStatic { None => { // No desired firmware description found for this host tracing::debug!( - "start_firmware_uploads_or_continue {}: No firmware info found", - endpoint.address + address = %endpoint.address, + "No firmware info found while starting firmware uploads" ); return Ok(false); @@ -662,8 +671,8 @@ impl PreingestionManagerStatic { } tracing::debug!( - "start_firmware_uploads_or_continue {}: No further updates needed", - endpoint.address + address = %endpoint.address, + "No further firmware updates needed" ); // Nothing needed to be updated, we're complete. @@ -688,8 +697,9 @@ impl PreingestionManagerStatic { match need_upgrade(endpoint, fw_info, fw_type) { None => { tracing::debug!( - "start_upgrade_if_needed {}: Upgrade of {fw_type:?} not needed", - endpoint.address + address = %endpoint.address, + ?fw_type, + "Firmware upgrade not needed" ); Ok((false, false)) } @@ -700,9 +710,9 @@ impl PreingestionManagerStatic { } let Ok(_active) = self.upload_limiter.try_acquire() else { tracing::debug!( - "Deferring installation of {:?} on {}, too many uploads already active", - to_install, - endpoint.address + address = %endpoint.address, + ?to_install, + "Deferring firmware installation because too many uploads are active" ); return Ok((true, true)); // Don't check others }; @@ -712,7 +722,11 @@ impl PreingestionManagerStatic { return Ok((true, false)); } - tracing::info!("Installing {:?} on {}", to_install, endpoint.address); + tracing::info!( + address = %endpoint.address, + ?to_install, + "Installing firmware" + ); self.initiate_update(endpoint, &to_install, &fw_type, 0, db) .await?; @@ -751,7 +765,11 @@ impl PreingestionManagerStatic { { Ok(redfish_client) => redfish_client, Err(e) => { - tracing::warn!("Redfish connection to {} failed: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Redfish connection failed" + ); return Ok(()); } }; @@ -764,10 +782,10 @@ impl PreingestionManagerStatic { | Some(TaskState::Running) | Some(TaskState::Pending) => { tracing::debug!( - "Upgrade task for {} not yet complete, current state {:?} message {:?}", - endpoint.address, - task_info.task_state, - task_info.messages, + address = %endpoint.address, + task_state = ?task_info.task_state, + task_messages = ?task_info.messages, + "Firmware upgrade task not yet complete" ); } Some(TaskState::Completed) => { @@ -793,10 +811,10 @@ impl PreingestionManagerStatic { firmware_number < selected_firmware.artifact_count() }) { tracing::info!( - "Installing {:?} chain step {} on {}", - selected_firmware, + address = %endpoint.address, + ?selected_firmware, firmware_number, - endpoint.address + "Installing firmware chain step" ); if self @@ -821,8 +839,8 @@ impl PreingestionManagerStatic { } } tracing::info!( - "Marking completion of Redfish task of firmware upgrade for {}", - &endpoint.address + address = %endpoint.address, + "Marking firmware upgrade Redfish task complete" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_reset_for_new_firmware( @@ -917,9 +935,9 @@ impl PreingestionManagerStatic { _ => { // Unexpected state tracing::warn!( - "Unrecognized task state for {}: {:?}", - endpoint.address, - task_info.task_state + address = %endpoint.address, + task_state = ?task_info.task_state, + "Unrecognized firmware upgrade task state" ); } }; @@ -934,8 +952,8 @@ impl PreingestionManagerStatic { && current_version == final_version { tracing::debug!( - "Marking completion of Redfish task of firmware upgrade for {} with missing task", - &endpoint.address + address = %endpoint.address, + "Marking missing firmware upgrade Redfish task complete" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_recheck_versions( @@ -959,7 +977,11 @@ impl PreingestionManagerStatic { } } _ => { - tracing::warn!("Getting Redfish task from {} failed: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Getting Redfish task failed" + ); } }, }; @@ -1006,7 +1028,11 @@ impl PreingestionManagerStatic { { Ok(redfish_client) => redfish_client, Err(e) => { - tracing::error!("Redfish connection to {} failed: {e}", endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Redfish connection failed" + ); return Ok(()); } }; @@ -1021,8 +1047,9 @@ impl PreingestionManagerStatic { && *delay_until > chrono::Utc::now().timestamp() { tracing::info!( - "Waiting after {last_power_drain_operation:?} of {}", - &endpoint.address + address = %endpoint.address, + ?last_power_drain_operation, + "Waiting after power drain operation" ); return Ok(()); } @@ -1031,14 +1058,17 @@ impl PreingestionManagerStatic { None | Some(PowerDrainState::On) => { // The 1000 is for unit tests; values above this will skip delays. if *power_drains_needed == 0 || *power_drains_needed == 1000 { - tracing::info!("Power drains for {} done", &endpoint.address); + tracing::info!( + address = %endpoint.address, + "Firmware upgrade power drains complete" + ); // This path, and only this path of the match, exits the match and lets us proceed. All others should return after updating state. need_wait = false; // We've reset multiple times already and should be reporting the new version } else { tracing::info!( - "Upgrade task has completed for {} but needs {} power drain(s), initiating one", - &endpoint.address, - *power_drains_needed + address = %endpoint.address, + power_drains_needed = *power_drains_needed, + "Firmware upgrade task complete; initiating required power drain" ); if let Err(e) = count_power_op( PowerOperation::ForceOff, @@ -1046,7 +1076,11 @@ impl PreingestionManagerStatic { ) .await { - tracing::error!("Failed to power off {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power off" + ); return Ok(()); } @@ -1074,7 +1108,10 @@ impl PreingestionManagerStatic { } Some(PowerDrainState::Off) => { if endpoint.report.vendor.unwrap_or_default().is_lenovo() { - tracing::info!("Doing powercycle now for {}", &endpoint.address); + tracing::info!( + address = %endpoint.address, + "Starting AC power cycle" + ); match redfish_client.get_power_state().await { Ok(power_state) if power_state != PowerState::Off => { tracing::warn!( @@ -1089,8 +1126,9 @@ impl PreingestionManagerStatic { .await { tracing::error!( - "Failed to force off {}: {e}", - &endpoint.address + address = %endpoint.address, + error = %e, + "Failed to force off" ); return Ok(()); } @@ -1117,8 +1155,9 @@ impl PreingestionManagerStatic { Ok(_) => {} Err(e) => { tracing::error!( - "Failed to get power state for {}: {e}", - &endpoint.address + address = %endpoint.address, + error = %e, + "Failed to get power state" ); return Ok(()); } @@ -1129,7 +1168,11 @@ impl PreingestionManagerStatic { ) .await { - tracing::error!("Failed to power cycle {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power cycle" + ); return Ok(()); } } @@ -1154,14 +1197,18 @@ impl PreingestionManagerStatic { return Ok(()); } Some(PowerDrainState::Powercycle) => { - tracing::info!("Turning back on {}", &endpoint.address); + tracing::info!(address = %endpoint.address, "Turning system back on"); if let Err(e) = count_power_op( PowerOperation::On, redfish_client.power(SystemPowerControl::On), ) .await { - tracing::error!("Failed to power on {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power on" + ); return Ok(()); } let delay = if *power_drains_needed < 1000 { @@ -1187,8 +1234,8 @@ impl PreingestionManagerStatic { }; } else if upgrade_type.is_uefi() { tracing::info!( - "Upgrade task has completed for {} but needs reboot, initiating one", - &endpoint.address + address = %endpoint.address, + "Firmware upgrade task complete; initiating required reboot" ); if let Err(e) = count_power_op( PowerOperation::ForceRestart, @@ -1196,7 +1243,11 @@ impl PreingestionManagerStatic { ) .await { - tracing::error!("Failed to reboot {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to reboot" + ); return Ok(()); } db.with_txn(|txn| { @@ -1219,13 +1270,17 @@ impl PreingestionManagerStatic { .unwrap_or(bmc_vendor::BMCVendor::Unknown); if upgrade_type.is_bmc() && (bmc_vendor.is_lenovo() || bmc_vendor.is_nvidia()) { tracing::info!( - "Upgrade task has completed for {} but needs BMC reboot, initiating one", - &endpoint.address + address = %endpoint.address, + "Firmware upgrade task complete; initiating required BMC reboot" ); if let Err(e) = count_power_op(PowerOperation::BmcReset, redfish_client.bmc_reset()).await { - tracing::error!("Failed to reboot {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to reboot BMC" + ); return Ok(()); } db.with_txn(|txn| { @@ -1254,7 +1309,11 @@ impl PreingestionManagerStatic { if let Err(e) = count_power_op(poweroff_style.into(), redfish_client.power(poweroff_style)).await { - tracing::error!("Failed to power off {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power off" + ); return Ok(()); } tokio::time::sleep(self.config.hgx_bmc_gpu_reboot_delay).await; @@ -1264,7 +1323,11 @@ impl PreingestionManagerStatic { ) .await { - tracing::error!("Failed to power on {}: {e}", &endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to power on" + ); return Ok(()); } // Does not need a wait @@ -1287,12 +1350,17 @@ impl PreingestionManagerStatic { Ok(()) => {} Err(e) if e.to_string().contains("is not supported") => { tracing::error!( - "Chassis reset is not supported by current CEC FW. Need to do host power cycle! BMC IP: {}", - endpoint.address + address = %endpoint.address, + error = %e, + "Chassis reset is not supported by current CEC firmware; host power cycle required" ); } Err(e) => { - tracing::error!("Failed to call chassis_reset: {e}"); + tracing::error!( + address = %endpoint.address, + error = %e, + "Failed to call chassis reset" + ); } } } @@ -1329,9 +1397,9 @@ impl PreingestionManagerStatic { && previous_reset_time + 30 * 60 <= Utc::now().timestamp() { tracing::info!( - "Upgrade for {} {:?} has taken more than 30 minutes to report new version; resetting again.", - &endpoint.address, - upgrade_type + address = %endpoint.address, + ?upgrade_type, + "Firmware upgrade has taken more than 30 minutes to report the new version; resetting again" ); let state = &PreingestionState::ResetForNewFirmware { final_version: final_version.to_string(), @@ -1351,23 +1419,26 @@ impl PreingestionManagerStatic { }) .await??; tracing::info!( - "Upgrade {} task has completed for {} but still reports version {current_version} (expected version: {final_version})", - upgrade_type, - &endpoint.address + address = %endpoint.address, + %upgrade_type, + current_version, + final_version, + "Firmware upgrade task complete but reported version has not updated" ); return Ok(()); } tracing::info!( - "Upgrade for {} now reports version {current_version}", - &endpoint.address + address = %endpoint.address, + current_version, + "Firmware upgrade now reports the new version" ); } else { // This path should only happen if something strange happened with the version definitions tracing::error!( - "in_upgrade_firmware_wait: Could not find current version {} {:?} {:?}", - &endpoint.address, - fw_info, - *upgrade_type + address = %endpoint.address, + ?fw_info, + ?upgrade_type, + "Could not find current version while waiting for firmware upgrade" ); // Make sure we wait for the new version db.with_txn(|txn| { @@ -1379,10 +1450,10 @@ impl PreingestionManagerStatic { } else { // This path should only happen if something strange happened with the version definitions tracing::error!( - "in_upgrade_firmware_wait: Could not find fw_info {} {:?} {:?}", - &endpoint.address, - endpoint.report.vendor, - endpoint.report.systems + address = %endpoint.address, + vendor = ?endpoint.report.vendor, + systems = ?endpoint.report.systems, + "Could not find firmware info while waiting for firmware upgrade" ); // Make sure we wait for the new version db.with_txn(|txn| { @@ -1420,9 +1491,8 @@ impl PreingestionManagerStatic { // before its own remediation. if self.is_ingested_host(db, endpoint).await { tracing::info!( - "{} BMC time is out of sync but the host is already ingested; \ - skipping time-sync remediation", - endpoint.address + address = %endpoint.address, + "BMC time is out of sync but the host is already ingested; skipping time-sync remediation" ); return self .check_firmware_versions_below_preingestion(db, endpoint) @@ -1430,8 +1500,8 @@ impl PreingestionManagerStatic { } // Time is not in sync, initiate reset sequence tracing::warn!( - "{} BMC time is out of sync, initiating reset to fix time synchronization", - endpoint.address + address = %endpoint.address, + "BMC time is out of sync; initiating reset to fix time synchronization" ); self.time_sync_resets(db, endpoint, &TimeSyncResetPhase::Start, None, 0) .await @@ -1439,8 +1509,9 @@ impl PreingestionManagerStatic { Err(e) => { if let PreingestionManagerError::Internal { message } = e { tracing::error!( - "{} internal error checking BMC time sync: {message}, failing preingestion", - endpoint.address + address = %endpoint.address, + error = %message, + "Internal error checking BMC time sync; failing preingestion" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_failed( @@ -1453,8 +1524,9 @@ impl PreingestionManagerStatic { .await??; } else { tracing::warn!( - "{} retryable error checking BMC time sync: {e}, will retry later", - endpoint.address + address = %endpoint.address, + error = %e, + "Retryable error checking BMC time sync; will retry later" ); } Ok(false) @@ -1478,8 +1550,9 @@ impl PreingestionManagerStatic { Ok(client) => client, Err(e) => { tracing::warn!( - "Redfish connection to {} failed: {e}; will retry initial bmc reset", - endpoint.address + address = %endpoint.address, + error = %e, + "Redfish connection failed; will retry initial BMC reset" ); return Ok(false); } @@ -1490,9 +1563,11 @@ impl PreingestionManagerStatic { let next = attempts + 1; if next >= INITIAL_BMC_RESET_MAX_ATTEMPTS { tracing::warn!( - "{} initial BMC reset failed {next} times: {e}; \ - proceeding with preingestion without it", - endpoint.address + address = %endpoint.address, + attempt = next, + max_attempts = INITIAL_BMC_RESET_MAX_ATTEMPTS, + error = %e, + "Initial BMC reset failed; proceeding with preingestion without it" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_set_ntp_servers( @@ -1507,9 +1582,11 @@ impl PreingestionManagerStatic { return Ok(false); } tracing::warn!( - "{} initial BMC reset attempt {next}/{INITIAL_BMC_RESET_MAX_ATTEMPTS} \ - failed: {e}; will retry", - endpoint.address + address = %endpoint.address, + attempt = next, + max_attempts = INITIAL_BMC_RESET_MAX_ATTEMPTS, + error = %e, + "Initial BMC reset failed; will retry" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_initial_bmc_reset( @@ -1523,8 +1600,8 @@ impl PreingestionManagerStatic { return Ok(false); } tracing::info!( - "{} initial BMC reset initiated; polling for BMC return", - endpoint.address + address = %endpoint.address, + "Initial BMC reset initiated; polling for BMC return" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_initial_bmc_reset( @@ -1546,8 +1623,9 @@ impl PreingestionManagerStatic { Ok(client) => client, Err(e) => { tracing::warn!( - "Redfish connection to {} failed: {e}; will retry waiting for BMC", - endpoint.address + address = %endpoint.address, + error = %e, + "Redfish connection failed; will retry waiting for BMC" ); return Ok(false); } @@ -1577,8 +1655,8 @@ impl PreingestionManagerStatic { }) .await??; tracing::info!( - "{} BMC came back after initial reset; awaiting fresh exploration report before continuing", - endpoint.address + address = %endpoint.address, + "BMC came back after initial reset; awaiting fresh exploration report before continuing" ); Ok(false) } @@ -1586,8 +1664,9 @@ impl PreingestionManagerStatic { // An unreachable BMC is never a reason to move on: keep // waiting and continue once it comes back. tracing::info!( - "Waiting for {} BMC to return after initial reset: {e}", - endpoint.address + address = %endpoint.address, + error = %e, + "Waiting for BMC to return after initial reset" ); Ok(false) } @@ -1597,8 +1676,8 @@ impl PreingestionManagerStatic { // Reached only once the refresh flag is cleared, i.e. site // explorer re-reads the BMC post-reset. tracing::info!( - "{} fresh exploration report received after initial BMC reset; running NTP / time-sync / firmware checks", - endpoint.address + address = %endpoint.address, + "Fresh exploration report received after initial BMC reset; running NTP, time-sync, and firmware checks" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_set_ntp_servers( @@ -1625,8 +1704,11 @@ impl PreingestionManagerStatic { ) -> PreingestionManagerResult { if self.ntp_servers.is_empty() || attempts >= SET_NTP_SERVERS_MAX_ATTEMPTS { tracing::info!( - "{} has no NTP servers configured or max attempts reached; running initial checks", - endpoint.address + address = %endpoint.address, + ntp_server_count = self.ntp_servers.len(), + attempts, + max_attempts = SET_NTP_SERVERS_MAX_ATTEMPTS, + "No NTP servers configured or maximum attempts reached; running initial checks" ); return self.run_initial_checks(db, endpoint).await; } @@ -1635,15 +1717,15 @@ impl PreingestionManagerStatic { let elapsed = Utc::now().signed_duration_since(*set_at); if elapsed < SET_NTP_SERVERS_CONVERGENCE_WAIT { tracing::info!( - "{} waiting for BMC NTP servers to converge before checking time sync", - endpoint.address + address = %endpoint.address, + "Waiting for BMC NTP servers to converge before checking time sync" ); return Ok(false); } tracing::info!( - "{} BMC NTP convergence wait complete; running initial checks", - endpoint.address + address = %endpoint.address, + "BMC NTP convergence wait complete; running initial checks" ); return self.run_initial_checks(db, endpoint).await; } @@ -1678,8 +1760,8 @@ impl PreingestionManagerStatic { } tracing::info!( - "{} set NTP servers; waiting for BMC time to converge", - endpoint.address + address = %endpoint.address, + "Set NTP servers; waiting for BMC time to converge" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_set_ntp_servers( @@ -1710,8 +1792,11 @@ impl PreingestionManagerStatic { let next = attempts + 1; if next >= SET_NTP_SERVERS_MAX_ATTEMPTS { tracing::warn!( - "{} failed to set NTP servers after {next} attempts: {error}; proceeding with initial checks", - endpoint.address + address = %endpoint.address, + attempt = next, + max_attempts = SET_NTP_SERVERS_MAX_ATTEMPTS, + error = %error, + "Failed to set NTP servers; proceeding with initial checks" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_set_ntp_servers( @@ -1727,8 +1812,11 @@ impl PreingestionManagerStatic { } tracing::warn!( - "{} failed to set NTP servers attempt {next}/{SET_NTP_SERVERS_MAX_ATTEMPTS}: {error}; will retry", - endpoint.address + address = %endpoint.address, + attempt = next, + max_attempts = SET_NTP_SERVERS_MAX_ATTEMPTS, + error = %error, + "Failed to set NTP servers; will retry" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_set_ntp_servers( @@ -1759,10 +1847,18 @@ impl PreingestionManagerStatic { Ok(()) => {} Err(e) if matches!(e, RedfishError::UnnecessaryOperation) => { // ignore because it is already off - tracing::debug!("Power off not needed on {}: {e}", endpoint.address); + tracing::debug!( + address = %endpoint.address, + error = %e, + "Power off not needed" + ); } Err(e) => { - tracing::warn!("Could not turn off power on {}: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Could not turn off power" + ); return false; } } @@ -1770,16 +1866,28 @@ impl PreingestionManagerStatic { let status = match redfish_client.get_power_state().await { Ok(status) => status, Err(e) => { - tracing::warn!("Could not get power of {}: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Could not get power state" + ); return false; } }; if status != PowerState::Off { - tracing::warn!("Host {} did not turn off when requested", endpoint.address); + tracing::warn!( + address = %endpoint.address, + %status, + "Host did not turn off when requested" + ); return false; } if let Err(e) = count_power_op(PowerOperation::BmcReset, redfish_client.bmc_reset()).await { - tracing::warn!("Could not reset BMC on {}: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Could not reset BMC" + ); return false; } true @@ -1794,8 +1902,9 @@ impl PreingestionManagerStatic { ) -> bool { if let Err(e) = redfish_client.get_tasks().await { tracing::info!( - "Waiting for {} BMC reset to complete: {e}", - endpoint.address + address = %endpoint.address, + error = %e, + "Waiting for BMC reset to complete" ); return false; } @@ -1809,10 +1918,18 @@ impl PreingestionManagerStatic { Ok(()) => {} Err(e) if matches!(e, RedfishError::UnnecessaryOperation) => { // ignore because it is already on - tracing::debug!("Power on not needed on {}: {e}", endpoint.address); + tracing::debug!( + address = %endpoint.address, + error = %e, + "Power on not needed" + ); } Err(e) => { - tracing::warn!("Could not turn on power on {}: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Could not turn on power" + ); return false; } } @@ -1820,12 +1937,20 @@ impl PreingestionManagerStatic { let status = match redfish_client.get_power_state().await { Ok(status) => status, Err(e) => { - tracing::warn!("Could not get power of {}: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Could not get power state" + ); return false; } }; if status != PowerState::On { - tracing::warn!("Host {} did not turn on when requested", endpoint.address); + tracing::warn!( + address = %endpoint.address, + %status, + "Host did not turn on when requested" + ); return false; } true @@ -1841,7 +1966,10 @@ impl PreingestionManagerStatic { if Utc::now().signed_duration_since(last_time.unwrap_or(&Utc::now())) < chrono::TimeDelta::minutes(20) { - tracing::trace!("Waiting for {} to complete boot sequence", endpoint.address); + tracing::trace!( + address = %endpoint.address, + "Waiting for host to complete boot sequence" + ); return false; } true @@ -1861,7 +1989,11 @@ impl PreingestionManagerStatic { { Ok(redfish_client) => redfish_client, Err(e) => { - tracing::warn!("Redfish connection to {} failed: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Redfish connection failed" + ); return Ok(()); } }; @@ -1874,7 +2006,10 @@ impl PreingestionManagerStatic { { return Ok(()); } - tracing::info!("{} initial reset BMC reset intiated", endpoint.address); + tracing::info!( + address = %endpoint.address, + "Initial reset BMC reset initiated" + ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_initial_reset( endpoint.address, @@ -1894,8 +2029,8 @@ impl PreingestionManagerStatic { return Ok(()); } tracing::info!( - "{} initial reset BMC reset complete, started host reset", - endpoint.address + address = %endpoint.address, + "Initial reset BMC reset complete; started host reset" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_initial_reset( @@ -1913,7 +2048,7 @@ impl PreingestionManagerStatic { return Ok(()); } // Now we can actually proceed with the upgrade. Go back to checking firmware so we don't have to store all of that info. - tracing::info!("{} initial reset complete", endpoint.address); + tracing::info!(address = %endpoint.address, "Initial reset complete"); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_recheck_versions(endpoint.address, txn) .boxed() @@ -1947,7 +2082,11 @@ impl PreingestionManagerStatic { { Ok(redfish_client) => redfish_client, Err(e) => { - tracing::warn!("Redfish connection to {} failed: {e}", endpoint.address); + tracing::warn!( + address = %endpoint.address, + error = %e, + "Redfish connection failed" + ); return Ok(false); } }; @@ -1955,7 +2094,11 @@ impl PreingestionManagerStatic { match phase { TimeSyncResetPhase::Start => { if let Err(e) = redfish_client.set_utc_timezone().await { - tracing::error!("Could not set UTC timezone on {}: {e}", endpoint.address); + tracing::error!( + address = %endpoint.address, + error = %e, + "Could not set UTC timezone" + ); return Err(PreingestionManagerError::RedfishError(e)); } if !self @@ -1964,7 +2107,10 @@ impl PreingestionManagerStatic { { return Ok(false); } - tracing::info!("{} time sync reset BMC reset initiated", endpoint.address); + tracing::info!( + address = %endpoint.address, + "Time-sync reset BMC reset initiated" + ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_time_sync_reset( endpoint.address, @@ -1985,8 +2131,8 @@ impl PreingestionManagerStatic { return Ok(false); } tracing::info!( - "{} time sync reset BMC reset complete, started host reset", - endpoint.address + address = %endpoint.address, + "Time-sync reset BMC reset complete; started host reset" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_time_sync_reset( @@ -2007,14 +2153,17 @@ impl PreingestionManagerStatic { // Host has booted, now check time sync again tracing::info!( - "{} time sync reset complete, checking time sync", - endpoint.address + address = %endpoint.address, + "Time-sync reset complete; checking time sync" ); match self.check_bmc_time_sync(db, endpoint).await { Ok(true) => { // Time is now in sync, proceed with firmware check - tracing::info!("{} BMC time is now in sync after reset", endpoint.address); + tracing::info!( + address = %endpoint.address, + "BMC time is now in sync after reset" + ); let delayed_upgrade = self .check_firmware_versions_below_preingestion(db, endpoint) .await?; @@ -2028,10 +2177,10 @@ impl PreingestionManagerStatic { let attempts_done = attempt + 1; if attempts_done < MAX_TIME_SYNC_RESET_ATTEMPTS { tracing::warn!( - "{} BMC time still out of sync after reset attempt {}/{}, retrying reset", - endpoint.address, - attempts_done, - MAX_TIME_SYNC_RESET_ATTEMPTS + address = %endpoint.address, + attempt = attempts_done, + max_attempts = MAX_TIME_SYNC_RESET_ATTEMPTS, + "BMC time still out of sync after reset; retrying reset" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_time_sync_reset( @@ -2047,9 +2196,10 @@ impl PreingestionManagerStatic { } tracing::error!( - "{} BMC time is still out of sync after {} reset attempts, failing preingestion", - endpoint.address, - attempts_done + address = %endpoint.address, + attempts = attempts_done, + max_attempts = MAX_TIME_SYNC_RESET_ATTEMPTS, + "BMC time is still out of sync after reset attempts; failing preingestion" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_failed( @@ -2068,8 +2218,9 @@ impl PreingestionManagerStatic { if let PreingestionManagerError::Internal { message } = e { // Error checking time sync after reset, fail now tracing::error!( - "{} internal error checking BMC time sync after reset: {message}, failing preingestion", - endpoint.address + address = %endpoint.address, + error = %message, + "Internal error checking BMC time sync after reset; failing preingestion" ); db.with_txn(|txn| { db::explored_endpoints::set_preingestion_failed( @@ -2082,8 +2233,9 @@ impl PreingestionManagerStatic { .await??; } else { tracing::warn!( - "{} retryable error checking BMC time sync after reset: {e}, will retry later", - endpoint.address + address = %endpoint.address, + error = %e, + "Retryable error checking BMC time sync after reset; will retry later" ); } Ok(false) @@ -2110,7 +2262,8 @@ impl PreingestionManagerStatic { let interface = db::machine_interface::find_by_ip(db, endpoint_address).await?; let Some(interface) = interface else { tracing::warn!( - "Unable to run update script for {address}: MAC address not retrievable" + address = address.as_str(), + "Unable to run update script; MAC address not retrievable" ); return Ok(()); }; @@ -2126,13 +2279,16 @@ impl PreingestionManagerStatic { }, Ok(None) => { tracing::warn!( - "Unable to run update script for {address}: No credentials exists" + address = address.as_str(), + "Unable to run update script; no credentials exist" ); return Ok(()); } Err(e) => { tracing::warn!( - "Unable to run update script for {address}: Unable to retrieve credentials due to error: {e}" + address = address.as_str(), + error = %e, + "Unable to run update script; could not retrieve credentials" ); return Ok(()); } @@ -2151,14 +2307,21 @@ impl PreingestionManagerStatic { { Ok(cmd) => cmd, Err(e) => { - tracing::error!("Upgrade script {address} command creation failed: {e}"); + tracing::error!( + address = address.as_str(), + error = %e, + "Upgrade script command creation failed" + ); upgrade_script_state.completed(address, false); return; } }; let Some(stdout) = cmd.stdout.take() else { - tracing::error!("Upgrade script {address} STDOUT creation failed"); + tracing::error!( + address = address.as_str(), + "Upgrade script stdout creation failed" + ); let _ = cmd.kill().await; let _ = cmd.wait().await; upgrade_script_state.completed(address, false); @@ -2167,7 +2330,10 @@ impl PreingestionManagerStatic { let stdout = tokio::io::BufReader::new(stdout); let Some(stderr) = cmd.stderr.take() else { - tracing::error!("Upgrade script {address} STDERR creation failed"); + tracing::error!( + address = address.as_str(), + "Upgrade script stderr creation failed" + ); let _ = cmd.kill().await; let _ = cmd.wait().await; upgrade_script_state.completed(address, false); @@ -2191,15 +2357,26 @@ impl PreingestionManagerStatic { match cmd.wait().await { Err(e) => { - tracing::info!("Upgrade script {address} FAILED: Wait failure {e}"); + tracing::info!( + address = address.as_str(), + error = %e, + "Upgrade script failed while waiting" + ); upgrade_script_state.completed(address, false); } Ok(errorcode) => { if errorcode.success() { - tracing::info!("Upgrade script {address} completed successfully"); + tracing::info!( + address = address.as_str(), + "Upgrade script completed successfully" + ); upgrade_script_state.completed(address, true); } else { - tracing::warn!("Upgrade script {address} FAILED: Exited with {errorcode}"); + tracing::warn!( + address = address.as_str(), + exit_status = %errorcode, + "Upgrade script exited unsuccessfully" + ); upgrade_script_state.completed(address, false); } } @@ -2249,7 +2426,7 @@ impl PreingestionManagerStatic { db: &PgPool, endpoint: &ExploredEndpoint, ) -> PreingestionManagerResult { - tracing::debug!("Checking BMC time sync for {:?}", endpoint); + tracing::debug!(?endpoint, "Checking BMC time sync"); let redfish_client = match self .redfish_client_pool .create_client_for_ingested_host(endpoint.address, db) @@ -2284,18 +2461,18 @@ impl PreingestionManagerStatic { if time_diff > NTP_DRIFT_THRESHOLD_SECONDS { tracing::warn!( - "BMC time for {} is out of sync: BMC time: {}, System time: {}, Difference: {} seconds", - endpoint.address, - bmc_time, - system_time, - time_diff + address = %endpoint.address, + %bmc_time, + %system_time, + difference_seconds = time_diff, + "BMC time is out of sync" ); Ok(false) } else { tracing::debug!( - "BMC time for {} is in sync: difference {} seconds", - endpoint.address, - time_diff + address = %endpoint.address, + difference_seconds = time_diff, + "BMC time is in sync" ); Ok(true) } @@ -2321,9 +2498,9 @@ impl PreingestionManagerStatic { Ok(machine_id) => machine_id.is_some(), Err(e) => { tracing::warn!( - "Could not determine if {} is an ingested host: {e}; \ - treating as ingested to skip time-sync remediation", - endpoint.address + address = %endpoint.address, + error = %e, + "Could not determine whether endpoint is an ingested host; treating as ingested to skip time-sync remediation" ); true } @@ -2372,11 +2549,6 @@ impl PreingestionManagerStatic { use model::site_explorer::BfbPlatformPowercyclePhase; let address = endpoint.address; - let label = if post_install { - "post-install" - } else { - "pre-copy" - }; match phase { BfbPlatformPowercyclePhase::PowerOff => { @@ -2387,19 +2559,36 @@ impl PreingestionManagerStatic { { Ok(c) => c, Err(e) => { - tracing::error!(%address, host_ip=%host_bmc_ip, error=%e, "{label}: failed to create Redfish client for host, will retry"); + tracing::error!( + %address, + host_ip = %host_bmc_ip, + post_install, + error = %e, + "Failed to create Redfish client for host during BFB power cycle; will retry" + ); return Ok(()); } }; - tracing::info!(%address, host_ip=%host_bmc_ip, "{label}: powering off host"); + tracing::info!( + %address, + host_ip = %host_bmc_ip, + post_install, + "Powering off host during BFB power cycle" + ); if let Err(e) = count_power_op( PowerOperation::ForceOff, redfish_client.power(SystemPowerControl::ForceOff), ) .await { - tracing::error!(%address, host_ip=%host_bmc_ip, error=%e, "{label}: failed to power off host, will retry"); + tracing::error!( + %address, + host_ip = %host_bmc_ip, + post_install, + error = %e, + "Failed to power off host during BFB power cycle; will retry" + ); return Ok(()); } @@ -2423,19 +2612,36 @@ impl PreingestionManagerStatic { { Ok(c) => c, Err(e) => { - tracing::error!(%address, host_ip=%host_bmc_ip, error=%e, "{label}: failed to create Redfish client for host, will retry"); + tracing::error!( + %address, + host_ip = %host_bmc_ip, + post_install, + error = %e, + "Failed to create Redfish client for host during BFB power cycle; will retry" + ); return Ok(()); } }; - tracing::info!(%address, host_ip=%host_bmc_ip, "{label}: powering on host"); + tracing::info!( + %address, + host_ip = %host_bmc_ip, + post_install, + "Powering on host during BFB power cycle" + ); if let Err(e) = count_power_op( PowerOperation::On, redfish_client.power(SystemPowerControl::On), ) .await { - tracing::error!(%address, host_ip=%host_bmc_ip, error=%e, "{label}: failed to power on host, will retry"); + tracing::error!( + %address, + host_ip = %host_bmc_ip, + post_install, + error = %e, + "Failed to power on host during BFB power cycle; will retry" + ); return Ok(()); } @@ -2485,7 +2691,11 @@ impl PreingestionManagerStatic { self.start_bfb_copy(db, endpoint, *host_bmc_ip).await?; } Err(_) => { - tracing::debug!(%address, "DPU BMC not yet reachable after {label} power-cycle"); + tracing::debug!( + %address, + post_install, + "DPU BMC not yet reachable after power cycle" + ); } } } @@ -2976,7 +3186,11 @@ impl PreingestionManagerStatic { ) { Ok(artifact) => artifact, Err(error) => { - tracing::error!("Failed to resolve firmware artifact: {error}"); + tracing::error!( + address = %endpoint_clone.address, + %error, + "Failed to resolve firmware artifact" + ); return Ok(false); } }; @@ -2986,9 +3200,10 @@ impl PreingestionManagerStatic { ResolvedFirmwareArtifactSource::Remote { url, sha256 } => { if !self.downloader.available(&artifact.local_path, url, sha256) { tracing::debug!( - "{} is being downloaded from {}, update deferred", - artifact.local_path.display(), - url + address = %endpoint_clone.address, + path = %artifact.local_path.display(), + %url, + "Firmware artifact is being downloaded; update deferred" ); return Ok(false); @@ -2997,8 +3212,9 @@ impl PreingestionManagerStatic { ResolvedFirmwareArtifactSource::Local => { if !artifact.local_path.exists() { tracing::error!( - "Firmware artifact {} is not present", - artifact.local_path.display() + address = %endpoint_clone.address, + path = %artifact.local_path.display(), + "Firmware artifact is not present" ); return Ok(false); } @@ -3015,16 +3231,17 @@ impl PreingestionManagerStatic { Ok(redfish_client) => redfish_client, Err(e) => { tracing::debug!( - "Failed to open redfish to {}: {e}", - endpoint_clone.address.to_string() + address = %endpoint_clone.address, + error = %e, + "Failed to open Redfish connection" ); return Ok(false); } }; tracing::debug!( - "initiate_update: Started upload of firmware to {}", - endpoint_clone.address + address = %endpoint_clone.address, + "Started firmware upload" ); let redfish_component_type: libredfish::model::update_service::ComponentType = @@ -3034,12 +3251,20 @@ impl PreingestionManagerStatic { }; let task = if is_bfb_artifact(&artifact.local_path) { - let _ = redfish_client.enable_rshim_bmc().await.map_err(|e| { - tracing::error!("initiate_update: Failed to call enable_rshim_bmc: {e}") - }); + redfish_client + .enable_rshim_bmc() + .await + .inspect_err(|e| { + tracing::error!( + address = %endpoint_clone.address, + error = %e, + "Failed to enable RSHIM on BMC" + ) + }) + .ok(); tracing::debug!( - "initiate_update: Using simple_update with image URI: {}", - artifact.bfb_image_uri + image_uri = artifact.bfb_image_uri.as_str(), + "Using simple update for firmware upload" ); match redfish_client .update_firmware_simple_update( @@ -3062,8 +3287,9 @@ impl PreingestionManagerStatic { outcome: Outcome::Error, }); tracing::error!( - "initiate_update: Failed to call update_firmware_simple_update {}: {e}", - endpoint_clone.address + address = %endpoint_clone.address, + error = %e, + "Simple firmware update failed" ); return Ok(false); } @@ -3091,12 +3317,18 @@ impl PreingestionManagerStatic { outcome: Outcome::Error, }); tracing::warn!( - "Multipart update is not supported: {err}. Trying to use HttpPushUri" + address = %endpoint_clone.address, + error = %err, + "Multipart firmware update is not supported; trying HttpPushUri" ); let file = match File::open(artifact.local_path.as_path()).await { Ok(f) => f, Err(e) => { - tracing::error!("Failed to open a file: {e}"); + tracing::error!( + address = %endpoint_clone.address, + error = %e, + "Failed to open firmware file" + ); return Ok(false); } }; @@ -3114,8 +3346,9 @@ impl PreingestionManagerStatic { outcome: Outcome::Error, }); tracing::error!( - "initiate_update: Failed uploading firmware to {}: {e}", - endpoint_clone.address + address = %endpoint_clone.address, + error = %e, + "Failed to upload firmware via HttpPushUri" ); return Ok(false); } @@ -3127,8 +3360,9 @@ impl PreingestionManagerStatic { outcome: Outcome::Error, }); tracing::warn!( - "initiate_update: Failed uploading firmware to {}: {e}", - endpoint_clone.address + address = %endpoint_clone.address, + error = %e, + "Failed to upload firmware via multipart update" ); return Ok(false); } @@ -3136,8 +3370,8 @@ impl PreingestionManagerStatic { }; tracing::debug!( - "initiate_update: Completed upload of firmware to {}", - endpoint_clone.address + address = %endpoint_clone.address, + "Completed firmware upload" ); db_pool diff --git a/crates/preingestion-manager/tests/integration/host_bmc_firmware.rs b/crates/preingestion-manager/tests/integration/host_bmc_firmware.rs index 8156cb07b6..9aba5ffafa 100644 --- a/crates/preingestion-manager/tests/integration/host_bmc_firmware.rs +++ b/crates/preingestion-manager/tests/integration/host_bmc_firmware.rs @@ -389,7 +389,7 @@ async fn test_preingestion_preupdate_powercycling( let underlay_segment = nc.create_underlay_segment(&domain).await; let config = default_config::get(); - tracing::debug!("{:?}", config.host_models); + tracing::debug!(host_models = ?config.host_models, "Using host models"); let mgr = PreingestionManager::new( pool.clone(), @@ -523,7 +523,7 @@ async fn test_preingestion_preupdate_powercycling( let endpoints = db::explored_endpoints::find_all(txn.as_mut()).await?; assert!(endpoints.len() == 1); let mut endpoint = endpoints.into_iter().next().unwrap(); - tracing::debug!("State should be {state:?}"); + tracing::debug!(?state, "Checking expected preingestion state"); match &endpoint.preingestion_state { PreingestionState::ResetForNewFirmware { delay_until, diff --git a/crates/rack-controller/src/created.rs b/crates/rack-controller/src/created.rs index 9f30d3738b..310eac6f35 100644 --- a/crates/rack-controller/src/created.rs +++ b/crates/rack-controller/src/created.rs @@ -93,11 +93,11 @@ pub async fn handle_created( } tracing::info!( - "Rack {} has all expected devices (compute={}, switch={}, power_shelf={}). Transitioning to Discovering.", - id, + rack_id = %id, compute_count, switch_count, - power_shelf_count + power_shelf_count, + "Rack has all expected devices (compute, switch, power_shelf). Transitioning to Discovering.", ); Ok(StateHandlerOutcome::transition(RackState::Discovering).with_txn(txn)) } diff --git a/crates/rack-controller/src/discovering.rs b/crates/rack-controller/src/discovering.rs index 961070db7d..a2926a6be8 100644 --- a/crates/rack-controller/src/discovering.rs +++ b/crates/rack-controller/src/discovering.rs @@ -107,8 +107,8 @@ pub async fn handle_discovering( } tracing::info!( - "Rack {} all devices ready, transitioning to Maintenance", - id + rack_id = %id, + "Rack all devices ready, transitioning to Maintenance", ); Ok(StateHandlerOutcome::transition(RackState::Maintenance { maintenance_state: RackMaintenanceState::FirmwareUpgrade { diff --git a/crates/rack-controller/src/error_state.rs b/crates/rack-controller/src/error_state.rs index 11c6267112..89f5b7d1d1 100644 --- a/crates/rack-controller/src/error_state.rs +++ b/crates/rack-controller/src/error_state.rs @@ -48,18 +48,18 @@ pub async fn handle_error( }; if scope.is_full_rack() { tracing::info!( - "Rack {} on-demand maintenance requested from Error state (full rack, activities: [{}]), transitioning to Maintenance", - id, - activities_desc, + rack_id = %id, + activities_desc = %activities_desc, + "Rack on-demand maintenance requested from Error state (full rack, activities), transitioning to Maintenance", ); } else { tracing::info!( - "Rack {} on-demand maintenance requested from Error state (partial: {} machines, {} switches, {} power shelves, activities: [{}]), transitioning to Maintenance", - id, - scope.machine_ids.len(), - scope.switch_ids.len(), - scope.power_shelf_ids.len(), - activities_desc, + rack_id = %id, + machine_id_count = scope.machine_ids.len(), + switch_id_count = scope.switch_ids.len(), + power_shelf_id_count = scope.power_shelf_ids.len(), + activities_desc = %activities_desc, + "Rack on-demand maintenance requested from Error state (partial: machines, switches, power shelves, activities), transitioning to Maintenance", ); } let txn = ctx.services.db_pool.begin().await?; @@ -71,14 +71,18 @@ pub async fn handle_error( if all_components_ready(id, ctx).await? { tracing::info!( - "Rack {} components all Ready, transitioning from Error back to Ready", - id + rack_id = %id, + "Rack components all Ready, transitioning from Error back to Ready", ); let txn = ctx.services.db_pool.begin().await?; return Ok(StateHandlerOutcome::transition(RackState::Ready).with_txn(txn)); } - tracing::error!("Rack {} is in error state: {}", id, cause); + tracing::error!( + rack_id = %id, + cause = %cause, + "Rack is in error state", + ); Ok(StateHandlerOutcome::wait(format!( "rack in error state: {}", cause diff --git a/crates/rack-controller/src/handler.rs b/crates/rack-controller/src/handler.rs index 18762e4611..39984b3d86 100644 --- a/crates/rack-controller/src/handler.rs +++ b/crates/rack-controller/src/handler.rs @@ -101,15 +101,19 @@ impl StateHandler for RackStateHandler { controller_state: &Self::ControllerState, ctx: &mut StateHandlerContext, ) -> Result, StateHandlerError> { - tracing::info!("Rack {} is in state {}", id, controller_state.to_string()); + tracing::info!( + rack_id = %id, + %controller_state, + "Rack is in state", + ); self.record_metrics(state, ctx); if state.deleted.is_some() && !matches!(controller_state, RackState::Deleting) { tracing::info!( - "Rack {} is marked as deleted, transitioning from {} to Deleting", - id, - controller_state + rack_id = %id, + %controller_state, + "Rack is marked as deleted, transitioning to Deleting", ); return Ok(StateHandlerOutcome::transition(RackState::Deleting)); } diff --git a/crates/rack-controller/src/lib.rs b/crates/rack-controller/src/lib.rs index c5c71580ef..39d5a7257c 100644 --- a/crates/rack-controller/src/lib.rs +++ b/crates/rack-controller/src/lib.rs @@ -75,7 +75,10 @@ pub(crate) fn resolve_profile<'a>( let rack_profile_id = match rack_profile_id { Some(rc) => rc, None => { - tracing::info!("Rack {} has no rack_profile_id configured", id); + tracing::info!( + rack_id = %id, + "Rack has no rack_profile_id configured", + ); return None; } }; @@ -89,9 +92,9 @@ pub(crate) fn resolve_profile<'a>( Some(profile) => Some(profile), None => { tracing::warn!( - "Rack {} has unknown rack_profile_id '{}'", - id, - rack_profile_id + rack_id = %id, + rack_profile_id = %rack_profile_id, + "Rack has unknown rack_profile_id", ); None } diff --git a/crates/rack-controller/src/maintenance.rs b/crates/rack-controller/src/maintenance.rs index a66c624307..85636715fb 100644 --- a/crates/rack-controller/src/maintenance.rs +++ b/crates/rack-controller/src/maintenance.rs @@ -1931,8 +1931,8 @@ pub async fn handle_maintenance( .await?; } else { tracing::error!( - "switch {} not found in database for NVOS update", - switch.mac + mac_address = %switch.mac, + "switch not found in database for NVOS update", ); } } @@ -2426,20 +2426,29 @@ pub async fn handle_maintenance( }, RackMaintenanceState::PowerSequence { rack_power } => match rack_power { RackPowerState::PoweringOn => { - tracing::info!("Rack {} power sequence (on) - stubbed", id); + tracing::info!( + rack_id = %id, + "Rack power sequence (on) - stubbed", + ); Ok(StateHandlerOutcome::transition(RackState::Maintenance { maintenance_state: RackMaintenanceState::Completed, })) } RackPowerState::PoweringOff => { - tracing::info!("Rack {} power sequence (off) - stubbed", id); + tracing::info!( + rack_id = %id, + "Rack power sequence (off) - stubbed", + ); Ok(StateHandlerOutcome::wait( "power sequence (off) in progress".into(), )) } RackPowerState::PowerReset => { - tracing::info!("Rack {} power sequence (reset) - stubbed", id); + tracing::info!( + rack_id = %id, + "Rack power sequence (reset) - stubbed", + ); Ok(StateHandlerOutcome::wait( "power sequence (reset) in progress".into(), )) diff --git a/crates/rack-controller/src/ready.rs b/crates/rack-controller/src/ready.rs index bb99015daa..f3e5c983d7 100644 --- a/crates/rack-controller/src/ready.rs +++ b/crates/rack-controller/src/ready.rs @@ -46,8 +46,8 @@ pub async fn handle_ready( ) -> Result, StateHandlerError> { if config.topology_changed { tracing::info!( - "Rack {} topology changed, transitioning from Ready to Discovering", - id + rack_id = %id, + "Rack topology changed, transitioning from Ready to Discovering", ); state.config.topology_changed = false; let mut txn = ctx.services.db_pool.begin().await?; @@ -57,8 +57,8 @@ pub async fn handle_ready( if config.reprovision_requested { tracing::info!( - "Rack {} reprovision requested, transitioning from Ready to Maintenance", - id + rack_id = %id, + "Rack reprovision requested, transitioning from Ready to Maintenance", ); state.config.reprovision_requested = false; state.config.maintenance_requested = None; @@ -85,18 +85,18 @@ pub async fn handle_ready( }; if scope.is_full_rack() { tracing::info!( - "Rack {} on-demand maintenance requested (full rack, activities: [{}]), transitioning to Maintenance", - id, - activities_desc, + rack_id = %id, + activities_desc = %activities_desc, + "Rack on-demand maintenance requested (full rack, activities), transitioning to Maintenance", ); } else { tracing::info!( - "Rack {} on-demand maintenance requested (partial: {} machines, {} switches, {} power shelves, activities: [{}]), transitioning to Maintenance", - id, - scope.machine_ids.len(), - scope.switch_ids.len(), - scope.power_shelf_ids.len(), - activities_desc, + rack_id = %id, + machine_id_count = scope.machine_ids.len(), + switch_id_count = scope.switch_ids.len(), + power_shelf_id_count = scope.power_shelf_ids.len(), + activities_desc = %activities_desc, + "Rack on-demand maintenance requested (partial: machines, switches, power shelves, activities), transitioning to Maintenance", ); } // Leave maintenance_requested set; the maintenance handler reads the @@ -109,7 +109,11 @@ pub async fn handle_ready( } if let Some(cause) = detect_component_failure(id, ctx).await? { - tracing::warn!("Rack {} transitioning from Ready to Error: {}", id, cause); + tracing::warn!( + rack_id = %id, + cause = %cause, + "Rack transitioning from Ready to Error", + ); let txn = ctx.services.db_pool.begin().await?; return Ok(StateHandlerOutcome::transition(RackState::Error { cause }).with_txn(txn)); } @@ -277,8 +281,8 @@ pub(super) async fn all_components_ready( let total = all_switches.len() + all_power_shelves.len() + all_machines.len(); if total == 0 { tracing::debug!( - "Rack {} has no components registered; not promoting out of Error", - rack_id + rack_id = %rack_id, + "Rack has no components registered; not promoting out of Error", ); return Ok(false); } @@ -289,14 +293,14 @@ pub(super) async fn all_components_ready( if !all_ready { tracing::debug!( - "Rack {} components not yet all Ready: switches {}/{}, power shelves {}/{}, machines {}/{}", - rack_id, - ready_switches.len(), - all_switches.len(), - ready_power_shelves.len(), - all_power_shelves.len(), - ready_machines.len(), - all_machines.len(), + rack_id = %rack_id, + ready_switch_count = ready_switches.len(), + all_switch_count = all_switches.len(), + ready_power_shelf_count = ready_power_shelves.len(), + all_power_shelf_count = all_power_shelves.len(), + ready_machine_count = ready_machines.len(), + all_machine_count = all_machines.len(), + "Rack components not yet all Ready: switches, power shelves, machines", ); } diff --git a/crates/rack-controller/src/validating.rs b/crates/rack-controller/src/validating.rs index a54769179c..2ef1afd3c8 100644 --- a/crates/rack-controller/src/validating.rs +++ b/crates/rack-controller/src/validating.rs @@ -197,7 +197,11 @@ pub(super) async fn load_partition_summary( let machines = super::get_machines_from_rack(rack, &mut txn).await?; txn.commit().await?; - tracing::debug!("Rack {} has {} machines", rack_id, machines.len()); + tracing::debug!( + rack_id = %rack_id, + machine_count = machines.len(), + "Rack has machines", + ); let partitions = RvPartitions::from_machines(machines, run_id)?; Ok(partitions.summarize()) @@ -219,7 +223,11 @@ pub(super) async fn find_rv_run_id( .into_iter() .find_map(|m| m.metadata.labels.get(run_label).cloned()); - tracing::debug!("Rack {} rv.run-id scan: {:?}", rack_id, found); + tracing::debug!( + rack_id = %rack_id, + found_run_id = ?found, + "Rack rv.run-id scan", + ); Ok(found) } @@ -320,7 +328,10 @@ pub async fn handle_validating( ctx: &mut StateHandlerContext<'_, RackStateHandlerContextObjects>, ) -> Result, StateHandlerError> { if !ctx.services.site_config.rack_validation_config.enabled { - tracing::info!("Rack {} validation disabled, skipping to Ready", id); + tracing::info!( + rack_id = %id, + "Rack validation disabled, skipping to Ready", + ); return Ok(StateHandlerOutcome::transition(RackState::Ready)); } @@ -329,9 +340,9 @@ pub async fn handle_validating( // Stay in Pending until RVS sets rv.run-id on at least one rack machine. if let Some(found_run_id) = find_rv_run_id(id, state, ctx).await? { tracing::info!( - "Rack {} validation run started (run_id={}), entering InProgress", - id, - found_run_id + rack_id = %id, + found_run_id = %found_run_id, + "Rack validation run started, entering InProgress", ); Ok(StateHandlerOutcome::transition(RackState::Validating { validating_state: RackValidationState::InProgress { @@ -340,8 +351,8 @@ pub async fn handle_validating( })) } else { tracing::debug!( - "Rack {} in Validating(Pending), waiting for RVS to set rv.run-id", - id + rack_id = %id, + "Rack in Validating(Pending), waiting for RVS to set rv.run-id", ); Ok(StateHandlerOutcome::do_nothing()) } @@ -356,32 +367,35 @@ pub async fn handle_validating( let summary = load_partition_summary(id, state, run_id, ctx).await?; tracing::debug!( - "Rack {} partition summary: total={}, pending={}, in_progress={}, validated={}, failed={}", - id, - summary.total_partitions, - summary.pending, - summary.in_progress, - summary.validated, - summary.failed + rack_id = %id, + total_partitions = summary.total_partitions, + pending = summary.pending, + in_progress = summary.in_progress, + validated = summary.validated, + failed = summary.failed, + "Rack partition validation summary", ); if let Some(next_vs) = compute_validation_transition(other, &summary) { tracing::info!( - "Rack {} validation transitioning from {} to {}", - id, - other, - next_vs + rack_id = %id, + previous_state = %other, + next_state = %next_vs, + "Rack validation state transition", ); Ok(StateHandlerOutcome::transition(RackState::Validating { validating_state: next_vs, })) } else if matches!(other, RackValidationState::Validated { .. }) { - tracing::info!("Rack {} fully validated, transitioning to Ready", id); + tracing::info!( + rack_id = %id, + "Rack fully validated, transitioning to Ready", + ); Ok(StateHandlerOutcome::transition(RackState::Ready)) } else if matches!(other, RackValidationState::Failed { .. }) { tracing::warn!( - "Rack {} is in Validating(Failed) state, requires intervention", - id + rack_id = %id, + "Rack is in Validating(Failed) state, requires intervention", ); Ok(StateHandlerOutcome::do_nothing()) } else { diff --git a/crates/rpc-utils/src/managed_host_display.rs b/crates/rpc-utils/src/managed_host_display.rs index 58552f3cc2..2ebb2154c9 100644 --- a/crates/rpc-utils/src/managed_host_display.rs +++ b/crates/rpc-utils/src/managed_host_display.rs @@ -68,7 +68,7 @@ impl ManagedHostMetadata { .await .map(|response| response.into_inner()) .map_err(|e| { - warn!("Failed to get site exploration report: {:?}", e); + warn!(error = ?e, "failed to get site exploration report"); }) .unwrap_or_default(); @@ -429,7 +429,10 @@ pub fn get_managed_host_output(source: ManagedHostMetadata) -> Vec(t: Option, machine_id: Option) -> Optio } Err(err) => { warn!( - "get_managed_host_output {}, invalid timestamp: {}", - machine_id + machine_id = %machine_id .map(|x| x.to_string()) .unwrap_or_else(|| "(no machine ID)".to_string()), - err + error = %err, + "managed host has an invalid timestamp" ); None } diff --git a/crates/rpc/src/errors.rs b/crates/rpc/src/errors.rs index 805cea6fc7..1128fdd665 100644 --- a/crates/rpc/src/errors.rs +++ b/crates/rpc/src/errors.rs @@ -124,7 +124,12 @@ impl From for tonic::Status { if f.len() == 2 { let handler = f[0].trim(); let location = f[1].trim().replace("at ", ""); - tracing::error!("{from} location={location} handler='{handler}'"); + tracing::error!( + error = %from, + error_location = %location, + handler, + "RPC error conversion", + ); true } else { false @@ -134,7 +139,7 @@ impl From for tonic::Status { }; if !printed { - tracing::error!("{from}"); + tracing::error!(error = %from, "RPC error conversion"); } Status::invalid_argument(from.to_string()) diff --git a/crates/rpc/src/forge_tls_client.rs b/crates/rpc/src/forge_tls_client.rs index cacd9d0d66..1b0e3f5889 100644 --- a/crates/rpc/src/forge_tls_client.rs +++ b/crates/rpc/src/forge_tls_client.rs @@ -383,9 +383,9 @@ impl<'a> ForgeTlsClient<'a> { .await .inspect_err(|err| { tracing::error!( - "error connecting client to forge api (url: {}), will retry: {}", - api_config.url, - format_error_chain(err) + url = %api_config.url, + error = %format_error_chain(err), + "Failed to connect client to Forge API; retrying", ); }) .map_err(|e| ForgeTlsClientError::Connection(format_error_chain(&e)))?; @@ -397,10 +397,10 @@ impl<'a> ForgeTlsClient<'a> { .await .inspect_err(|err| { tracing::error!( - "error connecting client to forge api (url: {}, attempts: {}): {}", - api_config.url, - api_config.retry_config.retries, - err + url = %api_config.url, + retries = api_config.retry_config.retries, + error = %err, + "Failed to connect client to Forge API after retries", ); }); @@ -688,9 +688,9 @@ impl<'a> ForgeTlsClient<'a> { .await .inspect_err(|err| { tracing::error!( - "error connecting client to forge api (url: {}), will retry: {}", - api_config.url, - format_error_chain(err) + url = %api_config.url, + error = %format_error_chain(err), + "Failed to connect client to NMX-C API; retrying", ); }) .map_err(|e| ForgeTlsClientError::Connection(format_error_chain(&e)))?; @@ -702,10 +702,10 @@ impl<'a> ForgeTlsClient<'a> { .await .inspect_err(|err| { tracing::error!( - "error connecting client to nmx-c api (url: {}, attempts: {}): {}", - api_config.url, - api_config.retry_config.retries, - err + url = %api_config.url, + retries = api_config.retry_config.retries, + error = %err, + "Failed to connect client to NMX-C API after retries", ); }); diff --git a/crates/rpc/src/model/dpu_remediation.rs b/crates/rpc/src/model/dpu_remediation.rs index 418928934e..5cb30be965 100644 --- a/crates/rpc/src/model/dpu_remediation.rs +++ b/crates/rpc/src/model/dpu_remediation.rs @@ -153,7 +153,11 @@ impl RpcTryFrom<(RevokeRemediationRequest, String)> for RevokeRemediation { "Request must contain a remediation id.", ))?; let revoked_by = value.1; - tracing::info!("Remediation: '{}' revoked by: '{}'", id, revoked_by); + tracing::info!( + remediation_id = %id, + revoked_by = %revoked_by, + "Remediation revoked", + ); Ok(Self { id }) } @@ -170,7 +174,11 @@ impl RpcTryFrom<(EnableRemediationRequest, String)> for EnableRemediation { "Request must contain a remediation id.", ))?; let enabled_by = value.1; - tracing::info!("Remediation: '{}' enabled by: '{}'", id, enabled_by); + tracing::info!( + remediation_id = %id, + enabled_by = %enabled_by, + "Remediation enabled", + ); Ok(Self { id }) } @@ -187,7 +195,11 @@ impl RpcTryFrom<(DisableRemediationRequest, String)> for DisableRemediation { "Request must contain a remediation id.", ))?; let disabled_by = value.1; - tracing::info!("Remediation: '{}' disabled by: '{}'", id, disabled_by); + tracing::info!( + remediation_id = %id, + disabled_by = %disabled_by, + "Remediation disabled", + ); Ok(Self { id }) } diff --git a/crates/scout/src/attestation.rs b/crates/scout/src/attestation.rs index 78a1eca3a0..72c1e52d20 100644 --- a/crates/scout/src/attestation.rs +++ b/crates/scout/src/attestation.rs @@ -159,10 +159,7 @@ pub(crate) fn activate_credential( let digest = ctx.activate_credential(*ak_handle, *ek_handle, cred_blob, encr_secret)?; - tracing::debug!( - "Activated credential with session key value of {:?} ", - digest.value() - ); + tracing::debug!("Activated credential"); ctx.flush_context(SessionHandle::from(ak_auth_session).into())?; ctx.flush_context(SessionHandle::from(ek_auth_session).into())?; @@ -175,14 +172,20 @@ fn detect_pcr_hash_algo(ctx: &mut Context) -> Result val, Err(err) => { - tracing::error!("Error probing hash SHA256, setting to FALSE: {}", err); + tracing::error!( + error = %err, + "Error probing hash SHA256; setting to false", + ); false } }; let is_sha384 = match probe_sample_pcr_value(ctx, HashingAlgorithm::Sha384) { Ok(val) => val, Err(err) => { - tracing::error!("Error probing hash SHA384, setting to FALSE: {}", err); + tracing::error!( + error = %err, + "Error probing hash SHA384; setting to false", + ); false } }; @@ -221,7 +224,7 @@ pub(crate) fn get_pcr_quote( // it used to be that PCR values would only be in SHA256, this can now // be in SHA384 also. We figure out which ones those are by probing them. let pcr_hash_algo = detect_pcr_hash_algo(ctx)?; - tracing::info!("Using PCR HASH {:?}", pcr_hash_algo); + tracing::info!(?pcr_hash_algo, "Using PCR hash"); let ak_auth_session_option = ctx.start_auth_session( None, @@ -279,8 +282,8 @@ pub(crate) fn get_pcr_quote( selection_list.clone(), )?; - tracing::debug!("Obtained attestation {:?}", attest); - tracing::debug!("Obtained signature {:?}", signature); + tracing::debug!(?attest, "Obtained attestation"); + tracing::debug!(?signature, "Obtained signature"); //verify_signature(ctx, &attest, &signature, &ak_handle); @@ -311,7 +314,7 @@ pub(crate) fn get_pcr_quote( ); } - tracing::debug!("Obtained pcr digests {:?}", digest_vec); + tracing::debug!(pcr_digests = ?digest_vec, "Obtained PCR digests"); Ok((attest, signature, digest_vec)) } @@ -348,15 +351,16 @@ pub(crate) fn get_tpm_eventlog() -> Option> { let output = match output_res { Ok(output) => output, Err(e) => { - tracing::error!("Could not retrieve TPM Event Log {0}", e.to_string()); + tracing::error!(error = %e, "Could not retrieve TPM event log"); return None; } }; if !output.status.success() { tracing::error!( - "Error retrieving TPM Event Log: {0}", - String::from_utf8(output.stderr).unwrap_or("".to_string()) + stderr = %String::from_utf8(output.stderr) + .unwrap_or("".to_string()), + "Error retrieving TPM event log", ); None } else { @@ -368,7 +372,10 @@ pub fn get_tpm_description(ctx: &mut Context) -> Option { let (capabilities, _more) = match ctx.get_capability(CapabilityType::TpmProperties, 0, 80) { Ok(tuple) => tuple, Err(e) => { - tracing::error!("GetTpmDescription: Could not get TPM capability data: {e}"); + tracing::error!( + error = %e, + "GetTpmDescription: Could not get TPM capability data", + ); return None; } }; @@ -423,14 +430,14 @@ pub fn get_tpm_description(ctx: &mut Context) -> Option { } } - tracing::debug!("family_indicator is {0}", spec_version); + tracing::debug!(family_indicator = %spec_version, "Read TPM family indicator"); let vendor = vendor_1.clone() + vendor_2.as_str(); - tracing::debug!("vendor is {0}", vendor); + tracing::debug!(%vendor, "Read TPM vendor"); let firmware_version = format!("0x{firmware_version_1:x}.0x{firmware_version_2:x}"); - tracing::debug!("firmware version is {0}", firmware_version); + tracing::debug!(%firmware_version, "Read TPM firmware version"); if firmware_version_1 == 0 && firmware_version_2 == 0 diff --git a/crates/scout/src/deprovision/scrabbing.rs b/crates/scout/src/deprovision/scrabbing.rs index f07e4bf74f..c6cda8149a 100644 --- a/crates/scout/src/deprovision/scrabbing.rs +++ b/crates/scout/src/deprovision/scrabbing.rs @@ -192,11 +192,12 @@ async fn get_best_lba_format(nvmename: &str) -> Result<(u8, u64), CarbideClientE // Try 512B format first (ds=9 means 2^9 = 512 bytes) if let Some((idx, lbaf)) = select_best_lba_format(&namespace_params.lbafs, 9) { tracing::info!( - "Selected FLBAS {} for {} with sector_size=512 bytes (ms={}, rp={})", - idx, - nvmename, - lbaf.ms, - lbaf.rp + flbas_index = idx, + device = nvmename, + sector_size_bytes = 512u64, + metadata_size = lbaf.ms, + relative_performance = lbaf.rp, + "Selected NVMe LBA format", ); return Ok((idx as u8, 512u64)); } @@ -204,26 +205,27 @@ async fn get_best_lba_format(nvmename: &str) -> Result<(u8, u64), CarbideClientE // Try 4K format (ds=12 means 2^12 = 4096 bytes) if let Some((idx, lbaf)) = select_best_lba_format(&namespace_params.lbafs, 12) { tracing::info!( - "Selected FLBAS {} for {} with sector_size=4096 bytes (ms={}, rp={})", - idx, - nvmename, - lbaf.ms, - lbaf.rp + flbas_index = idx, + device = nvmename, + sector_size_bytes = 4096u64, + metadata_size = lbaf.ms, + relative_performance = lbaf.rp, + "Selected NVMe LBA format", ); return Ok((idx as u8, 4096u64)); } // Fallback to FLBAS 0 - determine its actual sector size tracing::warn!( - "No 512B or 4K LBA format found for {}, falling back to FLBAS 0", - nvmename + device = nvmename, + "No 512B or 4K NVMe LBA format found; falling back to FLBAS 0", ); if let Some(lbaf0) = namespace_params.lbafs.first() { let sector_size = 1u64 << lbaf0.ds; tracing::warn!( - "FLBAS 0 has sector_size={} bytes (ds={})", - sector_size, - lbaf0.ds + sector_size_bytes = sector_size, + data_size_exponent = lbaf0.ds, + "NVMe FLBAS 0 sector size", ); Ok((0u8, sector_size)) } else { @@ -234,22 +236,22 @@ async fn get_best_lba_format(nvmename: &str) -> Result<(u8, u64), CarbideClientE } async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { - tracing::debug!("cleaning {}", nvmename); + tracing::debug!(device = %nvmename, "Cleaning NVMe device"); let nvme_drive_params = get_nvme_params(nvmename).await?; let namespaces_supported = nvme_drive_params.oacs & 0x8 == 0x8; tracing::debug!( - "nvme: device={} size={} cntlid={} oacs={} namespaces_supported={} sn={} mn={} fr={}", - nvmename, - nvme_drive_params.tnvmcap, - nvme_drive_params.cntlid, - nvme_drive_params.oacs, + device = %nvmename, + total_capacity = nvme_drive_params.tnvmcap, + controller_id = nvme_drive_params.cntlid, + optional_admin_commands = nvme_drive_params.oacs, namespaces_supported, - nvme_drive_params.sn, - nvme_drive_params.mn, - nvme_drive_params.fr + serial_number = %nvme_drive_params.sn, + model_number = %nvme_drive_params.mn, + firmware_revision = %nvme_drive_params.fr, + "Read NVMe device parameters", ); if nvme_drive_params.mn.trim() == "M.2 NVMe 2-Bay RAID Kit" { @@ -264,7 +266,7 @@ async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { )) })?; - tracing::info!("Using Lenovo mnv_cli at {}", lenovo_mnv_cli_prog); + tracing::info!(program = lenovo_mnv_cli_prog, "Using Lenovo mnv_cli",); let vd_out = cmdrun::run_prog(lenovo_mnv_cli_prog, ["info", "-o", "vd", "-i", "0"]).await?; @@ -322,7 +324,7 @@ async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { None => continue, }; let nsid = caps.get(1).map_or("", |m| m.as_str()); - tracing::debug!("namespace {}", nsid); + tracing::debug!(namespace_id = nsid, "Found NVMe namespace"); // format with "-s2" is secure erase match cmdrun::run_prog(NVME_CLI_PROG, ["format", nvmename, "-s2", "-f", "-n", nsid]) @@ -332,7 +334,7 @@ async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { Err(e) => { if namespaces_supported { // format can fail if there is a wrong params for namespace. We delete it anyway. - tracing::debug!("nvme format error: {}", e); + tracing::debug!(error = %e, "NVMe format error"); } else { return Err(e); } @@ -350,11 +352,11 @@ async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { let flbas_str = flbas_index.to_string(); tracing::debug!( - "Creating namespace on {} with flbas={}, sector_size={}, sectors={}", - nvmename, + device = %nvmename, flbas_index, - sector_size, - sectors + sector_size_bytes = sector_size, + sectors, + "Creating NVMe namespace", ); let line_created_ns_id = cmdrun::run_prog( @@ -393,7 +395,7 @@ async fn clean_this_nvme(nvmename: &String) -> Result<(), CarbideClientError> { .await?; } } - tracing::debug!("Cleanup completed for nvme device {}", nvmename); + tracing::debug!(device = %nvmename, "Cleanup completed for NVMe device"); Ok(()) } @@ -542,7 +544,7 @@ async fn try_ata_secure_erase(devpath: &str) -> Result<(), CarbideClientError> { cmdrun::run_prog(HDPARM_CLI_PROG, ["--security-set-pass", "p", devpath]).await?; - tracing::info!("Using enhanced erase for {}", devpath); + tracing::info!(device = devpath, "Using enhanced erase"); cmdrun::run_prog(HDPARM_CLI_PROG, ["--security-erase-enhanced", "p", devpath]).await?; Ok(()) @@ -679,10 +681,16 @@ fn block_device_cleanup_skip_reason(devname: &str) -> Option Result<(), CarbideClientError> { let devname = devpath.trim_start_matches("/dev/"); if is_sata_device(devname) { - tracing::info!("{} detected as SATA, using ATA Secure Erase", devpath); + tracing::info!( + device = devpath, + "Detected SATA device; using ATA Secure Erase", + ); try_ata_secure_erase(devpath).await } else { - tracing::info!("{} detected as SAS/SCSI, using SCSI Sanitize", devpath); + tracing::info!( + device = devpath, + "Detected SAS/SCSI device; using SCSI Sanitize", + ); try_scsi_sanitize(devpath).await } } @@ -919,12 +927,16 @@ async fn set_ib_link_up() -> Result<(), CarbideClientError> { { Ok(_) => { tracing::info!( - "set KEEP_IB_LINK_UP_P1=1 on IB device {} successfully.", - slot + device = %slot, + "Set KEEP_IB_LINK_UP_P1=1 on IB device successfully", ); } Err(e) => { - tracing::error!("{}", e); + tracing::error!( + device = %slot, + error = %e, + "Failed to set KEEP_IB_LINK_UP_P1=1 on IB device", + ); return Err(e); } } @@ -937,16 +949,16 @@ async fn set_ib_link_up() -> Result<(), CarbideClientError> { { Ok(_) => { tracing::info!( - "set KEEP_IB_LINK_UP_P2=1 on IB device {} successfully.", - slot + device = %slot, + "Set KEEP_IB_LINK_UP_P2=1 on IB device successfully", ); } Err(e) => { // P2 may not exist on single-port devices, ignore error tracing::debug!( - "KEEP_IB_LINK_UP_P2 not available on IB device {} (single-port): {}", - slot, - e + device = %slot, + error = %e, + "KEEP_IB_LINK_UP_P2 not available on IB device (single-port)", ); } } @@ -954,7 +966,7 @@ async fn set_ib_link_up() -> Result<(), CarbideClientError> { } } Err(e) => { - tracing::error!("{}", e); + tracing::error!(error = %e, "Failed to discover IB devices"); return Err(CarbideClientError::GenericError(format!( "Failed to get ibs: {e}" ))); @@ -976,10 +988,14 @@ async fn reset_ib_devices() -> Result<(), CarbideClientError> { let slot = p.slot.unwrap(); match cmdrun::run_prog("mlxconfig", ["-y", "-d", &slot, "reset"]).await { Ok(_) => { - tracing::info!("reset IB device {} successfully.", slot); + tracing::info!(device = %slot, "Reset IB device successfully"); } Err(e) => { - tracing::error!("{}", e); + tracing::error!( + device = %slot, + error = %e, + "Failed to reset IB device", + ); return Err(e); } } @@ -987,7 +1003,7 @@ async fn reset_ib_devices() -> Result<(), CarbideClientError> { } } Err(e) => { - tracing::error!("{}", e); + tracing::error!(error = %e, "Failed to discover IB devices"); return Err(CarbideClientError::GenericError(format!( "Failed to get ibs: {e}" ))); @@ -1025,7 +1041,7 @@ async fn do_cleanup(machine_id: &MachineId) -> CarbideClientResult { - tracing::error!("{}", e); + tracing::error!(error = %e, "NVMe cleanup failed"); cleanup_result.nvme = Some(rpc::machine_cleanup_info::CleanupStepResult { result: rpc::machine_cleanup_info::CleanupResult::Error as _, message: e.to_string(), @@ -1042,7 +1058,7 @@ async fn do_cleanup(machine_id: &MachineId) -> CarbideClientResult { - tracing::error!("{}", e); + tracing::error!(error = %e, "HDD/SAS cleanup failed"); cleanup_result.hdd = Some(rpc::machine_cleanup_info::CleanupStepResult { result: rpc::machine_cleanup_info::CleanupResult::Error as _, message: e.to_string(), @@ -1051,7 +1067,10 @@ async fn do_cleanup(machine_id: &MachineId) -> CarbideClientResult CarbideClientResult { - tracing::error!("{}", e); + tracing::error!(error = %e, "Memory overwrite check failed"); cleanup_result.mem_overwrite = Some(rpc::machine_cleanup_info::CleanupStepResult { result: rpc::machine_cleanup_info::CleanupResult::Error as _, message: e.to_string(), @@ -1099,7 +1118,7 @@ async fn do_cleanup(machine_id: &MachineId) -> CarbideClientResult { - tracing::error!("{}", e); + tracing::error!(error = %e, "IB device reset failed"); cleanup_result.ib = Some(rpc::machine_cleanup_info::CleanupStepResult { result: rpc::machine_cleanup_info::CleanupResult::Error as _, message: e.to_string(), diff --git a/crates/scout/src/firmware_upgrade.rs b/crates/scout/src/firmware_upgrade.rs index d1ab3e7de4..a3de6fe42c 100644 --- a/crates/scout/src/firmware_upgrade.rs +++ b/crates/scout/src/firmware_upgrade.rs @@ -62,9 +62,9 @@ async fn run_firmware_upgrade( task: &FirmwareUpgradeTask, ) -> Result> { tracing::info!( - "[firmware_upgrade] starting for component={} version={}", - task.component_type, - task.target_version, + component_type = %task.component_type, + target_version = %task.target_version, + "[firmware_upgrade] starting", ); let work_dir = tempfile::tempdir()?; @@ -96,8 +96,8 @@ async fn run_firmware_upgrade( .into()); } tracing::info!( - "[firmware_upgrade] script downloaded and verified: {:?}", - script_path + script_path = ?script_path, + "[firmware_upgrade] script downloaded and verified", ); // Download file artifacts and verify checksums. @@ -124,13 +124,16 @@ async fn run_firmware_upgrade( ) .into()); } - tracing::info!("[firmware_upgrade] checksum verified for {}", artifact.url); + tracing::info!( + artifact_url = %artifact.url, + "[firmware_upgrade] checksum verified", + ); downloaded_artifacts.push(dest); } tracing::info!( - "[firmware_upgrade] files downloaded. Executing script {:?}", - script_path, + script_path = ?script_path, + "[firmware_upgrade] files downloaded; executing script", ); // Execute the script with env vars for context. @@ -204,8 +207,11 @@ async fn download_file_with_retries( .on_retry(|attempt, next_delay, error| { let delay = next_delay.unwrap_or_default(); tracing::warn!( - "[firmware_upgrade] download attempt {attempt} failed for {}: {error}, retrying in {delay:?}", - url_owned + attempt, + url = %url_owned, + error = %error, + retry_delay = ?delay, + "[firmware_upgrade] download attempt failed; retrying", ); std::future::ready(()) }) @@ -233,7 +239,11 @@ async fn download_file( let dest = target_dir.join(filename); - tracing::info!("[firmware_upgrade] downloading {url} -> {dest:?}"); + tracing::info!( + url, + destination = ?dest, + "[firmware_upgrade] downloading", + ); let response = client.get(url).send().await?.error_for_status()?; let mut stream = response.bytes_stream(); diff --git a/crates/scout/src/main.rs b/crates/scout/src/main.rs index 10c334e89e..65573e2362 100644 --- a/crates/scout/src/main.rs +++ b/crates/scout/src/main.rs @@ -98,7 +98,11 @@ async fn main() -> Result<(), eyre::Report> { carbide_host_support::init_logging("nico-scout")?; - tracing::info!("Running as {}...{}", config.mode, config.version); + tracing::info!( + mode = %config.mode, + version_requested = config.version, + "Running scout", + ); match config.mode { Mode::Service => run_as_service(&config).await?, @@ -129,12 +133,12 @@ async fn initial_setup(config: &Options) -> Result<(uuid::Uuid, MachineId), eyre .custom_backoff(|_attempt, error: &CarbideClientError| { // we only want to retry if attestation has failed. In all other cases // just preserve the old behaviour by breaking from the retry loop - tracing::error!("Failed to register machine with error {}", error); + tracing::error!(error = %error, "Failed to register machine"); if !error.to_string().contains("Attestation failed") { tracing::info!("Not retrying registration as it is not an attestation error"); RetryPolicy::Break } else { - tracing::info!("Retrying registration again in {} seconds", retry.secs); + tracing::info!(retry_delay_seconds = retry.secs, "Retrying registration",); RetryPolicy::Delay(Duration::from_secs(retry.secs)) } }) @@ -185,17 +189,26 @@ async fn run_as_service(config: &Options) -> Result<(), eyre::Report> { // initial_setup (and after registration is complete). match mlx_device::create_device_report_request(machine_id) { Ok(request) => match mlx_device::publish_mlx_device_report(config, request).await { - Ok(response) => tracing::info!("recevied PublishMlxDeviceReportResponse: {response:?}"), - Err(e) => tracing::warn!("failed to publish PublishMlxDeviceReportRequest: {e:?}"), + Ok(response) => tracing::info!(?response, "received PublishMlxDeviceReportResponse",), + Err(e) => tracing::warn!( + error = ?e, + "failed to publish PublishMlxDeviceReportRequest", + ), }, - Err(e) => tracing::warn!("failed to create PublishMlxDeviceReportRequest: {e:?}"), + Err(e) => tracing::warn!( + error = ?e, + "failed to create PublishMlxDeviceReportRequest", + ), }; let mut scout_stream_started = false; loop { if is_time_to_check_certs_expiry(next_certs_check_time) { next_certs_check_time = get_next_certs_check_datetime()?; - tracing::info!("Renewed next certs check time to {}", next_certs_check_time); + tracing::info!( + %next_certs_check_time, + "Renewed next certificate check time", + ); if check_certs_validity(&client_cert)? { initial_setup(config).await?; @@ -211,8 +224,12 @@ async fn run_as_service(config: &Options) -> Result<(), eyre::Report> { if let Some(action) = controller_response.action { let action_str = action.as_str_name().to_owned(); match handle_action(action, &machine_id, machine_interface_id, config).await { - Ok(_) => tracing::info!("Successfully served {}", action_str), - Err(e) => tracing::info!("Failed to serve {}: Err {}", action_str, e), + Ok(_) => tracing::info!(action = %action_str, "Successfully served action"), + Err(e) => tracing::info!( + action = %action_str, + error = %e, + "Failed to serve action", + ), }; } else { tracing::warn!("API response did not contain an action, skipping."); @@ -350,7 +367,10 @@ async fn handle_action( fac::Action::Noop(_) => {} fac::Action::LogError(_) => match logerror_to_carbide(config, machine_interface_id).await { Ok(()) => (), - Err(e) => tracing::info!("Forge Scout logerror_to_carbide error: {}", e), + Err(e) => tracing::info!( + error = %e, + "Failed to report Scout error to Carbide", + ), }, fac::Action::Retry(_) => { panic!( @@ -424,19 +444,19 @@ async fn handle_firmware_upgrade_action( })?; tracing::info!( - "[firmware_upgrade] received upgrade task for component={} version={}", - task.component_type, - task.target_version, + component_type = %task.component_type, + target_version = %task.target_version, + "[firmware_upgrade] received upgrade task", ); let result = firmware_upgrade::handle_firmware_upgrade(&http_client, &task).await; tracing::info!( - "[firmware_upgrade] upgrade finished: success={} component={} version={} exit_code={}", - result.success, - task.component_type, - task.target_version, - result.exit_code, + success = result.success, + component_type = %task.component_type, + target_version = %task.target_version, + exit_code = result.exit_code, + "[firmware_upgrade] upgrade finished", ); report_firmware_upgrade_status(config, machine_id, task.upgrade_task_id, &result).await?; @@ -510,8 +530,9 @@ async fn handle_mlxreport_action( Ok(command) => Some((device_action.pci_name.clone(), command)), Err(e) => { tracing::error!( - "handle_mlxreport_action error decoding command {e} for dev: {:#?}", - device_action.pci_name + error = %e, + pci_name = %device_action.pci_name, + "handle_mlxreport_action error decoding command", ); None } @@ -542,8 +563,9 @@ async fn handle_mlxreport_commands( Ok(d) => d, Err(s) => { tracing::error!( - "handle_mlxreport_action Error from discover_device::from_str {s} for dev: {:#?}", - dev_pci_name + error = %s, + pci_name = %dev_pci_name, + "handle_mlxreport_action error from discover_device::from_str", ); continue; } @@ -564,8 +586,9 @@ async fn handle_mlxreport_commands( } Err(e) => { tracing::info!( - "handle_mlxreport_action Error from lock_device: {e} for dev: {:#?}", - dev_pci_name + error = %e, + pci_name = %dev_pci_name, + "handle_mlxreport_action error from lock_device", ); } }, @@ -640,8 +663,9 @@ async fn handle_mlxreport_commands( } Err(e) => { tracing::info!( - "handle_mlxreport_action Error from unlock_device: {e} for dev: {:#?}", - dev_pci_name + error = %e, + pci_name = %dev_pci_name, + "handle_mlxreport_action error from unlock_device", ); } }, @@ -656,7 +680,10 @@ async fn handle_mlxreport_commands( match mlx_device::publish_mlx_observation_report(config, req).await { Ok(_resp) => (), Err(e) => { - tracing::error!("Error from publish_mlx_observation_report {e}"); + tracing::error!( + error = %e, + "Error from publish_mlx_observation_report", + ); } } } @@ -724,9 +751,9 @@ async fn query_api( query_attempt: u64, ) -> CarbideClientResult { tracing::info!( - "Sending ForgeAgentControlRequest (attempt:{}.{})", action_attempt, query_attempt, + "Sending ForgeAgentControlRequest", ); let query = rpc_forge::ForgeAgentControlRequest { machine_id: Some(*machine_id), @@ -741,10 +768,10 @@ async fn query_api( .unwrap_or_default(); tracing::info!( - "Received ForgeAgentControlResponse (attempt:{}.{}, action:{})", action_attempt, query_attempt, - action_str, + action = %action_str, + "Received ForgeAgentControlResponse", ); Ok(response) } @@ -768,7 +795,7 @@ async fn query_api_with_retries( .on_retry(|_attempt, _next_delay, error: &CarbideClientError| { // We can't move the error, but CarbideClientError contains some results that are not clonable, so just do the format here let error = format!("{error}"); - async move { tracing::info!("ForgeAgentControlRequest failed: {error}") } + async move { tracing::info!(error = %error, "ForgeAgentControlRequest failed") } }); // State machine handler needs 1-2 cycles to update host_adminIP to leaf. @@ -825,9 +852,9 @@ fn is_time_to_check_certs_expiry(next_check_time: DateTime) -> bool { let diff = next_check_time - now; if diff < TimeDelta::minutes(2) { tracing::info!( - "Time to check certs expiry: time now is {}, certs check time is {}", - now, - next_check_time + %now, + %next_check_time, + "Time to check certificate expiry", ); return true; } @@ -876,16 +903,16 @@ fn check_certs_validity(client_cert_path: &str) -> CarbideClientResult { let diff = not_after_datetime - now; if diff < TimeDelta::days(2) { tracing::info!( - "Now timestamp is {}, NotAfter is {}, triggering certs regen", - now, - not_after_datetime + %now, + %not_after_datetime, + "Certificate expires soon; triggering regeneration", ); Ok(true) } else { tracing::info!( - "Now timestamp is {}, NotAfter is {}, NOT triggering certs regen", - now, - not_after_datetime + %now, + %not_after_datetime, + "Certificate does not expire soon; skipping regeneration", ); Ok(false) } diff --git a/crates/scout/src/mlx_device.rs b/crates/scout/src/mlx_device.rs index 0dcabec953..fc31d5a0c3 100644 --- a/crates/scout/src/mlx_device.rs +++ b/crates/scout/src/mlx_device.rs @@ -67,7 +67,7 @@ pub async fn publish_mlx_device_report( config: &Options, req: PublishMlxDeviceReportRequest, ) -> CarbideClientResult { - tracing::info!("sending PublishMlxDeviceReportRequest: {req:?}"); + tracing::info!(request = ?req, "sending PublishMlxDeviceReportRequest"); let request = tonic::Request::new(req); let mut client = client::create_forge_client(config).await?; let response = client @@ -81,7 +81,10 @@ pub async fn publish_mlx_observation_report( config: &Options, req: PublishMlxObservationReportRequest, ) -> CarbideClientResult { - tracing::info!("sending PublishMlxObservationReportRequest: {req:?}"); + tracing::info!( + request = ?req, + "sending PublishMlxObservationReportRequest", + ); let request = tonic::Request::new(req); let mut client = client::create_forge_client(config).await?; let response = client @@ -115,9 +118,9 @@ pub fn handle_profile_sync( request: mlx_device_pb::MlxDeviceProfileSyncRequest, ) -> mlx_device_pb::MlxDeviceProfileSyncResponse { tracing::info!( - "[scout_stream::mlx_device] profile sync to device requested (device_id:{}, profile_name:{})", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + "[scout_stream::mlx_device] profile sync to device requested", ); let Some(serializable_profile_pb) = request.serializable_profile else { @@ -136,7 +139,10 @@ pub fn handle_profile_sync( let serializable_profile: SerializableProfile = match serializable_profile_pb.try_into() { Ok(profile) => profile, Err(e) => { - tracing::error!("[scout_stream::mlx_device] failed to parse profile: {e}"); + tracing::error!( + error = %e, + "[scout_stream::mlx_device] failed to parse profile", + ); return mlx_device_pb::MlxDeviceProfileSyncResponse { reply: Some( mlx_device_pb::mlx_device_profile_sync_response::Reply::Error( @@ -153,9 +159,9 @@ pub fn handle_profile_sync( match load_and_sync_profile(&request.device_id, serializable_profile) { Ok(sync_result) => { tracing::info!( - "[scout_stream::mlx_device] profile sync to device successful (device_id:{}, profile_name:{})", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + "[scout_stream::mlx_device] profile sync to device successful", ); match sync_result.try_into() { @@ -168,7 +174,8 @@ pub fn handle_profile_sync( }, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] profile sync result failed to serialize: {e}" + error = %e, + "[scout_stream::mlx_device] profile sync result failed to serialize", ); mlx_device_pb::MlxDeviceProfileSyncResponse { reply: Some( @@ -186,9 +193,10 @@ pub fn handle_profile_sync( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] profile sync to device failed (device_id:{}, profile_name:{}): {e}", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + error = %e, + "[scout_stream::mlx_device] profile sync to device failed", ); mlx_device_pb::MlxDeviceProfileSyncResponse { reply: Some( @@ -208,9 +216,9 @@ pub fn handle_profile_compare( request: mlx_device_pb::MlxDeviceProfileCompareRequest, ) -> mlx_device_pb::MlxDeviceProfileCompareResponse { tracing::info!( - "[scout_stream::mlx_device] profile compare against device requested (device_id:{}, profile_name:{})", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + "[scout_stream::mlx_device] profile compare against device requested", ); let Some(serializable_profile_pb) = request.serializable_profile else { @@ -229,7 +237,10 @@ pub fn handle_profile_compare( let serializable_profile: SerializableProfile = match serializable_profile_pb.try_into() { Ok(profile) => profile, Err(e) => { - tracing::error!("[scout_stream::mlx_device] failed to parse profile: {e}"); + tracing::error!( + error = %e, + "[scout_stream::mlx_device] failed to parse profile", + ); return mlx_device_pb::MlxDeviceProfileCompareResponse { reply: Some( mlx_device_pb::mlx_device_profile_compare_response::Reply::Error( @@ -246,9 +257,9 @@ pub fn handle_profile_compare( match load_and_compare_profile(&request.device_id, serializable_profile) { Ok(comparison_result) => { tracing::info!( - "[scout_stream::mlx_device] profile compare against device successful (device_id:{}, profile_name:{})", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + "[scout_stream::mlx_device] profile compare against device successful", ); match comparison_result.try_into() { @@ -261,7 +272,8 @@ pub fn handle_profile_compare( }, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] profile compare result failed to serialize: {e}" + error = %e, + "[scout_stream::mlx_device] profile compare result failed to serialize", ); mlx_device_pb::MlxDeviceProfileCompareResponse { reply: Some( @@ -279,9 +291,10 @@ pub fn handle_profile_compare( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] profile compare against device failed (device_id:{}, profile_name:{}): {e}", - request.device_id, - request.profile_name + device_id = %request.device_id, + profile_name = %request.profile_name, + error = %e, + "[scout_stream::mlx_device] profile compare against device failed", ); mlx_device_pb::MlxDeviceProfileCompareResponse { reply: Some( @@ -302,15 +315,16 @@ pub fn handle_lockdown_lock( request: mlx_device_pb::MlxDeviceLockdownLockRequest, ) -> mlx_device_pb::MlxDeviceLockdownResponse { tracing::info!( - "[scout_stream::mlx_device] lockdown lock requested (device_id:{})", - request.device_id + device_id = %request.device_id, + "[scout_stream::mlx_device] lockdown lock requested", ); let manager = match LockdownManager::new() { Ok(m) => m, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown manager initialization failed: {e}" + error = %e, + "[scout_stream::mlx_device] lockdown manager initialization failed", ); return mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -326,8 +340,9 @@ pub fn handle_lockdown_lock( match manager.lock_device(&request.device_id, &request.key) { Ok(status) => { tracing::info!( - "[scout_stream::mlx_device] lockdown lock successful (device_id:{}, status:{status})", - request.device_id + device_id = %request.device_id, + %status, + "[scout_stream::mlx_device] lockdown lock successful", ); let report = StatusReport::new(request.device_id.clone(), status); mlx_device_pb::MlxDeviceLockdownResponse { @@ -338,8 +353,9 @@ pub fn handle_lockdown_lock( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown lock failed (device_id:{}): {e}", - request.device_id + device_id = %request.device_id, + error = %e, + "[scout_stream::mlx_device] lockdown lock failed", ); mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -358,15 +374,16 @@ pub fn handle_lockdown_unlock( request: mlx_device_pb::MlxDeviceLockdownUnlockRequest, ) -> mlx_device_pb::MlxDeviceLockdownResponse { tracing::info!( - "[scout_stream::mlx_device] lockdown unlock requested (device_id:{})", - request.device_id + device_id = %request.device_id, + "[scout_stream::mlx_device] lockdown unlock requested", ); let manager = match LockdownManager::new() { Ok(m) => m, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown manager initialization failed: {e}" + error = %e, + "[scout_stream::mlx_device] lockdown manager initialization failed", ); return mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -382,8 +399,9 @@ pub fn handle_lockdown_unlock( match manager.unlock_device(&request.device_id, &request.key) { Ok(status) => { tracing::info!( - "[scout_stream::mlx_device] lockdown unlock successful (device_id:{}, status:{status})", - request.device_id + device_id = %request.device_id, + %status, + "[scout_stream::mlx_device] lockdown unlock successful", ); let report = StatusReport::new(request.device_id.clone(), status); mlx_device_pb::MlxDeviceLockdownResponse { @@ -394,8 +412,9 @@ pub fn handle_lockdown_unlock( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown unlock failed (device_id:{}): {e}", - request.device_id + device_id = %request.device_id, + error = %e, + "[scout_stream::mlx_device] lockdown unlock failed", ); mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -414,15 +433,16 @@ pub fn handle_lockdown_status( request: mlx_device_pb::MlxDeviceLockdownStatusRequest, ) -> mlx_device_pb::MlxDeviceLockdownResponse { tracing::info!( - "[scout_stream::mlx_device] lockdown status check requested (device_id:{})", - request.device_id + device_id = %request.device_id, + "[scout_stream::mlx_device] lockdown status check requested", ); let manager = match LockdownManager::new() { Ok(m) => m, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown manager initialization failed: {e}" + error = %e, + "[scout_stream::mlx_device] lockdown manager initialization failed", ); return mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -438,8 +458,9 @@ pub fn handle_lockdown_status( match manager.get_status(&request.device_id) { Ok(status) => { tracing::info!( - "[scout_stream::mlx_device] lockdown status check successful (device_id:{}, status: {status})", - request.device_id + device_id = %request.device_id, + %status, + "[scout_stream::mlx_device] lockdown status check successful", ); let report = StatusReport::new(request.device_id.clone(), status); mlx_device_pb::MlxDeviceLockdownResponse { @@ -450,8 +471,9 @@ pub fn handle_lockdown_status( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] lockdown status check failed (device_id:{}): {e}", - request.device_id + device_id = %request.device_id, + error = %e, + "[scout_stream::mlx_device] lockdown status check failed", ); mlx_device_pb::MlxDeviceLockdownResponse { reply: Some(mlx_device_pb::mlx_device_lockdown_response::Reply::Error( @@ -469,15 +491,15 @@ pub fn handle_info_device( request: mlx_device_pb::MlxDeviceInfoDeviceRequest, ) -> mlx_device_pb::MlxDeviceInfoDeviceResponse { tracing::info!( - "[scout_stream::mlx_device] device info request (device_id:{})", - request.device_id + device_id = %request.device_id, + "[scout_stream::mlx_device] device info request", ); match discovery::discover_device(&request.device_id) { Ok(device_info) => { tracing::info!( - "[scout_stream::mlx_device] device info retrieved successfully (device_id:{})", - request.device_id + device_id = %request.device_id, + "[scout_stream::mlx_device] device info retrieved successfully", ); mlx_device_pb::MlxDeviceInfoDeviceResponse { reply: Some( @@ -489,8 +511,9 @@ pub fn handle_info_device( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] device info request failed (device_id:{}): {e}", - request.device_id + device_id = %request.device_id, + error = %e, + "[scout_stream::mlx_device] device info request failed", ); mlx_device_pb::MlxDeviceInfoDeviceResponse { reply: Some( @@ -516,7 +539,8 @@ pub fn handle_info_report( Ok(filters) => MlxDeviceReport::new().with_filter_set(filters), Err(e) => { tracing::error!( - "[scout_stream::mlx_device] device report request failed to parse filters: {e}" + error = %e, + "[scout_stream::mlx_device] device report request failed to parse filters", ); return mlx_device_pb::MlxDeviceInfoReportResponse { reply: Some( @@ -537,8 +561,8 @@ pub fn handle_info_report( match report.collect() { Ok(report) => { tracing::info!( - "[scout_stream::mlx_device] device report generated (device_count:{})", - report.devices.len() + device_count = report.devices.len(), + "[scout_stream::mlx_device] device report generated", ); mlx_device_pb::MlxDeviceInfoReportResponse { reply: Some( @@ -549,7 +573,10 @@ pub fn handle_info_report( } } Err(e) => { - tracing::error!("[scout_stream::mlx_device] device report generation failed: {e}"); + tracing::error!( + error = %e, + "[scout_stream::mlx_device] device report generation failed", + ); mlx_device_pb::MlxDeviceInfoReportResponse { reply: Some( mlx_device_pb::mlx_device_info_report_response::Reply::Error( @@ -585,16 +612,16 @@ pub fn handle_registry_show( request: mlx_device_pb::MlxDeviceRegistryShowRequest, ) -> mlx_device_pb::MlxDeviceRegistryShowResponse { tracing::info!( - "[scout_stream::mlx_device] variable registry details requested (registry_name:{})", - request.registry_name + registry_name = %request.registry_name, + "[scout_stream::mlx_device] variable registry details requested", ); match registries::get(&request.registry_name) { Some(registry) => { let registry_pb = registry.clone().into(); tracing::info!( - "[scout_stream::mlx_device] variable registry details generated (registry_name:{})", - request.registry_name + registry_name = %request.registry_name, + "[scout_stream::mlx_device] variable registry details generated", ); mlx_device_pb::MlxDeviceRegistryShowResponse { reply: Some( @@ -606,8 +633,8 @@ pub fn handle_registry_show( } None => { tracing::error!( - "[scout_stream::mlx_device] variable registry not found (registry_name:{})", - request.registry_name + registry_name = %request.registry_name, + "[scout_stream::mlx_device] variable registry not found", ); mlx_device_pb::MlxDeviceRegistryShowResponse { reply: Some( @@ -629,19 +656,19 @@ pub fn handle_config_query( request: mlx_device_pb::MlxDeviceConfigQueryRequest, ) -> mlx_device_pb::MlxDeviceConfigQueryResponse { tracing::info!( - "[scout_stream::mlx_device] config query requested (device_id:{}, registry_name:{}): {:?}", - request.device_id, - request.registry_name, - request.variables, + device_id = %request.device_id, + registry_name = %request.registry_name, + variables = ?request.variables, + "[scout_stream::mlx_device] config query requested", ); let registry = match registries::get(&request.registry_name) { Some(r) => r.clone(), None => { tracing::warn!( - "[scout_stream::mlx_device] config registry not found (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config registry not found", ); return mlx_device_pb::MlxDeviceConfigQueryResponse { reply: Some( @@ -672,9 +699,9 @@ pub fn handle_config_query( match result { Ok(query_result) => { tracing::info!( - "[scout_stream::mlx_device] config query against device successful (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config query against device successful", ); match query_result.try_into() { @@ -687,9 +714,10 @@ pub fn handle_config_query( }, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config query result failed to serialize (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config query result failed to serialize", ); mlx_device_pb::MlxDeviceConfigQueryResponse { reply: Some( @@ -710,9 +738,10 @@ pub fn handle_config_query( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config query against device failed (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config query against device failed", ); mlx_device_pb::MlxDeviceConfigQueryResponse { reply: Some( @@ -736,19 +765,19 @@ pub fn handle_config_set( request: mlx_device_pb::MlxDeviceConfigSetRequest, ) -> mlx_device_pb::MlxDeviceConfigSetResponse { tracing::info!( - "[scout_stream::mlx_device] config set assignment requested (device_id:{}, registry_name:{}): {:?}", - request.device_id, - request.registry_name, - request.assignments + device_id = %request.device_id, + registry_name = %request.registry_name, + assignments = ?request.assignments, + "[scout_stream::mlx_device] config set assignment requested", ); let registry = match registries::get(&request.registry_name) { Some(r) => r.clone(), None => { tracing::warn!( - "[scout_stream::mlx_device] config registry not found (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config registry not found", ); return mlx_device_pb::MlxDeviceConfigSetResponse { reply: Some(mlx_device_pb::mlx_device_config_set_response::Reply::Error( @@ -780,9 +809,9 @@ pub fn handle_config_set( match runner.set(assignments) { Ok(_) => { tracing::info!( - "[scout_stream::mlx_device] config set on device successfully (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config set on device successfully", ); mlx_device_pb::MlxDeviceConfigSetResponse { reply: Some( @@ -794,9 +823,10 @@ pub fn handle_config_set( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config set to device failed (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config set to device failed", ); mlx_device_pb::MlxDeviceConfigSetResponse { reply: Some(mlx_device_pb::mlx_device_config_set_response::Reply::Error( @@ -819,19 +849,19 @@ pub fn handle_config_sync( request: mlx_device_pb::MlxDeviceConfigSyncRequest, ) -> mlx_device_pb::MlxDeviceConfigSyncResponse { tracing::info!( - "[scout_stream::mlx_device] config sync requested (device_id:{}, registry_name:{}): {:?}", - request.device_id, - request.registry_name, - request.assignments + device_id = %request.device_id, + registry_name = %request.registry_name, + assignments = ?request.assignments, + "[scout_stream::mlx_device] config sync requested", ); let registry = match registries::get(&request.registry_name) { Some(r) => r.clone(), None => { tracing::warn!( - "[scout_stream::mlx_device] config registry not found (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config registry not found", ); return mlx_device_pb::MlxDeviceConfigSyncResponse { reply: Some( @@ -863,9 +893,9 @@ pub fn handle_config_sync( match runner.sync(assignments) { Ok(sync_result) => { tracing::info!( - "[scout_stream::mlx_device] config sync to device successful (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config sync to device successful", ); match sync_result.try_into() { @@ -878,9 +908,10 @@ pub fn handle_config_sync( }, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config sync result failed to serialize (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config sync result failed to serialize", ); mlx_device_pb::MlxDeviceConfigSyncResponse { reply: Some( @@ -901,9 +932,10 @@ pub fn handle_config_sync( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config sync to device failed (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config sync to device failed", ); mlx_device_pb::MlxDeviceConfigSyncResponse { reply: Some( @@ -928,19 +960,19 @@ pub fn handle_config_compare( request: mlx_device_pb::MlxDeviceConfigCompareRequest, ) -> mlx_device_pb::MlxDeviceConfigCompareResponse { tracing::info!( - "[scout_stream::mlx_device] config compare requested (device_id:{}, registry_name:{}): {:?}", - request.device_id, - request.registry_name, - request.assignments + device_id = %request.device_id, + registry_name = %request.registry_name, + assignments = ?request.assignments, + "[scout_stream::mlx_device] config compare requested", ); let registry = match registries::get(&request.registry_name) { Some(r) => r.clone(), None => { tracing::warn!( - "[scout_stream::mlx_device] config registry not found (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config registry not found", ); return mlx_device_pb::MlxDeviceConfigCompareResponse { reply: Some( @@ -972,9 +1004,9 @@ pub fn handle_config_compare( match runner.compare(assignments) { Ok(comparison_result) => { tracing::info!( - "[scout_stream::mlx_device] config compare against device successful (device_id:{}, registry_name:{})", - request.device_id, - request.registry_name, + device_id = %request.device_id, + registry_name = %request.registry_name, + "[scout_stream::mlx_device] config compare against device successful", ); match comparison_result.try_into() { @@ -987,9 +1019,10 @@ pub fn handle_config_compare( }, Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config compare result failed to serialize (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config compare result failed to serialize", ); mlx_device_pb::MlxDeviceConfigCompareResponse { reply: Some( @@ -1010,9 +1043,10 @@ pub fn handle_config_compare( } Err(e) => { tracing::error!( - "[scout_stream::mlx_device] config compare against device failed (device_id:{}, registry_name:{}): {e}", - request.device_id, - request.registry_name + device_id = %request.device_id, + registry_name = %request.registry_name, + error = %e, + "[scout_stream::mlx_device] config compare against device failed", ); mlx_device_pb::MlxDeviceConfigCompareResponse { reply: Some( diff --git a/crates/scout/src/register.rs b/crates/scout/src/register.rs index ea2aefe5b6..da3662c49f 100644 --- a/crates/scout/src/register.rs +++ b/crates/scout/src/register.rs @@ -106,7 +106,11 @@ pub async fn run( ) .await?; let machine_id = registration_data.machine_id; - info!("successfully discovered machine {machine_id} for interface {machine_interface_id:?}"); + info!( + %machine_id, + ?machine_interface_id, + "successfully discovered machine", + ); // If we are not on a DPU and have some post-registration things to do, // we do them here. @@ -124,9 +128,9 @@ pub async fn run( "Sent AttestKeyInfo and received AttestKeyBindChallenge, starting measurements ..." ); tracing::info!( - "cred_blob - {} bytes long, secret - {} bytes long", - attest_key_challenge.cred_blob.len(), - attest_key_challenge.encrypted_secret.len() + credential_blob_bytes = attest_key_challenge.cred_blob.len(), + encrypted_secret_bytes = attest_key_challenge.encrypted_secret.len(), + "Received attestation key challenge credential sizes", ); let Some(ek_handle) = endorsement_key_handle_opt else { diff --git a/crates/scout/src/stream.rs b/crates/scout/src/stream.rs index a2d7a4307b..b68e449f29 100644 --- a/crates/scout/src/stream.rs +++ b/crates/scout/src/stream.rs @@ -53,27 +53,32 @@ pub fn start_scout_stream(machine_id: MachineId, options: &Options) -> tokio::ta tokio::spawn(async move { loop { tracing::info!( - "scout stream starting (api:{}, machine_id:{machine_id})", - options.api + api_endpoint = %options.api, + %machine_id, + "scout stream starting", ); match run_scout_stream_loop(machine_id, &options).await { Ok(_) => { tracing::info!( - "scout stream closed (api:{}, machine_id:{machine_id})", - options.api + api_endpoint = %options.api, + %machine_id, + "scout stream closed", ); } Err(e) => { tracing::error!( - "scout stream error (api:{}, machine_id:{machine_id}): {e}", - options.api + api_endpoint = %options.api, + %machine_id, + error = %e, + "scout stream error", ); } } tracing::warn!( - "scout stream reconnecting (api:{}, machine_id:{machine_id}): 10s delay", - options.api + api_endpoint = %options.api, + %machine_id, + "scout stream reconnecting after 10s delay", ); tokio::time::sleep(Duration::from_secs(10)).await; } @@ -113,8 +118,9 @@ async fn run_scout_stream_loop( let mut response_stream = client.scout_stream(request_stream).await?.into_inner(); tracing::info!( - "scout stream connection established (api:{}, machine_id:{machine_id})", - options.api + api_endpoint = %options.api, + %machine_id, + "scout stream connection established", ); // ...and start processing streaming updates. @@ -137,8 +143,10 @@ async fn run_scout_stream_loop( // And then send the response back to carbide-api. if let Err(e) = tx.send(payload).await { tracing::error!( - "scout stream failed to send response (api:{}, machine_id:{machine_id}): {e}", - options.api + api_endpoint = %options.api, + %machine_id, + error = %e, + "scout stream failed to send response", ); break; } @@ -156,8 +164,8 @@ fn handle_scout_stream_api_bound_message( request: scout_stream_scout_bound_message::Payload, ) -> ScoutStreamApiBoundMessage { tracing::info!( - "[scout_stream] processing incoming request for flow_uuid: {}", - flow_uuid + %flow_uuid, + "[scout_stream] processing incoming request", ); match request { scout_stream_scout_bound_message::Payload::ScoutStreamAgentPingRequest(req) => { diff --git a/crates/scout/src/tpm.rs b/crates/scout/src/tpm.rs index 01dbe10b1f..4ddd6c0eec 100644 --- a/crates/scout/src/tpm.rs +++ b/crates/scout/src/tpm.rs @@ -38,24 +38,26 @@ pub(crate) fn set_tpm_max_auth_fail() -> Result<(), CarbideClientError> { CarbideClientError::TpmError(format!("tpm2_dictionarylockout call failed: {e}")) })?; tracing::info!( - "Tried setting TPM_PT_MAX_AUTH_FAIL to 256. Return code is: {0}", - output + return_code = %output .status .code() .map(|v| v.to_string()) - .unwrap_or_else(|| "NO RETURN CODE PRESENT".to_string()) + .unwrap_or_else(|| "NO RETURN CODE PRESENT".to_string()), + "Tried setting TPM_PT_MAX_AUTH_FAIL to 256", ); if !output.stderr.is_empty() { tracing::error!( - "TPM_PT_MAX_AUTH_FAIL stderr is {0}", - String::from_utf8(output.stderr).unwrap_or_else(|_| "Invalid UTF8".to_string()) + stderr = %String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Invalid UTF8".to_string()), + "TPM_PT_MAX_AUTH_FAIL command wrote to stderr", ); } if !output.stdout.is_empty() { tracing::info!( - "TPM_PT_MAX_AUTH_FAIL stdout is {0}", - String::from_utf8(output.stdout).unwrap_or_else(|_| "Invalid UTF8".to_string()) + stdout = %String::from_utf8(output.stdout) + .unwrap_or_else(|_| "Invalid UTF8".to_string()), + "TPM_PT_MAX_AUTH_FAIL command wrote to stdout", ); } diff --git a/crates/secrets/src/forge_vault.rs b/crates/secrets/src/forge_vault.rs index 4c9a4f2086..c429511835 100644 --- a/crates/secrets/src/forge_vault.rs +++ b/crates/secrets/src/forge_vault.rs @@ -91,14 +91,15 @@ fn resolve_vault_root_ca_path(configured_path: &str) -> Result Ok(env_path), Ok(env_path) => { tracing::error!( - "VAULT_CACERT={env_path} does not exist. Refusing to connect without TLS verification." + %env_path, + "VAULT_CACERT does not exist. Refusing to connect without TLS verification.", ); Err(eyre!("Vault root CA not found")) } Err(_) => { tracing::error!( - "Vault root CA not found at {}. Refusing to connect without TLS verification.", - configured_path + configured_path, + "Vault root CA not found. Refusing to connect without TLS verification.", ); Err(eyre!("Vault root CA not found")) } @@ -340,7 +341,10 @@ async fn vault_token_refresh( } }; - tracing::info!("successfully refreshed vault token, with lifetime: {vault_token_expiry_secs}"); + tracing::info!( + vault_token_expiry_secs, + "successfully refreshed vault token" + ); let vault_client_settings = create_vault_client_settings(vault_token, vault_client_config)?; let vault_client = VaultClient::new(vault_client_settings)?; @@ -517,15 +521,16 @@ impl VaultTask> for GetCredentialsHelper<'_, '_> { Some(404) => { // Not found errors are common and of no concern tracing::debug!( - "Credentials not found for key ({})", - self.key.to_key_str().as_ref() + credential_key = %self.key.to_key_str(), + "Credentials not found", ); Ok(None) } _ => { tracing::error!( - "Error getting credentials ({}). Error: {ce:?}", - self.key.to_key_str().as_ref() + credential_key = %self.key.to_key_str(), + error = ?ce, + "Error getting credentials", ); Err(SecretsError::GenericError(ce.into())) } @@ -601,7 +606,7 @@ impl VaultTask<()> for SetCredentialsHelper<'_, '_> { let _secret_version_metadata = vault_response.map_err(|err| { record_vault_client_error(&err, VaultRequestType::SetCredentials); - tracing::error!("Error setting credentials. Error: {err:?}"); + tracing::error!(error = ?err, "Error setting credentials"); err })?; @@ -640,7 +645,7 @@ impl VaultTask<()> for DeleteCredentialsHelper<'_, '_> { let _secret_version_metadata = vault_response.map_err(|err| { record_vault_client_error(&err, VaultRequestType::DeleteCredentials); - tracing::error!("Error deleting credentials. Error: {err:?}"); + tracing::error!(error = ?err, "Error deleting credentials"); err })?; diff --git a/crates/secrets/src/local_credentials/file.rs b/crates/secrets/src/local_credentials/file.rs index 553e799461..bf50adb384 100644 --- a/crates/secrets/src/local_credentials/file.rs +++ b/crates/secrets/src/local_credentials/file.rs @@ -83,12 +83,15 @@ impl FileCredentialsWatcher { move |res: notify::Result| match res { Ok(ref event) if event.kind.is_create() || event.kind.is_modify() => { if let Err(err) = tx_clone.blocking_send(res) { - tracing::warn!("failed to send static credential watch event: {err}"); + tracing::warn!( + error = %err, + "failed to send static credential watch event", + ); } } Ok(_) => {} Err(err) => { - tracing::warn!("primary static credential watcher error: {err}"); + tracing::warn!(error = %err, "primary static credential watcher error"); } }, notify::Config::default(), @@ -102,7 +105,10 @@ impl FileCredentialsWatcher { let mut secondary = PollWatcher::new( move |res| { if let Err(err) = tx.blocking_send(res) { - tracing::warn!("failed to send static credential poll event: {err}"); + tracing::warn!( + error = %err, + "failed to send static credential poll event", + ); } }, notify::Config::default() @@ -134,12 +140,18 @@ impl FileCredentialsWatcher { credentials_clone.store(Arc::new(updated)); } Err(err) => { - tracing::warn!("failed to reload credentials file: {err}"); + tracing::warn!( + error = %err, + "failed to reload credentials file", + ); } } } Err(err) => { - tracing::warn!("credentials file watcher event error: {err}"); + tracing::warn!( + error = %err, + "credentials file watcher event error", + ); } } } diff --git a/crates/site-explorer/src/bmc_endpoint_explorer.rs b/crates/site-explorer/src/bmc_endpoint_explorer.rs index efb0ddde0c..d7c2015096 100644 --- a/crates/site-explorer/src/bmc_endpoint_explorer.rs +++ b/crates/site-explorer/src/bmc_endpoint_explorer.rs @@ -302,12 +302,14 @@ impl BmcEndpointExplorer { (Ok(report), Ok(nv_report)) => warn_report_diff(report, nv_report), (Ok(_), Err(_)) => { tracing::warn!( - "libredfish returned success when nv-redfish error: {nvredfish:?}" + nvredfish = ?nvredfish, + "libredfish succeeded while nv-redfish returned an error" ); } (Err(_), Ok(_)) => { tracing::warn!( - "libredfish returned error: {libredfish:?}, when nv-redfish success" + libredfish = ?libredfish, + "libredfish returned an error while nv-redfish succeeded" ); } (Err(_), Err(_)) => (), @@ -445,7 +447,7 @@ impl BmcEndpointExplorer { ) { tracing::info!( %bmc_mac_address, - "Storing NVOS admin credentials in vault for switch {bmc_mac_address}" + "Storing NVOS admin credentials in vault" ); self.credential_client .set_bmc_nvos_admin_credentials( @@ -653,7 +655,11 @@ impl EndpointExplorer for BmcEndpointExplorer { let vendor = match self.redfish_client.get_redfish_vendor(bmc_ip_address).await { Ok(vendor) => vendor, Err(e) => { - tracing::error!(%bmc_ip_address, "Failed to probe Redfish service root endpoint: {e}"); + tracing::error!( + %bmc_ip_address, + error = %e, + "Failed to probe Redfish service root endpoint" + ); // Lite-On power shelf BMCs don't expose Vendor details in the // service root, so we fall back to probing the Chassis endpoint. @@ -684,7 +690,11 @@ impl EndpointExplorer for BmcEndpointExplorer { { Ok(v) => v, Err(chassis_err) => { - tracing::error!(%bmc_ip_address, "Failed to probe vendor from chassis: {chassis_err}"); + tracing::error!( + %bmc_ip_address, + error = %chassis_err, + "Failed to probe vendor from chassis" + ); return Err(e); } }; @@ -699,7 +709,11 @@ impl EndpointExplorer for BmcEndpointExplorer { } }; - tracing::info!(%bmc_ip_address, "Is a {vendor} BMC that supports Redfish"); + tracing::info!( + %bmc_ip_address, + %vendor, + "BMC supports Redfish" + ); // Authenticate and set the BMC root account credentials @@ -773,12 +787,18 @@ impl EndpointExplorer for BmcEndpointExplorer { tracing::info!( %bmc_ip_address, - "Site explorer could not find an entry in vault at 'bmc/{bmc_mac_address}/root' - this is expected if the BMC has never been seen before.", + %bmc_mac_address, + "Site explorer could not find a BMC root credential entry in vault - this is expected if the BMC has never been seen before.", ); let bmc_cred_data = match expected { Some(v) => { - tracing::info!(%bmc_ip_address, %bmc_mac_address, "Found an expected {} for this BMC mac address", v.name()); + tracing::info!( + %bmc_ip_address, + %bmc_mac_address, + expected_entity = v.name(), + "Found an expected entity" + ); v.bmc_credentials_data() } None => { @@ -863,13 +883,15 @@ impl EndpointExplorer for BmcEndpointExplorer { Ok(_) => { tracing::trace!( %bmc_ip_address, %bmc_mac_address, - "NVOS admin credentials already exist in vault for switch {bmc_mac_address}" + "NVOS admin credentials already exist in vault" ); } - Err(_) => { + Err(e) => { tracing::info!( - %bmc_ip_address, %bmc_mac_address, - "Site explorer could not find NVOS admin credentials in vault for switch {bmc_mac_address} - setting them up.", + %bmc_ip_address, + %bmc_mac_address, + error = %e, + "Failed to load NVOS admin credentials; attempting credential setup", ); self.set_sitewide_switch_nvos_admin_credentials( bmc_mac_address, @@ -896,8 +918,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "Site explorer does not support resetting the BMCs that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for BMC reset", ); Err(e) } @@ -935,8 +958,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "Site explorer cannot fetch live power state for an endpoint without credentials: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for fetching live power state", ); Err(e) } @@ -959,8 +983,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "Site explorer does not support rebooting the endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for power control", ); Err(e) } @@ -979,8 +1004,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support disabling secure boot for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for disabling secure boot", ); Err(e) } @@ -1000,8 +1026,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support lockdown for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for changing lockdown state", ); Err(e) } @@ -1020,8 +1047,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support lockdown status for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for checking lockdown status", ); Err(e) } @@ -1040,8 +1068,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support enabling infinite boot for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for enabling infinite boot", ); Err(e) } @@ -1063,8 +1092,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support checking infinite boot status for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for checking infinite boot status", ); Err(e) } @@ -1087,8 +1117,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support starting machine_setup for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for machine setup", ); Err(e) } @@ -1111,8 +1142,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support configuring the boot order on host BMCs that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for configuring boot order", ); Err(e) } @@ -1132,8 +1164,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support set_nic_mode for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for setting NIC mode", ); Err(e) } @@ -1152,8 +1185,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support is_viking for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for checking BMC hardware type", ); Err(e) } @@ -1172,8 +1206,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support clear_nvram for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for clearing NVRAM", ); Err(e) } @@ -1198,8 +1233,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support create_bmc_user for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for creating BMC user", ); Err(e) } @@ -1222,8 +1258,9 @@ impl EndpointExplorer for BmcEndpointExplorer { Err(e) => { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support delete_bmc_user for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for deleting BMC user", ); Err(e) } @@ -1238,12 +1275,15 @@ impl EndpointExplorer for BmcEndpointExplorer { ) -> Result<(), EndpointExplorationError> { let bmc_mac_address = interface.mac_address; - let current_credentials = - self.get_bmc_root_credentials(bmc_mac_address).await.inspect_err(|_| { + let current_credentials = self + .get_bmc_root_credentials(bmc_mac_address) + .await + .inspect_err(|e| { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support set_bmc_root_password for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for setting BMC root password", ); })?; @@ -1281,12 +1321,15 @@ impl EndpointExplorer for BmcEndpointExplorer { ) -> Result { let bmc_mac_address = interface.mac_address; - let credentials = - self.get_bmc_root_credentials(bmc_mac_address).await.inspect_err(|_| { + let credentials = self + .get_bmc_root_credentials(bmc_mac_address) + .await + .inspect_err(|e| { tracing::info!( %bmc_ip_address, - "BMC endpoint explorer does not support probe_bmc_vendor for endpoints that have not been authenticated: could not find an entry in vault at 'bmc/{}/root'.", - bmc_mac_address, + %bmc_mac_address, + error = %e, + "Failed to load BMC root credentials for probing BMC vendor", ); })?; @@ -1301,98 +1344,102 @@ impl EndpointExplorer for BmcEndpointExplorer { fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplorationReport) { if report1.endpoint_type != report2.endpoint_type { tracing::warn!( - "endpoint_type are not equal: {:?} != {:?}", - report1.endpoint_type, - report2.endpoint_type + libredfish_endpoint_type = ?report1.endpoint_type, + nvredfish_endpoint_type = ?report2.endpoint_type, + "endpoint types are not equal" ); } if report1.vendor != report2.vendor { tracing::warn!( - "vendors are not equal: {:?} != {:?}", - report1.vendor, - report2.vendor + libredfish_vendor = ?report1.vendor, + nvredfish_vendor = ?report2.vendor, + "vendors are not equal" ); } if report1.managers != report2.managers { tracing::warn!( - "managers are not equal: {:?} != {:?}", - report1.managers, - report2.managers + libredfish_managers = ?report1.managers, + nvredfish_managers = ?report2.managers, + "managers are not equal" ); } if report1.systems.len() != report2.systems.len() { tracing::warn!( - "reported different number of systems: {:?} != {:?}", - report1.systems.len(), - report2.systems.len(), + libredfish_system_count = report1.systems.len(), + nvredfish_system_count = report2.systems.len(), + "reported different number of systems", ); } for (s1, s2) in report1.systems.iter().zip(report2.systems.iter()) { if s1.id != s2.id { - tracing::warn!("systems.id are not equal: {:?} != {:?}", s1.id, s2.id); + tracing::warn!( + libredfish_system_id = ?s1.id, + nvredfish_system_id = ?s2.id, + "system IDs are not equal" + ); } else { if s1.ethernet_interfaces != s2.ethernet_interfaces { tracing::warn!( - "systems[{:?}].ethernet_interfaces are not equal: {:?} != {:?}", - s1.id, - s1.ethernet_interfaces, - s2.ethernet_interfaces + system_id = ?s1.id, + libredfish_ethernet_interfaces = ?s1.ethernet_interfaces, + nvredfish_ethernet_interfaces = ?s2.ethernet_interfaces, + "system Ethernet interfaces are not equal" ); } if s1.manufacturer != s2.manufacturer { tracing::warn!( - "systems[{:?}].manufacturer are not equal: {:?} != {:?}", - s1.id, - s1.manufacturer, - s2.manufacturer + system_id = ?s1.id, + libredfish_manufacturer = ?s1.manufacturer, + nvredfish_manufacturer = ?s2.manufacturer, + "system manufacturers are not equal" ); } if s1.model != s2.model { tracing::warn!( - "systems[{:?}].model are not equal: {:?} != {:?}", - s1.id, - s1.model, - s2.model + system_id = ?s1.id, + libredfish_model = ?s1.model, + nvredfish_model = ?s2.model, + "system models are not equal" ); } if s1.serial_number != s2.serial_number { tracing::warn!( - "systems[{:?}].serial_number are not equal: {:?} != {:?}", - s1.id, - s1.serial_number, - s2.serial_number + system_id = ?s1.id, + libredfish_serial_number = ?s1.serial_number, + nvredfish_serial_number = ?s2.serial_number, + "system serial numbers are not equal" ); } if s1.attributes != s2.attributes { tracing::warn!( - "systems[{:?}].attributes are not equal: {:?} != {:?}", - s1.id, - s1.attributes, - s2.attributes + system_id = ?s1.id, + libredfish_attributes = ?s1.attributes, + nvredfish_attributes = ?s2.attributes, + "system attributes are not equal" ); } if s1.pcie_devices != s2.pcie_devices { if s1.pcie_devices.len() != s2.pcie_devices.len() { tracing::warn!( - "systems[{:?}].pcie_devices.len() are not equal: ids1: {:?}, ids2: {:?}", - s1.id, - s1.pcie_devices + system_id = ?s1.id, + libredfish_pcie_device_ids = ?s1.pcie_devices .iter() .map(|v| v.id.as_ref()) .collect::>(), - s2.pcie_devices + nvredfish_pcie_device_ids = ?s2.pcie_devices .iter() .map(|v| v.id.as_ref()) .collect::>(), + "system PCIe device counts are not equal", ); } else { let s2devices = s2 @@ -1404,18 +1451,18 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo if let Some(s2dev) = s2devices.get(&s1dev.id) { if s1dev != *s2dev { tracing::warn!( - "systems[{:?}].pcie_devices[{:?}] devices not equal: {:?} != {:?}", - s1.id, - s1dev.id, - s1dev, - s2dev, + system_id = ?s1.id, + device_id = ?s1dev.id, + libredfish_pcie_device = ?s1dev, + nvredfish_pcie_device = ?s2dev, + "system PCIe devices are not equal", ); } } else { tracing::warn!( - "systems[{:?}].pcie_devices.len() device {:?} is not found in second report", - s1.id, - s1dev.id + system_id = ?s1.id, + device_id = ?s1dev.id, + "system PCIe device is missing from the second report" ); } } @@ -1424,37 +1471,37 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo if s1.base_mac != s2.base_mac { tracing::warn!( - "systems[{:?}].base_mac are not equal: {:?} != {:?}", - s1.id, - s1.base_mac, - s2.base_mac + system_id = ?s1.id, + libredfish_base_mac = ?s1.base_mac, + nvredfish_base_mac = ?s2.base_mac, + "system base MAC addresses are not equal" ); } if s1.power_state != s2.power_state { tracing::warn!( - "systems[{:?}].power_state are not equal: {:?} != {:?}", - s1.id, - s1.power_state, - s2.power_state + system_id = ?s1.id, + libredfish_power_state = ?s1.power_state, + nvredfish_power_state = ?s2.power_state, + "system power states are not equal" ); } if s1.sku != s2.sku { tracing::warn!( - "systems[{:?}].sku are not equal: {:?} != {:?}", - s1.id, - s1.sku, - s2.sku + system_id = ?s1.id, + libredfish_sku = ?s1.sku, + nvredfish_sku = ?s2.sku, + "system SKUs are not equal" ); } if s1.boot_order != s2.boot_order { tracing::warn!( - "systems[{:?}].boot_order are not equal: {:?} != {:?}", - s1.id, - s1.boot_order, - s2.boot_order + system_id = ?s1.id, + libredfish_boot_order = ?s1.boot_order, + nvredfish_boot_order = ?s2.boot_order, + "system boot orders are not equal" ); } } @@ -1462,34 +1509,52 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo if report1.chassis.len() != report2.chassis.len() { tracing::warn!( - "reported different number of chassis: {:?} != {:?}", - report1.chassis.len(), - report2.chassis.len(), + libredfish_chassis_count = report1.chassis.len(), + nvredfish_chassis_count = report2.chassis.len(), + "reported different number of chassis", ); } for (c1, c2) in report1.chassis.iter().zip(report2.chassis.iter()) { if c1.id != c2.id { - tracing::warn!("chassis.id are not equal: {:?} != {:?}", c1.id, c2.id); + tracing::warn!( + libredfish_chassis_id = ?c1.id, + nvredfish_chassis_id = ?c2.id, + "chassis IDs are not equal" + ); } else if c1 != c2 { - tracing::warn!("chassis[{:?}] are not equal: {:?} != {:?}", c1.id, c1, c2); + tracing::warn!( + chassis_id = ?c1.id, + libredfish_chassis = ?c1, + nvredfish_chassis = ?c2, + "chassis reports are not equal" + ); } } if report1.service.len() != report2.service.len() { tracing::warn!( - "reported different number of service: {:?} != {:?}", - report1.service.len(), - report2.service.len(), + libredfish_service_count = report1.service.len(), + nvredfish_service_count = report2.service.len(), + "reported different number of service", ); } for (s1, s2) in report1.service.iter().zip(report2.service.iter()) { if s1.id != s2.id { - tracing::warn!("service.id are not equal: {:?} != {:?}", s1.id, s2.id); + tracing::warn!( + libredfish_service_id = ?s1.id, + nvredfish_service_id = ?s2.id, + "service IDs are not equal" + ); } else { if s1.inventories.len() != s2.inventories.len() { - tracing::warn!("service[{:?}] are not equal: {:?} != {:?}", s1.id, s1, s2); + tracing::warn!( + service_id = ?s1.id, + libredfish_service = ?s1, + nvredfish_service = ?s2, + "service reports are not equal" + ); } // Stable ordering of FW by id. Dell PowerEdge R770 doesn't // provide stable order of FW versions. @@ -1514,10 +1579,10 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo .and_then(|v| if v == "00:00:00Z" { None } else { Some(v) }) { tracing::warn!( - "service[{:?}].inventories are not equal: {:?} != {:?}", - s1.id, - i1, - i2 + service_id = ?s1.id, + libredfish_inventory = ?i1, + nvredfish_inventory = ?i2, + "service inventories are not equal" ); } } @@ -1526,15 +1591,19 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo if report1.machine_setup_status.is_some() != report2.machine_setup_status.is_some() { tracing::warn!( - "forge_setup_status(es) are not equal: {:?} != {:?}", - report1.machine_setup_status, - report2.machine_setup_status, + libredfish_machine_setup_status = ?report1.machine_setup_status, + nvredfish_machine_setup_status = ?report2.machine_setup_status, + "machine setup statuses are not equal", ); } else if let Some(r1) = &report1.machine_setup_status && let Some(r2) = &report2.machine_setup_status { if r1.is_done != r2.is_done { - tracing::warn!("forge_setup_status(es) are not equal: {r1:?} != {r2:?}",); + tracing::warn!( + libredfish_machine_setup_status = ?r1, + nvredfish_machine_setup_status = ?r2, + "machine setup statuses are not equal" + ); } let mut sst1_idx = (0..r1.diffs.len()).collect::>(); @@ -1543,16 +1612,20 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo sst2_idx.sort_by_key(|i| &r2.diffs[*i].key); if sst1_idx.len() != sst2_idx.len() { tracing::warn!( - "machine_setup_status diffs are not equal: {:?} != {:?}", - r1.diffs, - r2.diffs + libredfish_machine_setup_diffs = ?r1.diffs, + nvredfish_machine_setup_diffs = ?r2.diffs, + "machine setup status differences are not equal" ); } else { for (i1, i2) in sst1_idx.into_iter().zip(sst2_idx) { let d1 = &r1.diffs[i1]; let d2 = &r2.diffs[i2]; if d1 != d2 { - tracing::warn!("machine_setup_status diffs are not equal: {d1:?} != {d2:?}"); + tracing::warn!( + libredfish_machine_setup_diff = ?d1, + nvredfish_machine_setup_diff = ?d2, + "machine setup status differences are not equal" + ); } } } @@ -1560,65 +1633,65 @@ fn warn_report_diff(report1: &EndpointExplorationReport, report2: &EndpointExplo if report1.secure_boot_status != report2.secure_boot_status { tracing::warn!( - "secure_boot_status(es) are not equal: {:?} != {:?}", - report1.secure_boot_status, - report2.secure_boot_status, + libredfish_secure_boot_status = ?report1.secure_boot_status, + nvredfish_secure_boot_status = ?report2.secure_boot_status, + "secure boot statuses are not equal", ); } if report1.lockdown_status != report2.lockdown_status { tracing::warn!( - "lockdown_status(es) are not equal: {:?} != {:?}", - report1.lockdown_status, - report2.lockdown_status, + libredfish_lockdown_status = ?report1.lockdown_status, + nvredfish_lockdown_status = ?report2.lockdown_status, + "lockdown statuses are not equal", ); } if report1.power_shelf_id != report2.power_shelf_id { tracing::warn!( - "power_shelf_id are not equal: {:?} != {:?}", - report1.power_shelf_id, - report2.power_shelf_id + libredfish_power_shelf_id = ?report1.power_shelf_id, + nvredfish_power_shelf_id = ?report2.power_shelf_id, + "power shelf IDs are not equal" ) } if report1.switch_id != report2.switch_id { tracing::warn!( - "switch_id are not equal: {:?} != {:?}", - report1.switch_id, - report2.switch_id + libredfish_switch_id = ?report1.switch_id, + nvredfish_switch_id = ?report2.switch_id, + "switch IDs are not equal" ) } if report1.physical_slot_number != report2.physical_slot_number { tracing::warn!( - "physical_slot_number are not equal: {:?} != {:?}", - report1.physical_slot_number, - report2.physical_slot_number + libredfish_physical_slot_number = ?report1.physical_slot_number, + nvredfish_physical_slot_number = ?report2.physical_slot_number, + "physical slot numbers are not equal" ) } if report1.compute_tray_index != report2.compute_tray_index { tracing::warn!( - "compute_tray_index are not equal: {:?} != {:?}", - report1.compute_tray_index, - report2.compute_tray_index + libredfish_compute_tray_index = ?report1.compute_tray_index, + nvredfish_compute_tray_index = ?report2.compute_tray_index, + "compute tray indexes are not equal" ) } if report1.topology_id != report2.topology_id { tracing::warn!( - "topology_id are not equal: {:?} != {:?}", - report1.topology_id, - report2.topology_id + libredfish_topology_id = ?report1.topology_id, + nvredfish_topology_id = ?report2.topology_id, + "topology IDs are not equal" ) } if report1.revision_id != report2.revision_id { tracing::warn!( - "revision_id are not equal: {:?} != {:?}", - report1.revision_id, - report2.revision_id + libredfish_revision_id = ?report1.revision_id, + nvredfish_revision_id = ?report2.revision_id, + "revision IDs are not equal" ) } } diff --git a/crates/site-explorer/src/explored_endpoint_index.rs b/crates/site-explorer/src/explored_endpoint_index.rs index 97e5fa5785..7b5797fad6 100644 --- a/crates/site-explorer/src/explored_endpoint_index.rs +++ b/crates/site-explorer/src/explored_endpoint_index.rs @@ -201,18 +201,18 @@ impl ExploredEndpointIndexBuilder { pub fn with_expected_power_shelves(mut self, shelves: Vec) -> Self { for shelf in shelves { tracing::info!( - "expected_power_shelf from DB: {} {}", - shelf.bmc_mac_address, - shelf.metadata.name + bmc_mac_address = %shelf.bmc_mac_address, + power_shelf_name = %shelf.metadata.name, + "loaded expected power shelf from database" ); if let Some(iface) = self .explored_underlay_interfaces .get(&shelf.bmc_mac_address) { tracing::info!( - "iface mac address {} expected power shelf mac address {}", - iface.mac_address, - shelf.bmc_mac_address + interface_mac_address = %iface.mac_address, + expected_power_shelf_mac_address = %shelf.bmc_mac_address, + "matched interface to expected power shelf" ); for addr in &iface.addresses { self.explored_power_shelves_addr_index @@ -229,18 +229,18 @@ impl ExploredEndpointIndexBuilder { // Create a mapping of expected switches by IP address and MAC address for switch in switches { tracing::info!( - "expected_switch from DB: {} {}", - switch.bmc_mac_address, - switch.metadata.name + bmc_mac_address = %switch.bmc_mac_address, + switch_name = %switch.metadata.name, + "loaded expected switch from database" ); if let Some(iface) = self .explored_underlay_interfaces .get(&switch.bmc_mac_address) { tracing::info!( - "iface mac address {} expected switch mac address {}", - iface.mac_address, - switch.bmc_mac_address + interface_mac_address = %iface.mac_address, + expected_switch_mac_address = %switch.bmc_mac_address, + "matched interface to expected switch" ); for addr in &iface.addresses { self.explored_switches_addr_index diff --git a/crates/site-explorer/src/lib.rs b/crates/site-explorer/src/lib.rs index d16886f741..bf10904cf4 100644 --- a/crates/site-explorer/src/lib.rs +++ b/crates/site-explorer/src/lib.rs @@ -138,8 +138,8 @@ pub fn enrich_endpoint_exploration_report( let components_without_version = report.parse_versions(&fw_info); if !components_without_version.is_empty() { tracing::debug!( - "Can not find firmware version for component(s): {:?}", - components_without_version + components = ?components_without_version, + "Can not find firmware version for component(s)" ); } } else { @@ -147,9 +147,9 @@ pub fn enrich_endpoint_exploration_report( // do not keep stale data. report.versions = HashMap::default(); tracing::debug!( - "Can not find firmware info for: vendor: {:?}; model: {:?}", - report.vendor, - report.model() + vendor = ?report.vendor, + model = ?report.model(), + "Can not find firmware info" ); } @@ -388,7 +388,7 @@ impl SiteExplorer { .boot_order_tracker .track_hosts(Instant::now(), &identified_hosts), Err(e) => { - tracing::warn!("SiteExplorer error: {}", e); + tracing::warn!(error = %e, "SiteExplorer error"); } } } else { @@ -573,7 +573,7 @@ impl SiteExplorer { explore_site_span.record("otel.status_code", "ok"); } Err(e) => { - tracing::error!("SiteExplorer run failed due to: {:?}", e); + tracing::error!(error = ?e, "SiteExplorer run failed"); explore_site_span.record("otel.status_code", "error"); // Writing this field will set the span status to error // Therefore we only write it on errors @@ -969,8 +969,8 @@ impl SiteExplorer { expected_endpoint_index.matched_expected_power_shelf(&endpoint.address) else { tracing::info!( - "No expected power shelf found for endpoint {:#?}", - endpoint.address + endpoint_address = ?endpoint.address, + "No expected power shelf found" ); continue; }; @@ -989,7 +989,11 @@ impl SiteExplorer { } Ok(false) => {} Err(error) => { - tracing::error!(%error, "Failed to create power shelf {:#?}", address) + tracing::error!( + %error, + bmc_ip_address = ?address, + "Failed to create power shelf" + ) } } } @@ -1009,8 +1013,8 @@ impl SiteExplorer { .map_err(|e| DatabaseError::new("begin load create_power_shelf", e))?; tracing::info!( - "creating power shelf for endpoint: {} ", - explored_endpoint.address + endpoint_address = %explored_endpoint.address, + "Creating power shelf" ); // Defense against the duplicate-power-shelves bug: if a power shelf @@ -1023,10 +1027,10 @@ impl SiteExplorer { .await? { tracing::warn!( - bmc_mac = %expected_shelf.bmc_mac_address, + bmc_mac_address = %expected_shelf.bmc_mac_address, existing_power_shelf_id = %existing.id, - endpoint = %explored_endpoint.address, - "Power shelf already exists for this BMC MAC; skipping discovery", + endpoint_address = %explored_endpoint.address, + "Power shelf already exists; skipping discovery", ); txn.rollback() .await @@ -1046,9 +1050,9 @@ impl SiteExplorer { for existing_ps in &existing_power_shelves { if existing_ps.config.name == expected_shelf.metadata.name { tracing::info!( - "Power shelf with name '{}' already exists, skipping creation for endpoint {}", - &expected_shelf.metadata.name, - explored_endpoint.address + power_shelf_name = %expected_shelf.metadata.name, + endpoint_address = %explored_endpoint.address, + "Power shelf already exists; skipping creation" ); txn.rollback() .await @@ -1073,7 +1077,7 @@ impl SiteExplorer { if power_shelf_chassis.is_none() { tracing::warn!( - endpoint = %explored_endpoint.address, + endpoint_address = %explored_endpoint.address, "No chassis reported for power shelf endpoint; falling back to defaults for id generation", ); } @@ -1147,9 +1151,9 @@ impl SiteExplorer { .map_err(|e| DatabaseError::new("end create_power_shelf", e))?; tracing::info!( - "Created power shelf {} for endpoint {}", - power_shelf_id, - explored_endpoint.address + %power_shelf_id, + endpoint_address = %explored_endpoint.address, + "Created power shelf" ); Ok(true) @@ -1474,8 +1478,11 @@ impl SiteExplorer { tracing::warn!( address = %ep.address, exploration_report = ?ep, - "cannot identify managed host because the site explorer has only discovered {} out of the {} attached DPUs (all_dpus_configured_properly_in_host={all_dpus_configured_properly_in_host}):\n{:#?}", - dpus_explored_for_host.len(), expected_managed_dpus_total, dpus_explored_for_host + discovered_dpus = dpus_explored_for_host.len(), + expected_managed_dpus_total, + all_dpus_configured_properly_in_host, + discovered_dpu_details = ?dpus_explored_for_host, + "cannot identify managed host because the site explorer has not discovered all attached DPUs" ); } @@ -1492,8 +1499,9 @@ impl SiteExplorer { ); if time_since_redfish_powercycle > self.config.reset_rate_limit { tracing::warn!( - "power cycling host {} to apply nic mode change for its incorrectly configured DPUs; time since last powercycle: {time_since_redfish_powercycle}", - ep.address, + address = %ep.address, + %time_since_redfish_powercycle, + "power cycling host to apply nic mode change for its incorrectly configured DPUs" ); metrics.increment_dpu_migration_signal( DpuMigrationSignal::ResetRequested, @@ -1501,8 +1509,9 @@ impl SiteExplorer { if let Err(err) = self.redfish_powercycle(ep.address).await { tracing::warn!( - "site explorer failed to power cycle host {} to apply DPU mode changes: {err}; a manual power cycle may be required", - ep.address + address = %ep.address, + error = %err, + "site explorer failed to power cycle host to apply DPU mode changes; a manual power cycle may be required" ); metrics.increment_host_dpu_pairing_blocker( PairingBlockerReason::ManualPowerCycleRequired, @@ -1619,8 +1628,10 @@ impl SiteExplorer { .join(","); tracing::error!( - "Could not find mac_address {mac_address} in discovered DPU's list {all_mac}, host bmc: {}.", - ep.address + %mac_address, + discovered_dpu_macs = %all_mac, + host_bmc_address = %ep.address, + "MAC address not found in discovered DPU list" ); metrics.increment_host_dpu_pairing_blocker( PairingBlockerReason::BootInterfaceMacMismatch, @@ -2248,7 +2259,7 @@ impl SiteExplorer { Some(guard) => guard, None => { tracing::info!( - address = %endpoint.address, + bmc_ip_address = %endpoint.address, "Skipping periodic endpoint exploration; endpoint already in progress" ); return Ok(None); @@ -2556,7 +2567,9 @@ impl SiteExplorer { // New endpoints haven't been explored yet, so pause_remediation defaults to false if endpoint.last_explored.is_some_and(|e| e.pause_remediation) { tracing::info!( - "Site explorer will not remediate error for {endpoint} because remediation is paused for this endpoint: {error}" + %endpoint, + %error, + "Site explorer will not remediate error because remediation is paused for this endpoint" ); return; } @@ -2575,8 +2588,10 @@ impl SiteExplorer { PreingestionState::Initial | PreingestionState::Complete ) { tracing::info!( - "Site explorer will not remediate error for {endpoint} because endpoint is in preingestion state {:?}: {error}", - endpoint.preingestion_state(), + %endpoint, + preingestion_state = ?endpoint.preingestion_state(), + %error, + "Site explorer will not remediate error because endpoint is in preingestion state", ); return; } @@ -2588,13 +2603,19 @@ impl SiteExplorer { Ok(managed_host_exists) => { if managed_host_exists { tracing::info!( - "Site explorer will not remediate error for {endpoint} because a managed host has already been created for this endpoint: {error}" + %endpoint, + %error, + "Site explorer will not remediate error because a managed host has already been created for this endpoint" ); return; } } Err(e) => { - tracing::error!(%e, "failed to retrieve whether managed host was created for endpoint: {endpoint}"); + tracing::error!( + %endpoint, + error = %e, + "Failed to determine whether managed host was created" + ); return; } }; @@ -2607,8 +2628,8 @@ impl SiteExplorer { && !matches!(power_state, PowerState::On) { tracing::warn!( - "Site Explorer found a host (bmc_ip_address: {}) that isnt on. Turning it on now.", - endpoint.address, + bmc_ip_address = %endpoint.address, + "Site Explorer found a host that isn't on. Turning it on now.", ); match self @@ -2650,13 +2671,23 @@ impl SiteExplorer { || time_since_ipmitool_bmc_reset.num_minutes() < min_time_since_last_action_mins { tracing::info!( - "waiting to remediate error {error} for {endpoint}; time_since_redfish_reboot: {time_since_redfish_reboot}; time_since_redfish_bmc_reset: {time_since_redfish_bmc_reset}; time_since_ipmitool_bmc_reset: {time_since_ipmitool_bmc_reset}" + %endpoint, + %error, + %time_since_redfish_reboot, + %time_since_redfish_bmc_reset, + %time_since_ipmitool_bmc_reset, + "waiting to remediate error" ); return; } tracing::info!( - "Site explorer captured an error for {endpoint}: {error};\n time_since_redfish_reboot: {time_since_redfish_reboot}; time_since_redfish_bmc_reset: {time_since_redfish_bmc_reset}; time_since_ipmitool_bmc_reset: {time_since_ipmitool_bmc_reset}'" + %endpoint, + %error, + %time_since_redfish_reboot, + %time_since_redfish_bmc_reset, + %time_since_ipmitool_bmc_reset, + "Site explorer captured an error" ); // If the endpoint is a DPU, and the error is that the BIOS attributes are coming up as empty for this DPU, @@ -2673,9 +2704,9 @@ impl SiteExplorer { .await .map_err(|err| { tracing::error!( - "Site Explorer failed to reboot {}: {}", - endpoint.address, - err + bmc_ip_address = %endpoint.address, + error = %err, + "Site Explorer failed to reboot" ) }) .is_ok() @@ -2692,9 +2723,9 @@ impl SiteExplorer { } Err(e) => { tracing::error!( - "Site Explorer failed to clear nvram {}: {}", - endpoint.address, - e + bmc_ip_address = %endpoint.address, + error = %e, + "Site Explorer failed to clear nvram" ) } } @@ -2706,9 +2737,9 @@ impl SiteExplorer { .await .map_err(|err| { tracing::error!( - "Site Explorer failed to reset BMC {} through redfish: {}", - endpoint.address, - err + bmc_ip_address = %endpoint.address, + error = %err, + "Site Explorer failed to reset BMC through redfish" ) }) .is_ok() @@ -2722,9 +2753,9 @@ impl SiteExplorer { .await .map_err(|err| { tracing::error!( - "Site Explorer failed to reset BMC {} through ipmitool: {}", - endpoint.address, - err + bmc_ip_address = %endpoint.address, + error = %err, + "Site Explorer failed to reset BMC through ipmitool" ) }) .ok(); @@ -2734,8 +2765,8 @@ impl SiteExplorer { pub async fn ipmitool_reset_bmc(&self, endpoint: &Endpoint<'_>) -> SiteExplorerResult<()> { tracing::info!( - "SiteExplorer is initiating a cold BMC reset through IPMI to IP {}", - endpoint.address + bmc_ip_address = %endpoint.address, + "SiteExplorer is initiating a cold BMC reset through IPMI" ); let bmc_target_port = self.config.override_target_port.unwrap_or(443); @@ -2764,8 +2795,8 @@ impl SiteExplorer { pub async fn redfish_reset_bmc(&self, endpoint: &Endpoint<'_>) -> SiteExplorerResult<()> { tracing::info!( - "SiteExplorer is initiating a BMC reset through Redfish to IP {}", - endpoint.address + bmc_ip_address = %endpoint.address, + "SiteExplorer is initiating a BMC reset through Redfish" ); let bmc_target_port = self.config.override_target_port.unwrap_or(443); let bmc_target_addr = SocketAddr::new(endpoint.address, bmc_target_port); @@ -2844,15 +2875,19 @@ impl SiteExplorer { { Ok(is_viking) => is_viking, Err(e) => { - tracing::warn!("could not retrieve vendor for {}: {e}", endpoint.address); + tracing::warn!( + bmc_ip_address = %endpoint.address, + error = %e, + "could not retrieve vendor" + ); false } } } pub async fn clear_nvram(&self, endpoint: &Endpoint<'_>) -> SiteExplorerResult<()> { tracing::info!( - "SiteExplorer is issuing a clean_nvram through Redfish to IP {}", - endpoint.address + bmc_ip_address = %endpoint.address, + "Site explorer is clearing NVRAM through Redfish" ); let bmc_target_port = self.config.override_target_port.unwrap_or(443); let bmc_target_addr = SocketAddr::new(endpoint.address, bmc_target_port); @@ -2898,7 +2933,11 @@ impl SiteExplorer { { Ok(managed_host_exists) => managed_host_exists, Err(e) => { - tracing::error!(%e, "failed to retrieve whether managed host was created for DPU endpoint: {dpu_endpoint}"); + tracing::error!( + %dpu_endpoint, + error = %e, + "Failed to determine whether managed host was created" + ); // return true by default true } @@ -2916,22 +2955,22 @@ impl SiteExplorer { // from the redfish response. Skip the next check because the DPUs // in NIC mode will not expose a pf0 interface to the host. tracing::info!( - "Site explorer found an uningested DPU (bmc ip: {}) in NIC mode", - dpu_endpoint.address + bmc_ip_address = %dpu_endpoint.address, + "Site explorer found an uningested DPU in NIC mode" ); return Ok(true); } Some(NicMode::Dpu) => {} None if dpu_endpoint.report.dpu_pairing_serial_number().is_some() => { tracing::warn!( - "Site explorer found an uningested DPU (bmc ip: {}) without a Redfish DPU/NIC mode; continuing because it has a host-pairing serial", - dpu_endpoint.address + bmc_ip_address = %dpu_endpoint.address, + "Site explorer found an uningested DPU without a Redfish DPU/NIC mode; continuing because it has a host-pairing serial" ); } None => { tracing::error!( - "Site explorer found an uningested DPU (bmc ip: {}) without being able to determine if it is in NIC mode", - dpu_endpoint.address + bmc_ip_address = %dpu_endpoint.address, + "Site explorer found an uningested DPU without being able to determine if it is in NIC mode" ); metrics.increment_host_dpu_pairing_blocker(PairingBlockerReason::DpuNicModeUnknown); return Ok(false); @@ -2944,7 +2983,11 @@ impl SiteExplorer { match find_host_pf_mac_address(dpu_endpoint) { Ok(_) => Ok(true), Err(error) => { - tracing::error!(%error, "Site explorer found an uningested DPU (bmc ip: {}): failed to find the MAC address of the pf0 interface that the DPU exposes to the host", dpu_endpoint.address); + tracing::error!( + %error, + bmc_ip_address = %dpu_endpoint.address, + "Site explorer found an uningested DPU: failed to find the MAC address of the pf0 interface that the DPU exposes to the host" + ); metrics.increment_host_dpu_pairing_blocker(PairingBlockerReason::DpuPf0MacMissing); Ok(false) } @@ -3060,7 +3103,11 @@ impl SiteExplorer { { Ok(managed_host_exists) => managed_host_exists, Err(e) => { - tracing::error!(%e, "failed to retrieve whether managed host was created for Host endpoint: {host_endpoint}"); + tracing::error!( + %host_endpoint, + error = %e, + "Failed to determine whether managed host was created" + ); // return true by default true } @@ -3075,8 +3122,8 @@ impl SiteExplorer { let bmc_target_addr = SocketAddr::new(host_endpoint.address, bmc_target_port); let Some(system) = host_endpoint.report.systems.first() else { tracing::warn!( - "Site Explorer could not find the system report for a host (bmc_ip_address: {})", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + "Site Explorer could not find the system report for a host", ); metrics .increment_host_dpu_pairing_blocker(PairingBlockerReason::HostSystemReportMissing); @@ -3087,8 +3134,8 @@ impl SiteExplorer { // then don't do it if host_endpoint.pause_ingestion_and_poweron { tracing::warn!( - "Host with bmc_ip_address: {} is configured to pause on ingestion", - host_endpoint.address + bmc_ip_address = %host_endpoint.address, + "Host is configured to pause on ingestion" ); return Ok(false); } @@ -3141,14 +3188,14 @@ impl SiteExplorer { if host_endpoint.pause_remediation { tracing::info!( - "Site Explorer found an uningested host (bmc_ip_address: {}) that is off, but remediation is paused — skipping power-on", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + "Site Explorer found an uningested host that is off, but remediation is paused — skipping power-on", ); } else if fresh_power_state.is_some() { tracing::warn!( - "Site Explorer found an uningested host (bmc_ip_address: {}) that isn't on: {:#?}", - host_endpoint.address, - effective_power_state + bmc_ip_address = %host_endpoint.address, + power_state = ?effective_power_state, + "Site Explorer found an uningested host that isn't on" ); if let Some(interface) = interface.as_ref() { @@ -3161,9 +3208,9 @@ impl SiteExplorer { .await .map_err(|err| { tracing::error!( - "Site Explorer failed to turn on host (bmc_ip_address: {}) through redfish: {}", - host_endpoint.address, - err + bmc_ip_address = %host_endpoint.address, + error = %err, + "Site Explorer failed to turn on host through redfish" ) }) .ok(); @@ -3174,8 +3221,8 @@ impl SiteExplorer { if host_endpoint.report.vendor.unwrap_or_default().is_nvidia() { let Some(manager) = host_endpoint.report.managers.first() else { tracing::warn!( - "Site Explorer could not find the system report for a Nvidia host (bmc_ip_address: {})", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + "Site Explorer could not find the manager report for an NVIDIA host", ); return Ok(false); @@ -3198,8 +3245,10 @@ impl SiteExplorer { Ok(is_cpldmb_version_at_expected) => { if !is_cpldmb_version_at_expected { tracing::warn!( - "Site Explorer found a Viking (bmc_ip_address: {}) with a CPLDMB_0 version of {current_cpldmb_0_version}, which is less than the expected version of {expected_cpldmb_0_version}. A DC Power Cycle may be needed", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + %current_cpldmb_0_version, + %expected_cpldmb_0_version, + "Site Explorer found a Viking whose CPLDMB_0 version does not match the expected version. A DC power cycle may be needed", ); metrics.increment_host_dpu_pairing_blocker( PairingBlockerReason::VikingCpldVersionIssue, @@ -3209,8 +3258,11 @@ impl SiteExplorer { } Err(e) => { tracing::warn!( - "Site Explorer found a Viking (bmc_ip_address: {}) with a CPLDMB_0 version of {current_cpldmb_0_version} and could not compare it to the current CPLDMB_0 version of {expected_cpldmb_0_version}: {e:#?}", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + %current_cpldmb_0_version, + %expected_cpldmb_0_version, + error = ?e, + "Site Explorer found a Viking with a CPLDMB_0 version and could not compare it to the current CPLDMB_0 version", ); metrics.increment_host_dpu_pairing_blocker( PairingBlockerReason::VikingCpldVersionIssue, @@ -3220,8 +3272,8 @@ impl SiteExplorer { } } else { tracing::warn!( - "Site Explorer could not find the CPLDMB_0 inventory for a Viking (bmc_ip_address: {})", - host_endpoint.address, + bmc_ip_address = %host_endpoint.address, + "Site Explorer could not find the CPLDMB_0 inventory for a Viking", ); metrics.increment_host_dpu_pairing_blocker( PairingBlockerReason::VikingCpldVersionIssue, @@ -3239,9 +3291,9 @@ impl SiteExplorer { .is_some_and(|status| !status) { tracing::warn!( - "Site Explorer found an uningested Lenovo (bmc_ip_address: {}) without infinite boot enabled; System Report: {:#?}", - host_endpoint.address, - system.attributes + bmc_ip_address = %host_endpoint.address, + system_report = ?system.attributes, + "Site Explorer found an uningested Lenovo without infinite boot enabled" ); let interface = self @@ -3253,11 +3305,12 @@ impl SiteExplorer { .await .inspect_err(|err| { tracing::error!( - "Site Explorer failed to call machine_setup against Lenovo (bmc_ip_address: {}): {}", - host_endpoint.address, - err + bmc_ip_address = %host_endpoint.address, + error = %err, + "Site explorer failed to run machine setup against Lenovo" ) - }).ok(); + }) + .ok(); self.endpoint_explorer .redfish_power_control( @@ -3268,11 +3321,12 @@ impl SiteExplorer { .await .inspect_err(|err| { tracing::error!( - "Site Explorer failed to restart Lenovo (bmc_ip_address: {}) after calling machine_setup: {}", - host_endpoint.address, - err + bmc_ip_address = %host_endpoint.address, + error = %err, + "Site explorer failed to restart Lenovo after running machine setup" ) - }).ok(); + }) + .ok(); ingest_host = false; } @@ -3351,9 +3405,9 @@ impl SiteExplorer { } None => { tracing::warn!( - "Site explorer cannot determine this DPU's mode {}: {:#?}", - dpu_ep.address, - dpu_ep.report + bmc_ip_address = %dpu_ep.address, + dpu_report = ?dpu_ep.report, + "Site explorer cannot determine this DPU's mode" ); Ok(true) } @@ -3702,8 +3756,8 @@ fn is_dpu_in_nic_mode(dpu_ep: &ExploredEndpoint, host_ep: &ExploredEndpoint) -> if nic_mode { tracing::info!( address = %dpu_ep.address, - "discovered bluefield in NIC mode attached to host {}", - host_ep.address + host_address = %host_ep.address, + "discovered bluefield in NIC mode attached to host" ); } nic_mode diff --git a/crates/site-explorer/src/machine_creator.rs b/crates/site-explorer/src/machine_creator.rs index f264ef54b2..9448c048da 100644 --- a/crates/site-explorer/src/machine_creator.rs +++ b/crates/site-explorer/src/machine_creator.rs @@ -108,7 +108,11 @@ impl MachineCreator { } } Ok(false) => {} - Err(error) => tracing::error!(%error, "Failed to create managed host {:#?}", host), + Err(error) => tracing::error!( + %error, + host = ?host, + "Failed to create managed host" + ), } } @@ -250,7 +254,12 @@ impl MachineCreator { if let Some(rack_id) = machine_data.and_then(|d| d.rack_id.as_ref()) { tracing::info!(%rack_id, %host_machine_id, "Ensuring rack exists for host machine"); if let Some(rack) = crate::ensure_rack_exists(&mut txn, rack_id).await? { - tracing::info!(%rack_id, "Rack exists for host machine {host_machine_id}: {rack:#?}"); + tracing::info!( + %rack_id, + %host_machine_id, + rack = ?rack, + "Rack exists" + ); rack_profile_id = rack.rack_profile_id; } } @@ -752,8 +761,9 @@ impl MachineCreator { && interface.machine_id.is_none() { tracing::info!( - "Updating machine interface {} with machine id {dpu_machine_id}.", - interface.id + interface_id = %interface.id, + machine_id = %dpu_machine_id, + "Associating machine interface with machine" ); db::machine_interface::associate_interface_with_machine( &interface.id, @@ -799,7 +809,7 @@ impl MachineCreator { .await { Ok(machine) => { - tracing::info!("Created DPU machine with id: {}", dpu_machine_id); + tracing::info!(machine_id = %dpu_machine_id, "Created DPU machine"); Ok(Some(machine)) } Err(e) => { diff --git a/crates/site-explorer/src/redfish.rs b/crates/site-explorer/src/redfish.rs index 2300c9a0a9..c468d63b03 100644 --- a/crates/site-explorer/src/redfish.rs +++ b/crates/site-explorer/src/redfish.rs @@ -721,7 +721,8 @@ async fn fetch_system(client: &dyn Redfish) -> Result Ok(interfaces), Err(e) if is_dpu => { tracing::warn!( - "Error getting system ethernet interfaces. The error will be ignored. ({e})" + error = %e, + "Failed to get system Ethernet interfaces; ignoring the error" ); Ok(Vec::default()) } @@ -748,14 +749,19 @@ async fn fetch_system(client: &dyn Redfish) -> Result base_mac.and_then(|v| { v.parse() .inspect_err(|err| { - tracing::warn!("Failed to parse BaseMAC: {err} (mac: {v})"); + tracing::warn!( + error = %err, + mac_address = %v, + "Failed to parse BaseMAC" + ); }) .ok() }), Err(error) => { tracing::info!( - "Could not use new method to retreive base mac address for DPU (serial number {:#?}): {error}", - system.serial_number + serial_number = ?system.serial_number, + %error, + "Could not use new method to retrieve base MAC address for DPU" ); None } @@ -878,8 +884,10 @@ async fn fetch_ethernet_interfaces( // ignore this error and create the interface with an empty mac address // in the exploration report tracing::debug!( - "could not parse MAC address for a disabled interface {iface_id} (link_status: {:#?}): {e}", - iface.link_status + interface_id = %iface_id, + link_status = ?iface.link_status, + error = %e, + "could not parse MAC address for a disabled interface" ); Ok(None) } else { diff --git a/crates/site-explorer/src/switch_creator.rs b/crates/site-explorer/src/switch_creator.rs index afabb91a9a..91d2ab9357 100644 --- a/crates/site-explorer/src/switch_creator.rs +++ b/crates/site-explorer/src/switch_creator.rs @@ -53,7 +53,7 @@ impl SwitchCreator { None => { tracing::info!( bmc_ip = %explored_managed_switch.bmc_ip, - "No expected switch found for explored switch endpoint" + "No expected switch found" ); continue; } @@ -78,8 +78,8 @@ impl SwitchCreator { Err(error) => { tracing::error!( %error, - "Failed to create managed switch {:#?}", - explored_managed_switch.bmc_ip + bmc_ip_address = ?explored_managed_switch.bmc_ip, + "Failed to create managed switch" ); } } @@ -163,8 +163,7 @@ impl SwitchCreator { if let Some(_existing_switch) = existing_switch { tracing::warn!( %switch_id, - "Switch already exists, skipping. {} for switch id", - switch_id.to_string() + "Switch already exists, skipping." ); return Ok(None); } diff --git a/crates/site-explorer/src/test_support/mock_endpoint_explorer.rs b/crates/site-explorer/src/test_support/mock_endpoint_explorer.rs index 9da66eb3bc..5621067bc1 100644 --- a/crates/site-explorer/src/test_support/mock_endpoint_explorer.rs +++ b/crates/site-explorer/src/test_support/mock_endpoint_explorer.rs @@ -149,7 +149,7 @@ impl EndpointExplorer for MockEndpointExplorer { _last_error: Option<&EndpointExplorationError>, _boot_interface_mac: Option, ) -> Result { - tracing::info!("Endpoint {bmc_ip_address} is getting explored"); + tracing::info!(%bmc_ip_address, "Endpoint is getting explored"); self.explore_endpoint_calls .lock() .unwrap() diff --git a/crates/site-explorer/tests/integration/power_shelf.rs b/crates/site-explorer/tests/integration/power_shelf.rs index 290808bc0c..6a06f49d2e 100644 --- a/crates/site-explorer/tests/integration/power_shelf.rs +++ b/crates/site-explorer/tests/integration/power_shelf.rs @@ -111,9 +111,9 @@ async fn test_site_explorer_power_shelf_discovery( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); // Create expected power shelf entry in the database @@ -221,9 +221,9 @@ async fn test_site_explorer_power_shelf_discovery_with_static_ip( let power_shelf = env.new_power_shelf("B8:3F:D2:90:97:B0", "192.0.1.180", "PS123456789"); tracing::info!( - "Static ip {} assigned to power shelf mac {}", - power_shelf.ip, - power_shelf.bmc_mac_address, + ip_address = %power_shelf.ip, + mac_address = %power_shelf.bmc_mac_address, + "Static ip assigned to power shelf mac", ); // Create expected power shelf via the RPC handler, which // pre-allocates a machine interface with the static IP. @@ -471,9 +471,9 @@ async fn test_site_explorer_power_shelf_with_expected_config( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); // Create an expected power shelf entry @@ -580,9 +580,9 @@ async fn test_site_explorer_power_shelf_creation_limit( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); } @@ -690,9 +690,9 @@ async fn test_site_explorer_power_shelf_disabled( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); @@ -794,9 +794,9 @@ async fn test_site_explorer_power_shelf_error_handling( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); // Create expected power shelf entry in the database @@ -892,9 +892,9 @@ async fn test_site_explorer_creates_power_shelf( .await? .into_inner(); tracing::info!( - "DHCP with mac {} assigned ip {}", - power_shelf.bmc_mac_address, - response.address + mac_address = %power_shelf.bmc_mac_address, + ip_address = %response.address, + "DHCP assigned ip" ); power_shelf.ip = response.address.clone(); // Create expected power shelf entry in the database @@ -1065,9 +1065,9 @@ async fn test_power_shelf_state_history(pool: PgPool) -> Result<(), Box x.version, Err(libredfish::RedfishError::NotSupported(msg)) => { tracing::info!( - "Device attestation is not supported for device (firmware version not available): {device_id}, machine: {machine_id}, msg: {msg}" + device_id = %device_id, + machine_id = %machine_id, + reason = %msg, + "device attestation is not supported because firmware version is unavailable" ); return Ok(StateHandlerOutcome::transition( SpdmAttestationState::Passed, @@ -238,9 +241,9 @@ impl StateHandler for SpdmAttestationStateHandler { task_state => { let err = task.messages.iter().map(|t| t.message.clone()).join("\n"); tracing::error!( - "Error while triggering measurement: {}: State: {:?}", - err, - task_state + error = %err, + task_state = ?task_state, + "measurement collection task entered an unexpected state" ); if *retry_count > 4 { Ok(StateHandlerOutcome::transition( diff --git a/crates/ssh-console-mock-api-server/src/lib.rs b/crates/ssh-console-mock-api-server/src/lib.rs index 278243ba23..75279d2c69 100644 --- a/crates/ssh-console-mock-api-server/src/lib.rs +++ b/crates/ssh-console-mock-api-server/src/lib.rs @@ -93,7 +93,7 @@ impl MockApiServer { rustls::crypto::aws_lc_rs::default_provider() .install_default() .inspect_err(|crypto_provider| { - tracing::warn!("Crypto provider already configured: {crypto_provider:?}") + tracing::warn!(?crypto_provider, "Crypto provider already configured") }) .ok(); // if something else is already default, ignore. diff --git a/crates/ssh-console/src/bmc/client.rs b/crates/ssh-console/src/bmc/client.rs index 89ebafbae3..e25b280b05 100644 --- a/crates/ssh-console/src/bmc/client.rs +++ b/crates/ssh-console/src/bmc/client.rs @@ -214,8 +214,11 @@ impl BmcClient { tracing::error!( %error, %machine_id, - "error spawning BMC connection, will retry in {}s", - next_retry.checked_duration_since(Instant::now()).unwrap_or_default().as_secs() + retry_delay_seconds = next_retry + .checked_duration_since(Instant::now()) + .unwrap_or_default() + .as_secs(), + "error spawning BMC connection, will retry" ); continue 'retry; } @@ -273,7 +276,11 @@ impl BmcClient { if recovered_conflicting_sol_session { tracing::debug!(%machine_id, "retrying immediately after IPMI SOL session recovery"); } else { - tracing::debug!(%machine_id, "last connection lasted {}s, resetting backoff to 0s", connection_time.as_secs()); + tracing::debug!( + %machine_id, + connection_duration_seconds = connection_time.as_secs(), + "last connection succeeded long enough; resetting backoff" + ); } next_retry = Instant::now(); } @@ -284,8 +291,11 @@ impl BmcClient { %machine_id, error = error_string, ?connection_time, - "connection to BMC closed, will retry in {}s", - next_retry.checked_duration_since(Instant::now()).unwrap_or_default().as_secs(), + retry_delay_seconds = next_retry + .checked_duration_since(Instant::now()) + .unwrap_or_default() + .as_secs(), + "connection to BMC closed, will retry", ); } } @@ -346,7 +356,11 @@ async fn wait_until_host_is_up( if do_log { do_log = false; - tracing::info!(%machine_id, %addr, "BMC is not listening on {addr}. Will wait until the port is open before attempting to connect.") + tracing::info!( + %machine_id, + bmc_address = %addr, + "BMC is not listening; waiting for the port to open before connecting" + ) } } } @@ -490,9 +504,9 @@ fn next_retry_backoff(config: &Config, prev: Duration) -> Duration { Duration::from_secs_f64(rand::random_range(prev.as_secs_f64()..upper)) }; tracing::debug!( - "next_retry_backoff, increasing from {}ms to {}ms", - prev.as_millis(), - duration.as_millis() + previous_backoff_milliseconds = prev.as_millis(), + next_backoff_milliseconds = duration.as_millis(), + "increasing connection retry backoff" ); duration } diff --git a/crates/ssh-console/src/bmc/connection.rs b/crates/ssh-console/src/bmc/connection.rs index fbfbe7e0ee..3700858c9e 100644 --- a/crates/ssh-console/src/bmc/connection.rs +++ b/crates/ssh-console/src/bmc/connection.rs @@ -120,7 +120,10 @@ pub async fn lookup( })), }; tracing::info!( - "Overriding bmc connection to {machine_or_instance_id} with {connection_details:?}" + %machine_or_instance_id, + bmc_address = %connection_details.addr(), + bmc_machine_id = %connection_details.machine_id(), + "Overriding BMC connection" ); return Ok(connection_details); } @@ -203,7 +206,9 @@ pub async fn lookup( let addr = if let Some(override_ssh_addr) = config.override_bmc_ssh_addr(port).await? { tracing::info!( - "Overriding bmc connection to {ip} with {override_ssh_addr} per configuration" + %ip, + %override_ssh_addr, + "Overriding BMC connection per configuration" ); override_ssh_addr } else { diff --git a/crates/ssh-console/src/bmc/connection_impl/ipmi.rs b/crates/ssh-console/src/bmc/connection_impl/ipmi.rs index 4d755d58bb..7f4f54a9f6 100644 --- a/crates/ssh-console/src/bmc/connection_impl/ipmi.rs +++ b/crates/ssh-console/src/bmc/connection_impl/ipmi.rs @@ -663,7 +663,11 @@ impl IpmitoolMessageProxy { return Ok(()); } other => { - tracing::debug!(%machine_id, "Not handling unknown SSH frontend message in ipmitool: {other:?}"); + tracing::debug!( + %machine_id, + ?other, + "Not handling unknown SSH frontend message in ipmitool" + ); } }; Ok(()) diff --git a/crates/ssh-console/src/bmc/connection_impl/ssh.rs b/crates/ssh-console/src/bmc/connection_impl/ssh.rs index 599c08a0c6..74af01dd60 100644 --- a/crates/ssh-console/src/bmc/connection_impl/ssh.rs +++ b/crates/ssh-console/src/bmc/connection_impl/ssh.rs @@ -104,7 +104,11 @@ pub async fn spawn( && ringbuf_contains(&output_ringbuf, bmc_prompt) { let mut ringbuf_str = String::new(); output_ringbuf.read_to_string(&mut ringbuf_str).ok(); - tracing::warn!(%machine_id, "BMC dropped to system prompt, exiting. output: {ringbuf_str:?}"); + tracing::warn!( + %machine_id, + output = ?ringbuf_str, + "BMC dropped to system prompt, exiting" + ); break; } } @@ -380,7 +384,7 @@ async fn make_authenticated_client( } } other => { - tracing::debug!(%machine_id, "Ignoring unsupported auth method {other:?}") + tracing::debug!(%machine_id, ?other, "Ignoring unsupported auth method") } } } @@ -565,7 +569,8 @@ async fn trigger_and_await_sol_console( msg => { tracing::debug!( %machine_id, - "message from BMC while activating serial prompt: {msg:?}" + ?msg, + "message from BMC while activating serial prompt" ) } } diff --git a/crates/ssh-console/src/bmc/message_proxy.rs b/crates/ssh-console/src/bmc/message_proxy.rs index 615037dff1..52473c20a7 100644 --- a/crates/ssh-console/src/bmc/message_proxy.rs +++ b/crates/ssh-console/src/bmc/message_proxy.rs @@ -248,7 +248,7 @@ where })?; } _ => { - tracing::debug!("Ignoring unknown channel message {channel_msg:?}"); + tracing::debug!(?channel_msg, "Ignoring unknown channel message"); } } diff --git a/crates/ssh-console/src/config.rs b/crates/ssh-console/src/config.rs index 17e7fabe79..fb07555169 100644 --- a/crates/ssh-console/src/config.rs +++ b/crates/ssh-console/src/config.rs @@ -392,7 +392,10 @@ role_separator = {cert_authorization_keyid_format_role_separator:?} pub fn make_forge_api_client(&self) -> ForgeApiClient { let carbide_uri_string = self.carbide_uri.to_string(); - tracing::info!("carbide_uri_string: {}", carbide_uri_string); + tracing::info!( + carbide_uri = carbide_uri_string.as_str(), + "Configured Carbide API URI" + ); // TODO: The API's for ClientCert/ForgeClientConfig/etc really ought to take PathBufs, not Strings. let client_cert = ClientCert { diff --git a/crates/ssh-console/src/console_logger.rs b/crates/ssh-console/src/console_logger.rs index fd05775241..f36a322de8 100644 --- a/crates/ssh-console/src/console_logger.rs +++ b/crates/ssh-console/src/console_logger.rs @@ -145,7 +145,11 @@ impl ConsoleLogger { } Err(broadcast::error::RecvError::Lagged(count)) => { let msg = format!("console logger is lagged by {count} messages (typically bytes). Data may be missing from log"); - tracing::warn!(machine_id=%self.machine_id,"{msg}"); + tracing::warn!( + machine_id = %self.machine_id, + lagged_message_count = count, + "console logger lagged; data may be missing from log" + ); log_file.write_all(format!("\n--- {msg} ---\n").as_bytes()).await.ok(); } }, @@ -197,7 +201,7 @@ impl RotatableLogFile { self.file = Self::open_log_file(&self.path).await?; } // If we couldn't rotate, just keep writing to this file. - Err(error) => tracing::error!("error rotating logs: {error}"), + Err(error) => tracing::error!(%error, "error rotating logs"), } } @@ -209,7 +213,7 @@ impl RotatableLogFile { } async fn rotate_logs(&mut self) -> Result<(), LogRotationError> { - tracing::info!("rotating logs for {}", self.path.display()); + tracing::info!(path = %self.path.display(), "rotating logs"); let log_path_as_str = self .path .to_str() @@ -231,12 +235,12 @@ impl RotatableLogFile { }; if !src_path.exists() { - tracing::debug!("no log file at {}", src_path.display()); + tracing::debug!(path = %src_path.display(), "no log file found"); continue; } let dst_path = if dst_num >= self.max_rotated_files { - tracing::debug!("deleting oldest log file at {}", src_path.display()); + tracing::debug!(path = %src_path.display(), "deleting oldest log file"); // Oldest log, more than max allowed rotated file count, delete it and continue tokio::fs::remove_file(src_path.as_path()) .await @@ -251,7 +255,11 @@ impl RotatableLogFile { .expect("BUG: known-good log path didn't parse") }; - tracing::debug!("renaming {} to {}", src_path.display(), dst_path.display()); + tracing::debug!( + source_path = %src_path.display(), + destination_path = %dst_path.display(), + "renaming log file" + ); tokio::fs::rename(src_path.as_path(), dst_path.as_path()) .await diff --git a/crates/ssh-console/src/frontend.rs b/crates/ssh-console/src/frontend.rs index 79b7d6c68e..e3ea10a39a 100644 --- a/crates/ssh-console/src/frontend.rs +++ b/crates/ssh-console/src/frontend.rs @@ -278,7 +278,9 @@ impl russh::server::Handler for Handler { ) { tracing::warn!( peer_addr = self.peer_addr, - "certificate auth failed for user {machine_string}, not in role {admin_certificate_role}", + machine = machine_string, + role = admin_certificate_role.as_str(), + "certificate auth failed because the required role is missing", ); self.last_auth_failure = Some(AuthFailureReason::Certificate { user, @@ -293,12 +295,17 @@ impl russh::server::Handler for Handler { if let Some(user) = &user { tracing::info!( peer_addr = self.peer_addr, - "certificate auth succeeded for user {user} to machine {machine_string}, in role {admin_certificate_role}" + user = user.as_str(), + machine = machine_string, + role = admin_certificate_role.as_str(), + "certificate auth succeeded" ); } else { tracing::info!( peer_addr = self.peer_addr, - "certificate auth succeeded to machine {machine_string}, in role {admin_certificate_role}" + machine = machine_string, + role = admin_certificate_role.as_str(), + "certificate auth succeeded" ); } self.authenticated_machine_string = Some(machine_string.to_owned()); @@ -335,8 +342,8 @@ impl russh::server::Handler for Handler { } else { tracing::debug!( peer_addr = self.peer_addr, - machine_string, - "rejecting public key for user {machine_string}" + ssh_username = machine_string, + "rejecting public key for user" ); false }; @@ -760,7 +767,8 @@ async fn pubkey_auth_tenant( Code::InvalidArgument => { // InvalidArgument can happen if the user is not a valid instance ID. tracing::warn!( - "InvalidArgument when calling carbide-api to validate pubkey for {user}" + user, + "InvalidArgument when validating public key via carbide-api" ); false } diff --git a/crates/ssh-console/src/metrics.rs b/crates/ssh-console/src/metrics.rs index fd26bd7a82..a46553c20a 100644 --- a/crates/ssh-console/src/metrics.rs +++ b/crates/ssh-console/src/metrics.rs @@ -43,7 +43,10 @@ pub async fn spawn( .await .map_err(SpawnError::Listen)?; - tracing::info!("metrics listening on {}", config.metrics_address); + tracing::info!( + metrics_address = %config.metrics_address, + "metrics service listening" + ); let join_handle = tokio::spawn(async move { loop { @@ -55,7 +58,7 @@ pub async fn spawn( res = listener.accept() => match res { Ok((stream, addr)) => { - tracing::info!("got metrics connection from {addr}"); + tracing::info!(peer_address = %addr, "accepted metrics connection"); tokio::task::spawn({ let metrics_state = metrics_state.clone(); async move { diff --git a/crates/ssh-console/src/ssh_cert_parsing.rs b/crates/ssh-console/src/ssh_cert_parsing.rs index 823289d44d..5d25ee833e 100644 --- a/crates/ssh-console/src/ssh_cert_parsing.rs +++ b/crates/ssh-console/src/ssh_cert_parsing.rs @@ -80,9 +80,8 @@ fn key_id_contains_role(key_id: &str, role: &str, key_id_format: &KeyIdFormat) - }) else { tracing::warn!( - "Could not find `{}=` substring in key_id: {:?}", - key_id_format.role_field, - key_id, + role_field = key_id_format.role_field.as_str(), + "Could not find role field in key ID" ); return false; }; @@ -108,9 +107,8 @@ fn get_user_from_key_id<'a>(key_id: &'a str, key_id_format: &KeyIdFormat) -> Opt }) else { tracing::warn!( - "Could not find `{}=` substring in key_id: {:?}", - key_id_format.user_field, - key_id + user_field = key_id_format.user_field.as_str(), + "Could not find user field in key ID" ); return None; }; diff --git a/crates/ssh-console/src/ssh_server.rs b/crates/ssh-console/src/ssh_server.rs index a3a150a6ec..7aa6e7a902 100644 --- a/crates/ssh-console/src/ssh_server.rs +++ b/crates/ssh-console/src/ssh_server.rs @@ -75,7 +75,7 @@ pub async fn spawn( addr: listen_address, error, })?; - tracing::info!("listening on {}", listen_address); + tracing::info!(%listen_address, "SSH server listening"); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let join_handle = tokio::spawn(server.run(listener, shutdown_rx)); diff --git a/crates/ssh-console/tests/util/ipmi_sim.rs b/crates/ssh-console/tests/util/ipmi_sim.rs index c316ec05d5..eb84465723 100644 --- a/crates/ssh-console/tests/util/ipmi_sim.rs +++ b/crates/ssh-console/tests/util/ipmi_sim.rs @@ -212,7 +212,9 @@ pub async fn run(prompt: String) -> eyre::Result { }; tracing::debug!( - "ipmi_sim will listen on port {ipmi_sim_lanplus_port} and {ipmi_sim_serial_port}" + ipmi_sim_lanplus_port, + ipmi_sim_serial_port, + "ipmi_sim will listen" ); let mock_serial_console_port = mock_serial_console.port; @@ -290,7 +292,7 @@ sol "telnet:127.0.0.1:{mock_serial_console_port}" 115200 pub async fn run_mock_serial_console(prompt: String) -> eyre::Result { let listener = TcpListener::bind("127.0.0.1:0").await?; let port = listener.local_addr()?.port(); - tracing::debug!("mock serial console: listening on port {port}"); + tracing::debug!(port, "mock serial console listening"); let (shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>(); tokio::spawn(async move { loop { @@ -298,7 +300,7 @@ pub async fn run_mock_serial_console(prompt: String) -> eyre::Result { match res { Ok((tcp_stream, addr)) => { - tracing::debug!("mock serial console: got connection from {addr}"); + tracing::debug!(peer_address = %addr, "mock serial console accepted connection"); tokio::spawn({ let prompt = prompt.clone(); async move { diff --git a/crates/ssh-console/tests/util/mod.rs b/crates/ssh-console/tests/util/mod.rs index 182982ff0a..8044f4ac21 100644 --- a/crates/ssh-console/tests/util/mod.rs +++ b/crates/ssh-console/tests/util/mod.rs @@ -192,7 +192,10 @@ pub async fn run_baseline_test_environment( .collect(), ); - tracing::debug!("baseline test mock hosts: {mock_hosts:?}"); + tracing::debug!( + mock_host_count = mock_hosts.len(), + "Configured baseline test mock hosts" + ); let api_server_handle = ssh_console_mock_api_server::MockApiServer { mock_hosts: mock_hosts.clone(), diff --git a/crates/state-controller/src/controller/periodic_enqueuer.rs b/crates/state-controller/src/controller/periodic_enqueuer.rs index c0cedb3ac9..ac7f4f400a 100644 --- a/crates/state-controller/src/controller/periodic_enqueuer.rs +++ b/crates/state-controller/src/controller/periodic_enqueuer.rs @@ -183,7 +183,11 @@ impl PeriodicEnqueuer { } }; - tracing::trace!(iteration_data = ?locked_controller_iteration.iteration_data, "Starting iteration with ID "); + tracing::trace!( + iteration_id = locked_controller_iteration.iteration_data.id.0, + iteration_data = ?locked_controller_iteration.iteration_data, + "Starting controller iteration" + ); iteration_metrics.iteration_id = Some(locked_controller_iteration.iteration_data.id); self.enqueue_objects(iteration_metrics).await?; diff --git a/crates/state-controller/src/controller/processor.rs b/crates/state-controller/src/controller/processor.rs index 88d63838c3..48c28a01ee 100644 --- a/crates/state-controller/src/controller/processor.rs +++ b/crates/state-controller/src/controller/processor.rs @@ -421,8 +421,8 @@ impl StateProcessor { Err(_) => { tracing::error!( controller = IO::LOG_SPAN_CONTROLLER_NAME, - "Can not convert queued object ID \"{}\" to IO::ObjectID format", - object.object_id + object_id = object.object_id.as_str(), + "Can not convert queued object ID to IO::ObjectID format" ); None } diff --git a/crates/switch-controller/src/certificate.rs b/crates/switch-controller/src/certificate.rs index 21e33a2022..00cd849088 100644 --- a/crates/switch-controller/src/certificate.rs +++ b/crates/switch-controller/src/certificate.rs @@ -61,8 +61,8 @@ pub async fn start_configure_switch_certificate( return Ok(match mode { ConfigureSwitchCertificateMode::BringUp => { tracing::info!( - "Switch {:?}: no rack association, skipping certificate configuration", - switch_id + switch_id = ?switch_id, + "Switch: no rack association, skipping certificate configuration", ); StartConfigureSwitchCertificateResult::EarlyTransition( StateHandlerOutcome::transition( @@ -88,8 +88,8 @@ pub async fn start_configure_switch_certificate( return Ok(match mode { ConfigureSwitchCertificateMode::BringUp => { tracing::info!( - "Switch {:?}: component manager is not configured, skipping certificate configuration", - switch_id + switch_id = ?switch_id, + "Switch: component manager is not configured, skipping certificate configuration", ); StartConfigureSwitchCertificateResult::EarlyTransition( StateHandlerOutcome::transition( @@ -155,8 +155,8 @@ pub async fn start_configure_switch_certificate( tracing::info!( %job_id, ?mode, - "Switch {:?}: started switch certificate configuration", - switch_id + switch_id = ?switch_id, + "Switch: started switch certificate configuration", ); Ok(StartConfigureSwitchCertificateResult::JobStarted(job_id)) diff --git a/crates/switch-controller/src/configuring.rs b/crates/switch-controller/src/configuring.rs index ef84f93bdb..b94358bbbc 100644 --- a/crates/switch-controller/src/configuring.rs +++ b/crates/switch-controller/src/configuring.rs @@ -70,9 +70,9 @@ async fn handle_rotate_os_password( ctx.services.credential_manager.get_credentials(&key).await { tracing::info!( - "Switch {:?}: NVOS admin credentials already exist in vault for BMC MAC {}", - switch_id, - bmc_mac_address + switch_id = ?switch_id, + bmc_mac_address = %bmc_mac_address, + "Switch: NVOS admin credentials already exist in vault", ); return Ok(StateHandlerOutcome::transition( SwitchControllerState::FetchInfo, @@ -139,20 +139,18 @@ async fn handle_rotate_os_password( } }; - let (username, password) = match ( - expected_switch.nvos_username, - expected_switch.nvos_password, - ) { - (Some(username), Some(password)) => (username, password), - _ => { - tracing::info!( - "Switch {:?}: no NVOS credentials in vault or expected switch for BMC MAC {}, skipping", - switch_id, - bmc_mac_address - ); - return Ok(outcome); - } - }; + let (username, password) = + match (expected_switch.nvos_username, expected_switch.nvos_password) { + (Some(username), Some(password)) => (username, password), + _ => { + tracing::info!( + switch_id = ?switch_id, + bmc_mac_address = %bmc_mac_address, + "Switch: no NVOS credentials in vault or expected switch; skipping", + ); + return Ok(outcome); + } + }; let credentials = Credentials::UsernamePassword { username, password }; @@ -169,9 +167,9 @@ async fn handle_rotate_os_password( })?; tracing::info!( - "Switch {:?}: stored NVOS admin credentials from expected switch into vault for BMC MAC {}", - switch_id, - bmc_mac_address + switch_id = ?switch_id, + bmc_mac_address = %bmc_mac_address, + "Switch: stored NVOS admin credentials from expected switch into vault", ); Ok(outcome) @@ -228,8 +226,8 @@ async fn handle_configure_certificate_wait_for_complete( crate::certificate::ConfigureSwitchCertificatePollOutcome::Completed => { tracing::info!( %job_id, - "Switch {:?}: switch certificate configuration completed", - switch_id + switch_id = ?switch_id, + "Switch: switch certificate configuration completed", ); Ok(StateHandlerOutcome::transition( SwitchControllerState::Configuring { diff --git a/crates/switch-controller/src/created.rs b/crates/switch-controller/src/created.rs index d43f26ab97..23083a911a 100644 --- a/crates/switch-controller/src/created.rs +++ b/crates/switch-controller/src/created.rs @@ -31,8 +31,8 @@ pub async fn handle_created( _ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { tracing::info!( - "Switch {:?} created, transitioning to Initializing", - switch_id + switch_id = ?switch_id, + "Switch created, transitioning to Initializing", ); Ok(StateHandlerOutcome::transition( SwitchControllerState::Initializing { diff --git a/crates/switch-controller/src/deleting.rs b/crates/switch-controller/src/deleting.rs index 70cf1d5472..99d0b2ea34 100644 --- a/crates/switch-controller/src/deleting.rs +++ b/crates/switch-controller/src/deleting.rs @@ -33,7 +33,10 @@ pub async fn handle_deleting( _state: &mut Switch, ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { - tracing::info!("Deleting Switch {}", switch_id.to_string()); + tracing::info!( + %switch_id, + "Deleting switch", + ); let mut txn = ctx.services.db_pool.begin().await?; db_switch::final_delete(*switch_id, &mut txn).await?; Ok(StateHandlerOutcome::deleted().with_txn(txn)) diff --git a/crates/switch-controller/src/error_state.rs b/crates/switch-controller/src/error_state.rs index 96e26377d8..0ec8454fb4 100644 --- a/crates/switch-controller/src/error_state.rs +++ b/crates/switch-controller/src/error_state.rs @@ -34,7 +34,10 @@ pub async fn handle_error( state: &mut Switch, _ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { - tracing::info!("Switch is in error state {}", _switch_id.to_string()); + tracing::info!( + switch_id = %_switch_id, + "Switch is in error state", + ); if state.is_marked_as_deleted() { return Ok(StateHandlerOutcome::transition( SwitchControllerState::Deleting, diff --git a/crates/switch-controller/src/initializing.rs b/crates/switch-controller/src/initializing.rs index 53bf17f5b8..bfe63edd9f 100644 --- a/crates/switch-controller/src/initializing.rs +++ b/crates/switch-controller/src/initializing.rs @@ -68,9 +68,9 @@ async fn handle_wait_for_os_machine_interface( Some(es) => es, None => { tracing::info!( - "Switch {:?}: no expected switch found for BMC MAC {}, waiting", - switch_id, - bmc_mac_address + switch_id = %switch_id, + bmc_mac_address = %bmc_mac_address, + "No expected switch found; transitioning to error", ); return Ok(StateHandlerOutcome::transition( SwitchControllerState::Error { @@ -83,10 +83,10 @@ async fn handle_wait_for_os_machine_interface( let nvos_mac_addresses = &expected_switch.nvos_mac_addresses; if nvos_mac_addresses.is_empty() { tracing::warn!( - "Switch {:?}: no NVOS MAC addresses on expected switch for serial {}, BMC MAC {}", - switch_id, - bmc_mac_address, - expected_switch.bmc_mac_address + switch_id = %switch_id, + bmc_mac_address = %bmc_mac_address, + expected_bmc_mac_address = %expected_switch.bmc_mac_address, + "Switch has no NVOS MAC addresses on expected switch", ); return Ok(StateHandlerOutcome::transition( SwitchControllerState::Error { @@ -111,10 +111,10 @@ async fn handle_wait_for_os_machine_interface( if let Some(existing_switch_id) = interface.switch_id { if existing_switch_id != *switch_id { tracing::warn!( - "Switch {:?}: NVOS MAC {} already associated with switch {}", - switch_id, - mac_address, - existing_switch_id + switch_id = %switch_id, + nvos_mac_address = %mac_address, + existing_switch_id = %existing_switch_id, + "Switch: NVOS MAC already associated with switch", ); return Ok(StateHandlerOutcome::transition( SwitchControllerState::Error { @@ -136,10 +136,10 @@ async fn handle_wait_for_os_machine_interface( ) .await?; tracing::info!( - "Switch {:?}: associated NVOS interface {} (MAC {})", - switch_id, - interface.id, - mac_address + switch_id = %switch_id, + interface_id = %interface.id, + nvos_mac_address = %mac_address, + "Switch associated NVOS interface", ); associated_count += 1; } @@ -147,17 +147,17 @@ async fn handle_wait_for_os_machine_interface( txn.commit().await?; tracing::info!( - "Switch {:?}: associated {} NVOS interfaces for BMC MAC {}", - switch_id, - associated_count, - bmc_mac_address + switch_id = %switch_id, + associated_nvos_interface_count = associated_count, + bmc_mac_address = %bmc_mac_address, + "Switch associated NVOS interfaces", ); if associated_count >= 1 { tracing::info!( - "Switch {:?}: at least one NVOS interface associated ({}/{}), transitioning to Configuring", - switch_id, - associated_count, - total + switch_id = %switch_id, + associated_nvos_interface_count = associated_count, + total_nvos_interface_count = total, + "Switch has at least one NVOS interface associated, transitioning to Configuring", ); Ok(StateHandlerOutcome::transition( SwitchControllerState::Configuring { @@ -168,10 +168,10 @@ async fn handle_wait_for_os_machine_interface( )) } else { tracing::info!( - "Switch {:?}: {}/{} NVOS interfaces associated, waiting", - switch_id, - associated_count, - total + switch_id = %switch_id, + associated_nvos_interface_count = associated_count, + total_nvos_interface_count = total, + "Switch has no NVOS interfaces associated, waiting", ); Ok(StateHandlerOutcome::wait(format!( "{}/{} NVOS interfaces associated, waiting", diff --git a/crates/switch-controller/src/ready.rs b/crates/switch-controller/src/ready.rs index 478dd50c68..06a3def884 100644 --- a/crates/switch-controller/src/ready.rs +++ b/crates/switch-controller/src/ready.rs @@ -67,8 +67,8 @@ pub async fn handle_ready( } tracing::warn!( - "unknown initiator for switch reprovisioning request: {}", - req.initiator + initiator = %req.initiator, + "Unknown initiator for switch reprovisioning request", ); return Ok(StateHandlerOutcome::transition( SwitchControllerState::Error { diff --git a/crates/switch-controller/src/validating.rs b/crates/switch-controller/src/validating.rs index 3522c50f11..b087ebe9dd 100644 --- a/crates/switch-controller/src/validating.rs +++ b/crates/switch-controller/src/validating.rs @@ -32,7 +32,10 @@ pub async fn handle_validating( state: &mut Switch, _ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { - tracing::info!("Validating Switch {:?}", switch_id); + tracing::info!( + switch_id = ?switch_id, + "Validating switch", + ); let validating_state = match &state.controller_state.value { SwitchControllerState::Validating { validating_state } => validating_state, _ => unreachable!("handle_validating called with non-Validating state"),