From da8fb6d9bf8631bbaaf3862b7edf5bd28a79e575 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Fri, 4 Apr 2025 17:16:36 -0400 Subject: [PATCH 01/25] chore: ddcommon fips feature --- Cargo.lock | 52 +++++++++++++++++-------------------- ddcommon/Cargo.toml | 2 ++ local-linux.Dockerfile | 5 +++- scripts/verify_fips_deps.sh | 34 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 29 deletions(-) create mode 100755 scripts/verify_fips_deps.sh diff --git a/Cargo.lock b/Cargo.lock index 405d78d0de..a6682e16a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,31 +379,42 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-fips-sys" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9c2e952a1f57e8cbc78b058a968639e70c4ce8b9c0a5e6363d4e5670eed795" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "regex", +] + [[package]] name = "aws-lc-rs" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" dependencies = [ + "aws-lc-fips-sys", "aws-lc-sys", - "mirai-annotations", - "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.22.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" dependencies = [ "bindgen", "cc", "cmake", "dunce", "fs_extra", - "libc", - "paste", ] [[package]] @@ -526,7 +537,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "log", @@ -3149,15 +3160,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -3263,7 +3265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3460,12 +3462,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "msvc-demangler" version = "0.10.1" @@ -4069,7 +4065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.87", @@ -4530,9 +4526,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "aws-lc-rs", "once_cell", diff --git a/ddcommon/Cargo.toml b/ddcommon/Cargo.toml index 579ee909d2..8d24425803 100644 --- a/ddcommon/Cargo.toml +++ b/ddcommon/Cargo.toml @@ -76,3 +76,5 @@ use_webpki_roots = ["hyper-rustls/webpki-roots"] # Enable this feature to enable stubbing of cgroup # php directly import this crate and uses functions gated by this feature for their test cgroup_testing = [] +# FIPS mode uses the FIPS-compliant cryptographic provider +fips = ["https", "hyper-rustls/fips"] diff --git a/local-linux.Dockerfile b/local-linux.Dockerfile index e4c1a79bc8..c2587a7699 100644 --- a/local-linux.Dockerfile +++ b/local-linux.Dockerfile @@ -13,7 +13,10 @@ RUN apt-get update && \ protobuf-compiler \ docker.io \ sudo \ - && rm -rf /var/lib/apt/lists/* + wget \ + && rm -rf /var/lib/apt/lists/* \ + && wget -O go1.24.2.linux-arm64.tar.gz https://go.dev/dl/go1.24.2.linux-arm64.tar.gz \ + && tar -C /usr/local -xzf go1.24.2.linux-arm64.tar.gz # Docker-in-Docker configuration (necessary for integration tests) RUN mkdir -p /var/lib/docker diff --git a/scripts/verify_fips_deps.sh b/scripts/verify_fips_deps.sh new file mode 100755 index 0000000000..204509a7b0 --- /dev/null +++ b/scripts/verify_fips_deps.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# verify_fips_deps.sh +# Script to verify that the fips feature doesn't include ring and uses the proper crypto library +# Usage: ./verify_fips_deps.sh [package_name] (defaults to ddcommon if not specified) + +set -e + +# Default to ddcommon if no package is specified +PACKAGE=${1:-ddcommon} + +echo "Checking ${PACKAGE} with fips feature..." + +# Check if aws-lc-fips-sys is included with FIPS feature +FIPS_SYS_COUNT=$(cargo tree -p ${PACKAGE} --features fips | grep -c "aws-lc-fips-sys" || true) + +if [ "$FIPS_SYS_COUNT" -eq 0 ]; then + echo "❌ ERROR: aws-lc-fips-sys is not included when fips feature is enabled" + exit 1 +else + echo "✅ aws-lc-fips-sys is correctly included with fips feature" +fi + +# Check if ring is included with FIPS feature (should not be) +RING_COUNT=$(cargo tree -p ${PACKAGE} --features fips | grep -c "ring" || true) + +if [ "$RING_COUNT" -eq 0 ]; then + echo "✅ ring is correctly NOT included with fips feature" +else + echo "❌ ERROR: ring is included when fips feature is enabled" + exit 1 +fi + +echo "All checks passed! ${PACKAGE} FIPS feature doesn't include ring." +exit 0 \ No newline at end of file From c359b473f7eb1465495f262507e56cd718642fcb Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Mon, 7 Apr 2025 12:28:44 -0400 Subject: [PATCH 02/25] chore: propagate ddcommon/fips to trace-utils --- trace-utils/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trace-utils/Cargo.toml b/trace-utils/Cargo.toml index cee040b0b2..8c4aa492ce 100644 --- a/trace-utils/Cargo.toml +++ b/trace-utils/Cargo.toml @@ -76,3 +76,5 @@ test-utils = [ ] proxy = ["hyper-http-proxy"] compression = ["zstd", "flate2"] +# FIPS mode uses the FIPS-compliant cryptographic provider +fips = ["ddcommon/fips"] From d4c5263b474fbeb399aba6256ac879a1bcac0aaf Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Mon, 7 Apr 2025 12:42:19 -0400 Subject: [PATCH 03/25] chore: update verify_fips_deps script to account for additional features --- scripts/verify_fips_deps.sh | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/scripts/verify_fips_deps.sh b/scripts/verify_fips_deps.sh index 204509a7b0..f936a790a9 100755 --- a/scripts/verify_fips_deps.sh +++ b/scripts/verify_fips_deps.sh @@ -1,34 +1,49 @@ #!/bin/bash # verify_fips_deps.sh # Script to verify that the fips feature doesn't include ring and uses the proper crypto library -# Usage: ./verify_fips_deps.sh [package_name] (defaults to ddcommon if not specified) +# Usage: ./verify_fips_deps.sh [package_name] [additional_features] +# Examples: +# ./verify_fips_deps.sh # Checks ddcommon with fips feature +# ./verify_fips_deps.sh datadog-trace-utils # Checks trace-utils with fips feature +# ./verify_fips_deps.sh datadog-trace-utils compression # Checks trace-utils with fips,compression features set -e # Default to ddcommon if no package is specified PACKAGE=${1:-ddcommon} +shift 2>/dev/null || true -echo "Checking ${PACKAGE} with fips feature..." +# Additional features to include +ADDITIONAL_FEATURES="$@" +FEATURES="fips" -# Check if aws-lc-fips-sys is included with FIPS feature -FIPS_SYS_COUNT=$(cargo tree -p ${PACKAGE} --features fips | grep -c "aws-lc-fips-sys" || true) +# Add additional features if specified +if [ -n "$ADDITIONAL_FEATURES" ]; then + FEATURES="$FEATURES,$ADDITIONAL_FEATURES" +fi + +echo "Checking ${PACKAGE} with features: ${FEATURES}..." + +# Check if aws-lc-fips-sys is included +FIPS_SYS_COUNT=$(cargo tree -p ${PACKAGE} --features ${FEATURES} | grep -c "aws-lc-fips-sys" || true) if [ "$FIPS_SYS_COUNT" -eq 0 ]; then echo "❌ ERROR: aws-lc-fips-sys is not included when fips feature is enabled" exit 1 else - echo "✅ aws-lc-fips-sys is correctly included with fips feature" + echo "✅ aws-lc-fips-sys is correctly included with features: ${FEATURES}" fi -# Check if ring is included with FIPS feature (should not be) -RING_COUNT=$(cargo tree -p ${PACKAGE} --features fips | grep -c "ring" || true) +# Check if ring is included (should not be for runtime dependencies) +RING_COUNT=$(cargo tree -p ${PACKAGE} --features ${FEATURES} -e=no-dev -i ring | grep -c "ring" || true) if [ "$RING_COUNT" -eq 0 ]; then - echo "✅ ring is correctly NOT included with fips feature" + echo "✅ ring is correctly NOT included with features: ${FEATURES} (in runtime dependencies)" else - echo "❌ ERROR: ring is included when fips feature is enabled" + echo "❌ ERROR: ring is included with features: ${FEATURES} (in runtime dependencies)" + cargo tree -p ${PACKAGE} --features ${FEATURES} --no-dev-dependencies -i ring exit 1 fi -echo "All checks passed! ${PACKAGE} FIPS feature doesn't include ring." -exit 0 \ No newline at end of file +echo "All checks passed! ${PACKAGE} with features ${FEATURES} doesn't include ring in runtime dependencies." +exit 0 From ec8fcac102b7dd6e260bb29abfc1c32f14e9beb3 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Tue, 15 Apr 2025 13:34:00 -0400 Subject: [PATCH 04/25] chore: do not initialize crypto in fips mode, it should be done already --- ddcommon/src/connector/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 8d378348fd..8c7294e903 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -96,11 +96,12 @@ mod https { /// sometimes this is done as a side-effect of other operations, but we need to ensure it /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 + /// In fips mode we expect someone to have done this already. fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); INIT_CRYPTO_PROVIDER.get_or_init(|| { - #[cfg(unix)] + #[cfg(all(unix, not(feature = "fips")))] #[allow(clippy::expect_used)] rustls::crypto::aws_lc_rs::default_provider() .install_default() From e0ec4f6acf1a5294bcd2025363dba23aed09befd Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 16 Apr 2025 10:30:56 -0400 Subject: [PATCH 05/25] chore: remove the verify script which is funky anyway --- scripts/verify_fips_deps.sh | 49 ------------------------------------- 1 file changed, 49 deletions(-) delete mode 100755 scripts/verify_fips_deps.sh diff --git a/scripts/verify_fips_deps.sh b/scripts/verify_fips_deps.sh deleted file mode 100755 index f936a790a9..0000000000 --- a/scripts/verify_fips_deps.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# verify_fips_deps.sh -# Script to verify that the fips feature doesn't include ring and uses the proper crypto library -# Usage: ./verify_fips_deps.sh [package_name] [additional_features] -# Examples: -# ./verify_fips_deps.sh # Checks ddcommon with fips feature -# ./verify_fips_deps.sh datadog-trace-utils # Checks trace-utils with fips feature -# ./verify_fips_deps.sh datadog-trace-utils compression # Checks trace-utils with fips,compression features - -set -e - -# Default to ddcommon if no package is specified -PACKAGE=${1:-ddcommon} -shift 2>/dev/null || true - -# Additional features to include -ADDITIONAL_FEATURES="$@" -FEATURES="fips" - -# Add additional features if specified -if [ -n "$ADDITIONAL_FEATURES" ]; then - FEATURES="$FEATURES,$ADDITIONAL_FEATURES" -fi - -echo "Checking ${PACKAGE} with features: ${FEATURES}..." - -# Check if aws-lc-fips-sys is included -FIPS_SYS_COUNT=$(cargo tree -p ${PACKAGE} --features ${FEATURES} | grep -c "aws-lc-fips-sys" || true) - -if [ "$FIPS_SYS_COUNT" -eq 0 ]; then - echo "❌ ERROR: aws-lc-fips-sys is not included when fips feature is enabled" - exit 1 -else - echo "✅ aws-lc-fips-sys is correctly included with features: ${FEATURES}" -fi - -# Check if ring is included (should not be for runtime dependencies) -RING_COUNT=$(cargo tree -p ${PACKAGE} --features ${FEATURES} -e=no-dev -i ring | grep -c "ring" || true) - -if [ "$RING_COUNT" -eq 0 ]; then - echo "✅ ring is correctly NOT included with features: ${FEATURES} (in runtime dependencies)" -else - echo "❌ ERROR: ring is included with features: ${FEATURES} (in runtime dependencies)" - cargo tree -p ${PACKAGE} --features ${FEATURES} --no-dev-dependencies -i ring - exit 1 -fi - -echo "All checks passed! ${PACKAGE} with features ${FEATURES} doesn't include ring in runtime dependencies." -exit 0 From c4f7c778f8ae22847198f09077dad73902cdfbb3 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Tue, 22 Apr 2025 17:10:49 -0400 Subject: [PATCH 06/25] chore: rearrange fips feature for tests --- ddcommon/src/connector/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 8c7294e903..bc027bcaec 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,11 +97,12 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. + #[cfg(not(feature = "fips"))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); INIT_CRYPTO_PROVIDER.get_or_init(|| { - #[cfg(all(unix, not(feature = "fips")))] + #[cfg(unix)] #[allow(clippy::expect_used)] rustls::crypto::aws_lc_rs::default_provider() .install_default() @@ -109,6 +110,10 @@ mod https { }); } + // This this actually needs to be done by the user somewhere in their own main. + #[cfg(feature = "fips")] + fn ensure_crypto_provider_initialized() {} + #[cfg(feature = "use_webpki_roots")] pub(super) fn build_https_connector_with_webpki_roots() -> anyhow::Result< hyper_rustls::HttpsConnector, From 83af1e5ed3e1d206d9478e81b9d4f4a300d7bb93 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Tue, 22 Apr 2025 17:30:48 -0400 Subject: [PATCH 07/25] chore: make the fips feature unix-only --- ddcommon/Cargo.toml | 2 +- ddcommon/src/connector/mod.rs | 5 +++-- trace-utils/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ddcommon/Cargo.toml b/ddcommon/Cargo.toml index 8d24425803..3ca248704c 100644 --- a/ddcommon/Cargo.toml +++ b/ddcommon/Cargo.toml @@ -76,5 +76,5 @@ use_webpki_roots = ["hyper-rustls/webpki-roots"] # Enable this feature to enable stubbing of cgroup # php directly import this crate and uses functions gated by this feature for their test cgroup_testing = [] -# FIPS mode uses the FIPS-compliant cryptographic provider +# FIPS mode uses the FIPS-compliant cryptographic provider (Unix only) fips = ["https", "hyper-rustls/fips"] diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index bc027bcaec..407365de04 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,7 +97,7 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. - #[cfg(not(feature = "fips"))] + #[cfg(not(all(unix, feature = "fips")))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); @@ -111,7 +111,8 @@ mod https { } // This this actually needs to be done by the user somewhere in their own main. - #[cfg(feature = "fips")] + // This will only be active on Unix platforms + #[cfg(all(unix, feature = "fips"))] fn ensure_crypto_provider_initialized() {} #[cfg(feature = "use_webpki_roots")] diff --git a/trace-utils/Cargo.toml b/trace-utils/Cargo.toml index 8c4aa492ce..212cae21f3 100644 --- a/trace-utils/Cargo.toml +++ b/trace-utils/Cargo.toml @@ -76,5 +76,5 @@ test-utils = [ ] proxy = ["hyper-http-proxy"] compression = ["zstd", "flate2"] -# FIPS mode uses the FIPS-compliant cryptographic provider +# FIPS mode uses the FIPS-compliant cryptographic provider (Unix only) fips = ["ddcommon/fips"] From 630dccc7984b27d592673789c9760764e611aab9 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 10:22:57 -0400 Subject: [PATCH 08/25] chore: more details from test errors, please --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10faa1c2b2..ce673af2cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,19 +56,19 @@ jobs: # Run doc tests with cargo test and run tests with nextest and generate junit.xml run: cargo test --workspace --exclude builder --doc --verbose && cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)' env: - RUST_BACKTRACE: 1 + RUST_BACKTRACE: full - name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --exclude builder --profile ci --test-threads=1 --verbose -E 'test(tracing_integration_tests::)'" if: runner.os == 'Linux' shell: bash run: cargo nextest run --workspace --exclude builder --profile ci --test-threads=1 --verbose -E 'test(tracing_integration_tests::)' env: - RUST_BACKTRACE: 1 + RUST_BACKTRACE: full - name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo nextest run --package test_spawn_from_lib --features prefer-dynamic -E '!test(tracing_integration_tests::)'" shell: bash run: cargo nextest run --package test_spawn_from_lib --features prefer-dynamic -E '!test(tracing_integration_tests::)' env: RUSTFLAGS: "-C prefer-dynamic" - RUST_BACKTRACE: 1 + RUST_BACKTRACE: full - name: Report Test Results if: success() || failure() uses: mikepenz/action-junit-report@db71d41eb79864e25ab0337e395c352e84523afe # 4.3.1 From fff172d696741cc4ad5b5618caca56231bd8ac63 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 12:00:15 -0400 Subject: [PATCH 09/25] chore: back to a plain fips check for ensure_crypto_provider_initialized --- ddcommon/src/connector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 407365de04..549e269e97 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,7 +97,7 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. - #[cfg(not(all(unix, feature = "fips")))] + #[cfg(not(feature = "fips"))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); @@ -112,7 +112,7 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms - #[cfg(all(unix, feature = "fips"))] + #[cfg(feature = "fips")] fn ensure_crypto_provider_initialized() {} #[cfg(feature = "use_webpki_roots")] From e530cd25a454e675a1f7279006c2f19b3d870bf2 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 12:21:05 -0400 Subject: [PATCH 10/25] chore: try without nasm on windows --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 296ebbd1c8..e44c89fc84 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,7 +45,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) + $([ ${{ matrix.platform }} = windows-latest ] && echo AWS_LC_FIPS_SYS_NO_ASM=1 ) cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" From f63aca169d5f0f3b47a728412f214bf7e304fb56 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 13:18:56 -0400 Subject: [PATCH 11/25] chore: env var differently --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e44c89fc84..f6502c7a67 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,7 +45,10 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - $([ ${{ matrix.platform }} = windows-latest ] && echo AWS_LC_FIPS_SYS_NO_ASM=1 ) cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) + if [[ "${{ matrix.platform ||" = "windows-latest" ]]; then + AWS_LC_FIPS_SYS_NO_ASM=1 + fi + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" From 088f47c33950bb5387efcfe3e5a45ded93709fc3 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 13:48:05 -0400 Subject: [PATCH 12/25] chore: fix typo --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f6502c7a67..669f9de16f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,7 +45,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - if [[ "${{ matrix.platform ||" = "windows-latest" ]]; then + if [[ "${{ matrix.platform }}" = "windows-latest" ]]; then AWS_LC_FIPS_SYS_NO_ASM=1 fi cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) From 95ba19fff54bc6d793e669020c1d0c03ee38a0cc Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 13:50:16 -0400 Subject: [PATCH 13/25] chore: maybe we export it --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 669f9de16f..f9e410a2bb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,7 +46,7 @@ jobs: run: | # shellcheck disable=SC2046 if [[ "${{ matrix.platform }}" = "windows-latest" ]]; then - AWS_LC_FIPS_SYS_NO_ASM=1 + export AWS_LC_FIPS_SYS_NO_ASM=1 fi cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: From 1a29663788ec29d2bf5bbd5e38c1c368b201df52 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 13:53:12 -0400 Subject: [PATCH 14/25] chore: fix equal --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f9e410a2bb..ff5ef00114 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,7 +45,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - if [[ "${{ matrix.platform }}" = "windows-latest" ]]; then + if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then export AWS_LC_FIPS_SYS_NO_ASM=1 fi cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) From 9c3cb329a78717fd69a00f4bc747395d26a95f9d Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 13:56:07 -0400 Subject: [PATCH 15/25] chore: which quoting?? --- .github/workflows/lint.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ff5ef00114..ecc8242f5a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,9 +46,10 @@ jobs: run: | # shellcheck disable=SC2046 if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then - export AWS_LC_FIPS_SYS_NO_ASM=1 - fi - cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) + export AWS_LC_FIPS_SYS_NO_ASM=1; + fi; + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call); + licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" From 9b455f3e870004cec7acda2b9c3e3168108c86ba Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 14:08:32 -0400 Subject: [PATCH 16/25] chore: move shellcheck line --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ecc8242f5a..16441dc8b5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -44,10 +44,10 @@ jobs: - name: Run clippy on ${{ matrix.platform }} ${{ matrix.rust_version }} shell: bash run: | - # shellcheck disable=SC2046 if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then export AWS_LC_FIPS_SYS_NO_ASM=1; fi; + # shellcheck disable=SC2046 cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call); licensecheck: From ec00d9a8f847a66435016e99c1ce36b7d1ad8002 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 14:34:17 -0400 Subject: [PATCH 17/25] chore: we want crypto providers in tests --- ddcommon/src/connector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 549e269e97..d0efb121ec 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,7 +97,7 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. - #[cfg(not(feature = "fips"))] + #[cfg(any(not(feature = "fips"), feature = "test"))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); @@ -112,7 +112,7 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms - #[cfg(feature = "fips")] + #[cfg(all(feature = "fips", not(feature = "test")))] fn ensure_crypto_provider_initialized() {} #[cfg(feature = "use_webpki_roots")] From 4f89ae4c5fb4050cb94d51c91d6d6a8491731ea2 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 14:46:26 -0400 Subject: [PATCH 18/25] chore: is test a cfg and not a feature --- ddcommon/src/connector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index d0efb121ec..5f3faac561 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,7 +97,7 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. - #[cfg(any(not(feature = "fips"), feature = "test"))] + #[cfg(any(not(feature = "fips"), test))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); @@ -112,7 +112,7 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms - #[cfg(all(feature = "fips", not(feature = "test")))] + #[cfg(all(feature = "fips", not(test)))] fn ensure_crypto_provider_initialized() {} #[cfg(feature = "use_webpki_roots")] From f1198245a3ca556f558c12d5146d6e8fe64e708c Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 14:58:19 -0400 Subject: [PATCH 19/25] chore: are we building the wrong thing? --- ddcommon/src/connector/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 5f3faac561..f26c963eb0 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -113,7 +113,9 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms #[cfg(all(feature = "fips", not(test)))] - fn ensure_crypto_provider_initialized() {} + fn ensure_crypto_provider_initialized() { + compile_error!("is this what we are trying to compile in coverage?"); + } #[cfg(feature = "use_webpki_roots")] pub(super) fn build_https_connector_with_webpki_roots() -> anyhow::Result< From cba7c85c7d71713b44b7bc63fd702133411f75aa Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 15:05:31 -0400 Subject: [PATCH 20/25] chore: coverage job suggests maybe we need the coverage cfg actually --- ddcommon/src/connector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index f26c963eb0..3af938c230 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -97,7 +97,7 @@ mod https { /// happens here. On non-unix platforms, ddcommon uses `ring` instead, which handles this /// at rustls initialization. TODO: Move to the more ergonomic LazyLock when MSRV is 1.80 /// In fips mode we expect someone to have done this already. - #[cfg(any(not(feature = "fips"), test))] + #[cfg(any(not(feature = "fips"), coverage))] fn ensure_crypto_provider_initialized() { use std::sync::OnceLock; static INIT_CRYPTO_PROVIDER: OnceLock<()> = OnceLock::new(); @@ -112,7 +112,7 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms - #[cfg(all(feature = "fips", not(test)))] + #[cfg(all(feature = "fips", not(coverage)))] fn ensure_crypto_provider_initialized() { compile_error!("is this what we are trying to compile in coverage?"); } From d71a033d08153598ebd63c96b4b15c53f5f0b323 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 15:20:04 -0400 Subject: [PATCH 21/25] chore: revert compile error now that coverage is working --- ddcommon/src/connector/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 3af938c230..0bd988c1f3 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -113,9 +113,7 @@ mod https { // This this actually needs to be done by the user somewhere in their own main. // This will only be active on Unix platforms #[cfg(all(feature = "fips", not(coverage)))] - fn ensure_crypto_provider_initialized() { - compile_error!("is this what we are trying to compile in coverage?"); - } + fn ensure_crypto_provider_initialized() {} #[cfg(feature = "use_webpki_roots")] pub(super) fn build_https_connector_with_webpki_roots() -> anyhow::Result< From 3894ba194d2301ade060ee4864c520a972188540 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 15:27:39 -0400 Subject: [PATCH 22/25] chore: coverage is an okay config --- ddcommon/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ddcommon/Cargo.toml b/ddcommon/Cargo.toml index 3ca248704c..dfaf4962c9 100644 --- a/ddcommon/Cargo.toml +++ b/ddcommon/Cargo.toml @@ -78,3 +78,6 @@ use_webpki_roots = ["hyper-rustls/webpki-roots"] cgroup_testing = [] # FIPS mode uses the FIPS-compliant cryptographic provider (Unix only) fips = ["https", "hyper-rustls/fips"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] } From fcaa39f736fb737253039d9d9770ff5f19bb2d9b Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Wed, 23 Apr 2025 16:06:32 -0400 Subject: [PATCH 23/25] chore: updating 3rd party license file --- LICENSE-3rdparty.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 9df97a1fa5..423984455c 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -2492,7 +2492,7 @@ third_party_libraries: IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - package_name: aws-lc-rs - package_version: 1.10.0 + package_version: 1.13.0 repository: https://github.com/aws/aws-lc-rs license: ISC AND (Apache-2.0 OR ISC) licenses: @@ -2905,7 +2905,7 @@ third_party_libraries: ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - package_name: aws-lc-sys - package_version: 0.22.0 + package_version: 0.28.0 repository: https://github.com/aws/aws-lc-rs license: ISC AND (Apache-2.0 OR ISC) AND OpenSSL licenses: @@ -15402,9 +15402,9 @@ third_party_libraries: limitations under the License. - package_name: itertools - package_version: 0.12.1 + package_version: 0.10.5 repository: https://github.com/rust-itertools/itertools - license: MIT OR Apache-2.0 + license: MIT/Apache-2.0 licenses: - license: MIT text: | @@ -17827,13 +17827,6 @@ third_party_libraries: LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -- package_name: mirai-annotations - package_version: 1.12.0 - repository: https://github.com/facebookexperimental/MIRAI - license: MIT - licenses: - - license: MIT - text: NOT FOUND - package_name: msvc-demangler package_version: 0.10.1 repository: https://github.com/mstange/msvc-demangler-rust @@ -23406,7 +23399,7 @@ third_party_libraries: IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - package_name: rustls - package_version: 0.23.18 + package_version: 0.23.23 repository: https://github.com/rustls/rustls license: Apache-2.0 OR ISC OR MIT licenses: From ef22180987980d27ca6f41577697f621414f502d Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Thu, 24 Apr 2025 11:40:46 -0400 Subject: [PATCH 24/25] chore: review fixes --- .github/workflows/lint.yml | 6 +++--- ddcommon/Cargo.toml | 8 ++++++++ ddcommon/src/connector/mod.rs | 4 ++-- local-linux.Dockerfile | 6 ++++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 16441dc8b5..9e6e15ecab 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,10 +45,10 @@ jobs: shell: bash run: | if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then - export AWS_LC_FIPS_SYS_NO_ASM=1; - fi; + export AWS_LC_FIPS_SYS_NO_ASM=1 + fi # shellcheck disable=SC2046 - cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call); + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: runs-on: ubuntu-latest diff --git a/ddcommon/Cargo.toml b/ddcommon/Cargo.toml index dfaf4962c9..19e42526af 100644 --- a/ddcommon/Cargo.toml +++ b/ddcommon/Cargo.toml @@ -80,4 +80,12 @@ cgroup_testing = [] fips = ["https", "hyper-rustls/fips"] [lints.rust] +# We run coverage checks in our github actions. These checks are run with +# --all-features which is incompatible with our fips feature. The crypto +# provider default needs to be set by the caller in fips mode. For now, we want +# to make sure that the coverage tests use the non-fips version of the crypto +# provider initialization logic, so we added a coverage cfg check on the +# function in src/connector/mod.rs. The coverage config is actually not used in +# normal environments, so we need to let the rust linter know that it is in +# fact a real thing, though one that shows up only in some situations. unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] } diff --git a/ddcommon/src/connector/mod.rs b/ddcommon/src/connector/mod.rs index 0bd988c1f3..5fcc06014a 100644 --- a/ddcommon/src/connector/mod.rs +++ b/ddcommon/src/connector/mod.rs @@ -110,8 +110,8 @@ mod https { }); } - // This this actually needs to be done by the user somewhere in their own main. - // This will only be active on Unix platforms + // This actually needs to be done by the user somewhere in their own main. This will only + // be active on Unix platforms #[cfg(all(feature = "fips", not(coverage)))] fn ensure_crypto_provider_initialized() {} diff --git a/local-linux.Dockerfile b/local-linux.Dockerfile index c2587a7699..a09f30473a 100644 --- a/local-linux.Dockerfile +++ b/local-linux.Dockerfile @@ -14,8 +14,10 @@ RUN apt-get update && \ docker.io \ sudo \ wget \ - && rm -rf /var/lib/apt/lists/* \ - && wget -O go1.24.2.linux-arm64.tar.gz https://go.dev/dl/go1.24.2.linux-arm64.tar.gz \ + && rm -rf /var/lib/apt/lists/* + +# We need go in order to build aws-lc-fips-sys +RUN wget -O go1.24.2.linux-arm64.tar.gz https://go.dev/dl/go1.24.2.linux-arm64.tar.gz \ && tar -C /usr/local -xzf go1.24.2.linux-arm64.tar.gz # Docker-in-Docker configuration (necessary for integration tests) From 1a60690538fd32384622404a9134110159ebe551 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Thu, 24 Apr 2025 12:46:21 -0400 Subject: [PATCH 25/25] chore: rebase and fix lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a6682e16a1..378cb33667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4045,7 +4045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.10.5", "log", "multimap", "once_cell",