From a5032919866649ee07d46352480ede820e0fdf40 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 05:20:28 +0000 Subject: [PATCH 1/6] simd(neon): native U32x16 = [U32x4;4] ARX lane + fix aarch64 stable compile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Native NEON U32x16 (the ChaCha20/BLAKE ARX lane), mirroring the wasm `[U32x4; 4]` shape so one ChaCha20 backend source compiles unchanged on every tier: - U32x4 gains `bitxor` (veorq_u32) + `rotate_left` (shift-or via the variable-count vshlq_u32; n%32 early-return). Rotate amount is a public ARX constant. - New U32x16([U32x4;4]) with Add / BitXor / rotate_left / splat / from_array / to_array, fanned over the 4 native lanes. - aarch64_simd F32x16::to_bits/from_bits switch to U32x16::from_array / to_array (native is [U32x4;4], not a [u32;16] tuple). - simd.rs aarch64 arm re-exports the native u32x16/U32x16 from simd_neon instead of the scalar fallback (drops them from the scalar list). Also fixes the pre-existing aarch64 *stable* compile breakage (the target never built on stable — no aarch64 CI caught it), a prerequisite for the cross-parity CI job: - Missing `pub type u16x8 = U16x8;` alias (simd.rs imported a symbol that did not exist). - `dot_i8x16_neon` / `codebook_gather_i8_dotprod` used the nightly-only `vdotq_s32` intrinsic (issue #117224). Rewritten with stable widening NEON (vmull_s8 + vpaddlq_s16 reduce; vmovl_s8 + vaddw_s16 accumulate) — bit-identical result, runs on all aarch64, no `dotprod` feature needed. Also fixes a latent bug in the gather (acc1..3 were never written, v1 aliased v0). Host ARX parity test green; aarch64 lib now `cargo check`-clean on stable. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- .claude/blackboard.md | 24 ++++++ src/simd.rs | 8 +- src/simd_neon.rs | 177 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 187 insertions(+), 22 deletions(-) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 4aff62da..cb31b860 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -630,3 +630,27 @@ is superseded by the matryoshka once the fork lands. 4. **Matryoshka execution:** fork `chacha20`, clone `avx2.rs` backend → rewire over `ndarray::simd::U32x16`, `[patch]` into the encryption stack (transitive accel), gate vs RustCrypto `soft`, retire `simd_crypto.rs`. Full plan in the doc above. + +--- + +## 2026-07-12 — PR #240 CI fully green (no_std/MSRV fix chain closed) + +Tip `5a914c37`. All 20 checks pass; the three previously-red failures are resolved: + +- **261f736a** — `simd_crypto` dispatcher: `is_x86_feature_detected!` → compile-time + `#[cfg(all(target_arch="x86_64", target_feature="avx512f"))]` (no runtime detection; + the workspace SIMD-dispatch rule). Unblocked `blas-msrv` + `nostd/thumbv6m`. +- **5a914c37** — three no_std/bare-`--no-default-features` test-build fixes: + - `simd_crypto` tests: `vec![[0u8;64]; n]` → fixed arrays `[[0u8;64]; 40]` / `[[0u8;64]; 16]` + (no `vec!` under no_std). + - `tests/chacha20_rustcrypto_parity.rs`: added `#![cfg(feature = "std")]` (imports + std-gated `ndarray::simd`; no-ops under `--no-default-features`). + - `src/tri.rs`: `Array2::::zeros(...)` (serde_json dep-drift added + `impl PartialEq for i32`, making bare `Array2::zeros` element-type ambiguous). + +Green jobs of note: `tests/{stable,beta,1.95.0}`, `blas-msrv`, `nostd/thumbv6m-none-eabi`, +`clippy/1.95.0`, `format/stable`, `native-backend/stable`, `tier4-avx512-check`, +`wasm-simd/parity-node` (new gate), `hpc-stream-parallel/rayon`, CodeRabbit. + +Deferred (task #30, token-limit call): native neon `U32x16=[U32x4;4]` + aarch64 cross +parity CI + the matryoshka chacha20 fork. Plan in `.claude/CHACHA20_MATRYOSHKA_PLAN.md`. diff --git a/src/simd.rs b/src/simd.rs index 4b1552d7..91fc7482 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -383,10 +383,14 @@ pub use crate::simd_neon::{ // U16x8 on aarch64 comes from simd_neon (backed by uint16x8_t) #[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] pub use crate::simd_neon::{u16x8, U16x8}; +// U32x16 (native `[U32x4; 4]`) — the ARX lane the ChaCha20 backend rides. Comes +// from simd_neon, not the scalar fallback, so it carries Add/BitXor/rotate_left. +#[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] +pub use crate::simd_neon::{u32x16, U32x16}; #[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] pub use scalar::{ - f32x8, f64x4, i32x16, i32x8, i64x4, i64x8, u16x16, u32x16, u32x8, u64x4, u64x8, u8x64, F32x8, F64x4, I32x16, I32x8, - I64x4, I64x8, U16x16, U16x32, U32x16, U32x8, U64x4, U64x8, U8x64, + f32x8, f64x4, i32x16, i32x8, i64x4, i64x8, u16x16, u32x8, u64x4, u64x8, u8x64, F32x8, F64x4, I32x16, I32x8, I64x4, + I64x8, U16x16, U16x32, U32x8, U64x4, U64x8, U8x64, }; // wasm32 + simd128: the native v128 float hot path (F32x16 / F64x8 + masks) diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 2d347035..625e0112 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -183,24 +183,34 @@ pub unsafe fn codebook_gather_f32x4_a72(centroids: &[f32], indices: &[u8], dim: // Tier 3: A76 DotProd + FP16 (Pi 5, Orange Pi 5) // ═══════════════════════════════════════════════════════════════════════════ -/// SDOT: 4×(4×i8 · 4×i8) → 4×i32 in ONE instruction. -/// ARMv8.2+ dotprod. 4× throughput vs manual widening multiply. -/// Core of int8 quantized codebook inference on Pi 5. +/// int8 dot product of two 16-lane chunks → i32. +/// +/// Stable-Rust widening path: `vmull_s8` (8×(i8·i8)→i16 per half) then +/// `vpaddlq_s16` (pairwise widen i16→i32) and a horizontal `vaddvq_s32`. +/// Bit-identical result to the ARMv8.2 `SDOT` instruction, but compiles on +/// stable (the `vdotq_s32` intrinsic is nightly-only, issue #117224) and runs +/// on **all** aarch64 — no `dotprod` feature required. Max |product| = 16384, +/// pairwise sums stay well inside i32, so no overflow for any i8 input. #[cfg(target_arch = "aarch64")] -#[target_feature(enable = "dotprod")] +#[inline(always)] pub unsafe fn dot_i8x16_neon(a: &[i8; 16], b: &[i8; 16]) -> i32 { let va = vld1q_s8(a.as_ptr()); let vb = vld1q_s8(b.as_ptr()); - let acc = vdupq_n_s32(0); - let result = vdotq_s32(acc, va, vb); - // Horizontal sum of 4×i32 - vaddvq_s32(result) + // 8×i16 products for the low and high halves. + let plo = vmull_s8(vget_low_s8(va), vget_low_s8(vb)); + let phi = vmull_s8(vget_high_s8(va), vget_high_s8(vb)); + // Widen i16→i32 (pairwise) so the accumulation never overflows, then reduce. + vaddvq_s32(vaddq_s32(vpaddlq_s16(plo), vpaddlq_s16(phi))) } -/// Quantized codebook gather via SDOT (Pi 5 only). -/// Centroids stored as i8, accumulated as i32. 4× throughput vs f32 path. +/// Quantized codebook gather: element-wise widen-accumulate the selected i8 +/// centroids into an i32 output of length `dim` (`dim` a multiple of 16). +/// +/// The i32 counterpart of [`codebook_gather_f32x4_neon`]: for each output lane +/// `k`, `output_i32[k] = Σ_idx centroids_i8[idx*dim + k]`, widened i8→i32 via +/// `vmovl_s8` + `vaddw_s16`. Stable on all aarch64 (no `dotprod` intrinsic). #[cfg(target_arch = "aarch64")] -#[target_feature(enable = "dotprod")] +#[inline(always)] pub unsafe fn codebook_gather_i8_dotprod( centroids_i8: &[i8], // quantized centroids: N × dim (i8) indices: &[u8], @@ -208,9 +218,11 @@ pub unsafe fn codebook_gather_i8_dotprod( output_i32: &mut [i32], // accumulated i32 (dequantize later) ) { debug_assert!(dim % 16 == 0); + debug_assert!(output_i32.len() >= dim); let chunks = dim / 16; for c in 0..chunks { + // Four i32x4 accumulators cover the 16 i8 lanes of this chunk. let mut acc0 = vdupq_n_s32(0); let mut acc1 = vdupq_n_s32(0); let mut acc2 = vdupq_n_s32(0); @@ -218,14 +230,15 @@ pub unsafe fn codebook_gather_i8_dotprod( for &idx in indices { let base = idx as usize * dim + c * 16; - let v0 = vld1q_s8(centroids_i8[base..].as_ptr()); - let v1 = vld1q_s8(centroids_i8[base..].as_ptr()); - // dotprod: each vdotq_s32 does 4×(4×i8·4×i8)→4×i32 - let ones = vdupq_n_s8(1); // identity for accumulation - acc0 = vdotq_s32(acc0, v0, ones); + let v = vld1q_s8(centroids_i8[base..].as_ptr()); // 16 i8 + let lo = vmovl_s8(vget_low_s8(v)); // lanes 0..8 → i16 + let hi = vmovl_s8(vget_high_s8(v)); // lanes 8..16 → i16 + acc0 = vaddw_s16(acc0, vget_low_s16(lo)); + acc1 = vaddw_s16(acc1, vget_high_s16(lo)); + acc2 = vaddw_s16(acc2, vget_low_s16(hi)); + acc3 = vaddw_s16(acc3, vget_high_s16(hi)); } - // Store 4 i32 results vst1q_s32(output_i32[c * 16..].as_mut_ptr(), acc0); vst1q_s32(output_i32[c * 16 + 4..].as_mut_ptr(), acc1); vst1q_s32(output_i32[c * 16 + 8..].as_mut_ptr(), acc2); @@ -467,7 +480,11 @@ pub mod aarch64_simd { // Integer types come from the scalar fallback in simd.rs — they aren't on // the perf-critical f32 BLAS-1 / VML path that this module accelerates. - pub use crate::simd::scalar::{I32x16, U32x16, U64x8}; + // `U32x16` is the exception: it carries the ARX vocabulary (Add/BitXor/ + // rotate_left) the ChaCha20 lane needs, so it is the native `[U32x4; 4]` + // defined at the top of this file (mirroring `simd_wasm::wasm32_simd`). + pub use super::U32x16; + pub use crate::simd::scalar::{I32x16, U64x8}; /// 16×f32 backed by 4× NEON `float32x4_t` registers (paired loads). #[derive(Copy, Clone)] @@ -674,13 +691,14 @@ pub mod aarch64_simd { for i in 0..16 { o[i] = a[i].to_bits(); } - U32x16(o) + U32x16::from_array(o) } #[inline(always)] pub fn from_bits(bits: U32x16) -> Self { + let b = bits.to_array(); let mut o = [0.0f32; 16]; for i in 0..16 { - o[i] = f32::from_bits(bits.0[i]); + o[i] = f32::from_bits(b[i]); } Self::from_array(o) } @@ -1490,6 +1508,11 @@ impl U16x8 { } } +// Lowercase alias (consumer-API parity — `simd.rs` re-exports `u16x8`). +#[cfg(target_arch = "aarch64")] +#[allow(non_camel_case_types)] +pub type u16x8 = U16x8; + #[cfg(target_arch = "aarch64")] #[derive(Copy, Clone)] #[repr(transparent)] @@ -1542,8 +1565,122 @@ impl U32x4 { pub fn max(self, other: Self) -> Self { Self(unsafe { vmaxq_u32(self.0, other.0) }) } + /// Lane-wise XOR — the ARX `⊕` (`veorq_u32`). + #[inline(always)] + pub fn bitxor(self, other: Self) -> Self { + Self(unsafe { veorq_u32(self.0, other.0) }) + } + /// Lane-wise left-rotate by `n` bits — the ARX rotate (matches + /// `u32::rotate_left`). NEON has no rotate op, so this is the shift-or via + /// the variable-count `vshlq_u32` (a signed per-lane count: `+n` shifts + /// left by `n`, `n-32 < 0` shifts logical-right by `32-n`). The `n % 32 == 0` + /// early-return avoids the ambiguous full-width shift. Rotate amount is a + /// public ARX constant. + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let n = n % 32; + if n == 0 { + return self; + } + unsafe { + let l = vshlq_u32(self.0, vdupq_n_s32(n as i32)); + let r = vshlq_u32(self.0, vdupq_n_s32(n as i32 - 32)); + Self(vorrq_u32(l, r)) + } + } } +/// 16×u32 as `[U32x4; 4]` — the NEON-native 16-wide ARX lane (ChaCha20 / +/// BLAKE). `U32x4` is the dispatched native unit (`uint32x4_t`); `U32x16` fans +/// each op over the 4 sub-lanes. Consumer API (`Add` / `BitXor` / `rotate_left`) +/// matches `simd_avx512::U32x16` exactly, so the ChaCha20 backend compiles +/// unchanged on every tier (the wasm arm uses the identical `[U32x4; 4]` shape). +#[cfg(target_arch = "aarch64")] +#[derive(Copy, Clone)] +#[repr(align(64))] +pub struct U32x16(pub [U32x4; 4]); + +#[cfg(target_arch = "aarch64")] +impl U32x16 { + pub const LANES: usize = 16; + + #[inline(always)] + pub fn splat(v: u32) -> Self { + Self([U32x4::splat(v); 4]) + } + + #[inline(always)] + pub fn from_array(a: [u32; 16]) -> Self { + Self([ + U32x4::from_array([a[0], a[1], a[2], a[3]]), + U32x4::from_array([a[4], a[5], a[6], a[7]]), + U32x4::from_array([a[8], a[9], a[10], a[11]]), + U32x4::from_array([a[12], a[13], a[14], a[15]]), + ]) + } + + #[inline(always)] + pub fn to_array(self) -> [u32; 16] { + let mut o = [0u32; 16]; + for i in 0..4 { + o[i * 4..i * 4 + 4].copy_from_slice(&self.0[i].to_array()); + } + o + } + + /// Lane-wise left-rotate by `n` bits (ARX rotate), fanned over 4 lanes. + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + Self([ + self.0[0].rotate_left(n), + self.0[1].rotate_left(n), + self.0[2].rotate_left(n), + self.0[3].rotate_left(n), + ]) + } +} + +#[cfg(target_arch = "aarch64")] +impl core::ops::Add for U32x16 { + type Output = Self; + #[inline(always)] + fn add(self, r: Self) -> Self { + Self([self.0[0].add(r.0[0]), self.0[1].add(r.0[1]), self.0[2].add(r.0[2]), self.0[3].add(r.0[3])]) + } +} + +#[cfg(target_arch = "aarch64")] +impl core::ops::BitXor for U32x16 { + type Output = Self; + #[inline(always)] + fn bitxor(self, r: Self) -> Self { + Self([ + self.0[0].bitxor(r.0[0]), + self.0[1].bitxor(r.0[1]), + self.0[2].bitxor(r.0[2]), + self.0[3].bitxor(r.0[3]), + ]) + } +} + +#[cfg(target_arch = "aarch64")] +impl core::fmt::Debug for U32x16 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "U32x16({:?})", self.to_array()) + } +} + +#[cfg(target_arch = "aarch64")] +impl PartialEq for U32x16 { + fn eq(&self, other: &Self) -> bool { + self.to_array() == other.to_array() + } +} + +#[cfg(target_arch = "aarch64")] +#[allow(non_camel_case_types)] +pub type u32x16 = U32x16; + #[cfg(target_arch = "aarch64")] #[derive(Copy, Clone)] #[repr(transparent)] From 83d2ca70b3b545f900295e98c42d4b6d2efce104 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 05:39:46 +0000 Subject: [PATCH 2/6] crypto(matryoshka): vendor chacha20 fork over ndarray::simd::U32x16, retire simd_crypto.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AdaWorldAPI ChaCha20 "matryoshka": RustCrypto owns the cipher; ndarray owns only the U32x16 ARX lane. The standalone from-scratch `src/simd_crypto.rs` (raw _mm512_*/u32x4_* intrinsics) is RETIRED and superseded by a one-backend fork. vendor/chacha20/ — fork of RustCrypto chacha20 0.9.1 (name/version kept for [patch]; own [workspace] table, excluded from the parent workspace). Upstream cipher verbatim (state schedule, counter, XChaCha, RNG, soft/sse2/avx2/neon). The ONE delta: src/backends/ndarray_simd.rs — the transpose block16 (16 blocks in parallel, working vector w holds word w of every block across its 16 lanes, counter word carries lane-index 0..=15) expressed over ndarray::simd::U32x16: every quarter-round is a pure U32x16 + / ^ / rotate_left, NO cross-lane shuffle, NO raw intrinsics, NO unsafe (all of that lives once inside ndarray::simd). Wrapped in RustCrypto's StreamBackend (ParBlocksSize = U16), generic over R. Selected at compile time under cfg(all(target_arch="x86_64", target_feature= "avx512f")) via four cfg branches in backends.rs + lib.rs (Tokens / new / dispatch). The ndarray dep is target.'cfg(target_arch="x86_64")' + default-features=false, features=["std"] so wasm/aarch64 fork builds never pull ndarray. [patch.crates-io] chacha20 = { path = "vendor/chacha20" } folds it transitively under chacha20poly1305 -> encryption with zero change to the AEAD. Parity — triple-gated, GREEN: - the fork's own RustCrypto RFC 8439 vectors (chacha20/xchacha20 encryption + keystream, core, seek) pass THROUGH ndarray_simd under -Ctarget-cpu=x86-64-v4; - cargo test -p encryption (23 AEAD tests incl. XChaCha20-Poly1305 round-trip, bit-flip-fails, wrong-key-fails) passes on the default v3 build (RustCrypto avx2 fallback) and the avx512 build (ndarray_simd). Retired: deleted src/simd_crypto.rs + tests/chacha20_rustcrypto_parity.rs + the chacha20="0.9" dev-dep; removed the ndarray::simd::{chacha20_block, chacha20_keystream, chacha20_state} surface (zero downstream consumers). aead.rs doc rewritten to the transitive-acceleration framing. Note: the workspace default is target-cpu=x86-64-v3 (avx2), so ndarray_simd activates only on an avx512 build; the wasm-fork backend + cross-repo [patch] propagation (MedCare-rs) are documented follow-ups in .claude/CHACHA20_MATRYOSHKA_PLAN.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- .claude/CHACHA20_MATRYOSHKA_PLAN.md | 52 +- Cargo.lock | 4 +- Cargo.toml | 19 +- crates/encryption/src/aead.rs | 22 +- src/lib.rs | 10 +- src/simd.rs | 10 +- src/simd_crypto.rs | 484 ------------------ tests/chacha20_rustcrypto_parity.rs | 138 ----- vendor/chacha20/.gitignore | 2 + vendor/chacha20/CHANGELOG.md | 255 +++++++++ vendor/chacha20/Cargo.toml | 60 +++ vendor/chacha20/LICENSE-APACHE | 201 ++++++++ vendor/chacha20/LICENSE-MIT | 25 + vendor/chacha20/README.md | 117 +++++ vendor/chacha20/src/backends.rs | 28 + vendor/chacha20/src/backends/avx2.rs | 251 +++++++++ vendor/chacha20/src/backends/ndarray_simd.rs | 110 ++++ vendor/chacha20/src/backends/neon.rs | 356 +++++++++++++ vendor/chacha20/src/backends/soft.rs | 73 +++ vendor/chacha20/src/backends/sse2.rs | 180 +++++++ vendor/chacha20/src/legacy.rs | 76 +++ vendor/chacha20/src/lib.rs | 329 ++++++++++++ vendor/chacha20/src/xchacha.rs | 194 +++++++ .../chacha20/tests/data/chacha20-legacy.blb | Bin 0 -> 1045 bytes vendor/chacha20/tests/data/chacha20.blb | Bin 0 -> 812 bytes vendor/chacha20/tests/mod.rs | 223 ++++++++ 26 files changed, 2561 insertions(+), 658 deletions(-) delete mode 100644 src/simd_crypto.rs delete mode 100644 tests/chacha20_rustcrypto_parity.rs create mode 100644 vendor/chacha20/.gitignore create mode 100644 vendor/chacha20/CHANGELOG.md create mode 100644 vendor/chacha20/Cargo.toml create mode 100644 vendor/chacha20/LICENSE-APACHE create mode 100644 vendor/chacha20/LICENSE-MIT create mode 100644 vendor/chacha20/README.md create mode 100644 vendor/chacha20/src/backends.rs create mode 100644 vendor/chacha20/src/backends/avx2.rs create mode 100644 vendor/chacha20/src/backends/ndarray_simd.rs create mode 100644 vendor/chacha20/src/backends/neon.rs create mode 100644 vendor/chacha20/src/backends/soft.rs create mode 100644 vendor/chacha20/src/backends/sse2.rs create mode 100644 vendor/chacha20/src/legacy.rs create mode 100644 vendor/chacha20/src/lib.rs create mode 100644 vendor/chacha20/src/xchacha.rs create mode 100644 vendor/chacha20/tests/data/chacha20-legacy.blb create mode 100644 vendor/chacha20/tests/data/chacha20.blb create mode 100644 vendor/chacha20/tests/mod.rs diff --git a/.claude/CHACHA20_MATRYOSHKA_PLAN.md b/.claude/CHACHA20_MATRYOSHKA_PLAN.md index bb6c961d..1027db98 100644 --- a/.claude/CHACHA20_MATRYOSHKA_PLAN.md +++ b/.claude/CHACHA20_MATRYOSHKA_PLAN.md @@ -32,10 +32,54 @@ CI: `wasm_simd` job (`.github/workflows/ci.yaml` + `scripts/wasm-parity.sh` + lane-by-lane parity selfcheck under **node** — the standing guard for the wasm tier (invisible to the x86 `cargo test`). Extend its per-lane blocks as lanes land. -The stand-alone `src/simd_crypto.rs` (`chacha20_block`/`chacha20_keystream`, -scalar+AVX-512+wasm128, RustCrypto-parity-proven) is the *interim* primitive; the -matryoshka below **supersedes it** (RustCrypto supplies the rounds, so the -from-scratch cipher retires once the fork lands). +The stand-alone `src/simd_crypto.rs` was the *interim* primitive; the matryoshka +below **superseded it** — it is now **RETIRED** (2026-07-12, see below). + +## ✅ SHIPPED 2026-07-12 — fork landed + `simd_crypto.rs` retired + +- **Native NEON `U32x16 = [U32x4; 4]`** — `U32x4` gained `bitxor`(veorq_u32) + + `rotate_left`(vshlq_u32 shift-or); composed `U32x16` with Add/BitXor/ + rotate_left. aarch64 now `cargo check`-clean on stable (also fixed the + pre-existing `u16x8` alias + the nightly-only `vdotq_s32` breakage → stable + widening NEON). Every tier of the ARX lane is now native. +- **The fork** lives at `vendor/chacha20/` (name/version kept `chacha20`/`0.9.1`, + excluded from the workspace, own `[workspace]` table). Upstream cipher verbatim; + the ONE delta is `src/backends/ndarray_simd.rs` — the transpose block16 (16 + blocks ∥, word→16-lane vector, counter lane-index) over `ndarray::simd::U32x16`, + pure `+`/`^`/`rotate_left`, **no raw intrinsics, no `unsafe`**. Selected at + compile time under `cfg(all(x86_64, avx512f))` in `backends.rs` + + `lib.rs::process_with_backend` (Tokens/new/dispatch — four cfg branches). The + `ndarray` dep is `target.'cfg(target_arch="x86_64")'` + `default-features=false, + features=["std"]` so wasm/aarch64 fork builds never pull ndarray. +- **`[patch.crates-io] chacha20 = { path = "vendor/chacha20" }`** in the ndarray + root `Cargo.toml` folds it under `chacha20poly1305 → encryption` transparently. +- **Parity — triple-gated, GREEN:** the fork's own RustCrypto RFC 8439 vectors + (`chacha20_encryption`/`_keystream`, `xchacha20_*`, `chacha20_core`, seek) pass + **through `ndarray_simd`** under `-Ctarget-cpu=x86-64-v4`; `cargo test -p + encryption` (23 AEAD tests incl. XChaCha20-Poly1305 round-trip / bit-flip / + wrong-key) passes on both the default (v3→RustCrypto avx2 fallback) and avx512 + (→`ndarray_simd`) builds. +- **RETIRED:** deleted `src/simd_crypto.rs` + `tests/chacha20_rustcrypto_parity.rs` + + the `chacha20 = "0.9"` dev-dep; removed the `ndarray::simd::{chacha20_block, + chacha20_keystream, chacha20_state}` surface. RustCrypto owns the cipher; ndarray + owns only the `U32x16` lane. `aead.rs` doc updated to the transitive-acceleration + framing. + +### Reality note + remaining follow-ups +- **The workspace default is `target-cpu=x86-64-v3` (AVX2), NOT v4** — the CLAUDE.md + "v4 mandatory" line is stale. So `ndarray_simd` activates only on an **avx512 + build** (`.cargo/config-avx512.toml` / `-Ctarget-cpu=x86-64-v4`); the default v3 + build uses RustCrypto's own avx2 backend (fast, vetted). This is correct and + safe — the matryoshka is the *server-avx512* accelerator. +- **wasm browser backend (next):** add a sibling `#[cfg(all(target_arch="wasm32", + target_feature="simd128"))]` branch selecting an `ndarray_simd` backend (same + `U32x16` source; the wasm arm is native `[U32x4;4]`) + a `target.'cfg(wasm32)'` + ndarray dep. Gated on verifying `ndarray/std` builds inside the encryption + `cdylib` for `wasm32-unknown-unknown`. +- **cross-repo `[patch]` (next):** `[patch]` does not transit, so MedCare-rs (and + any other consumer building `encryption` for avx512) needs its own + `[patch.crates-io] chacha20 = { path = "vendor/chacha20" }` pointing at a vendored + copy of the fork, to inherit the acceleration. Documented, low-risk. ## DEFERRED — TO-DO (next session) diff --git a/Cargo.lock b/Cargo.lock index bc704f14..5564d8ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,12 +271,11 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "ndarray", ] [[package]] @@ -1157,7 +1156,6 @@ dependencies = [ "approx", "blake3", "cblas-sys", - "chacha20", "cranelift-codegen", "cranelift-frontend", "cranelift-jit", diff --git a/Cargo.toml b/Cargo.toml index 0e826fa7..5e8ce633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -217,11 +217,6 @@ approx = { workspace = true, default-features = true } itertools = { workspace = true } ndarray-gen = { workspace = true } criterion = { version = "0.5", features = ["html_reports"] } -# The vetted RustCrypto ChaCha20 stream cipher — used ONLY as the independent -# oracle for the `chacha20_rustcrypto_parity` integration test that pins -# `ndarray::simd::chacha20_keystream` byte-for-byte against a trusted reference. -# Same version chacha20poly1305 0.10 pulls transitively (see Cargo.lock). -chacha20 = "0.9" [[bench]] name = "append" @@ -407,7 +402,7 @@ members = [ "ndarray-rand", "crates/*", ] -exclude = ["crates/burn", "crates/wasm-simd-parity"] +exclude = ["crates/burn", "crates/wasm-simd-parity", "vendor/chacha20"] default-members = [ ".", "ndarray-rand", @@ -449,3 +444,15 @@ tag-name = "{{version}}" features = ["approx", "serde", "rayon"] # Define the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] + +# ── ChaCha20 matryoshka (AdaWorldAPI fork) ────────────────────────────────── +# Fold the AVX-512/ndarray::simd-accelerated `chacha20` fork over the registry +# crate. This transitively accelerates the `encryption` crate's XChaCha20-Poly1305 +# keystream (chacha20poly1305 -> chacha20) with zero change to the AEAD, on any +# x86_64+avx512f build (the workspace's `target-cpu=x86-64-v4`). Non-x86 builds of +# the fork fall through to RustCrypto's own backends and never pull ndarray. +# NOTE: a "Patch `chacha20` was not used in the crate graph" note appears on +# scoped builds that don't pull `encryption` (e.g. `-p ndarray`); it is benign — +# the patch is exercised by `cargo test -p encryption` and the full workspace. +[patch.crates-io] +chacha20 = { path = "vendor/chacha20" } diff --git a/crates/encryption/src/aead.rs b/crates/encryption/src/aead.rs index 777c987e..49c2b187 100644 --- a/crates/encryption/src/aead.rs +++ b/crates/encryption/src/aead.rs @@ -6,18 +6,18 @@ //! and a server. Callers normally use [`crate::envelope`], which manages //! nonce generation and layout; this module is the raw primitive. //! -//! ## Deliberately NOT hand-composed from `ndarray::simd::chacha20_keystream` +//! ## Acceleration is transitive (the matryoshka), never hand-composed //! -//! `ndarray` now ships a hardware-accelerated ChaCha20 keystream -//! (`ndarray::simd::chacha20_keystream`, AVX-512 / wasm128, byte-parity-proven -//! against RustCrypto). It is tempting to swap this AEAD's keystream to that -//! primitive for speed — **do not.** XChaCha20-Poly1305 is not just a keystream: -//! it is HChaCha20 subkey derivation + the Poly1305 one-time-key + MAC framing + -//! the AAD/length encoding. Re-wiring only the keystream means re-implementing -//! that authenticated composition by hand, which is exactly the "roll your own -//! AEAD" footgun the stack forbids. The vetted RustCrypto `XChaCha20Poly1305` -//! construction stays here. The accelerated primitive is for *raw-stream* use -//! sites whose caller already owns a vetted MAC/framing — not this module. +//! This AEAD stays 100% vetted RustCrypto `XChaCha20Poly1305` — HChaCha20 subkey +//! derivation + Poly1305 one-time-key + MAC framing + AAD/length encoding — and +//! we do **not** hand-wire a raw keystream into it (that would be the "roll your +//! own AEAD" footgun the stack forbids). Hardware acceleration instead arrives +//! *underneath*, transparently: the AdaWorldAPI `chacha20` fork (`vendor/chacha20/`) +//! replaces one backend of the transitive `chacha20` dependency with a keystream +//! double-round expressed over `ndarray::simd::U32x16` (the AVX-512 lane), folded +//! in via `[patch.crates-io]`. So `chacha20poly1305 -> chacha20 -> ndarray::simd` +//! accelerates on an AVX-512 build with zero change to this module, and every +//! RustCrypto RFC 8439 test vector (incl. XChaCha20) still passes through it. use chacha20poly1305::aead::{Aead, KeyInit, Payload}; use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce}; diff --git a/src/lib.rs b/src/lib.rs index 098b92cc..2c787693 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -246,12 +246,10 @@ pub(crate) mod simd_avx512; #[allow(clippy::all, missing_docs, dead_code, unused_variables, unused_imports)] pub mod simd_avx2; -// Crypto ARX primitives (ChaCha20 block; Poly1305 / Blake2 to follow) — the -// hardware-acceleration layer the Ada `encryption` crate draws its hot-path -// keystream from. Scalar reference + RFC 8439 KAT land first (arch-agnostic, -// no cfg gate); the AVX-512 / wasm128 / NEON vectorized backends slot in behind -// the same signature, parity-checked against the KAT (W1a contract). -pub mod simd_crypto; +// (The standalone `simd_crypto` ChaCha20 primitive was RETIRED — superseded by +// the AdaWorldAPI `chacha20` fork whose backend rides `ndarray::simd::U32x16`, +// `[patch]`ed under the `encryption` AEAD. RustCrypto owns the cipher; ndarray +// owns only the U32x16 ARX lane. See `vendor/chacha20/` + `.claude/CHACHA20_MATRYOSHKA_PLAN.md`.) // Portable-SIMD backend — nightly-only. Wraps `core::simd::*` so miri can // execute the polyfill paths (intrinsic-based backends are opaque to diff --git a/src/simd.rs b/src/simd.rs index 91fc7482..ce869ba1 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -682,12 +682,10 @@ pub use crate::simd_ops::{ sub_f32_inplace, }; -// ChaCha20 ARX keystream — the crypto hot path the `encryption` AEAD draws from. -// `chacha20_block` is the scalar reference (RFC 8439 KAT); `chacha20_keystream` -// is the runtime-dispatched accelerated fill (AVX-512 16-block strides + scalar -// tail), byte-parity-pinned to the reference. Consumers pull from -// `ndarray::simd::*` per the W1a contract. -pub use crate::simd_crypto::{chacha20_block, chacha20_keystream, chacha20_state}; +// ChaCha20 keystream is NO LONGER an `ndarray::simd` surface: the AdaWorldAPI +// `chacha20` fork (`vendor/chacha20/`) carries the accelerated backend, riding +// the `U32x16` ARX lane above, and is `[patch]`ed transitively under the +// `encryption` AEAD. RustCrypto owns the cipher; ndarray exposes only the lane. // ============================================================================ // Tests diff --git a/src/simd_crypto.rs b/src/simd_crypto.rs deleted file mode 100644 index 32d7d022..00000000 --- a/src/simd_crypto.rs +++ /dev/null @@ -1,484 +0,0 @@ -//! Crypto ARX primitives for the `ndarray::simd` polyfill — the hardware -//! acceleration layer the Ada `encryption` crate (Argon2id / XChaCha20-Poly1305 -//! / Ed25519 / SHA-384) draws its hot-path keystream from. -//! -//! # Why these live in `ndarray::simd` -//! -//! The workspace invariant (`AdaWorldAPI/lance-graph` `simd-savant`): **all SIMD -//! comes from `ndarray::simd` via the polyfill — `simd.rs` + `simd_ops.rs` > -//! `simd_{arch}.rs`**. The crypto keystream is a SIMD hot path like any other, so -//! its accelerated form belongs here (server AVX-512 / browser wasm128), not -//! re-hidden inside a per-consumer copy. The consumer-facing crypto surface -//! (`ogar-encryption` → the `encryption` crate) composes AEAD/KDF framing on top; -//! this module supplies the vectorizable core. -//! -//! # Why a SIMD ChaCha20 is *safe* to hand-vectorize (unlike, say, AES) -//! -//! ChaCha20 is an **ARX** cipher: every operation is a 32-bit `wrapping_add`, -//! `xor`, or fixed-distance `rotate_left`. There are **no secret-dependent -//! branches and no secret-dependent memory indices** (no S-boxes, no T-tables), -//! so the block function is **constant-time by construction** — a property a -//! straight-line SIMD implementation preserves automatically. That is exactly -//! why RFC 8439 chose ChaCha20 for constant-time software, and why an -//! `ndarray::simd` vectorization does not introduce a timing side channel the -//! scalar path lacked. (AES in software, by contrast, is NOT safe to hand-roll -//! this way — its table lookups are secret-indexed; that is deliberately out of -//! scope here, and browsers already expose hardware AES via WebCrypto.) -//! -//! # Backend ladder (W1a consumer contract) -//! -//! | Backend | ChaCha20 block strategy | -//! |------------------|----------------------------------------------------------| -//! | scalar | the RFC 8439 reference double-round — the CORRECTNESS anchor | -//! | AVX-512 (server) | 16 blocks in parallel, word-sliced `__m512i` lanes (DONE) | -//! | wasm128 (browser)| 4 blocks in parallel, word-sliced `v128` u32x4 lanes (DONE) | -//! | NEON (edge) | `uint32x4_t` lanes (next) | -//! -//! The scalar reference lands FIRST with the RFC 8439 §2.3.2 Known-Answer-Test: -//! every future vectorized backend is a drop-in that MUST reproduce this exact -//! keystream (parity test), so correctness is pinned before performance. This is -//! the "reference + KAT, then vectorize with parity" discipline the W1a contract -//! mandates for any new `ndarray::simd` primitive. - -/// The ChaCha20 constants — the ASCII of `"expand 32-byte k"` as four -/// little-endian `u32` words (RFC 8439 §2.3). -const CHACHA20_CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574]; - -/// One ChaCha20 quarter-round on four working words (RFC 8439 §2.1). Pure ARX: -/// `wrapping_add` / `xor` / `rotate_left` — no branch, no memory index, so it is -/// constant-time regardless of the (secret) word values. -#[inline(always)] -fn quarter_round(state: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize) { - state[a] = state[a].wrapping_add(state[b]); - state[d] = (state[d] ^ state[a]).rotate_left(16); - state[c] = state[c].wrapping_add(state[d]); - state[b] = (state[b] ^ state[c]).rotate_left(12); - state[a] = state[a].wrapping_add(state[b]); - state[d] = (state[d] ^ state[a]).rotate_left(8); - state[c] = state[c].wrapping_add(state[d]); - state[b] = (state[b] ^ state[c]).rotate_left(7); -} - -/// Compute one 64-byte ChaCha20 keystream block from a fully-populated 16-word -/// input `state` (RFC 8439 §2.3.1): 20 rounds (10 column + diagonal -/// double-rounds), then add the original input state, then serialize each word -/// little-endian. -/// -/// `state` is the caller-assembled block state — words `0..4` the constants, -/// `4..12` the 256-bit key, word `12` the block counter, `13..16` the 96-bit -/// nonce. Building that state (and the XChaCha `HChaCha20` nonce extension) is -/// the caller's job; this is the pure, vectorizable block core. -/// -/// **Constant-time:** straight-line ARX, no data-dependent control flow. -/// This scalar form is the reference every SIMD backend is parity-checked -/// against (see the module KAT). -#[must_use] -pub fn chacha20_block(state: &[u32; 16]) -> [u8; 64] { - let mut w = *state; - // 10 double-rounds = 20 rounds. - for _ in 0..10 { - // Column round. - quarter_round(&mut w, 0, 4, 8, 12); - quarter_round(&mut w, 1, 5, 9, 13); - quarter_round(&mut w, 2, 6, 10, 14); - quarter_round(&mut w, 3, 7, 11, 15); - // Diagonal round. - quarter_round(&mut w, 0, 5, 10, 15); - quarter_round(&mut w, 1, 6, 11, 12); - quarter_round(&mut w, 2, 7, 8, 13); - quarter_round(&mut w, 3, 4, 9, 14); - } - let mut out = [0u8; 64]; - for (i, word) in w.iter().enumerate() { - let sum = word.wrapping_add(state[i]); - out[i * 4..i * 4 + 4].copy_from_slice(&sum.to_le_bytes()); - } - out -} - -/// Assemble a ChaCha20 block state from a 256-bit `key`, a 32-bit block -/// `counter`, and a 96-bit `nonce` (RFC 8439 §2.3). Little-endian word packing. -/// Convenience for callers and for the KAT; the vectorized backends operate on -/// the assembled `[u32; 16]` from [`chacha20_block`]. -#[must_use] -pub fn chacha20_state(key: &[u8; 32], counter: u32, nonce: &[u8; 12]) -> [u32; 16] { - let mut s = [0u32; 16]; - s[0..4].copy_from_slice(&CHACHA20_CONSTANTS); - for i in 0..8 { - s[4 + i] = u32::from_le_bytes([key[i * 4], key[i * 4 + 1], key[i * 4 + 2], key[i * 4 + 3]]); - } - s[12] = counter; - for i in 0..3 { - s[13 + i] = u32::from_le_bytes([nonce[i * 4], nonce[i * 4 + 1], nonce[i * 4 + 2], nonce[i * 4 + 3]]); - } - s -} - -/// Fill `out` with successive ChaCha20 keystream blocks: block `b` uses block -/// counter `counter.wrapping_add(b)`, key and nonce fixed (RFC 8439 CTR mode). -/// -/// Accelerated entry point for the `encryption` AEAD keystream. Compile-time -/// tier dispatch (the `ndarray::simd` model — no `is_x86_feature_detected!`): -/// the AVX-512 16-block backend when built `+avx512f`, the wasm128 4-block -/// backend when built `+simd128`, else the scalar [`chacha20_block`] reference, -/// which also fills the ragged tail. Every backend is byte-parity-pinned to the -/// scalar reference (`chacha20_keystream_dispatch_parity_scalar`). -/// -/// Constant-time: straight-line ARX (add / xor / rotate), no secret-dependent -/// control flow or memory indexing. -pub fn chacha20_keystream(key: &[u8; 32], counter: u32, nonce: &[u8; 12], out: &mut [[u8; 64]]) { - let mut i = 0usize; - - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - while i + 16 <= out.len() { - let state = chacha20_state(key, counter.wrapping_add(i as u32), nonce); - let blocks = chacha20_block16_avx512(&state); - out[i..i + 16].copy_from_slice(&blocks); - i += 16; - } - } - - // wasm128 (browser) backend: 4-block strides. `simd128` is a compile-time - // feature, so this arm is present only when the module is built for it. - #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] - { - while i + 4 <= out.len() { - let state = chacha20_state(key, counter.wrapping_add(i as u32), nonce); - let blocks = chacha20_block4_wasm(&state); - out[i..i + 4].copy_from_slice(&blocks); - i += 4; - } - } - - // Scalar reference: the ragged tail, and the whole slice on scalar targets. - while i < out.len() { - let state = chacha20_state(key, counter.wrapping_add(i as u32), nonce); - out[i] = chacha20_block(&state); - i += 1; - } -} - -/// AVX-512 backend: 16 ChaCha20 blocks in parallel, word-sliced (lane `l` = block -/// `l`), so each round is pure SIMD add/xor/rotate and only the final LE serialize -/// transposes. cfg-gated to an `+avx512f` build → the intrinsics are statically -/// available (no `#[target_feature]`, no runtime detect). -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -fn chacha20_block16_avx512(state: &[u32; 16]) -> [[u8; 64]; 16] { - use core::arch::x86_64::*; - // SAFETY: avx512f is statically enabled (cfg gate), so every AVX-512F - // intrinsic below is available; all lane indices are fixed constants. - unsafe { - // One AVX-512 vertical quarter-round: same word indices as the scalar - // `quarter_round`, applied across all 16 blocks at once. Pure ARX. - macro_rules! qr512 { - ($v:expr, $a:expr, $b:expr, $c:expr, $d:expr) => {{ - $v[$a] = _mm512_add_epi32($v[$a], $v[$b]); - $v[$d] = _mm512_rol_epi32::<16>(_mm512_xor_si512($v[$d], $v[$a])); - $v[$c] = _mm512_add_epi32($v[$c], $v[$d]); - $v[$b] = _mm512_rol_epi32::<12>(_mm512_xor_si512($v[$b], $v[$c])); - $v[$a] = _mm512_add_epi32($v[$a], $v[$b]); - $v[$d] = _mm512_rol_epi32::<8>(_mm512_xor_si512($v[$d], $v[$a])); - $v[$c] = _mm512_add_epi32($v[$c], $v[$d]); - $v[$b] = _mm512_rol_epi32::<7>(_mm512_xor_si512($v[$b], $v[$c])); - }}; - } - - // Build the 16 initial working vectors. Every word is broadcast identically - // across the 16 lanes EXCEPT the counter (word 12), which gets lane-index - // 0..=15 added (u32 wrapping) — the 16 consecutive block counters. - let lane_index = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let mut orig = [_mm512_setzero_si512(); 16]; - for (w, o) in orig.iter_mut().enumerate() { - *o = _mm512_set1_epi32(state[w] as i32); - } - orig[12] = _mm512_add_epi32(_mm512_set1_epi32(state[12] as i32), lane_index); - - let mut v = orig; - // 10 double-rounds = 20 rounds, identical schedule to the scalar reference. - for _ in 0..10 { - qr512!(v, 0, 4, 8, 12); - qr512!(v, 1, 5, 9, 13); - qr512!(v, 2, 6, 10, 14); - qr512!(v, 3, 7, 11, 15); - qr512!(v, 0, 5, 10, 15); - qr512!(v, 1, 6, 11, 12); - qr512!(v, 2, 7, 8, 13); - qr512!(v, 3, 4, 9, 14); - } - - // Add the original input state back (RFC 8439 §2.3.1), then de-vectorize: - // `words[w][l]` = final word `w` of block `l`. - let mut words = [[0u32; 16]; 16]; - for (w, dst) in words.iter_mut().enumerate() { - let summed = _mm512_add_epi32(v[w], orig[w]); - let mut tmp = [0i32; 16]; - _mm512_storeu_si512(tmp.as_mut_ptr().cast(), summed); - for (l, slot) in dst.iter_mut().enumerate() { - *slot = tmp[l] as u32; - } - } - - // Serialize each block little-endian: block `l`, word `w` → bytes 4w..4w+4. - let mut out = [[0u8; 64]; 16]; - for (l, block) in out.iter_mut().enumerate() { - for w in 0..16 { - block[w * 4..w * 4 + 4].copy_from_slice(&words[w][l].to_le_bytes()); - } - } - out - } -} - -/// wasm128 (browser) backend: compute 4 consecutive ChaCha20 keystream blocks in -/// parallel using the WebAssembly SIMD `v128` register as four `u32x4` lanes. -/// Same word-sliced ("vertical") layout as the AVX-512 backend — lane `l` of -/// working vector `w` holds word `w` of block `l` — so every round is a pure -/// `u32x4` add/xor/rotate with no lane shuffles. -/// -/// Requires the `simd128` target feature (a compile-time WASM feature, not -/// runtime-detected). Uses **no `unsafe`**: the wasm arithmetic and -/// lane-extraction intrinsics are safe under the enabled feature; output is read -/// with `u32x4_extract_lane` rather than a raw `v128_store`. -#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] -fn chacha20_block4_wasm(state: &[u32; 16]) -> [[u8; 64]; 4] { - use core::arch::wasm32::*; - - // Fixed-distance left-rotate of each u32 lane: `(x << n) | (x >> (32-n))`. - macro_rules! rotl { - ($x:expr, $n:expr) => { - v128_or(u32x4_shl($x, $n), u32x4_shr($x, 32 - $n)) - }; - } - // One vertical quarter-round across all 4 blocks. Same word indices as the - // scalar `quarter_round`. Pure ARX. - macro_rules! qr128 { - ($v:expr, $a:expr, $b:expr, $c:expr, $d:expr) => {{ - $v[$a] = u32x4_add($v[$a], $v[$b]); - $v[$d] = rotl!(v128_xor($v[$d], $v[$a]), 16); - $v[$c] = u32x4_add($v[$c], $v[$d]); - $v[$b] = rotl!(v128_xor($v[$b], $v[$c]), 12); - $v[$a] = u32x4_add($v[$a], $v[$b]); - $v[$d] = rotl!(v128_xor($v[$d], $v[$a]), 8); - $v[$c] = u32x4_add($v[$c], $v[$d]); - $v[$b] = rotl!(v128_xor($v[$b], $v[$c]), 7); - }}; - } - - // Initial working vectors: every word broadcast across the 4 lanes EXCEPT the - // counter (word 12), which gets lane-index 0..=3 added (u32 wrapping). - let lane_index = u32x4(0, 1, 2, 3); - let mut orig = [u32x4_splat(0); 16]; - for (w, o) in orig.iter_mut().enumerate() { - *o = u32x4_splat(state[w]); - } - orig[12] = u32x4_add(u32x4_splat(state[12]), lane_index); - - let mut v = orig; - for _ in 0..10 { - qr128!(v, 0, 4, 8, 12); - qr128!(v, 1, 5, 9, 13); - qr128!(v, 2, 6, 10, 14); - qr128!(v, 3, 7, 11, 15); - qr128!(v, 0, 5, 10, 15); - qr128!(v, 1, 6, 11, 12); - qr128!(v, 2, 7, 8, 13); - qr128!(v, 3, 4, 9, 14); - } - - // Add the original input state back, then de-vectorize via lane extraction. - let mut words = [[0u32; 4]; 16]; - for (w, dst) in words.iter_mut().enumerate() { - let summed = u32x4_add(v[w], orig[w]); - dst[0] = u32x4_extract_lane::<0>(summed); - dst[1] = u32x4_extract_lane::<1>(summed); - dst[2] = u32x4_extract_lane::<2>(summed); - dst[3] = u32x4_extract_lane::<3>(summed); - } - - let mut out = [[0u8; 64]; 4]; - for (l, block) in out.iter_mut().enumerate() { - for w in 0..16 { - block[w * 4..w * 4 + 4].copy_from_slice(&words[w][l].to_le_bytes()); - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The RFC 8439 §2.3.2 canonical keystream block (key = `00,01,…,1f`, block - /// counter = 1, nonce = `00 00 00 09 00 00 00 4a 00 00 00 00`). The scalar - /// reference and every SIMD backend MUST reproduce this exact 64-byte block. - const RFC8439_KAT_BLOCK: [u8; 64] = [ - 0x10, 0xf1, 0xe7, 0xe4, 0xd1, 0x3b, 0x59, 0x15, 0x50, 0x0f, 0xdd, 0x1f, 0xa3, 0x20, 0x71, 0xc4, 0xc7, 0xd1, - 0xf4, 0xc7, 0x33, 0xc0, 0x68, 0x03, 0x04, 0x22, 0xaa, 0x9a, 0xc3, 0xd4, 0x6c, 0x4e, 0xd2, 0x82, 0x64, 0x46, - 0x07, 0x9f, 0xaa, 0x09, 0x14, 0xc2, 0xd7, 0x05, 0xd9, 0x8b, 0x02, 0xa2, 0xb5, 0x12, 0x9c, 0xd1, 0xde, 0x16, - 0x4e, 0xb9, 0xcb, 0xd0, 0x83, 0xe8, 0xa2, 0x50, 0x3c, 0x4e, - ]; - - /// The RFC 8439 §2.3.2 key material (key `00..1f`, counter 1, the §2.3.2 nonce). - fn rfc8439_kat_inputs() -> ([u8; 32], u32, [u8; 12]) { - let mut key = [0u8; 32]; - for (i, b) in key.iter_mut().enumerate() { - *b = i as u8; - } - let nonce: [u8; 12] = [0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00]; - (key, 1, nonce) - } - - /// RFC 8439 §2.3.2 Known-Answer-Test on the scalar reference block. This pins - /// the scalar reference; every SIMD backend added later MUST reproduce this - /// exact 64-byte keystream (the parity gate). - #[test] - fn chacha20_block_rfc8439_kat() { - let (key, counter, nonce) = rfc8439_kat_inputs(); - let state = chacha20_state(&key, counter, &nonce); - let block = chacha20_block(&state); - assert_eq!(block, RFC8439_KAT_BLOCK, "ChaCha20 block must match RFC 8439 §2.3.2"); - } - - /// The quarter-round test vector (RFC 8439 §2.1.1) — the ARX core in - /// isolation, so a backend can be debugged at the round level. - #[test] - fn quarter_round_rfc8439_vector() { - // §2.1.1: inputs a,b,c,d and expected outputs, placed in a 16-word state - // at indices 0..4 so `quarter_round` operates on them. - let mut s = [0u32; 16]; - s[0] = 0x1111_1111; - s[1] = 0x0102_0304; - s[2] = 0x9b8d_6f43; - s[3] = 0x0123_4567; - quarter_round(&mut s, 0, 1, 2, 3); - assert_eq!( - [s[0], s[1], s[2], s[3]], - [0xea2a_92f4, 0xcb1c_f8ce, 0x4581_472e, 0x5881_c4bb], - "quarter-round must match RFC 8439 §2.1.1" - ); - } - - /// The mandatory SIMD-crypto correctness gate: every block produced by the - /// `chacha20_keystream` dispatcher (AVX-512 backend when `avx512f` is present, - /// scalar tail always) MUST be byte-identical to the scalar - /// [`chacha20_block`] reference for the same key/counter/nonce. `n = 40` - /// spans two full 16-block AVX-512 strides plus an 8-block scalar tail, so - /// both the vectorized body and the ragged-tail path are exercised. If this - /// test is red, no SIMD crypto backend may be committed. - #[test] - fn chacha20_keystream_dispatch_parity_scalar() { - let mut key = [0u8; 32]; - for (i, b) in key.iter_mut().enumerate() { - *b = (i as u8).wrapping_mul(7).wrapping_add(3); - } - let nonce: [u8; 12] = [0xde, 0xad, 0xbe, 0xef, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]; - let base = 0x0102_0304u32; - - // 40 blocks: two AVX-512 strides (16 each) + an 8-block scalar tail. - // Fixed-size array (not `vec!`) so the test compiles under `no_std`. - let mut ks = [[0u8; 64]; 40]; - chacha20_keystream(&key, base, &nonce, &mut ks); - - for (b, got) in ks.iter().enumerate() { - let state = chacha20_state(&key, base.wrapping_add(b as u32), &nonce); - let want = chacha20_block(&state); - assert_eq!(*got, want, "keystream block {b} must equal the scalar reference"); - } - } - - /// The dispatcher must reproduce the RFC 8439 §2.3.2 KAT at block 0. When the - /// host CPU has AVX-512F this drives the RFC vector THROUGH the AVX-512 - /// backend (16 blocks per stride, block 0 uses counter 1), directly pinning - /// the vectorized path to the published answer — not just to the scalar path. - #[test] - fn chacha20_keystream_dispatch_matches_rfc8439_kat() { - let (key, counter, nonce) = rfc8439_kat_inputs(); - // 16 blocks forces the AVX-512 stride when available; block 0 == the KAT. - let mut ks = [[0u8; 64]; 16]; - chacha20_keystream(&key, counter, &nonce, &mut ks); - assert_eq!(ks[0], RFC8439_KAT_BLOCK, "dispatcher block 0 must match RFC 8439 §2.3.2"); - } - - /// A native scalar mirror of `chacha20_block4_wasm`: it emulates each wasm - /// `v128` as a `[u32; 4]` lane array and applies the *identical* operation - /// sequence (word-sliced layout, counter-lane `0..=3`, the 8-quarter-round - /// double-round schedule, add-original, per-lane LE serialization). The wasm - /// intrinsics (`u32x4_add`/`v128_xor`/`u32x4_shl`/`u32x4_shr`/`u32x4_extract_lane`) - /// compute exactly these lane-wise u32 ops, so this validates the wasm arm's - /// *logic* on a target that can't run wasm; the wasm intrinsic API itself is - /// covered by the `wasm32-unknown-unknown` + `simd128` compile-check, and the - /// same word-sliced algorithm is byte-parity-proven 16-wide by the AVX-512 - /// tests above. - fn chacha20_block4_wasm_logic_ref(state: &[u32; 16]) -> [[u8; 64]; 4] { - // Lane-wise emulations of the wasm ops. - let add = |a: [u32; 4], b: [u32; 4]| [0, 1, 2, 3].map(|l| a[l].wrapping_add(b[l])); - let xor = |a: [u32; 4], b: [u32; 4]| [0, 1, 2, 3].map(|l| a[l] ^ b[l]); - let rotl = |a: [u32; 4], n: u32| [0, 1, 2, 3].map(|l| a[l].rotate_left(n)); - let splat = |x: u32| [x; 4]; - - macro_rules! qr { - ($v:expr, $a:expr, $b:expr, $c:expr, $d:expr) => {{ - $v[$a] = add($v[$a], $v[$b]); - $v[$d] = rotl(xor($v[$d], $v[$a]), 16); - $v[$c] = add($v[$c], $v[$d]); - $v[$b] = rotl(xor($v[$b], $v[$c]), 12); - $v[$a] = add($v[$a], $v[$b]); - $v[$d] = rotl(xor($v[$d], $v[$a]), 8); - $v[$c] = add($v[$c], $v[$d]); - $v[$b] = rotl(xor($v[$b], $v[$c]), 7); - }}; - } - - let lane_index = [0u32, 1, 2, 3]; - let mut orig = [[0u32; 4]; 16]; - for (w, o) in orig.iter_mut().enumerate() { - *o = splat(state[w]); - } - orig[12] = add(splat(state[12]), lane_index); - - let mut v = orig; - for _ in 0..10 { - qr!(v, 0, 4, 8, 12); - qr!(v, 1, 5, 9, 13); - qr!(v, 2, 6, 10, 14); - qr!(v, 3, 7, 11, 15); - qr!(v, 0, 5, 10, 15); - qr!(v, 1, 6, 11, 12); - qr!(v, 2, 7, 8, 13); - qr!(v, 3, 4, 9, 14); - } - - let mut words = [[0u32; 4]; 16]; - for (w, dst) in words.iter_mut().enumerate() { - *dst = add(v[w], orig[w]); - } - let mut out = [[0u8; 64]; 4]; - for (l, block) in out.iter_mut().enumerate() { - for w in 0..16 { - block[w * 4..w * 4 + 4].copy_from_slice(&words[w][l].to_le_bytes()); - } - } - out - } - - /// The wasm arm's 4-block word-sliced logic must equal the scalar reference - /// for the 4 consecutive counters it covers — the native proxy for the wasm - /// backend's correctness (see `chacha20_block4_wasm_logic_ref`). - #[test] - fn chacha20_wasm_logic_matches_scalar() { - let mut key = [0u8; 32]; - for (i, b) in key.iter_mut().enumerate() { - *b = (i as u8).wrapping_mul(11).wrapping_add(5); - } - let nonce: [u8; 12] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98]; - let base = 0x1000_0000u32; - - let state = chacha20_state(&key, base, &nonce); - let four = chacha20_block4_wasm_logic_ref(&state); - for (l, got) in four.iter().enumerate() { - let want = chacha20_block(&chacha20_state(&key, base + l as u32, &nonce)); - assert_eq!(*got, want, "wasm-logic block {l} must equal the scalar reference"); - } - } -} diff --git a/tests/chacha20_rustcrypto_parity.rs b/tests/chacha20_rustcrypto_parity.rs deleted file mode 100644 index f3aa3b8e..00000000 --- a/tests/chacha20_rustcrypto_parity.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Cross-implementation trust gate for `ndarray::simd::chacha20_keystream`. -//! -//! The module KAT (`src/simd_crypto.rs`) pins the scalar reference to the single -//! RFC 8439 §2.3.2 vector, and `chacha20_keystream_dispatch_parity_scalar` pins -//! the AVX-512 backend to that scalar reference. This integration test closes the -//! remaining gap: it proves the *whole dispatcher* is byte-for-byte identical to -//! an **independent, vetted implementation** — RustCrypto's `chacha20::ChaCha20` -//! (the IETF 96-bit-nonce / 32-bit-counter variant, RFC 8439) — across a wide -//! space of random keys, nonces, counters, and lengths. This is the -//! "reference + KAT, then vectorize with parity" discipline extended to a second -//! oracle: no consumer (the `encryption` AEAD, a future XChaCha vault) should -//! draw keystream from the accelerated primitive until it is proven equivalent -//! to the trusted implementation everywhere, not just at one KAT point. -//! -//! ChaCha20 is ARX, so the accelerated path is constant-time by construction; -//! this test guards *correctness*, which is the property a hand-vectorization -//! can actually get wrong. -//! -//! Gated to `std`: `ndarray::simd` is `#[cfg(feature = "std")]`, so this test is -//! a no-op in `--no-default-features` builds. -#![cfg(feature = "std")] - -use chacha20::cipher::{KeyIvInit, StreamCipher, StreamCipherSeek}; -use chacha20::ChaCha20; -use ndarray::simd::{chacha20_block, chacha20_keystream, chacha20_state}; - -/// Deterministic SplitMix64 — a stand-in for a PRNG so the vectors are fixed and -/// reproducible without a `rand` dev-dependency (and without `Math.random`-style -/// nondeterminism that would make a failure impossible to bisect). -struct SplitMix64(u64); - -impl SplitMix64 { - fn next_u64(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - fn fill(&mut self, buf: &mut [u8]) { - for chunk in buf.chunks_mut(8) { - let bytes = self.next_u64().to_le_bytes(); - chunk.copy_from_slice(&bytes[..chunk.len()]); - } - } -} - -/// The trusted oracle: `n_blocks` of RustCrypto ChaCha20 keystream starting at -/// block `counter` (produced by encrypting a zero buffer). -fn rustcrypto_keystream(key: &[u8; 32], counter: u32, nonce: &[u8; 12], n_blocks: usize) -> Vec { - let mut cipher = ChaCha20::new_from_slices(key, nonce).expect("valid key/nonce lengths"); - // Seek to block `counter` (position is in bytes; each block is 64 bytes). - cipher.seek(u64::from(counter) * 64); - let mut buf = vec![0u8; n_blocks * 64]; - cipher.apply_keystream(&mut buf); - buf -} - -/// The primitive under test, flattened to a byte stream for comparison. -fn ndarray_keystream(key: &[u8; 32], counter: u32, nonce: &[u8; 12], n_blocks: usize) -> Vec { - let mut blocks = vec![[0u8; 64]; n_blocks]; - chacha20_keystream(key, counter, nonce, &mut blocks); - blocks.concat() -} - -/// The dispatcher (AVX-512 backend when present, scalar otherwise) must equal the -/// RustCrypto oracle across a broad, deterministic sample of the parameter space. -#[test] -fn keystream_matches_rustcrypto_across_vectors() { - let mut rng = SplitMix64(0x0DDB_1A5E_5EED_1234); - - // Lengths chosen to exercise: sub-stride, exactly one AVX-512 stride (16), - // multi-stride, and every ragged-tail size in between. - let block_counts = [1usize, 2, 7, 15, 16, 17, 31, 33, 40, 64]; - - for &n in &block_counts { - // A handful of independent (key, nonce, counter) draws per length. - for _ in 0..8 { - let mut key = [0u8; 32]; - let mut nonce = [0u8; 12]; - rng.fill(&mut key); - rng.fill(&mut nonce); - // Keep counter + n well inside u32 so the RustCrypto oracle (which - // refuses to wrap its 32-bit counter) stays in its valid range; the - // wrapping edge is covered separately by the scalar-parity unit test. - let counter = (rng.next_u64() as u32) & 0x00FF_FFFF; - - let got = ndarray_keystream(&key, counter, &nonce, n); - let want = rustcrypto_keystream(&key, counter, &nonce, n); - assert_eq!( - got, want, - "keystream mismatch vs RustCrypto: n={n} blocks, counter={counter}, key[0]={}, nonce[0]={}", - key[0], nonce[0] - ); - } - } -} - -/// The single-block scalar primitive `chacha20_block` (fed via `chacha20_state`) -/// must equal RustCrypto's first block for the same inputs — pins the low-level -/// entry point, not just the batched dispatcher. -#[test] -fn single_block_matches_rustcrypto() { - let mut rng = SplitMix64(0xC0FF_EE00_1357_9BDF); - for _ in 0..64 { - let mut key = [0u8; 32]; - let mut nonce = [0u8; 12]; - rng.fill(&mut key); - rng.fill(&mut nonce); - let counter = (rng.next_u64() as u32) & 0x00FF_FFFF; - - let got = chacha20_block(&chacha20_state(&key, counter, &nonce)); - let want = rustcrypto_keystream(&key, counter, &nonce, 1); - assert_eq!(&got[..], &want[..], "single-block mismatch vs RustCrypto at counter={counter}"); - } -} - -/// Sanity-check the oracle itself is the IETF RFC 8439 variant (not the 64-bit -/// legacy nonce cipher) by reproducing the §2.3.2 KAT through RustCrypto. If this -/// fails, the oracle is misconfigured and the parity tests above prove nothing. -#[test] -fn rustcrypto_oracle_is_rfc8439_ietf() { - let mut key = [0u8; 32]; - for (i, b) in key.iter_mut().enumerate() { - *b = i as u8; - } - let nonce: [u8; 12] = [0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x00]; - let block = rustcrypto_keystream(&key, 1, &nonce, 1); - - let expected: [u8; 64] = [ - 0x10, 0xf1, 0xe7, 0xe4, 0xd1, 0x3b, 0x59, 0x15, 0x50, 0x0f, 0xdd, 0x1f, 0xa3, 0x20, 0x71, 0xc4, 0xc7, 0xd1, - 0xf4, 0xc7, 0x33, 0xc0, 0x68, 0x03, 0x04, 0x22, 0xaa, 0x9a, 0xc3, 0xd4, 0x6c, 0x4e, 0xd2, 0x82, 0x64, 0x46, - 0x07, 0x9f, 0xaa, 0x09, 0x14, 0xc2, 0xd7, 0x05, 0xd9, 0x8b, 0x02, 0xa2, 0xb5, 0x12, 0x9c, 0xd1, 0xde, 0x16, - 0x4e, 0xb9, 0xcb, 0xd0, 0x83, 0xe8, 0xa2, 0x50, 0x3c, 0x4e, - ]; - assert_eq!(&block[..], &expected[..], "RustCrypto oracle must reproduce RFC 8439 §2.3.2"); -} diff --git a/vendor/chacha20/.gitignore b/vendor/chacha20/.gitignore new file mode 100644 index 00000000..96ef6c0b --- /dev/null +++ b/vendor/chacha20/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/vendor/chacha20/CHANGELOG.md b/vendor/chacha20/CHANGELOG.md new file mode 100644 index 00000000..29510534 --- /dev/null +++ b/vendor/chacha20/CHANGELOG.md @@ -0,0 +1,255 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## 0.9.1 (2023-04-01) +### Added +- NEON support via `chacha20_force_neon` cfg attribute ([#310], [#317]) + +[#310]: https://github.com/RustCrypto/stream-ciphers/pull/310 +[#317]: https://github.com/RustCrypto/stream-ciphers/pull/317 + +## 0.9.0 (2022-02-21) +### Added +- `chacha20_force_soft`, `chacha20_force_sse2`, and `chacha20_force_avx2` +configuration flags ([#293]) + +### Changed +- Bump `cipher` dependency to v0.4 ([#276]) + +### Fixed +- Minimal versions build ([#290]) + +### Removed +- `neon`, `force-soft`, `expose-core`, `hchacha`, `legacy`, and `rng` features ([#276], [#293]) + +[#276]: https://github.com/RustCrypto/stream-ciphers/pull/276 +[#290]: https://github.com/RustCrypto/stream-ciphers/pull/290 +[#293]: https://github.com/RustCrypto/stream-ciphers/pull/293 + +## 0.8.2 (2022-07-07) +### Changed +- Unpin `zeroize` dependency ([#301]) + +[#301]: https://github.com/RustCrypto/stream-ciphers/pull/301 + +## 0.8.1 (2021-08-30) +### Added +- NEON implementation for aarch64 ([#274]) + +[#274]: https://github.com/RustCrypto/stream-ciphers/pull/274 + +## 0.8.0 (2021-08-29) +### Added +- SSE2 autodetection support ([#270]) + +### Changed +- AVX2 performance improvements ([#267], [#267]) +- MSRV 1.51+ ([#267]) +- Lock to `zeroize` <1.5 ([#269]) + +### Removed +- `xchacha` feature: all `XChaCha*` types are now available by-default ([#271]) + +[#267]: https://github.com/RustCrypto/stream-ciphers/pull/267 +[#269]: https://github.com/RustCrypto/stream-ciphers/pull/269 +[#270]: https://github.com/RustCrypto/stream-ciphers/pull/270 +[#271]: https://github.com/RustCrypto/stream-ciphers/pull/271 + +## 0.7.3 (2021-08-27) +### Changed +- Improve AVX2 performance ([#261]) +- Bump `cpufeatures` to v0.2 ([#265]) + +[#261]: https://github.com/RustCrypto/stream-ciphers/pull/261 +[#265]: https://github.com/RustCrypto/stream-ciphers/pull/265 + +## 0.7.2 (2021-07-20) +### Changed +- Pin `zeroize` dependency to v1.3 ([#256]) + +[#256]: https://github.com/RustCrypto/stream-ciphers/pull/256 + +## 0.7.1 (2021-04-29) +### Added +- `hchacha` feature ([#234]) + +[#234]: https://github.com/RustCrypto/stream-ciphers/pull/234 + +## 0.7.0 (2021-04-29) [YANKED] +### Added +- AVX2 detection; MSRV 1.49+ ([#200], [#212]) +- `XChaCha8` and `XChaCha12` ([#215]) + +### Changed +- Full 64-bit counters ([#217]) +- Bump `cipher` crate dependency to v0.3 release ([#226]) + +### Fixed +- `rng` feature on big endian platforms ([#202]) +- Stream-length overflow check ([#216]) + +### Removed +- `Clone` impls on RNGs ([#220]) + +[#200]: https://github.com/RustCrypto/stream-ciphers/pull/200 +[#202]: https://github.com/RustCrypto/stream-ciphers/pull/202 +[#212]: https://github.com/RustCrypto/stream-ciphers/pull/212 +[#215]: https://github.com/RustCrypto/stream-ciphers/pull/215 +[#216]: https://github.com/RustCrypto/stream-ciphers/pull/216 +[#217]: https://github.com/RustCrypto/stream-ciphers/pull/217 +[#220]: https://github.com/RustCrypto/stream-ciphers/pull/220 +[#226]: https://github.com/RustCrypto/stream-ciphers/pull/226 + +## 0.6.0 (2020-10-16) +### Changed +- Rename `Cipher` to `ChaCha` ([#177]) +- Replace `block-cipher`/`stream-cipher` with `cipher` crate ([#177]) + +[#177]: https://github.com/RustCrypto/stream-ciphers/pull/177 + +## 0.5.0 (2020-08-25) +### Changed +- Bump `stream-cipher` dependency to v0.7 ([#161], [#164]) + +[#161]: https://github.com/RustCrypto/stream-ciphers/pull/161 +[#164]: https://github.com/RustCrypto/stream-ciphers/pull/164 + +## 0.4.3 (2020-06-11) +### Changed +- Documentation improvements ([#153], [#154], [#155]) + +[#153]: https://github.com/RustCrypto/stream-ciphers/pull/155 +[#154]: https://github.com/RustCrypto/stream-ciphers/pull/155 +[#155]: https://github.com/RustCrypto/stream-ciphers/pull/155 + +## 0.4.2 (2020-06-11) +### Added +- Documentation improvements ([#149]) +- `Key`, `Nonce`, `XNonce`, and `LegacyNonce` type aliases ([#147]) + +[#149]: https://github.com/RustCrypto/stream-ciphers/pull/149 +[#147]: https://github.com/RustCrypto/stream-ciphers/pull/147 + +## 0.4.1 (2020-06-06) +### Fixed +- Links in documentation ([#142]) + +[#142]: https://github.com/RustCrypto/stream-ciphers/pull/142 + +## 0.4.0 (2020-06-06) +### Changed +- Upgrade to the `stream-cipher` v0.4 crate ([#121], [#138]) + +[#138]: https://github.com/RustCrypto/stream-ciphers/pull/138 +[#121]: https://github.com/RustCrypto/stream-ciphers/pull/121 + +## 0.3.4 (2020-03-02) +### Fixed +- Avoid accidental `alloc` and `std` linking ([#105]) + +[#105]: https://github.com/RustCrypto/stream-ciphers/pull/105 + +## 0.3.3 (2020-01-18) +### Changed +- Replace macros with `Rounds` trait + generics ([#100]) + +### Fixed +- Fix warnings when building with `rng` feature alone ([#99]) + +[#99]: https://github.com/RustCrypto/stream-ciphers/pull/99 +[#100]: https://github.com/RustCrypto/stream-ciphers/pull/100 + +## 0.3.2 (2020-01-17) +### Added +- `CryptoRng` marker on all `ChaCha*Rng` types ([#91]) + +[#91]: https://github.com/RustCrypto/stream-ciphers/pull/91 + +## 0.3.1 (2020-01-16) +### Added +- Parallelize AVX2 backend ([#87]) +- Benchmark for `ChaCha20Rng` ([#87]) + +### Fixed +- Fix broken buffering logic ([#86]) + +[#86]: https://github.com/RustCrypto/stream-ciphers/pull/86 +[#87]: https://github.com/RustCrypto/stream-ciphers/pull/87 + +## 0.3.0 (2020-01-15) [YANKED] + +NOTE: This release was yanked due to a showstopper bug in the newly added +buffering logic which when seeking in the keystream could result in plaintexts +being clobbered with the keystream instead of XOR'd correctly. + +The bug was addressed in v0.3.1 ([#86]). + +### Added +- AVX2 accelerated implementation ([#83]) +- ChaCha8 and ChaCha20 reduced round variants ([#84]) + +### Changed +- Simplify portable implementation ([#76]) +- Make 2018 edition crate; MSRV 1.34+ ([#77]) +- Replace `salsa20-core` dependency with `ctr`-derived buffering ([#81]) + +### Removed +- `byteorder` dependency ([#80]) + +[#76]: https://github.com/RustCrypto/stream-ciphers/pull/76 +[#77]: https://github.com/RustCrypto/stream-ciphers/pull/77 +[#80]: https://github.com/RustCrypto/stream-ciphers/pull/80 +[#81]: https://github.com/RustCrypto/stream-ciphers/pull/81 +[#83]: https://github.com/RustCrypto/stream-ciphers/pull/83 +[#84]: https://github.com/RustCrypto/stream-ciphers/pull/84 + +## 0.2.3 (2019-10-23) +### Security +- Ensure block counter < MAX_BLOCKS ([#68]) + +[#68]: https://github.com/RustCrypto/stream-ciphers/pull/68 + +## 0.2.2 (2019-10-22) +### Added +- SSE2 accelerated implementation ([#61]) + +[#61]: https://github.com/RustCrypto/stream-ciphers/pull/61 + +## 0.2.1 (2019-08-19) +### Added +- Add `MAX_BLOCKS` and `BLOCK_SIZE` constants ([#47]) + +[#47]: https://github.com/RustCrypto/stream-ciphers/pull/47 + +## 0.2.0 (2019-08-18) +### Added +- `impl SyncStreamCipher` ([#39]) +- `XChaCha20` ([#36]) +- Support for 12-byte nonces ala RFC 8439 ([#19]) + +### Changed +- Refactor around a `ctr`-like type ([#44]) +- Extract and encapsulate `Cipher` type ([#43]) +- Switch tests to use `new_sync_test!` ([#42]) +- Refactor into `ChaCha20` and `ChaCha20Legacy` ([#25]) + +### Fixed +- Fix `zeroize` cargo feature ([#21]) +- Fix broken Cargo feature attributes ([#21]) + +[#44]: https://github.com/RustCrypto/stream-ciphers/pull/44 +[#43]: https://github.com/RustCrypto/stream-ciphers/pull/43 +[#42]: https://github.com/RustCrypto/stream-ciphers/pull/42 +[#39]: https://github.com/RustCrypto/stream-ciphers/pull/39 +[#36]: https://github.com/RustCrypto/stream-ciphers/pull/36 +[#25]: https://github.com/RustCrypto/stream-ciphers/pull/25 +[#21]: https://github.com/RustCrypto/stream-ciphers/pull/21 +[#19]: https://github.com/RustCrypto/stream-ciphers/pull/19 + +## 0.1.0 (2019-06-24) + +- Initial release diff --git a/vendor/chacha20/Cargo.toml b/vendor/chacha20/Cargo.toml new file mode 100644 index 00000000..df9680ce --- /dev/null +++ b/vendor/chacha20/Cargo.toml @@ -0,0 +1,60 @@ +# AdaWorldAPI fork of RustCrypto `chacha20` v0.9.1 (the "matryoshka"). +# +# Verbatim upstream cipher — state schedule, counter, XChaCha, RNG, and the +# soft/sse2/avx2/neon backends are UNTOUCHED. The only delta is one added +# backend, `src/backends/ndarray_simd.rs`, which expresses the keystream +# double-round over `ndarray::simd::U32x16` and is compile-time-selected on an +# AVX-512 x86_64 build (see `src/backends.rs` + `src/lib.rs::process_with_backend`). +# +# The crate name/version stay `chacha20` / `0.9.1` so `[patch.crates-io]` folds +# this over the registry crate, transitively accelerating `chacha20poly1305 -> +# encryption -> ogar-encryption -> consumers` with zero change to any of them. +# Re-sync on a RustCrypto advisory: bump the vendored source, re-apply the single +# `ndarray_simd.rs` backend + the four cfg branches, re-run the vectors. +[package] +edition = "2021" +rust-version = "1.95" +name = "chacha20" +version = "0.9.1" +authors = ["RustCrypto Developers"] +description = "The ChaCha20 stream cipher (RFC 8439) — AdaWorldAPI fork with an ndarray::simd::U32x16 AVX-512 backend." +documentation = "https://docs.rs/chacha20" +readme = "README.md" +keywords = ["crypto", "stream-cipher", "chacha8", "chacha12", "xchacha20"] +categories = ["cryptography", "no-std"] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/AdaWorldAPI/ndarray" +resolver = "1" + +[dependencies.cfg-if] +version = "1" + +[dependencies.cipher] +version = "0.4.4" + +[dev-dependencies.cipher] +version = "0.4.4" +features = ["dev"] + +[dev-dependencies.hex-literal] +version = "0.3.3" + +# Standalone: this is a `[patch]` target folded into the parent workspace, not a +# member of it (the empty table makes it its own workspace root). +[workspace] + +[features] +std = ["cipher/std"] +zeroize = ["cipher/zeroize"] + +[target."cfg(any(target_arch = \"x86_64\", target_arch = \"x86\"))".dependencies.cpufeatures] +version = "0.2" + +# The matryoshka's one non-upstream dependency: the AVX-512 backend rides +# `ndarray::simd::U32x16`. Scoped to x86_64 so wasm / aarch64 / other builds of +# this crate never pull ndarray (they fall through to the upstream backends); +# `simd` needs only ndarray's `std` feature, not the full HPC default set. +[target."cfg(target_arch = \"x86_64\")".dependencies.ndarray] +path = "../.." +default-features = false +features = ["std"] diff --git a/vendor/chacha20/LICENSE-APACHE b/vendor/chacha20/LICENSE-APACHE new file mode 100644 index 00000000..78173fa2 --- /dev/null +++ b/vendor/chacha20/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/chacha20/LICENSE-MIT b/vendor/chacha20/LICENSE-MIT new file mode 100644 index 00000000..2889a9a8 --- /dev/null +++ b/vendor/chacha20/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2019-2023 The RustCrypto Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/vendor/chacha20/README.md b/vendor/chacha20/README.md new file mode 100644 index 00000000..2ff7d5bb --- /dev/null +++ b/vendor/chacha20/README.md @@ -0,0 +1,117 @@ +# RustCrypto: ChaCha20 + +[![Crate][crate-image]][crate-link] +[![Docs][docs-image]][docs-link] +![Apache2/MIT licensed][license-image] +![Rust Version][rustc-image] +[![Project Chat][chat-image]][chat-link] +[![Build Status][build-image]][build-link] +[![HAZMAT][hazmat-image]][hazmat-link] + +Pure Rust implementation of the [ChaCha20 Stream Cipher][1]. + +[Documentation][docs-link] + + + +## About + +[ChaCha20][1] is a [stream cipher][2] which is designed to support +high-performance software implementations. + +It improves upon the previous [Salsa20][3] stream cipher with increased +per-round diffusion at no cost to performance. + +This crate also contains an implementation of [XChaCha20][4]: a variant +of ChaCha20 with an extended 192-bit (24-byte) nonce, gated under the +`chacha20` Cargo feature (on-by-default). + +## Implementations + +This crate contains the following implementations of ChaCha20, all of which +work on stable Rust with the following `RUSTFLAGS`: + +- `x86` / `x86_64` + - `avx2`: (~1.4cpb) `-Ctarget-cpu=haswell -Ctarget-feature=+avx2` + - `sse2`: (~2.5cpb) `-Ctarget-feature=+sse2` (on by default on x86 CPUs) +- `aarch64` + - `neon` (~2-3x faster than `soft`) requires Rust 1.61+ and the `neon` feature enabled +- Portable + - `soft`: (~5 cpb on x86/x86_64) + +NOTE: cpb = cycles per byte (smaller is better) + +## Security + +### ⚠️ Warning: [Hazmat!][hazmat-link] + +This crate does not ensure ciphertexts are authentic (i.e. by using a MAC to +verify ciphertext integrity), which can lead to serious vulnerabilities +if used incorrectly! + +To avoid this, use an [AEAD][5] mode based on ChaCha20, i.e. [ChaCha20Poly1305][6]. +See the [RustCrypto/AEADs][7] repository for more information. + +USE AT YOUR OWN RISK! + +### Notes + +This crate has received one [security audit by NCC Group][8], with no significant +findings. We would like to thank [MobileCoin][9] for funding the audit. + +All implementations contained in the crate (along with the underlying ChaCha20 +stream cipher itself) are designed to execute in constant time. + +## Minimum Supported Rust Version + +Rust **1.56** or higher. + +Minimum supported Rust version can be changed in the future, but it will be +done with a minor version bump. + +## SemVer Policy + +- All on-by-default features of this library are covered by SemVer +- MSRV is considered exempt from SemVer as noted above + +## License + +Licensed under either of: + +- [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) +- [MIT license](http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + +[//]: # (badges) + +[crate-image]: https://img.shields.io/crates/v/chacha20.svg +[crate-link]: https://crates.io/crates/chacha20 +[docs-image]: https://docs.rs/chacha20/badge.svg +[docs-link]: https://docs.rs/chacha20/ +[license-image]: https://img.shields.io/badge/license-Apache2.0/MIT-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.56+-blue.svg +[chat-image]: https://img.shields.io/badge/zulip-join_chat-blue.svg +[chat-link]: https://rustcrypto.zulipchat.com/#narrow/stream/260049-stream-ciphers +[build-image]: https://github.com/RustCrypto/stream-ciphers/workflows/chacha20/badge.svg?branch=master&event=push +[build-link]: https://github.com/RustCrypto/stream-ciphers/actions?query=workflow%3Achacha20 +[hazmat-image]: https://img.shields.io/badge/crypto-hazmat%E2%9A%A0-red.svg +[hazmat-link]: https://github.com/RustCrypto/meta/blob/master/HAZMAT.md + +[//]: # (footnotes) + +[1]: https://en.wikipedia.org/wiki/Salsa20#ChaCha_variant +[2]: https://en.wikipedia.org/wiki/Stream_cipher +[3]: https://en.wikipedia.org/wiki/Salsa20 +[4]: https://tools.ietf.org/html/draft-arciszewski-xchacha-02 +[5]: https://en.wikipedia.org/wiki/Authenticated_encryption +[6]: https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305 +[7]: https://github.com/RustCrypto/AEADs +[8]: https://research.nccgroup.com/2020/02/26/public-report-rustcrypto-aes-gcm-and-chacha20poly1305-implementation-review/ +[9]: https://www.mobilecoin.com/ diff --git a/vendor/chacha20/src/backends.rs b/vendor/chacha20/src/backends.rs new file mode 100644 index 00000000..39fa8da3 --- /dev/null +++ b/vendor/chacha20/src/backends.rs @@ -0,0 +1,28 @@ +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(chacha20_force_soft)] { + pub(crate) mod soft; + } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + // AdaWorldAPI matryoshka: on an AVX-512 build the keystream double-round + // rides `ndarray::simd::U32x16` (the polyfill lowers it to `__m512i`). + // Takes precedence over the runtime-detected AVX2/SSE2 path below. + pub(crate) mod ndarray_simd; + } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cfg_if! { + if #[cfg(chacha20_force_avx2)] { + pub(crate) mod avx2; + } else if #[cfg(chacha20_force_sse2)] { + pub(crate) mod sse2; + } else { + pub(crate) mod soft; + pub(crate) mod avx2; + pub(crate) mod sse2; + } + } + } else if #[cfg(all(chacha20_force_neon, target_arch = "aarch64", target_feature = "neon"))] { + pub(crate) mod neon; + } else { + pub(crate) mod soft; + } +} diff --git a/vendor/chacha20/src/backends/avx2.rs b/vendor/chacha20/src/backends/avx2.rs new file mode 100644 index 00000000..5a097425 --- /dev/null +++ b/vendor/chacha20/src/backends/avx2.rs @@ -0,0 +1,251 @@ +use crate::{Block, StreamClosure, Unsigned, STATE_WORDS}; +use cipher::{ + consts::{U4, U64}, + BlockSizeUser, ParBlocks, ParBlocksSizeUser, StreamBackend, +}; +use core::marker::PhantomData; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +/// Number of blocks processed in parallel. +const PAR_BLOCKS: usize = 4; +/// Number of `__m256i` to store parallel blocks. +const N: usize = PAR_BLOCKS / 2; + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamClosure, +{ + let state_ptr = state.as_ptr() as *const __m128i; + let v = [ + _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(0))), + _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(1))), + _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(2))), + ]; + let mut c = _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(3))); + c = _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 0)); + let mut ctr = [c; N]; + for i in 0..N { + ctr[i] = c; + c = _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 2, 0, 0, 0, 2)); + } + let mut backend = Backend:: { + v, + ctr, + _pd: PhantomData, + }; + + f.call(&mut backend); + + state[12] = _mm256_extract_epi32(backend.ctr[0], 0) as u32; +} + +struct Backend { + v: [__m256i; 3], + ctr: [__m256i; N], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend { + type ParBlocksSize = U4; +} + +impl StreamBackend for Backend { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + unsafe { + let res = rounds::(&self.v, &self.ctr); + for c in self.ctr.iter_mut() { + *c = _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 1)); + } + + let res0: [__m128i; 8] = core::mem::transmute(res[0]); + + let block_ptr = block.as_mut_ptr() as *mut __m128i; + for i in 0..4 { + _mm_storeu_si128(block_ptr.add(i), res0[2 * i]); + } + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks) { + unsafe { + let vs = rounds::(&self.v, &self.ctr); + + let pb = PAR_BLOCKS as i32; + for c in self.ctr.iter_mut() { + *c = _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, pb, 0, 0, 0, pb)); + } + + let mut block_ptr = blocks.as_mut_ptr() as *mut __m128i; + for v in vs { + let t: [__m128i; 8] = core::mem::transmute(v); + for i in 0..4 { + _mm_storeu_si128(block_ptr.add(i), t[2 * i]); + _mm_storeu_si128(block_ptr.add(4 + i), t[2 * i + 1]); + } + block_ptr = block_ptr.add(8); + } + } + } +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn rounds(v: &[__m256i; 3], c: &[__m256i; N]) -> [[__m256i; 4]; N] { + let mut vs: [[__m256i; 4]; N] = [[_mm256_setzero_si256(); 4]; N]; + for i in 0..N { + vs[i] = [v[0], v[1], v[2], c[i]]; + } + for _ in 0..R::USIZE { + double_quarter_round(&mut vs); + } + + for i in 0..N { + for j in 0..3 { + vs[i][j] = _mm256_add_epi32(vs[i][j], v[j]); + } + vs[i][3] = _mm256_add_epi32(vs[i][3], c[i]); + } + + vs +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn double_quarter_round(v: &mut [[__m256i; 4]; N]) { + add_xor_rot(v); + rows_to_cols(v); + add_xor_rot(v); + cols_to_rows(v); +} + +/// The goal of this function is to transform the state words from: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c0, c1, c2, c3] [ 8, 9, 10, 11] +/// [d0, d1, d2, d3] [12, 13, 14, 15] +/// ``` +/// +/// to: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b1, b2, b3, b0] == [ 5, 6, 7, 4] +/// [c2, c3, c0, c1] [10, 11, 8, 9] +/// [d3, d0, d1, d2] [15, 12, 13, 14] +/// ``` +/// +/// so that we can apply [`add_xor_rot`] to the resulting columns, and have it compute the +/// "diagonal rounds" (as defined in RFC 7539) in parallel. In practice, this shuffle is +/// non-optimal: the last state word to be altered in `add_xor_rot` is `b`, so the shuffle +/// blocks on the result of `b` being calculated. +/// +/// We can optimize this by observing that the four quarter rounds in `add_xor_rot` are +/// data-independent: they only access a single column of the state, and thus the order of +/// the columns does not matter. We therefore instead shuffle the other three state words, +/// to obtain the following equivalent layout: +/// ```text +/// [a3, a0, a1, a2] [ 3, 0, 1, 2] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c1, c2, c3, c0] [ 9, 10, 11, 8] +/// [d2, d3, d0, d1] [14, 15, 12, 13] +/// ``` +/// +/// See https://github.com/sneves/blake2-avx2/pull/4 for additional details. The earliest +/// known occurrence of this optimization is in floodyberry's SSE4 ChaCha code from 2014: +/// - https://github.com/floodyberry/chacha-opt/blob/0ab65cb99f5016633b652edebaf3691ceb4ff753/chacha_blocks_ssse3-64.S#L639-L643 +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn rows_to_cols(vs: &mut [[__m256i; 4]; N]) { + // c >>>= 32; d >>>= 64; a >>>= 96; + for [a, _, c, d] in vs { + *c = _mm256_shuffle_epi32(*c, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1) + *d = _mm256_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2) + *a = _mm256_shuffle_epi32(*a, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3) + } +} + +/// The goal of this function is to transform the state words from: +/// ```text +/// [a3, a0, a1, a2] [ 3, 0, 1, 2] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c1, c2, c3, c0] [ 9, 10, 11, 8] +/// [d2, d3, d0, d1] [14, 15, 12, 13] +/// ``` +/// +/// to: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c0, c1, c2, c3] [ 8, 9, 10, 11] +/// [d0, d1, d2, d3] [12, 13, 14, 15] +/// ``` +/// +/// reversing the transformation of [`rows_to_cols`]. +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn cols_to_rows(vs: &mut [[__m256i; 4]; N]) { + // c <<<= 32; d <<<= 64; a <<<= 96; + for [a, _, c, d] in vs { + *c = _mm256_shuffle_epi32(*c, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3) + *d = _mm256_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2) + *a = _mm256_shuffle_epi32(*a, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1) + } +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn add_xor_rot(vs: &mut [[__m256i; 4]; N]) { + let rol16_mask = _mm256_set_epi64x( + 0x0d0c_0f0e_0908_0b0a, + 0x0504_0706_0100_0302, + 0x0d0c_0f0e_0908_0b0a, + 0x0504_0706_0100_0302, + ); + let rol8_mask = _mm256_set_epi64x( + 0x0e0d_0c0f_0a09_080b, + 0x0605_0407_0201_0003, + 0x0e0d_0c0f_0a09_080b, + 0x0605_0407_0201_0003, + ); + + // a += b; d ^= a; d <<<= (16, 16, 16, 16); + for [a, b, _, d] in vs.iter_mut() { + *a = _mm256_add_epi32(*a, *b); + *d = _mm256_xor_si256(*d, *a); + *d = _mm256_shuffle_epi8(*d, rol16_mask); + } + + // c += d; b ^= c; b <<<= (12, 12, 12, 12); + for [_, b, c, d] in vs.iter_mut() { + *c = _mm256_add_epi32(*c, *d); + *b = _mm256_xor_si256(*b, *c); + *b = _mm256_xor_si256(_mm256_slli_epi32(*b, 12), _mm256_srli_epi32(*b, 20)); + } + + // a += b; d ^= a; d <<<= (8, 8, 8, 8); + for [a, b, _, d] in vs.iter_mut() { + *a = _mm256_add_epi32(*a, *b); + *d = _mm256_xor_si256(*d, *a); + *d = _mm256_shuffle_epi8(*d, rol8_mask); + } + + // c += d; b ^= c; b <<<= (7, 7, 7, 7); + for [_, b, c, d] in vs.iter_mut() { + *c = _mm256_add_epi32(*c, *d); + *b = _mm256_xor_si256(*b, *c); + *b = _mm256_xor_si256(_mm256_slli_epi32(*b, 7), _mm256_srli_epi32(*b, 25)); + } +} diff --git a/vendor/chacha20/src/backends/ndarray_simd.rs b/vendor/chacha20/src/backends/ndarray_simd.rs new file mode 100644 index 00000000..7f9be4e3 --- /dev/null +++ b/vendor/chacha20/src/backends/ndarray_simd.rs @@ -0,0 +1,110 @@ +//! ChaCha backend riding `ndarray::simd::U32x16` — the AdaWorldAPI matryoshka. +//! +//! RustCrypto owns the cipher (state schedule, counter, XChaCha, the Rng and +//! Poly1305 framing above); this file expresses *only* the keystream double-round +//! over `ndarray::simd::U32x16`, which `ndarray`'s compile-time polyfill lowers to +//! the AVX-512 lane on this build. There are **no raw intrinsics and no `unsafe` +//! here** — all of that lives once, audited, inside `ndarray::simd`. +//! +//! Layout: the *transpose* (vertical) form — 16 keystream blocks in parallel, +//! working vector `w` holds word `w` of every block across its 16 lanes, and the +//! counter word (12) carries lane-index `0..=15`. Every quarter-round is then a +//! pure `U32x16` `+` / `^` / `rotate_left` with **no cross-lane shuffle** (unlike +//! RustCrypto's row-interleaved AVX2 backend, whose `_mm256_shuffle_*` has no +//! `U32x16` equivalent). Bit-identical to the `soft` backend by construction; the +//! crate's RFC 8439 `tests/mod.rs` runs the published vectors through it. + +use crate::{Block, ChaChaCore, Unsigned, STATE_WORDS}; +use cipher::{ + consts::{U16, U64}, + BlockSizeUser, ParBlocks, ParBlocksSizeUser, StreamBackend, +}; +use ndarray::simd::U32x16; + +pub(crate) struct Backend<'a, R: Unsigned>(pub(crate) &'a mut ChaChaCore); + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U16; +} + +impl StreamBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + // Single block: reuse the 16-wide core, take lane 0 (counter == state[12]), + // advance the counter by one — exactly the `soft` backend's contract. + let ks = ks16::(&self.0.state); + block.copy_from_slice(&ks[0]); + self.0.state[12] = self.0.state[12].wrapping_add(1); + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks) { + let ks = ks16::(&self.0.state); + for (dst, src) in blocks.iter_mut().zip(ks.iter()) { + dst.copy_from_slice(src); + } + self.0.state[12] = self.0.state[12].wrapping_add(16); + } +} + +/// 16 consecutive ChaCha keystream blocks — counters `state[12] ..= state[12]+15` +/// (per-lane `wrapping_add`) — in the transpose layout over `U32x16`. `R::USIZE` +/// double-rounds (10 for ChaCha20, 6 for ChaCha12, 4 for ChaCha8). +#[inline(always)] +fn ks16(state: &[u32; STATE_WORDS]) -> [[u8; 64]; 16] { + // One vertical quarter-round across all 16 blocks — the RFC 8439 §2.1 word + // indices, applied to the 16-lane vectors. Pure ARX (`+` / `^` / rotate). + #[inline(always)] + fn qr(v: &mut [U32x16; 16], a: usize, b: usize, c: usize, d: usize) { + v[a] = v[a] + v[b]; + v[d] = (v[d] ^ v[a]).rotate_left(16); + v[c] = v[c] + v[d]; + v[b] = (v[b] ^ v[c]).rotate_left(12); + v[a] = v[a] + v[b]; + v[d] = (v[d] ^ v[a]).rotate_left(8); + v[c] = v[c] + v[d]; + v[b] = (v[b] ^ v[c]).rotate_left(7); + } + + // Initial working vectors: every word broadcast across the 16 lanes EXCEPT the + // counter (word 12), which gets lane-index `0..=15` added (u32 wrapping) — the + // 16 consecutive block counters. + let lane_index = U32x16::from_array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let mut orig = [U32x16::splat(0); STATE_WORDS]; + for (w, o) in orig.iter_mut().enumerate() { + *o = U32x16::splat(state[w]); + } + orig[12] = orig[12] + lane_index; + + let mut v = orig; + for _ in 0..R::USIZE { + qr(&mut v, 0, 4, 8, 12); + qr(&mut v, 1, 5, 9, 13); + qr(&mut v, 2, 6, 10, 14); + qr(&mut v, 3, 7, 11, 15); + qr(&mut v, 0, 5, 10, 15); + qr(&mut v, 1, 6, 11, 12); + qr(&mut v, 2, 7, 8, 13); + qr(&mut v, 3, 4, 9, 14); + } + + // Add the original state back (RFC 8439 §2.3.1), then de-vectorize: + // `words[w][l]` = final word `w` of block `l`. + let mut words = [[0u32; 16]; STATE_WORDS]; + for (w, dst) in words.iter_mut().enumerate() { + *dst = (v[w] + orig[w]).to_array(); + } + + // Serialize each block little-endian: block `l`, word `w` → bytes `4w..4w+4`. + let mut out = [[0u8; 64]; 16]; + for (l, block) in out.iter_mut().enumerate() { + for w in 0..STATE_WORDS { + block[w * 4..w * 4 + 4].copy_from_slice(&words[w][l].to_le_bytes()); + } + } + out +} diff --git a/vendor/chacha20/src/backends/neon.rs b/vendor/chacha20/src/backends/neon.rs new file mode 100644 index 00000000..5c6d0aeb --- /dev/null +++ b/vendor/chacha20/src/backends/neon.rs @@ -0,0 +1,356 @@ +//! NEON-optimized implementation for aarch64 CPUs. +//! +//! Adapted from the Crypto++ `chacha_simd` implementation by Jack Lloyd and +//! Jeffrey Walton (public domain). + +use crate::{Block, StreamClosure, Unsigned, STATE_WORDS}; +use cipher::{ + consts::{U4, U64}, + BlockSizeUser, ParBlocks, ParBlocksSizeUser, StreamBackend, +}; +use core::{arch::aarch64::*, marker::PhantomData}; + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamClosure, +{ + let mut backend = Backend:: { + state: [ + vld1q_u32(state.as_ptr().offset(0)), + vld1q_u32(state.as_ptr().offset(4)), + vld1q_u32(state.as_ptr().offset(8)), + vld1q_u32(state.as_ptr().offset(12)), + ], + _pd: PhantomData, + }; + + f.call(&mut backend); + + vst1q_u32(state.as_mut_ptr().offset(12), backend.state[3]); +} + +struct Backend { + state: [uint32x4_t; 4], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend { + type ParBlocksSize = U4; +} + +macro_rules! add64 { + ($a:expr, $b:expr) => { + vreinterpretq_u32_u64(vaddq_u64( + vreinterpretq_u64_u32($a), + vreinterpretq_u64_u32($b), + )) + }; +} + +impl StreamBackend for Backend { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + let state3 = self.state[3]; + let mut par = ParBlocks::::default(); + self.gen_par_ks_blocks(&mut par); + *block = par[0]; + unsafe { + self.state[3] = add64!(state3, vld1q_u32([1, 0, 0, 0].as_ptr())); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks) { + macro_rules! rotate_left { + ($v:ident, 8) => {{ + let maskb = [3u8, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14]; + let mask = vld1q_u8(maskb.as_ptr()); + + vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32($v), mask)) + }}; + ($v:ident, 16) => { + vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32($v))) + }; + ($v:ident, $r:literal) => { + vorrq_u32(vshlq_n_u32($v, $r), vshrq_n_u32($v, 32 - $r)) + }; + } + + macro_rules! extract { + ($v:ident, $s:literal) => { + vextq_u32($v, $v, $s) + }; + } + + unsafe { + let ctrs = [ + vld1q_u32([1, 0, 0, 0].as_ptr()), + vld1q_u32([2, 0, 0, 0].as_ptr()), + vld1q_u32([3, 0, 0, 0].as_ptr()), + vld1q_u32([4, 0, 0, 0].as_ptr()), + ]; + + let mut r0_0 = self.state[0]; + let mut r0_1 = self.state[1]; + let mut r0_2 = self.state[2]; + let mut r0_3 = self.state[3]; + + let mut r1_0 = self.state[0]; + let mut r1_1 = self.state[1]; + let mut r1_2 = self.state[2]; + let mut r1_3 = add64!(r0_3, ctrs[0]); + + let mut r2_0 = self.state[0]; + let mut r2_1 = self.state[1]; + let mut r2_2 = self.state[2]; + let mut r2_3 = add64!(r0_3, ctrs[1]); + + let mut r3_0 = self.state[0]; + let mut r3_1 = self.state[1]; + let mut r3_2 = self.state[2]; + let mut r3_3 = add64!(r0_3, ctrs[2]); + + for _ in 0..R::USIZE { + r0_0 = vaddq_u32(r0_0, r0_1); + r1_0 = vaddq_u32(r1_0, r1_1); + r2_0 = vaddq_u32(r2_0, r2_1); + r3_0 = vaddq_u32(r3_0, r3_1); + + r0_3 = veorq_u32(r0_3, r0_0); + r1_3 = veorq_u32(r1_3, r1_0); + r2_3 = veorq_u32(r2_3, r2_0); + r3_3 = veorq_u32(r3_3, r3_0); + + r0_3 = rotate_left!(r0_3, 16); + r1_3 = rotate_left!(r1_3, 16); + r2_3 = rotate_left!(r2_3, 16); + r3_3 = rotate_left!(r3_3, 16); + + r0_2 = vaddq_u32(r0_2, r0_3); + r1_2 = vaddq_u32(r1_2, r1_3); + r2_2 = vaddq_u32(r2_2, r2_3); + r3_2 = vaddq_u32(r3_2, r3_3); + + r0_1 = veorq_u32(r0_1, r0_2); + r1_1 = veorq_u32(r1_1, r1_2); + r2_1 = veorq_u32(r2_1, r2_2); + r3_1 = veorq_u32(r3_1, r3_2); + + r0_1 = rotate_left!(r0_1, 12); + r1_1 = rotate_left!(r1_1, 12); + r2_1 = rotate_left!(r2_1, 12); + r3_1 = rotate_left!(r3_1, 12); + + r0_0 = vaddq_u32(r0_0, r0_1); + r1_0 = vaddq_u32(r1_0, r1_1); + r2_0 = vaddq_u32(r2_0, r2_1); + r3_0 = vaddq_u32(r3_0, r3_1); + + r0_3 = veorq_u32(r0_3, r0_0); + r1_3 = veorq_u32(r1_3, r1_0); + r2_3 = veorq_u32(r2_3, r2_0); + r3_3 = veorq_u32(r3_3, r3_0); + + r0_3 = rotate_left!(r0_3, 8); + r1_3 = rotate_left!(r1_3, 8); + r2_3 = rotate_left!(r2_3, 8); + r3_3 = rotate_left!(r3_3, 8); + + r0_2 = vaddq_u32(r0_2, r0_3); + r1_2 = vaddq_u32(r1_2, r1_3); + r2_2 = vaddq_u32(r2_2, r2_3); + r3_2 = vaddq_u32(r3_2, r3_3); + + r0_1 = veorq_u32(r0_1, r0_2); + r1_1 = veorq_u32(r1_1, r1_2); + r2_1 = veorq_u32(r2_1, r2_2); + r3_1 = veorq_u32(r3_1, r3_2); + + r0_1 = rotate_left!(r0_1, 7); + r1_1 = rotate_left!(r1_1, 7); + r2_1 = rotate_left!(r2_1, 7); + r3_1 = rotate_left!(r3_1, 7); + + r0_1 = extract!(r0_1, 1); + r0_2 = extract!(r0_2, 2); + r0_3 = extract!(r0_3, 3); + + r1_1 = extract!(r1_1, 1); + r1_2 = extract!(r1_2, 2); + r1_3 = extract!(r1_3, 3); + + r2_1 = extract!(r2_1, 1); + r2_2 = extract!(r2_2, 2); + r2_3 = extract!(r2_3, 3); + + r3_1 = extract!(r3_1, 1); + r3_2 = extract!(r3_2, 2); + r3_3 = extract!(r3_3, 3); + + r0_0 = vaddq_u32(r0_0, r0_1); + r1_0 = vaddq_u32(r1_0, r1_1); + r2_0 = vaddq_u32(r2_0, r2_1); + r3_0 = vaddq_u32(r3_0, r3_1); + + r0_3 = veorq_u32(r0_3, r0_0); + r1_3 = veorq_u32(r1_3, r1_0); + r2_3 = veorq_u32(r2_3, r2_0); + r3_3 = veorq_u32(r3_3, r3_0); + + r0_3 = rotate_left!(r0_3, 16); + r1_3 = rotate_left!(r1_3, 16); + r2_3 = rotate_left!(r2_3, 16); + r3_3 = rotate_left!(r3_3, 16); + + r0_2 = vaddq_u32(r0_2, r0_3); + r1_2 = vaddq_u32(r1_2, r1_3); + r2_2 = vaddq_u32(r2_2, r2_3); + r3_2 = vaddq_u32(r3_2, r3_3); + + r0_1 = veorq_u32(r0_1, r0_2); + r1_1 = veorq_u32(r1_1, r1_2); + r2_1 = veorq_u32(r2_1, r2_2); + r3_1 = veorq_u32(r3_1, r3_2); + + r0_1 = rotate_left!(r0_1, 12); + r1_1 = rotate_left!(r1_1, 12); + r2_1 = rotate_left!(r2_1, 12); + r3_1 = rotate_left!(r3_1, 12); + + r0_0 = vaddq_u32(r0_0, r0_1); + r1_0 = vaddq_u32(r1_0, r1_1); + r2_0 = vaddq_u32(r2_0, r2_1); + r3_0 = vaddq_u32(r3_0, r3_1); + + r0_3 = veorq_u32(r0_3, r0_0); + r1_3 = veorq_u32(r1_3, r1_0); + r2_3 = veorq_u32(r2_3, r2_0); + r3_3 = veorq_u32(r3_3, r3_0); + + r0_3 = rotate_left!(r0_3, 8); + r1_3 = rotate_left!(r1_3, 8); + r2_3 = rotate_left!(r2_3, 8); + r3_3 = rotate_left!(r3_3, 8); + + r0_2 = vaddq_u32(r0_2, r0_3); + r1_2 = vaddq_u32(r1_2, r1_3); + r2_2 = vaddq_u32(r2_2, r2_3); + r3_2 = vaddq_u32(r3_2, r3_3); + + r0_1 = veorq_u32(r0_1, r0_2); + r1_1 = veorq_u32(r1_1, r1_2); + r2_1 = veorq_u32(r2_1, r2_2); + r3_1 = veorq_u32(r3_1, r3_2); + + r0_1 = rotate_left!(r0_1, 7); + r1_1 = rotate_left!(r1_1, 7); + r2_1 = rotate_left!(r2_1, 7); + r3_1 = rotate_left!(r3_1, 7); + + r0_1 = extract!(r0_1, 3); + r0_2 = extract!(r0_2, 2); + r0_3 = extract!(r0_3, 1); + + r1_1 = extract!(r1_1, 3); + r1_2 = extract!(r1_2, 2); + r1_3 = extract!(r1_3, 1); + + r2_1 = extract!(r2_1, 3); + r2_2 = extract!(r2_2, 2); + r2_3 = extract!(r2_3, 1); + + r3_1 = extract!(r3_1, 3); + r3_2 = extract!(r3_2, 2); + r3_3 = extract!(r3_3, 1); + } + + r0_0 = vaddq_u32(r0_0, self.state[0]); + r0_1 = vaddq_u32(r0_1, self.state[1]); + r0_2 = vaddq_u32(r0_2, self.state[2]); + r0_3 = vaddq_u32(r0_3, self.state[3]); + + r1_0 = vaddq_u32(r1_0, self.state[0]); + r1_1 = vaddq_u32(r1_1, self.state[1]); + r1_2 = vaddq_u32(r1_2, self.state[2]); + r1_3 = vaddq_u32(r1_3, self.state[3]); + r1_3 = add64!(r1_3, ctrs[0]); + + r2_0 = vaddq_u32(r2_0, self.state[0]); + r2_1 = vaddq_u32(r2_1, self.state[1]); + r2_2 = vaddq_u32(r2_2, self.state[2]); + r2_3 = vaddq_u32(r2_3, self.state[3]); + r2_3 = add64!(r2_3, ctrs[1]); + + r3_0 = vaddq_u32(r3_0, self.state[0]); + r3_1 = vaddq_u32(r3_1, self.state[1]); + r3_2 = vaddq_u32(r3_2, self.state[2]); + r3_3 = vaddq_u32(r3_3, self.state[3]); + r3_3 = add64!(r3_3, ctrs[2]); + + vst1q_u8(blocks[0].as_mut_ptr().offset(0), vreinterpretq_u8_u32(r0_0)); + vst1q_u8( + blocks[0].as_mut_ptr().offset(16), + vreinterpretq_u8_u32(r0_1), + ); + vst1q_u8( + blocks[0].as_mut_ptr().offset(2 * 16), + vreinterpretq_u8_u32(r0_2), + ); + vst1q_u8( + blocks[0].as_mut_ptr().offset(3 * 16), + vreinterpretq_u8_u32(r0_3), + ); + + vst1q_u8(blocks[1].as_mut_ptr().offset(0), vreinterpretq_u8_u32(r1_0)); + vst1q_u8( + blocks[1].as_mut_ptr().offset(16), + vreinterpretq_u8_u32(r1_1), + ); + vst1q_u8( + blocks[1].as_mut_ptr().offset(2 * 16), + vreinterpretq_u8_u32(r1_2), + ); + vst1q_u8( + blocks[1].as_mut_ptr().offset(3 * 16), + vreinterpretq_u8_u32(r1_3), + ); + + vst1q_u8(blocks[2].as_mut_ptr().offset(0), vreinterpretq_u8_u32(r2_0)); + vst1q_u8( + blocks[2].as_mut_ptr().offset(16), + vreinterpretq_u8_u32(r2_1), + ); + vst1q_u8( + blocks[2].as_mut_ptr().offset(2 * 16), + vreinterpretq_u8_u32(r2_2), + ); + vst1q_u8( + blocks[2].as_mut_ptr().offset(3 * 16), + vreinterpretq_u8_u32(r2_3), + ); + + vst1q_u8(blocks[3].as_mut_ptr().offset(0), vreinterpretq_u8_u32(r3_0)); + vst1q_u8( + blocks[3].as_mut_ptr().offset(16), + vreinterpretq_u8_u32(r3_1), + ); + vst1q_u8( + blocks[3].as_mut_ptr().offset(2 * 16), + vreinterpretq_u8_u32(r3_2), + ); + vst1q_u8( + blocks[3].as_mut_ptr().offset(3 * 16), + vreinterpretq_u8_u32(r3_3), + ); + + self.state[3] = add64!(self.state[3], ctrs[3]); + } + } +} diff --git a/vendor/chacha20/src/backends/soft.rs b/vendor/chacha20/src/backends/soft.rs new file mode 100644 index 00000000..3eabb659 --- /dev/null +++ b/vendor/chacha20/src/backends/soft.rs @@ -0,0 +1,73 @@ +//! Portable implementation which does not rely on architecture-specific +//! intrinsics. + +use crate::{Block, ChaChaCore, Unsigned, STATE_WORDS}; +use cipher::{ + consts::{U1, U64}, + BlockSizeUser, ParBlocksSizeUser, StreamBackend, +}; + +pub(crate) struct Backend<'a, R: Unsigned>(pub(crate) &'a mut ChaChaCore); + +impl<'a, R: Unsigned> BlockSizeUser for Backend<'a, R> { + type BlockSize = U64; +} + +impl<'a, R: Unsigned> ParBlocksSizeUser for Backend<'a, R> { + type ParBlocksSize = U1; +} + +impl<'a, R: Unsigned> StreamBackend for Backend<'a, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + let res = run_rounds::(&self.0.state); + self.0.state[12] = self.0.state[12].wrapping_add(1); + + for (chunk, val) in block.chunks_exact_mut(4).zip(res.iter()) { + chunk.copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { + let mut res = *state; + + for _ in 0..R::USIZE { + // column rounds + quarter_round(0, 4, 8, 12, &mut res); + quarter_round(1, 5, 9, 13, &mut res); + quarter_round(2, 6, 10, 14, &mut res); + quarter_round(3, 7, 11, 15, &mut res); + + // diagonal rounds + quarter_round(0, 5, 10, 15, &mut res); + quarter_round(1, 6, 11, 12, &mut res); + quarter_round(2, 7, 8, 13, &mut res); + quarter_round(3, 4, 9, 14, &mut res); + } + + for (s1, s0) in res.iter_mut().zip(state.iter()) { + *s1 = s1.wrapping_add(*s0); + } + res +} + +/// The ChaCha20 quarter round function +fn quarter_round(a: usize, b: usize, c: usize, d: usize, state: &mut [u32; STATE_WORDS]) { + state[a] = state[a].wrapping_add(state[b]); + state[d] ^= state[a]; + state[d] = state[d].rotate_left(16); + + state[c] = state[c].wrapping_add(state[d]); + state[b] ^= state[c]; + state[b] = state[b].rotate_left(12); + + state[a] = state[a].wrapping_add(state[b]); + state[d] ^= state[a]; + state[d] = state[d].rotate_left(8); + + state[c] = state[c].wrapping_add(state[d]); + state[b] ^= state[c]; + state[b] = state[b].rotate_left(7); +} diff --git a/vendor/chacha20/src/backends/sse2.rs b/vendor/chacha20/src/backends/sse2.rs new file mode 100644 index 00000000..82692c01 --- /dev/null +++ b/vendor/chacha20/src/backends/sse2.rs @@ -0,0 +1,180 @@ +use crate::{Block, StreamClosure, Unsigned, STATE_WORDS}; +use cipher::{ + consts::{U1, U64}, + BlockSizeUser, ParBlocksSizeUser, StreamBackend, +}; +use core::marker::PhantomData; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +#[inline] +#[target_feature(enable = "sse2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamClosure, +{ + let state_ptr = state.as_ptr() as *const __m128i; + let mut backend = Backend:: { + v: [ + _mm_loadu_si128(state_ptr.add(0)), + _mm_loadu_si128(state_ptr.add(1)), + _mm_loadu_si128(state_ptr.add(2)), + _mm_loadu_si128(state_ptr.add(3)), + ], + _pd: PhantomData, + }; + + f.call(&mut backend); + + state[12] = _mm_cvtsi128_si32(backend.v[3]) as u32; +} + +struct Backend { + v: [__m128i; 4], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend { + type ParBlocksSize = U1; +} + +impl StreamBackend for Backend { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + unsafe { + let res = rounds::(&self.v); + self.v[3] = _mm_add_epi32(self.v[3], _mm_set_epi32(0, 0, 0, 1)); + + let block_ptr = block.as_mut_ptr() as *mut __m128i; + for i in 0..4 { + _mm_storeu_si128(block_ptr.add(i), res[i]); + } + } + } +} + +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn rounds(v: &[__m128i; 4]) -> [__m128i; 4] { + let mut res = *v; + for _ in 0..R::USIZE { + double_quarter_round(&mut res); + } + + for i in 0..4 { + res[i] = _mm_add_epi32(res[i], v[i]); + } + + res +} + +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn double_quarter_round(v: &mut [__m128i; 4]) { + add_xor_rot(v); + rows_to_cols(v); + add_xor_rot(v); + cols_to_rows(v); +} + +/// The goal of this function is to transform the state words from: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c0, c1, c2, c3] [ 8, 9, 10, 11] +/// [d0, d1, d2, d3] [12, 13, 14, 15] +/// ``` +/// +/// to: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b1, b2, b3, b0] == [ 5, 6, 7, 4] +/// [c2, c3, c0, c1] [10, 11, 8, 9] +/// [d3, d0, d1, d2] [15, 12, 13, 14] +/// ``` +/// +/// so that we can apply [`add_xor_rot`] to the resulting columns, and have it compute the +/// "diagonal rounds" (as defined in RFC 7539) in parallel. In practice, this shuffle is +/// non-optimal: the last state word to be altered in `add_xor_rot` is `b`, so the shuffle +/// blocks on the result of `b` being calculated. +/// +/// We can optimize this by observing that the four quarter rounds in `add_xor_rot` are +/// data-independent: they only access a single column of the state, and thus the order of +/// the columns does not matter. We therefore instead shuffle the other three state words, +/// to obtain the following equivalent layout: +/// ```text +/// [a3, a0, a1, a2] [ 3, 0, 1, 2] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c1, c2, c3, c0] [ 9, 10, 11, 8] +/// [d2, d3, d0, d1] [14, 15, 12, 13] +/// ``` +/// +/// See https://github.com/sneves/blake2-avx2/pull/4 for additional details. The earliest +/// known occurrence of this optimization is in floodyberry's SSE4 ChaCha code from 2014: +/// - https://github.com/floodyberry/chacha-opt/blob/0ab65cb99f5016633b652edebaf3691ceb4ff753/chacha_blocks_ssse3-64.S#L639-L643 +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn rows_to_cols([a, _, c, d]: &mut [__m128i; 4]) { + // c >>>= 32; d >>>= 64; a >>>= 96; + *c = _mm_shuffle_epi32(*c, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1) + *d = _mm_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2) + *a = _mm_shuffle_epi32(*a, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3) +} + +/// The goal of this function is to transform the state words from: +/// ```text +/// [a3, a0, a1, a2] [ 3, 0, 1, 2] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c1, c2, c3, c0] [ 9, 10, 11, 8] +/// [d2, d3, d0, d1] [14, 15, 12, 13] +/// ``` +/// +/// to: +/// ```text +/// [a0, a1, a2, a3] [ 0, 1, 2, 3] +/// [b0, b1, b2, b3] == [ 4, 5, 6, 7] +/// [c0, c1, c2, c3] [ 8, 9, 10, 11] +/// [d0, d1, d2, d3] [12, 13, 14, 15] +/// ``` +/// +/// reversing the transformation of [`rows_to_cols`]. +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn cols_to_rows([a, _, c, d]: &mut [__m128i; 4]) { + // c <<<= 32; d <<<= 64; a <<<= 96; + *c = _mm_shuffle_epi32(*c, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3) + *d = _mm_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2) + *a = _mm_shuffle_epi32(*a, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1) +} + +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn add_xor_rot([a, b, c, d]: &mut [__m128i; 4]) { + // a += b; d ^= a; d <<<= (16, 16, 16, 16); + *a = _mm_add_epi32(*a, *b); + *d = _mm_xor_si128(*d, *a); + *d = _mm_xor_si128(_mm_slli_epi32(*d, 16), _mm_srli_epi32(*d, 16)); + + // c += d; b ^= c; b <<<= (12, 12, 12, 12); + *c = _mm_add_epi32(*c, *d); + *b = _mm_xor_si128(*b, *c); + *b = _mm_xor_si128(_mm_slli_epi32(*b, 12), _mm_srli_epi32(*b, 20)); + + // a += b; d ^= a; d <<<= (8, 8, 8, 8); + *a = _mm_add_epi32(*a, *b); + *d = _mm_xor_si128(*d, *a); + *d = _mm_xor_si128(_mm_slli_epi32(*d, 8), _mm_srli_epi32(*d, 24)); + + // c += d; b ^= c; b <<<= (7, 7, 7, 7); + *c = _mm_add_epi32(*c, *d); + *b = _mm_xor_si128(*b, *c); + *b = _mm_xor_si128(_mm_slli_epi32(*b, 7), _mm_srli_epi32(*b, 25)); +} diff --git a/vendor/chacha20/src/legacy.rs b/vendor/chacha20/src/legacy.rs new file mode 100644 index 00000000..2541079c --- /dev/null +++ b/vendor/chacha20/src/legacy.rs @@ -0,0 +1,76 @@ +//! Legacy version of ChaCha20 with a 64-bit nonce + +use super::{ChaChaCore, Key, Nonce}; +use cipher::{ + consts::{U10, U32, U64, U8}, + generic_array::GenericArray, + BlockSizeUser, IvSizeUser, KeyIvInit, KeySizeUser, StreamCipherCore, StreamCipherCoreWrapper, + StreamCipherSeekCore, StreamClosure, +}; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::ZeroizeOnDrop; + +/// Nonce type used by [`ChaCha20Legacy`]. +pub type LegacyNonce = GenericArray; + +/// The ChaCha20 stream cipher (legacy "djb" construction with 64-bit nonce). +/// +/// **WARNING:** this implementation uses 32-bit counter, while the original +/// implementation uses 64-bit counter. In other words, it does +/// not allow encrypting of more than 256 GiB of data. +pub type ChaCha20Legacy = StreamCipherCoreWrapper; + +/// The ChaCha20 stream cipher (legacy "djb" construction with 64-bit nonce). +pub struct ChaCha20LegacyCore(ChaChaCore); + +impl KeySizeUser for ChaCha20LegacyCore { + type KeySize = U32; +} + +impl IvSizeUser for ChaCha20LegacyCore { + type IvSize = U8; +} + +impl BlockSizeUser for ChaCha20LegacyCore { + type BlockSize = U64; +} + +impl KeyIvInit for ChaCha20LegacyCore { + #[inline(always)] + fn new(key: &Key, iv: &LegacyNonce) -> Self { + let mut padded_iv = Nonce::default(); + padded_iv[4..].copy_from_slice(iv); + ChaCha20LegacyCore(ChaChaCore::new(key, &padded_iv)) + } +} + +impl StreamCipherCore for ChaCha20LegacyCore { + #[inline(always)] + fn remaining_blocks(&self) -> Option { + self.0.remaining_blocks() + } + + #[inline(always)] + fn process_with_backend(&mut self, f: impl StreamClosure) { + self.0.process_with_backend(f); + } +} + +impl StreamCipherSeekCore for ChaCha20LegacyCore { + type Counter = u32; + + #[inline(always)] + fn get_block_pos(&self) -> u32 { + self.0.get_block_pos() + } + + #[inline(always)] + fn set_block_pos(&mut self, pos: u32) { + self.0.set_block_pos(pos); + } +} + +#[cfg(feature = "zeroize")] +#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))] +impl ZeroizeOnDrop for ChaCha20LegacyCore {} diff --git a/vendor/chacha20/src/lib.rs b/vendor/chacha20/src/lib.rs new file mode 100644 index 00000000..3d1824a0 --- /dev/null +++ b/vendor/chacha20/src/lib.rs @@ -0,0 +1,329 @@ +//! Implementation of the [ChaCha] family of stream ciphers. +//! +//! Cipher functionality is accessed using traits from re-exported [`cipher`] crate. +//! +//! ChaCha stream ciphers are lightweight and amenable to fast, constant-time +//! implementations in software. It improves upon the previous [Salsa] design, +//! providing increased per-round diffusion with no cost to performance. +//! +//! This crate contains the following variants of the ChaCha20 core algorithm: +//! +//! - [`ChaCha20`]: standard IETF variant with 96-bit nonce +//! - [`ChaCha8`] / [`ChaCha12`]: reduced round variants of ChaCha20 +//! - [`XChaCha20`]: 192-bit extended nonce variant +//! - [`XChaCha8`] / [`XChaCha12`]: reduced round variants of XChaCha20 +//! - [`ChaCha20Legacy`]: "djb" variant with 64-bit nonce. +//! **WARNING:** This implementation internally uses 32-bit counter, +//! while the original implementation uses 64-bit counter. In other words, +//! it does not allow encryption of more than 256 GiB of data. +//! +//! # ⚠️ Security Warning: Hazmat! +//! +//! This crate does not ensure ciphertexts are authentic, which can lead to +//! serious vulnerabilities if used incorrectly! +//! +//! If in doubt, use the [`chacha20poly1305`] crate instead, which provides +//! an authenticated mode on top of ChaCha20. +//! +//! **USE AT YOUR OWN RISK!** +//! +//! # Diagram +//! +//! This diagram illustrates the ChaCha quarter round function. +//! Each round consists of four quarter-rounds: +//! +//! +//! +//! Legend: +//! +//! - ⊞ add +//! - ‹‹‹ rotate +//! - ⊕ xor +//! +//! # Example +//! ``` +//! use chacha20::ChaCha20; +//! // Import relevant traits +//! use chacha20::cipher::{KeyIvInit, StreamCipher, StreamCipherSeek}; +//! use hex_literal::hex; +//! +//! let key = [0x42; 32]; +//! let nonce = [0x24; 12]; +//! let plaintext = hex!("00010203 04050607 08090A0B 0C0D0E0F"); +//! let ciphertext = hex!("e405626e 4f1236b3 670ee428 332ea20e"); +//! +//! // Key and IV must be references to the `GenericArray` type. +//! // Here we use the `Into` trait to convert arrays into it. +//! let mut cipher = ChaCha20::new(&key.into(), &nonce.into()); +//! +//! let mut buffer = plaintext.clone(); +//! +//! // apply keystream (encrypt) +//! cipher.apply_keystream(&mut buffer); +//! assert_eq!(buffer, ciphertext); +//! +//! let ciphertext = buffer.clone(); +//! +//! // ChaCha ciphers support seeking +//! cipher.seek(0u32); +//! +//! // decrypt ciphertext by applying keystream again +//! cipher.apply_keystream(&mut buffer); +//! assert_eq!(buffer, plaintext); +//! +//! // stream ciphers can be used with streaming messages +//! cipher.seek(0u32); +//! for chunk in buffer.chunks_mut(3) { +//! cipher.apply_keystream(chunk); +//! } +//! assert_eq!(buffer, ciphertext); +//! ``` +//! +//! # Configuration Flags +//! +//! You can modify crate using the following configuration flags: +//! +//! - `chacha20_force_avx2`: force AVX2 backend on x86/x86_64 targets. +//! Requires enabled AVX2 target feature. Ignored on non-x86(-64) targets. +//! - `chacha20_force_neon`: force NEON backend on ARM targets. +//! Requires enabled NEON target feature. Ignored on non-ARM targets. Nightly-only. +//! - `chacha20_force_soft`: force software backend. +//! - `chacha20_force_sse2`: force SSE2 backend on x86/x86_64 targets. +//! Requires enabled SSE2 target feature. Ignored on non-x86(-64) targets. +//! +//! The flags can be enabled using `RUSTFLAGS` environmental variable +//! (e.g. `RUSTFLAGS="--cfg chacha20_force_avx2"`) or by modifying `.cargo/config`. +//! +//! You SHOULD NOT enable several `force` flags simultaneously. +//! +//! [ChaCha]: https://tools.ietf.org/html/rfc8439 +//! [Salsa]: https://en.wikipedia.org/wiki/Salsa20 +//! [`chacha20poly1305`]: https://docs.rs/chacha20poly1305 + +#![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc( + html_logo_url = "https://raw.githubusercontent.com/RustCrypto/media/8f1a9894/logo.svg", + html_favicon_url = "https://raw.githubusercontent.com/RustCrypto/media/8f1a9894/logo.svg" +)] +#![allow(clippy::needless_range_loop)] +// This is a vendored fork of chacha20 0.9.1 (pre cargo check-cfg): the +// `chacha20_force_{soft,avx2,sse2,neon}` cfgs are real RUSTFLAGS knobs but are +// not declared for check-cfg, so silence the resulting `unexpected_cfgs` note. +#![allow(unexpected_cfgs)] +#![warn(missing_docs, rust_2018_idioms, trivial_casts, unused_qualifications)] + +pub use cipher; + +use cfg_if::cfg_if; +use cipher::{ + consts::{U10, U12, U32, U4, U6, U64}, + generic_array::{typenum::Unsigned, GenericArray}, + BlockSizeUser, IvSizeUser, KeyIvInit, KeySizeUser, StreamCipherCore, StreamCipherCoreWrapper, + StreamCipherSeekCore, StreamClosure, +}; +use core::marker::PhantomData; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; + +mod backends; +mod legacy; +mod xchacha; + +pub use legacy::{ChaCha20Legacy, ChaCha20LegacyCore, LegacyNonce}; +pub use xchacha::{hchacha, XChaCha12, XChaCha20, XChaCha8, XChaChaCore, XNonce}; + +/// State initialization constant ("expand 32-byte k") +const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574]; + +/// Number of 32-bit words in the ChaCha state +const STATE_WORDS: usize = 16; + +/// Block type used by all ChaCha variants. +type Block = GenericArray; + +/// Key type used by all ChaCha variants. +pub type Key = GenericArray; + +/// Nonce type used by ChaCha variants. +pub type Nonce = GenericArray; + +/// ChaCha8 stream cipher (reduced-round variant of [`ChaCha20`] with 8 rounds) +pub type ChaCha8 = StreamCipherCoreWrapper>; + +/// ChaCha12 stream cipher (reduced-round variant of [`ChaCha20`] with 12 rounds) +pub type ChaCha12 = StreamCipherCoreWrapper>; + +/// ChaCha20 stream cipher (RFC 8439 version with 96-bit nonce) +pub type ChaCha20 = StreamCipherCoreWrapper>; + +cfg_if! { + if #[cfg(chacha20_force_soft)] { + type Tokens = (); + } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + // AdaWorldAPI matryoshka backend: compile-time selected, no runtime probe. + type Tokens = (); + } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cfg_if! { + if #[cfg(chacha20_force_avx2)] { + #[cfg(not(target_feature = "avx2"))] + compile_error!("You must enable `avx2` target feature with \ + `chacha20_force_avx2` configuration option"); + type Tokens = (); + } else if #[cfg(chacha20_force_sse2)] { + #[cfg(not(target_feature = "sse2"))] + compile_error!("You must enable `sse2` target feature with \ + `chacha20_force_sse2` configuration option"); + type Tokens = (); + } else { + cpufeatures::new!(avx2_cpuid, "avx2"); + cpufeatures::new!(sse2_cpuid, "sse2"); + type Tokens = (avx2_cpuid::InitToken, sse2_cpuid::InitToken); + } + } + } else { + type Tokens = (); + } +} + +/// The ChaCha core function. +pub struct ChaChaCore { + /// Internal state of the core function + state: [u32; STATE_WORDS], + /// CPU target feature tokens + #[allow(dead_code)] + tokens: Tokens, + /// Number of rounds to perform + rounds: PhantomData, +} + +impl KeySizeUser for ChaChaCore { + type KeySize = U32; +} + +impl IvSizeUser for ChaChaCore { + type IvSize = U12; +} + +impl BlockSizeUser for ChaChaCore { + type BlockSize = U64; +} + +impl KeyIvInit for ChaChaCore { + #[inline] + fn new(key: &Key, iv: &Nonce) -> Self { + let mut state = [0u32; STATE_WORDS]; + state[0..4].copy_from_slice(&CONSTANTS); + let key_chunks = key.chunks_exact(4); + for (val, chunk) in state[4..12].iter_mut().zip(key_chunks) { + *val = u32::from_le_bytes(chunk.try_into().unwrap()); + } + let iv_chunks = iv.chunks_exact(4); + for (val, chunk) in state[13..16].iter_mut().zip(iv_chunks) { + *val = u32::from_le_bytes(chunk.try_into().unwrap()); + } + + cfg_if! { + if #[cfg(chacha20_force_soft)] { + let tokens = (); + } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + let tokens = (); + } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cfg_if! { + if #[cfg(chacha20_force_avx2)] { + let tokens = (); + } else if #[cfg(chacha20_force_sse2)] { + let tokens = (); + } else { + let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); + } + } + } else { + let tokens = (); + } + } + + Self { + state, + tokens, + rounds: PhantomData, + } + } +} + +impl StreamCipherCore for ChaChaCore { + #[inline(always)] + fn remaining_blocks(&self) -> Option { + let rem = u32::MAX - self.get_block_pos(); + rem.try_into().ok() + } + + fn process_with_backend(&mut self, f: impl StreamClosure) { + cfg_if! { + if #[cfg(chacha20_force_soft)] { + f.call(&mut backends::soft::Backend(self)); + } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + // AdaWorldAPI matryoshka: keystream via ndarray::simd::U32x16. + f.call(&mut backends::ndarray_simd::Backend(self)); + } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cfg_if! { + if #[cfg(chacha20_force_avx2)] { + unsafe { + backends::avx2::inner::(&mut self.state, f); + } + } else if #[cfg(chacha20_force_sse2)] { + unsafe { + backends::sse2::inner::(&mut self.state, f); + } + } else { + let (avx2_token, sse2_token) = self.tokens; + if avx2_token.get() { + unsafe { + backends::avx2::inner::(&mut self.state, f); + } + } else if sse2_token.get() { + unsafe { + backends::sse2::inner::(&mut self.state, f); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } + } + } else if #[cfg(all(chacha20_force_neon, target_arch = "aarch64", target_feature = "neon"))] { + unsafe { + backends::neon::inner::(&mut self.state, f); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } + } +} + +impl StreamCipherSeekCore for ChaChaCore { + type Counter = u32; + + #[inline(always)] + fn get_block_pos(&self) -> u32 { + self.state[12] + } + + #[inline(always)] + fn set_block_pos(&mut self, pos: u32) { + self.state[12] = pos; + } +} + +#[cfg(feature = "zeroize")] +#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))] +impl Drop for ChaChaCore { + fn drop(&mut self) { + self.state.zeroize(); + } +} + +#[cfg(feature = "zeroize")] +#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))] +impl ZeroizeOnDrop for ChaChaCore {} diff --git a/vendor/chacha20/src/xchacha.rs b/vendor/chacha20/src/xchacha.rs new file mode 100644 index 00000000..f32b7270 --- /dev/null +++ b/vendor/chacha20/src/xchacha.rs @@ -0,0 +1,194 @@ +//! XChaCha is an extended nonce variant of ChaCha + +use super::{ChaChaCore, Key, Nonce, CONSTANTS, STATE_WORDS}; +use cipher::{ + consts::{U10, U16, U24, U32, U4, U6, U64}, + generic_array::{typenum::Unsigned, GenericArray}, + BlockSizeUser, IvSizeUser, KeyIvInit, KeySizeUser, StreamCipherCore, StreamCipherCoreWrapper, + StreamCipherSeekCore, StreamClosure, +}; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::ZeroizeOnDrop; + +/// Nonce type used by XChaCha variants. +pub type XNonce = GenericArray; + +/// XChaCha is a ChaCha20 variant with an extended 192-bit (24-byte) nonce. +/// +/// The construction is an adaptation of the same techniques used by +/// XChaCha as described in the paper "Extending the Salsa20 Nonce", +/// applied to the 96-bit nonce variant of ChaCha20, and derive a +/// separate subkey/nonce for each extended nonce: +/// +/// +/// +/// No authoritative specification exists for XChaCha20, however the +/// construction has "rough consensus and running code" in the form of +/// several interoperable libraries and protocols (e.g. libsodium, WireGuard) +/// and is documented in an (expired) IETF draft: +/// +/// +pub type XChaCha20 = StreamCipherCoreWrapper>; +/// XChaCha12 stream cipher (reduced-round variant of [`XChaCha20`] with 12 rounds) +pub type XChaCha12 = StreamCipherCoreWrapper>; +/// XChaCha8 stream cipher (reduced-round variant of [`XChaCha20`] with 8 rounds) +pub type XChaCha8 = StreamCipherCoreWrapper>; + +/// The XChaCha core function. +pub struct XChaChaCore(ChaChaCore); + +impl KeySizeUser for XChaChaCore { + type KeySize = U32; +} + +impl IvSizeUser for XChaChaCore { + type IvSize = U24; +} + +impl BlockSizeUser for XChaChaCore { + type BlockSize = U64; +} + +impl KeyIvInit for XChaChaCore { + fn new(key: &Key, iv: &XNonce) -> Self { + let subkey = hchacha::(key, iv[..16].as_ref().into()); + let mut padded_iv = Nonce::default(); + padded_iv[4..].copy_from_slice(&iv[16..]); + XChaChaCore(ChaChaCore::new(&subkey, &padded_iv)) + } +} + +impl StreamCipherCore for XChaChaCore { + #[inline(always)] + fn remaining_blocks(&self) -> Option { + self.0.remaining_blocks() + } + + #[inline(always)] + fn process_with_backend(&mut self, f: impl StreamClosure) { + self.0.process_with_backend(f); + } +} + +impl StreamCipherSeekCore for XChaChaCore { + type Counter = u32; + + #[inline(always)] + fn get_block_pos(&self) -> u32 { + self.0.get_block_pos() + } + + #[inline(always)] + fn set_block_pos(&mut self, pos: u32) { + self.0.set_block_pos(pos); + } +} + +#[cfg(feature = "zeroize")] +#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))] +impl ZeroizeOnDrop for XChaChaCore {} + +/// The HChaCha function: adapts the ChaCha core function in the same +/// manner that HSalsa adapts the Salsa function. +/// +/// HChaCha takes 512-bits of input: +/// +/// - Constants: `u32` x 4 +/// - Key: `u32` x 8 +/// - Nonce: `u32` x 4 +/// +/// It produces 256-bits of output suitable for use as a ChaCha key +/// +/// For more information on HSalsa on which HChaCha is based, see: +/// +/// +pub fn hchacha(key: &Key, input: &GenericArray) -> GenericArray { + let mut state = [0u32; STATE_WORDS]; + state[..4].copy_from_slice(&CONSTANTS); + + let key_chunks = key.chunks_exact(4); + for (v, chunk) in state[4..12].iter_mut().zip(key_chunks) { + *v = u32::from_le_bytes(chunk.try_into().unwrap()); + } + let input_chunks = input.chunks_exact(4); + for (v, chunk) in state[12..16].iter_mut().zip(input_chunks) { + *v = u32::from_le_bytes(chunk.try_into().unwrap()); + } + + // R rounds consisting of R/2 column rounds and R/2 diagonal rounds + for _ in 0..R::USIZE { + // column rounds + quarter_round(0, 4, 8, 12, &mut state); + quarter_round(1, 5, 9, 13, &mut state); + quarter_round(2, 6, 10, 14, &mut state); + quarter_round(3, 7, 11, 15, &mut state); + + // diagonal rounds + quarter_round(0, 5, 10, 15, &mut state); + quarter_round(1, 6, 11, 12, &mut state); + quarter_round(2, 7, 8, 13, &mut state); + quarter_round(3, 4, 9, 14, &mut state); + } + + let mut output = GenericArray::default(); + + for (chunk, val) in output[..16].chunks_exact_mut(4).zip(&state[..4]) { + chunk.copy_from_slice(&val.to_le_bytes()); + } + + for (chunk, val) in output[16..].chunks_exact_mut(4).zip(&state[12..]) { + chunk.copy_from_slice(&val.to_le_bytes()); + } + + output +} + +/// The ChaCha20 quarter round function +// for simplicity this function is copied from the software backend +fn quarter_round(a: usize, b: usize, c: usize, d: usize, state: &mut [u32; STATE_WORDS]) { + state[a] = state[a].wrapping_add(state[b]); + state[d] ^= state[a]; + state[d] = state[d].rotate_left(16); + + state[c] = state[c].wrapping_add(state[d]); + state[b] ^= state[c]; + state[b] = state[b].rotate_left(12); + + state[a] = state[a].wrapping_add(state[b]); + state[d] ^= state[a]; + state[d] = state[d].rotate_left(8); + + state[c] = state[c].wrapping_add(state[d]); + state[b] ^= state[c]; + state[b] = state[b].rotate_left(7); +} + +#[cfg(test)] +mod hchacha20_tests { + use super::*; + use hex_literal::hex; + + /// Test vectors from: + /// https://tools.ietf.org/id/draft-arciszewski-xchacha-03.html#rfc.section.2.2.1 + #[test] + fn test_vector() { + const KEY: [u8; 32] = hex!( + "000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + ); + + const INPUT: [u8; 16] = hex!("000000090000004a0000000031415927"); + + const OUTPUT: [u8; 32] = hex!( + "82413b4227b27bfed30e42508a877d73" + "a0f9e4d58a74a853c12ec41326d3ecdc" + ); + + let actual = hchacha::( + GenericArray::from_slice(&KEY), + &GenericArray::from_slice(&INPUT), + ); + assert_eq!(actual.as_slice(), &OUTPUT); + } +} diff --git a/vendor/chacha20/tests/data/chacha20-legacy.blb b/vendor/chacha20/tests/data/chacha20-legacy.blb new file mode 100644 index 0000000000000000000000000000000000000000..905ae84a1084d53af69f741e1519baf7d7b3dc83 GIT binary patch literal 1045 zcmZQ}V4x5v5Yo>9Q^3s1*uYS>z>-ISev3I>D3Vud&jei+byp)d+!Do zY5j<7W&xO46);DVLs#6Jv#a{^wRvBkx7|seZl7w!?fF1-?f&n={-STA|No+H3!Z z&2Al+68v2`gWfvpI_`e7VB4$Gv`b4bK7YjPv|cCp`lO9V>|Zt3&AL>&E+jMAfsu)s zg_Vt+gOiJ!hnJ6EKu}0nL{v;%LQ+avMpjNlbxn_JZL@SP$Xwj= zz>tBx=iqhI1-JeB-Tj_viX{A7NNlp03WRjAGBP(Xl0N3Ys|1oYOwCA~k;!akICRX{ zyyvQ!xz{6>o$n{xpOxxX>ezgL$+V)?Wziw;)Q@*f=i{3iBl3AgUy1beLv7#V`wz{0 zA<~@mOR~!{So&?Q)I^P*cE3X*pOp?RoNmGsy4!BH<&5w5lsP@JxVKDMeTipI+|N0} z1=r-tzi_Mx75OrWY0t5yURQ^?x*}pf)(K>@y}lG86sG3CM09V+X`zQ#T;|;fnz?ZM zBG2vd-%iZdv)AJi?vQ5kZb + // + + const KEY: [u8; 32] = hex!("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"); + + const IV: [u8; 12] = hex!("000000000000004a00000000"); + + const PLAINTEXT: [u8; 114] = hex!( + " + 4c616469657320616e642047656e746c + 656d656e206f662074686520636c6173 + 73206f66202739393a20496620492063 + 6f756c64206f6666657220796f75206f + 6e6c79206f6e652074697020666f7220 + 746865206675747572652c2073756e73 + 637265656e20776f756c642062652069 + 742e + " + ); + + const KEYSTREAM: [u8; 114] = hex!( + " + 224f51f3401bd9e12fde276fb8631ded8c131f823d2c06 + e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b + 9334794cba40c63e34cdea212c4cf07d41b769a6749f3f + 630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53a + c40c5945398b6eda1a832c89c167eacd901d7e2bf363 + " + ); + + const CIPHERTEXT: [u8; 114] = hex!( + " + 6e2e359a2568f98041ba0728dd0d6981 + e97e7aec1d4360c20a27afccfd9fae0b + f91b65c5524733ab8f593dabcd62b357 + 1639d624e65152ab8f530c359f0861d8 + 07ca0dbf500d6a6156a38e088a22b65e + 52bc514d16ccf806818ce91ab7793736 + 5af90bbf74a35be6b40b8eedf2785e42 + 874d + " + ); + + #[test] + fn chacha20_keystream() { + let mut cipher = ChaCha20::new(&Key::from(KEY), &Nonce::from(IV)); + + // The test vectors omit the first 64-bytes of the keystream + let mut prefix = [0u8; 64]; + cipher.apply_keystream(&mut prefix); + + let mut buf = [0u8; 114]; + cipher.apply_keystream(&mut buf); + assert_eq!(&buf[..], &KEYSTREAM[..]); + } + + #[test] + fn chacha20_encryption() { + let mut cipher = ChaCha20::new(&Key::from(KEY), &Nonce::from(IV)); + let mut buf = PLAINTEXT.clone(); + + // The test vectors omit the first 64-bytes of the keystream + let mut prefix = [0u8; 64]; + cipher.apply_keystream(&mut prefix); + + cipher.apply_keystream(&mut buf); + assert_eq!(&buf[..], &CIPHERTEXT[..]); + } +} + +#[rustfmt::skip] +mod xchacha20 { + use chacha20::{Key, XChaCha20, XNonce}; + use cipher::{KeyIvInit, StreamCipher}; + use hex_literal::hex; + + cipher::stream_cipher_seek_test!(xchacha20_seek, XChaCha20); + + // + // XChaCha20 test vectors from: + // + // + + const KEY: [u8; 32] = hex!(" + 808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f + "); + + const IV: [u8; 24] = hex!(" + 404142434445464748494a4b4c4d4e4f5051525354555658 + "); + + const PLAINTEXT: [u8; 304] = hex!(" + 5468652064686f6c65202870726f6e6f756e6365642022646f6c652229206973 + 20616c736f206b6e6f776e2061732074686520417369617469632077696c6420 + 646f672c2072656420646f672c20616e642077686973746c696e6720646f672e + 2049742069732061626f7574207468652073697a65206f662061204765726d61 + 6e20736865706865726420627574206c6f6f6b73206d6f7265206c696b652061 + 206c6f6e672d6c656767656420666f782e205468697320686967686c7920656c + 757369766520616e6420736b696c6c6564206a756d70657220697320636c6173 + 736966696564207769746820776f6c7665732c20636f796f7465732c206a6163 + 6b616c732c20616e6420666f78657320696e20746865207461786f6e6f6d6963 + 2066616d696c792043616e696461652e + "); + + const KEYSTREAM: [u8; 304] = hex!(" + 29624b4b1b140ace53740e405b2168540fd7d630c1f536fecd722fc3cddba7f4 + cca98cf9e47e5e64d115450f9b125b54449ff76141ca620a1f9cfcab2a1a8a25 + 5e766a5266b878846120ea64ad99aa479471e63befcbd37cd1c22a221fe46221 + 5cf32c74895bf505863ccddd48f62916dc6521f1ec50a5ae08903aa259d9bf60 + 7cd8026fba548604f1b6072d91bc91243a5b845f7fd171b02edc5a0a84cf28dd + 241146bc376e3f48df5e7fee1d11048c190a3d3deb0feb64b42d9c6fdeee290f + a0e6ae2c26c0249ea8c181f7e2ffd100cbe5fd3c4f8271d62b15330cb8fdcf00 + b3df507ca8c924f7017b7e712d15a2eb5c50484451e54e1b4b995bd8fdd94597 + bb94d7af0b2c04df10ba0890899ed9293a0f55b8bafa999264035f1d4fbe7fe0 + aafa109a62372027e50e10cdfecca127 + "); + + const CIPHERTEXT: [u8; 304] = hex!(" + 7d0a2e6b7f7c65a236542630294e063b7ab9b555a5d5149aa21e4ae1e4fbce87 + ecc8e08a8b5e350abe622b2ffa617b202cfad72032a3037e76ffdcdc4376ee05 + 3a190d7e46ca1de04144850381b9cb29f051915386b8a710b8ac4d027b8b050f + 7cba5854e028d564e453b8a968824173fc16488b8970cac828f11ae53cabd201 + 12f87107df24ee6183d2274fe4c8b1485534ef2c5fbc1ec24bfc3663efaa08bc + 047d29d25043532db8391a8a3d776bf4372a6955827ccb0cdd4af403a7ce4c63 + d595c75a43e045f0cce1f29c8b93bd65afc5974922f214a40b7c402cdb91ae73 + c0b63615cdad0480680f16515a7ace9d39236464328a37743ffc28f4ddb324f4 + d0f5bbdc270c65b1749a6efff1fbaa09536175ccd29fb9e6057b307320d31683 + 8a9c71f70b5b5907a66f7ea49aadc409 + "); + + #[test] + fn xchacha20_keystream() { + let mut cipher = XChaCha20::new(&Key::from(KEY), &XNonce::from(IV)); + + // The test vectors omit the first 64-bytes of the keystream + let mut prefix = [0u8; 64]; + cipher.apply_keystream(&mut prefix); + + let mut buf = [0u8; 304]; + cipher.apply_keystream(&mut buf); + assert_eq!(&buf[..], &KEYSTREAM[..]); + } + + #[test] + fn xchacha20_encryption() { + let mut cipher = XChaCha20::new(&Key::from(KEY), &XNonce::from(IV)); + let mut buf = PLAINTEXT.clone(); + + // The test vectors omit the first 64-bytes of the keystream + let mut prefix = [0u8; 64]; + cipher.apply_keystream(&mut prefix); + + cipher.apply_keystream(&mut buf); + assert_eq!(&buf[..], &CIPHERTEXT[..]); + } +} + +// Legacy "djb" version of ChaCha20 (64-bit nonce) +#[cfg(feature = "legacy")] +#[rustfmt::skip] +mod legacy { + use chacha20::{ChaCha20Legacy, Key, LegacyNonce}; + use cipher::{NewCipher, StreamCipher, StreamCipherSeek}; + use hex_literal::hex; + + cipher::stream_cipher_test!(chacha20_legacy_core, ChaCha20Legacy, "chacha20-legacy"); + cipher::stream_cipher_seek_test!(chacha20_legacy_seek, ChaCha20Legacy); + + const KEY_LONG: [u8; 32] = hex!(" + 0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20 + "); + + const IV_LONG: [u8; 8] = hex!("0301040105090206"); + + const EXPECTED_LONG: [u8; 256] = hex!(" + deeb6b9d06dff3e091bf3ad4f4d492b6dd98246f69691802e466e03bad235787 + 0f1c6c010b6c2e650c4bf58d2d35c72ab639437069a384e03100078cc1d735a0 + db4e8f474ee6291460fd9197c77ed87b4c64e0d9ac685bd1c56cce021f3819cd + 13f49c9a3053603602582a060e59c2fbee90ab0bf7bb102d819ced03969d3bae + 71034fe598246583336aa744d8168e5dfff5c6d10270f125a4130e719717e783 + c0858b6f7964437173ea1d7556c158bc7a99e74a34d93da6bf72ac9736a215ac + aefd4ec031f3f13f099e3d811d83a2cf1d544a68d2752409cc6be852b0511a2e + 32f69aa0be91b30981584a1c56ce7546cca24d8cfdfca525d6b15eea83b6b686 + "); + + #[test] + #[ignore] + fn chacha20_offsets() { + for idx in 0..256 { + for middle in idx..256 { + for last in middle..256 { + let mut cipher = + ChaCha20Legacy::new(&Key::from(KEY_LONG), &LegacyNonce::from(IV_LONG)); + let mut buf = [0; 256]; + + cipher.seek(idx as u64); + cipher.apply_keystream(&mut buf[idx..middle]); + cipher.apply_keystream(&mut buf[middle..last]); + + for k in idx..last { + assert_eq!(buf[k], EXPECTED_LONG[k]) + } + } + } + } + } +} From 7781028a4079ea2e5e3991b0c9faa5a61083bafd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 05:43:26 +0000 Subject: [PATCH 3/6] ci(neon): aarch64 NEON SIMD parity gate under qemu (twin of wasm_simd) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The x86 `cargo test` suite never compiles `simd_neon.rs` (cfg'd to aarch64), so the native NEON SIMD lanes — including the new `U32x16` ARX lane the ChaCha20 backend rides — had no automated runtime verification. This closes that blind spot, mirroring the `wasm_simd`/node gate. - crates/neon-simd-parity — excluded bin that calls the REAL `ndarray::simd` aarch64 types (no copy → no drift) and self-checks each lane against scalar: U32x16 ARX (Add / BitXor / rotate_left over 0,1,7,8,12,16,24,31), F32x16 (splat / roundtrip / add / reduce_sum), I8x16 (roundtrip / add). Exit code points at the exact lane+op on mismatch. - scripts/neon-parity.sh — cross-build for aarch64-unknown-linux-gnu (via aarch64-linux-gnu-gcc) then run under qemu-aarch64-static. Exercises both integration (ndarray compiles for aarch64 on stable) and parity. - .github/workflows/ci.yaml — new `neon_simd` (neon-simd/parity-qemu) job installing gcc-aarch64-linux-gnu + qemu-user-static; added to `conclusion`. Verified GREEN locally: cross-built + run under qemu-aarch64-static → "U32x16 / F32x16 / I8x16 lanes bit-identical to scalar". Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- .claude/CHACHA20_MATRYOSHKA_PLAN.md | 32 ++++---- .claude/blackboard.md | 25 ++++++ .github/workflows/ci.yaml | 23 ++++++ Cargo.toml | 2 +- crates/neon-simd-parity/.gitignore | 2 + crates/neon-simd-parity/Cargo.toml | 29 +++++++ crates/neon-simd-parity/src/main.rs | 123 ++++++++++++++++++++++++++++ scripts/neon-parity.sh | 33 ++++++++ 8 files changed, 253 insertions(+), 16 deletions(-) create mode 100644 crates/neon-simd-parity/.gitignore create mode 100644 crates/neon-simd-parity/Cargo.toml create mode 100644 crates/neon-simd-parity/src/main.rs create mode 100755 scripts/neon-parity.sh diff --git a/.claude/CHACHA20_MATRYOSHKA_PLAN.md b/.claude/CHACHA20_MATRYOSHKA_PLAN.md index 1027db98..5efe6dc8 100644 --- a/.claude/CHACHA20_MATRYOSHKA_PLAN.md +++ b/.claude/CHACHA20_MATRYOSHKA_PLAN.md @@ -81,21 +81,23 @@ below **superseded it** — it is now **RETIRED** (2026-07-12, see below). `[patch.crates-io] chacha20 = { path = "vendor/chacha20" }` pointing at a vendored copy of the fork, to inherit the acceleration. Documented, low-risk. -## DEFERRED — TO-DO (next session) - -1. **Native neon `U32x16 = [U32x4; 4]`.** `simd_neon.rs` has native - `U32x4(uint32x4_t)` with add/sub/min/max; add `bitxor` (`veorq_u32`) + - `rotate_left` (shift-or via `vshlq_u32` with variable count vectors, `n%32` - guard), then compose `U32x16([U32x4; 4])` with `impl Add`/`impl BitXor`/ - `rotate_left` (fan over 4 lanes, mirror the wasm shape in `simd_wasm.rs`), fix - `F32x16::to_bits/from_bits` to `from_array`/`to_array`, and swap the `simd.rs` - aarch64 arm's scalar `U32x16` re-export for the native one. -2. **aarch64 cross parity CI.** Generalize `wasm-simd-parity` into a shared - `simd-parity` harness (or add a sibling) run under **qemu** via `cross` in a - new `neon_simd` CI job — same selfcheck, closing NEON's identical x86-suite - blind spot (its `F32x16`/`I8x16` are unverified in CI today too). -3. **avx2 native `U32x16`** (2×`__m256i`) — the TD-SIMD-3 lowering; optional, the - scalar polyfill is correct meanwhile. +## DONE / DEFERRED + +1. ~~**Native neon `U32x16 = [U32x4; 4]`.**~~ **DONE 2026-07-12** — `U32x4` gained + `bitxor`(veorq_u32) + `rotate_left`(vshlq_u32 shift-or, n%32 guard); composed + native `U32x16([U32x4;4])` with Add/BitXor/rotate_left; `F32x16::to_bits/ + from_bits` → from_array/to_array; `simd.rs` aarch64 arm re-exports the native + lane. (+ fixed the pre-existing aarch64 stable-compile breakage.) +2. ~~**aarch64 cross parity CI.**~~ **DONE 2026-07-12** — `crates/neon-simd-parity` + (excluded bin, real `ndarray::simd` aarch64 types) + `scripts/neon-parity.sh` + (cross-build aarch64 + run under `qemu-aarch64-static`) + CI `neon_simd` + (`neon-simd/parity-qemu`) job, added to the `conclusion` needs. Runtime-verifies + U32x16 ARX (rotate 16/12/8/7 + edges) / F32x16 / I8x16 == scalar. Twin of + `wasm_simd`. **Green locally under qemu.** +3. **avx2 native `U32x16`** (2×`__m256i`) — the TD-SIMD-3 lowering; still optional, + the scalar polyfill is correct meanwhile. +4. **wasm matryoshka backend + cross-repo `[patch]`** — see the SHIPPED section's + follow-ups above. ## The MATRYOSHKA — how the lane gets USED (the finalization) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index cb31b860..2f5b1cca 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -654,3 +654,28 @@ Green jobs of note: `tests/{stable,beta,1.95.0}`, `blas-msrv`, `nostd/thumbv6m-n Deferred (task #30, token-limit call): native neon `U32x16=[U32x4;4]` + aarch64 cross parity CI + the matryoshka chacha20 fork. Plan in `.claude/CHACHA20_MATRYOSHKA_PLAN.md`. + +--- + +## 2026-07-12 — Matryoshka finalized + NEON cross-CI (deferred #30 CLOSED) + +- **Native NEON `U32x16 = [U32x4;4]` ARX lane** (commit 06a61bf9): bitxor/ + rotate_left on U32x4, composed U32x16 Add/BitXor/rotate_left; also fixed the + pre-existing aarch64 stable-compile breakage (`u16x8` alias; nightly-only + `vdotq_s32` → stable widening NEON). aarch64 now `cargo check`-clean on stable. +- **ChaCha20 matryoshka + `simd_crypto.rs` RETIRED** (commit 20dc6c3f): + `vendor/chacha20/` fork (name/version kept, own [workspace]); the ONE delta is + `backends/ndarray_simd.rs` — the transpose block16 over `ndarray::simd::U32x16` + (pure +/^/rotate_left, no intrinsics, no unsafe), compile-time-selected under + cfg(x86_64+avx512f), `[patch.crates-io]`-folded under encryption. Triple parity + gate GREEN (fork RFC 8439 vectors through ndarray_simd @ v4; encryption 23 AEAD + tests @ v3+v4). Deleted src/simd_crypto.rs + the parity test + the chacha20 dev-dep + + the ndarray::simd::chacha20_* surface. +- **NEON cross parity CI**: `crates/neon-simd-parity` (excluded bin) + + `scripts/neon-parity.sh` (cross-build aarch64 + run under qemu-aarch64-static) + + CI `neon_simd` job (added to conclusion needs). Runtime-verifies U32x16 ARX / + F32x16 / I8x16 == scalar on real aarch64. Green locally under qemu. + +Follow-ups (documented in `.claude/CHACHA20_MATRYOSHKA_PLAN.md`): wasm matryoshka +backend (simd128 branch); cross-repo `[patch]` for MedCare-rs; the workspace +default is x86-64-v3 (avx2) so ndarray_simd activates on avx512 builds only. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0dd21577..060daa4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -135,6 +135,28 @@ jobs: - name: wasm SIMD parity (build + node run) run: ./scripts/wasm-parity.sh + neon_simd: + # The x86 `cargo test` suite never touches `simd_neon.rs` (it is cfg'd to + # aarch64), so NEON SIMD correctness is otherwise unverified. Cross-build the + # real `ndarray::simd` aarch64 types via the excluded `neon-simd-parity` bin + # and run its numeric selfcheck under qemu — both integration (the lib + # compiles for aarch64 on stable) and parity (each lane == scalar). The + # aarch64 twin of the wasm_simd job above. + runs-on: ubuntu-latest + name: neon-simd/parity-qemu + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-unknown-linux-gnu + # rust-toolchain.toml pins 1.95.0 — install the aarch64 target for the + # pinned toolchain too (dtolnay installs for `stable`, which may differ). + - run: rustup target add aarch64-unknown-linux-gnu + - name: install aarch64 cross toolchain + qemu-user + run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user-static + - name: NEON SIMD parity (cross-build + qemu run) + run: ./scripts/neon-parity.sh + tests: runs-on: ubuntu-latest needs: pass-msrv @@ -359,6 +381,7 @@ jobs: - format - nostd - wasm_simd + - neon_simd - tests - native-backend - hpc-stream-parallel diff --git a/Cargo.toml b/Cargo.toml index 5e8ce633..472bc1f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -402,7 +402,7 @@ members = [ "ndarray-rand", "crates/*", ] -exclude = ["crates/burn", "crates/wasm-simd-parity", "vendor/chacha20"] +exclude = ["crates/burn", "crates/wasm-simd-parity", "crates/neon-simd-parity", "vendor/chacha20"] default-members = [ ".", "ndarray-rand", diff --git a/crates/neon-simd-parity/.gitignore b/crates/neon-simd-parity/.gitignore new file mode 100644 index 00000000..96ef6c0b --- /dev/null +++ b/crates/neon-simd-parity/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/crates/neon-simd-parity/Cargo.toml b/crates/neon-simd-parity/Cargo.toml new file mode 100644 index 00000000..331ed22c --- /dev/null +++ b/crates/neon-simd-parity/Cargo.toml @@ -0,0 +1,29 @@ +# neon-simd-parity — the run-under-qemu gate for the aarch64 NEON SIMD tier. +# +# The x86 `cargo test` suite never touches `simd_neon.rs` (it is `cfg`'d to +# aarch64), so NEON SIMD correctness is otherwise unverified by CI — exactly the +# blind spot `wasm-simd-parity` closes for wasm. This bin calls the REAL +# `ndarray::simd` aarch64 types (no copy → no drift) and self-checks each lane's +# arithmetic against scalar in-process; it is cross-built for +# `aarch64-unknown-linux-gnu` and executed under `qemu-aarch64-static`, exiting +# non-zero on any lane mismatch. +# +# EXCLUDED from the workspace (see root Cargo.toml `exclude`) so it has zero +# effect on the x86 jobs; the CI `neon_simd` job builds it via --manifest-path. +[package] +name = "neon-simd-parity" +version = "0.0.0" +edition = "2021" +publish = false + +[[bin]] +name = "neon-simd-parity" +path = "src/main.rs" + +[dependencies] +# `simd` needs only ndarray's `std` feature, not the full HPC default set. +ndarray = { path = "../..", default-features = false, features = ["std"] } + +[profile.release] +panic = "abort" +opt-level = "z" diff --git a/crates/neon-simd-parity/src/main.rs b/crates/neon-simd-parity/src/main.rs new file mode 100644 index 00000000..9477ac27 --- /dev/null +++ b/crates/neon-simd-parity/src/main.rs @@ -0,0 +1,123 @@ +//! Run-under-qemu parity gate for the aarch64 NEON SIMD tier. +//! +//! `selfcheck()` runs each NEON SIMD lane's arithmetic against the scalar +//! `u32`/`f32`/`i8` reference **in the same process**, returning `0` iff every +//! lane is bit-identical. `scripts/neon-parity.sh` cross-builds this bin for +//! `aarch64-unknown-linux-gnu` and runs it under `qemu-aarch64-static`, asserting +//! exit code `0`; the CI `neon_simd` job runs the script. This is the NEON twin +//! of `wasm-simd-parity` (the wasm tier's node gate) — it tests the REAL +//! `ndarray::simd` aarch64 types, so a regression in `simd_neon.rs` fails here +//! even though the x86 `cargo test` suite never compiles that file. Extend the +//! per-lane blocks below whenever a new `ndarray::simd` NEON lane lands. + +fn main() { + #[cfg(target_arch = "aarch64")] + { + let rc = checks::selfcheck(); + if rc != 0 { + eprintln!("neon-simd-parity FAILED: lane/op code = {rc}"); + std::process::exit(rc as i32); + } + println!("neon-simd-parity OK: U32x16 / F32x16 / I8x16 lanes bit-identical to scalar"); + } + #[cfg(not(target_arch = "aarch64"))] + { + eprintln!("neon-simd-parity is aarch64-only; nothing to check on this target"); + } +} + +#[cfg(target_arch = "aarch64")] +mod checks { + use ndarray::simd::{F32x16, I8x16, U32x16}; + + /// Distinct nonzero return codes make a CI failure point at the exact lane+op. + pub fn selfcheck() -> u32 { + if let Err(code) = check_u32x16() { + return code; + } + if let Err(code) = check_f32x16() { + return code; + } + if let Err(code) = check_i8x16() { + return code; + } + 0 + } + + /// `U32x16` ARX triple (Add / BitXor / rotate_left) — the ChaCha20/BLAKE lane + /// (native `[U32x4; 4]` on aarch64). + fn check_u32x16() -> Result<(), u32> { + let a_arr: [u32; 16] = [ + 0x0000_0000, 0xFFFF_FFFF, 0x0000_0001, 0x8000_0000, 0x1234_5678, 0x9ABC_DEF0, 0xDEAD_BEEF, 0xCAFE_BABE, + 0x0F0F_0F0F, 0xF0F0_F0F0, 0x5555_5555, 0xAAAA_AAAA, 0x0000_00FF, 0xFF00_0000, 0x0101_0101, 0x8080_8080, + ]; + let b_arr: [u32; 16] = [ + 0x9E37_79B9, 0x1111_1111, 0xDEAD_C0DE, 0x0BAD_F00D, 0x7FFF_FFFF, 0x0000_0000, 0xFFFF_FFFF, 0x1357_9BDF, + 0x2468_ACE0, 0xFEDC_BA98, 0x0000_0010, 0x0000_001F, 0xABCD_EF01, 0x1020_4080, 0x0F0F_F0F0, 0xC0DE_CAFE, + ]; + if U32x16::from_array(a_arr).to_array() != a_arr { + return Err(10); + } + let a = U32x16::from_array(a_arr); + let b = U32x16::from_array(b_arr); + let add = (a + b).to_array(); + let xor = (a ^ b).to_array(); + for i in 0..16 { + if add[i] != a_arr[i].wrapping_add(b_arr[i]) { + return Err(11); + } + if xor[i] != a_arr[i] ^ b_arr[i] { + return Err(12); + } + } + // ARX rotate — ChaCha20 uses 16/12/8/7; edges (0, 1, 24, 31) included. + for &n in &[0u32, 1, 7, 8, 12, 16, 24, 31] { + let r = a.rotate_left(n).to_array(); + for i in 0..16 { + if r[i] != a_arr[i].rotate_left(n) { + return Err(13); + } + } + } + Ok(()) + } + + /// `F32x16` — the float hot-path lane (splat / roundtrip / add / reduce_sum). + fn check_f32x16() -> Result<(), u32> { + let data: [f32; 16] = [ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + ]; + if F32x16::from_array(data).to_array() != data { + return Err(20); + } + let a = F32x16::from_array(data); + let b = F32x16::splat(2.0); + let sum = (a + b).to_array(); + for i in 0..16 { + if sum[i] != data[i] + 2.0 { + return Err(21); + } + } + // 0+1+…+15 = 120. + if F32x16::from_array(data).reduce_sum() != 120.0 { + return Err(22); + } + Ok(()) + } + + /// `I8x16` — the byte lane (roundtrip / add). + fn check_i8x16() -> Result<(), u32> { + let a_arr: [i8; 16] = [-128, -1, 0, 1, 127, 2, -2, 3, -3, 42, -42, 100, -100, 7, -7, 64]; + let b_arr: [i8; 16] = [1, 1, 1, 1, 1, -1, -1, -1, 5, -5, 10, -10, 25, -25, 63, -64]; + if I8x16::from_array(a_arr).to_array() != a_arr { + return Err(30); + } + let sum = I8x16::from_array(a_arr).add(I8x16::from_array(b_arr)).to_array(); + for i in 0..16 { + if sum[i] != a_arr[i].wrapping_add(b_arr[i]) { + return Err(31); + } + } + Ok(()) + } +} diff --git a/scripts/neon-parity.sh b/scripts/neon-parity.sh new file mode 100755 index 00000000..eef9b324 --- /dev/null +++ b/scripts/neon-parity.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# NEON SIMD parity gate — cross-build the real ndarray::simd aarch64 types (via +# the `neon-simd-parity` bin) for aarch64-unknown-linux-gnu, then run the numeric +# selfcheck under qemu-aarch64. Exercises BOTH: +# - correctness/integration: the full ndarray lib compiles for aarch64 on stable +# (the harness path-deps ndarray, so building it builds ndarray for aarch64); +# - parity: each NEON SIMD lane (U32x16 ARX / F32x16 / I8x16) is bit-identical +# to scalar, run under qemu. +# The x86 `cargo test` suite never touches simd_neon.rs, so this is the only +# automated guard for the NEON SIMD tier (the twin of scripts/wasm-parity.sh). +# +# Requirements (Ubuntu CI): gcc-aarch64-linux-gnu (cross linker + sysroot), +# qemu-user-static (qemu-aarch64-static), and `rustup target add +# aarch64-unknown-linux-gnu`. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MANIFEST="$ROOT/crates/neon-simd-parity/Cargo.toml" +TARGET="aarch64-unknown-linux-gnu" +QEMU="${QEMU_AARCH64:-qemu-aarch64-static}" +SYSROOT="${AARCH64_SYSROOT:-/usr/aarch64-linux-gnu}" + +echo "==> building neon-simd-parity (ndarray::simd aarch64 types) for $TARGET" +CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="aarch64-linux-gnu-gcc" \ + cargo build --release --manifest-path "$MANIFEST" --target "$TARGET" + +BIN="$ROOT/crates/neon-simd-parity/target/$TARGET/release/neon-simd-parity" +# Excluded crate → its own target dir under the crate; fall back to workspace target. +[ -f "$BIN" ] || BIN="$ROOT/target/$TARGET/release/neon-simd-parity" + +echo "==> running selfcheck under $QEMU (sysroot $SYSROOT): $BIN" +"$QEMU" -L "$SYSROOT" "$BIN" +echo "==> NEON SIMD parity: PASS" From 37306f2fb094a8cf70e042f37d270e379f7b5c83 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 05:58:45 +0000 Subject: [PATCH 4/6] =?UTF-8?q?crypto(matryoshka):=20wasm32+simd128=20back?= =?UTF-8?q?end=20=E2=80=94=20same=20ndarray::simd::U32x16=20source,=20brow?= =?UTF-8?q?ser=20lane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the matryoshka's browser half. The fork's `ndarray_simd` backend cfg gates widen from x86_64+avx512f to `any(all(x86_64, avx512f), all(wasm32, simd128))` (backends.rs module decl + lib.rs::process_with_backend dispatch), and the target-scoped ndarray dep to `cfg(any(target_arch="x86_64", target_arch="wasm32"))`. The SAME src/backends/ndarray_simd.rs source now drives BOTH hardware lanes — the AVX-512 `__m512i` lane on a server build and the native `[U32x4; 4]` lane on a wasm32+simd128 browser build — because the backend only ever uses `U32x16` ops, and ndarray's polyfill lowers `U32x16` to the right lane per target. aarch64 and other fork builds still never pull ndarray (they use RustCrypto's backends). Verified: - `cargo build --manifest-path vendor/chacha20/Cargo.toml --target wasm32-unknown-unknown --lib` with `-Ctarget-feature=+simd128` compiles ndarray-for-wasm (std, minimal features) + the wasm backend clean; - x86 avx512 RustCrypto vectors still 9/9 green through ndarray_simd after the cfg widening. Bit-exact by composition: the ndarray_simd source is RFC-8439-proven on x86, and the wasm U32x16 [U32x4;4] lane is node-proven by the existing wasm_simd CI job. CI: added a compile-guard step to the wasm_simd job (build the fork for wasm32+simd128) so a future ndarray::simd change that broke the wasm backend fails there. A consumer building `encryption` as a wasm cdylib with +simd128 now gets the accelerated XChaCha20-Poly1305 keystream transparently — the browser goal. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- .claude/CHACHA20_MATRYOSHKA_PLAN.md | 17 ++++++++++++----- .github/workflows/ci.yaml | 6 ++++++ vendor/chacha20/Cargo.toml | 11 ++++++----- vendor/chacha20/src/backends.rs | 11 ++++++++--- vendor/chacha20/src/lib.rs | 8 ++++++-- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/.claude/CHACHA20_MATRYOSHKA_PLAN.md b/.claude/CHACHA20_MATRYOSHKA_PLAN.md index 5efe6dc8..582a470d 100644 --- a/.claude/CHACHA20_MATRYOSHKA_PLAN.md +++ b/.claude/CHACHA20_MATRYOSHKA_PLAN.md @@ -71,11 +71,18 @@ below **superseded it** — it is now **RETIRED** (2026-07-12, see below). build** (`.cargo/config-avx512.toml` / `-Ctarget-cpu=x86-64-v4`); the default v3 build uses RustCrypto's own avx2 backend (fast, vetted). This is correct and safe — the matryoshka is the *server-avx512* accelerator. -- **wasm browser backend (next):** add a sibling `#[cfg(all(target_arch="wasm32", - target_feature="simd128"))]` branch selecting an `ndarray_simd` backend (same - `U32x16` source; the wasm arm is native `[U32x4;4]`) + a `target.'cfg(wasm32)'` - ndarray dep. Gated on verifying `ndarray/std` builds inside the encryption - `cdylib` for `wasm32-unknown-unknown`. +- ~~**wasm browser backend.**~~ **DONE 2026-07-12** — the fork's `ndarray_simd` + cfg gates were widened to `any(all(x86_64, avx512f), all(wasm32, simd128))` in + `backends.rs` + `lib.rs::process_with_backend`, and the ndarray dep to + `cfg(any(x86_64, wasm32))`. The SAME `ndarray_simd.rs` source now drives the + AVX-512 `__m512i` lane (server) and the native wasm `[U32x4;4]` lane (browser). + Verified: `cargo build --manifest-path vendor/chacha20/Cargo.toml --target + wasm32-unknown-unknown --lib` with `+simd128` compiles ndarray-for-wasm + the + wasm backend clean; x86 avx512 vectors still 9/9 green after the widening. Bit- + exact by composition (the `ndarray_simd` source is RFC-proven on x86; the wasm + `U32x16` lane is node-proven by the `wasm_simd` CI job). A CI compile-guard step + was added to the `wasm_simd` job. A consumer building `encryption` as a wasm + cdylib with `+simd128` now gets the accelerated keystream transparently. - **cross-repo `[patch]` (next):** `[patch]` does not transit, so MedCare-rs (and any other consumer building `encryption` for avx512) needs its own `[patch.crates-io] chacha20 = { path = "vendor/chacha20" }` pointing at a vendored diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 060daa4a..f2fbcab2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -134,6 +134,12 @@ jobs: node-version: "22" - name: wasm SIMD parity (build + node run) run: ./scripts/wasm-parity.sh + # Guard the wasm matryoshka: the chacha20 fork's ndarray_simd backend must + # keep compiling for wasm32+simd128 (it rides the same U32x16 lane the node + # parity step above proves bit-exact). A future ndarray::simd change that + # broke the wasm backend build would fail here. + - name: wasm matryoshka backend (chacha20 fork) compiles for wasm32+simd128 + run: RUSTFLAGS="-C target-feature=+simd128" cargo build --manifest-path vendor/chacha20/Cargo.toml --target wasm32-unknown-unknown --lib neon_simd: # The x86 `cargo test` suite never touches `simd_neon.rs` (it is cfg'd to diff --git a/vendor/chacha20/Cargo.toml b/vendor/chacha20/Cargo.toml index df9680ce..e4002cc6 100644 --- a/vendor/chacha20/Cargo.toml +++ b/vendor/chacha20/Cargo.toml @@ -50,11 +50,12 @@ zeroize = ["cipher/zeroize"] [target."cfg(any(target_arch = \"x86_64\", target_arch = \"x86\"))".dependencies.cpufeatures] version = "0.2" -# The matryoshka's one non-upstream dependency: the AVX-512 backend rides -# `ndarray::simd::U32x16`. Scoped to x86_64 so wasm / aarch64 / other builds of -# this crate never pull ndarray (they fall through to the upstream backends); -# `simd` needs only ndarray's `std` feature, not the full HPC default set. -[target."cfg(target_arch = \"x86_64\")".dependencies.ndarray] +# The matryoshka's one non-upstream dependency: the ndarray_simd backend rides +# `ndarray::simd::U32x16` (AVX-512 lane on x86_64, native [U32x4;4] on wasm32). +# Scoped to x86_64 + wasm32 so aarch64 / other builds of this crate never pull +# ndarray (they fall through to the upstream backends); `simd` needs only +# ndarray's `std` feature, not the full HPC default set. +[target."cfg(any(target_arch = \"x86_64\", target_arch = \"wasm32\"))".dependencies.ndarray] path = "../.." default-features = false features = ["std"] diff --git a/vendor/chacha20/src/backends.rs b/vendor/chacha20/src/backends.rs index 39fa8da3..014b7346 100644 --- a/vendor/chacha20/src/backends.rs +++ b/vendor/chacha20/src/backends.rs @@ -3,9 +3,14 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(chacha20_force_soft)] { pub(crate) mod soft; - } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { - // AdaWorldAPI matryoshka: on an AVX-512 build the keystream double-round - // rides `ndarray::simd::U32x16` (the polyfill lowers it to `__m512i`). + } else if #[cfg(any( + all(target_arch = "x86_64", target_feature = "avx512f"), + all(target_arch = "wasm32", target_feature = "simd128"), + ))] { + // AdaWorldAPI matryoshka: the keystream double-round rides + // `ndarray::simd::U32x16` — the polyfill lowers it to `__m512i` on an + // AVX-512 build (server) and to the native `[U32x4; 4]` lane on a + // wasm32+simd128 build (browser). Same source, two hardware lanes. // Takes precedence over the runtime-detected AVX2/SSE2 path below. pub(crate) mod ndarray_simd; } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { diff --git a/vendor/chacha20/src/lib.rs b/vendor/chacha20/src/lib.rs index 3d1824a0..cf197bd7 100644 --- a/vendor/chacha20/src/lib.rs +++ b/vendor/chacha20/src/lib.rs @@ -263,8 +263,12 @@ impl StreamCipherCore for ChaChaCore { cfg_if! { if #[cfg(chacha20_force_soft)] { f.call(&mut backends::soft::Backend(self)); - } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { - // AdaWorldAPI matryoshka: keystream via ndarray::simd::U32x16. + } else if #[cfg(any( + all(target_arch = "x86_64", target_feature = "avx512f"), + all(target_arch = "wasm32", target_feature = "simd128"), + ))] { + // AdaWorldAPI matryoshka: keystream via ndarray::simd::U32x16 + // (AVX-512 lane on x86_64, native [U32x4; 4] lane on wasm32). f.call(&mut backends::ndarray_simd::Backend(self)); } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cfg_if! { From 7849e37e6cd1ed8c039fcd6d6669e2979c1e293e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 06:05:23 +0000 Subject: [PATCH 5/6] fix(palette_codec): gate bytemuck_cast_u64_to_u8 to x86_64 (dead on aarch64 under -D warnings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new `neon_simd` CI job builds ndarray for aarch64 with the workflow-global `RUSTFLAGS: -D warnings`. `bytemuck_cast_u64_to_u8`'s only caller is `unpack_4bit_avx2` (`#[cfg(target_arch = "x86_64")]`), so on aarch64 the function is dead code → `-D dead-code` build error. Gate the function with the same `#[cfg(target_arch = "x86_64")]` as its caller. x86_64 is unchanged (both compile and the function is used); reproduced green locally with `RUSTFLAGS="-D warnings" cargo build --target aarch64-unknown-linux-gnu`. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- src/palette_codec.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/palette_codec.rs b/src/palette_codec.rs index 7447a76f..33683b4e 100644 --- a/src/palette_codec.rs +++ b/src/palette_codec.rs @@ -399,6 +399,10 @@ unsafe fn unpack_4bit_avx2(packed: &[u64], count: usize) -> Vec { } /// Reinterpret &[u64] as &[u8] (little-endian safe). +// Only reachable from the x86_64 AVX2 unpack path (`unpack_4bit_avx2`); gated to +// match so it isn't dead code on non-x86 targets under `-D warnings` (the +// aarch64 neon-parity + cross builds compile this file). +#[cfg(target_arch = "x86_64")] fn bytemuck_cast_u64_to_u8(words: &[u64]) -> &[u8] { // SAFETY: u64 and u8 have compatible layouts on little-endian unsafe { core::slice::from_raw_parts(words.as_ptr() as *const u8, words.len() * 8) } From 2539cad7885fe690de7a0056fbfdc2ba3269bc7c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 06:13:58 +0000 Subject: [PATCH 6/6] crypto(matryoshka): honor chacha20_force_avx2/_sse2 before ndarray_simd auto-selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses codex P2. The ndarray_simd branch was ordered before the x86 force-flag subtree, so on an x86-64-v4 build with `--cfg chacha20_force_avx2` (or `_sse2`) it matched first and silently ignored the force cfg — making A/B benchmarks and a rollback off the new backend impossible. Gate the x86_64 disjunct of the ndarray_simd auto-selection on `not(chacha20_force_avx2)` + `not(chacha20_force_sse2)` at all four cfg sites (backends.rs module decl, lib.rs Tokens type / new() tokens / process_with_backend) so a force cfg falls through to the existing AVX2/SSE2 subtree. The wasm32 disjunct stays unguarded — those cfgs are documented as ignored on non-x86. Verified under -Ctarget-cpu=x86-64-v4: default (no force) → ndarray_simd, 9/9 RustCrypto vectors green; `--cfg chacha20_force_avx2` → avx2 backend, 9/9 green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K6pT32kk6pnuAAqR3JiYqu --- vendor/chacha20/src/backends.rs | 12 ++++++++++-- vendor/chacha20/src/lib.rs | 24 +++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/vendor/chacha20/src/backends.rs b/vendor/chacha20/src/backends.rs index 014b7346..c8afa435 100644 --- a/vendor/chacha20/src/backends.rs +++ b/vendor/chacha20/src/backends.rs @@ -4,14 +4,22 @@ cfg_if! { if #[cfg(chacha20_force_soft)] { pub(crate) mod soft; } else if #[cfg(any( - all(target_arch = "x86_64", target_feature = "avx512f"), + all( + target_arch = "x86_64", + target_feature = "avx512f", + not(chacha20_force_avx2), + not(chacha20_force_sse2), + ), all(target_arch = "wasm32", target_feature = "simd128"), ))] { // AdaWorldAPI matryoshka: the keystream double-round rides // `ndarray::simd::U32x16` — the polyfill lowers it to `__m512i` on an // AVX-512 build (server) and to the native `[U32x4; 4]` lane on a // wasm32+simd128 build (browser). Same source, two hardware lanes. - // Takes precedence over the runtime-detected AVX2/SSE2 path below. + // AUTO-selection only: `chacha20_force_avx2`/`_sse2` still win on x86_64 + // (they fall through to the force subtree below) so A/B benchmarks and a + // rollback stay possible on v4 builds; the force cfgs are documented as + // ignored on non-x86, so the wasm arm is unguarded. pub(crate) mod ndarray_simd; } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cfg_if! { diff --git a/vendor/chacha20/src/lib.rs b/vendor/chacha20/src/lib.rs index cf197bd7..5372068d 100644 --- a/vendor/chacha20/src/lib.rs +++ b/vendor/chacha20/src/lib.rs @@ -161,8 +161,14 @@ pub type ChaCha20 = StreamCipherCoreWrapper>; cfg_if! { if #[cfg(chacha20_force_soft)] { type Tokens = (); - } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + } else if #[cfg(all( + target_arch = "x86_64", + target_feature = "avx512f", + not(chacha20_force_avx2), + not(chacha20_force_sse2), + ))] { // AdaWorldAPI matryoshka backend: compile-time selected, no runtime probe. + // Guarded so `chacha20_force_avx2`/`_sse2` fall through to the x86 subtree. type Tokens = (); } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cfg_if! { @@ -227,7 +233,12 @@ impl KeyIvInit for ChaChaCore { cfg_if! { if #[cfg(chacha20_force_soft)] { let tokens = (); - } else if #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] { + } else if #[cfg(all( + target_arch = "x86_64", + target_feature = "avx512f", + not(chacha20_force_avx2), + not(chacha20_force_sse2), + ))] { let tokens = (); } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cfg_if! { @@ -264,11 +275,18 @@ impl StreamCipherCore for ChaChaCore { if #[cfg(chacha20_force_soft)] { f.call(&mut backends::soft::Backend(self)); } else if #[cfg(any( - all(target_arch = "x86_64", target_feature = "avx512f"), + all( + target_arch = "x86_64", + target_feature = "avx512f", + not(chacha20_force_avx2), + not(chacha20_force_sse2), + ), all(target_arch = "wasm32", target_feature = "simd128"), ))] { // AdaWorldAPI matryoshka: keystream via ndarray::simd::U32x16 // (AVX-512 lane on x86_64, native [U32x4; 4] lane on wasm32). + // AUTO-selection only — `chacha20_force_avx2`/`_sse2` fall through + // to the x86 force subtree below (rollback/A-B stays possible). f.call(&mut backends::ndarray_simd::Backend(self)); } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cfg_if! {