Skip to content

Aarch64 asm: Have software fallback and CPU id checks #10754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SparkiDev wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Aarch64 asm: Have software fallback and CPU id checks #10754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion tests/api/test_tls.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1829,7 +1829,9 @@ int test_tls12_corrupted_finished(void)
}
else {
ExpectIntGE(finishedSz, finishedLen);
XMEMCPY(finishedMsg, test_ctx.s_buff + finishedOffInMsg, finishedLen);
if (EXPECT_SUCCESS()) {
XMEMCPY(finishedMsg, test_ctx.s_buff + finishedOffInMsg, finishedLen);
}
finishedSz = finishedLen;
ExpectIntEQ(test_memio_modify_message_len(&test_ctx, 0,
finishedMsgPos, finishedOffInMsg), 0);
Expand Down
123 changes: 90 additions & 33 deletions wolfcrypt/src/chacha.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,38 @@ Public domain.
static cpuid_flags_t cpuidFlags = WC_CPUID_INITIALIZER;
#endif

/* The aarch64 ChaCha assembly is NEON-only. When NEON might be absent, also
* build the C implementation: dispatch on ASIMD at runtime when NEON is
* compiled in, or use only the C path when NEON is disabled at build time. */
#if defined(USE_ARM_CHACHA_SPEEDUP) && defined(__aarch64__)
#ifdef WOLFSSL_ARMASM_NO_NEON
#define WOLFSSL_ARM_CHACHA_C_ONLY
#else
#define WOLFSSL_ARM_CHACHA_NEON_FALLBACK
#endif
#endif
#if defined(WOLFSSL_ARM_CHACHA_NEON_FALLBACK) || \
defined(WOLFSSL_ARM_CHACHA_C_ONLY)
#define WOLFSSL_ARM_CHACHA_NEED_C
#endif

#ifdef WOLFSSL_ARM_CHACHA_NEON_FALLBACK
static cpuid_flags_t chacha_cpuid_flags = WC_CPUID_INITIALIZER;
/* Return non-zero when NEON/ASIMD is present and the asm path should run. */
static WC_INLINE int chacha_use_neon(void)
{
cpuid_get_flags_ex(&chacha_cpuid_flags);
return IS_AARCH64_ASIMD(chacha_cpuid_flags);
}
#endif

/**
* Set up iv(nonce). Earlier versions used 64 bits instead of 96, this version
* uses the typical AEAD 96 bit nonce and can do record sizes of 256 GB.
*/
int wc_Chacha_SetIV(ChaCha* ctx, const byte* inIv, word32 counter)
{
#if !defined(USE_ARM_CHACHA_SPEEDUP)
#if !defined(USE_ARM_CHACHA_SPEEDUP) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
word32 temp[CHACHA_IV_WORDS];/* used for alignment of memory */
#endif

Expand All @@ -124,24 +149,31 @@ int wc_Chacha_SetIV(ChaCha* ctx, const byte* inIv, word32 counter)

ctx->left = 0; /* resets state */

#if !defined(USE_ARM_CHACHA_SPEEDUP)
XMEMCPY(temp, inIv, CHACHA_IV_BYTES);
/* block counter */
ctx->X[CHACHA_MATRIX_CNT_IV+0] = counter;
/* fixed variable from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+1] = LITTLE32(temp[0]);
/* counter from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+2] = LITTLE32(temp[1]);
/* counter from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+3] = LITTLE32(temp[2]);
#else
#ifdef WOLFSSL_ARM_CHACHA_NEON_FALLBACK
if (chacha_use_neon())
wc_chacha_setiv(ctx->X, inIv, counter);
else
#elif defined(USE_ARM_CHACHA_SPEEDUP) && !defined(WOLFSSL_ARM_CHACHA_C_ONLY)
wc_chacha_setiv(ctx->X, inIv, counter);
#endif
#if !defined(USE_ARM_CHACHA_SPEEDUP) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
{
XMEMCPY(temp, inIv, CHACHA_IV_BYTES);
/* block counter */
ctx->X[CHACHA_MATRIX_CNT_IV+0] = counter;
/* fixed variable from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+1] = LITTLE32(temp[0]);
/* counter from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+2] = LITTLE32(temp[1]);
/* counter from nonce */
ctx->X[CHACHA_MATRIX_CNT_IV+3] = LITTLE32(temp[2]);
}
#endif

return 0;
}

#if !defined(USE_ARM_CHACHA_SPEEDUP)
#if !defined(USE_ARM_CHACHA_SPEEDUP) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
/* "expand 32-byte k" as unsigned 32 byte */
static const word32 sigma[4] = {0x61707865, 0x3320646e, 0x79622d32, 0x6b206574};
/* "expand 16-byte k" as unsigned 16 byte */
Expand All @@ -153,7 +185,7 @@ static const word32 tau[4] = {0x61707865, 0x3120646e, 0x79622d36, 0x6b206574};
*/
int wc_Chacha_SetKey(ChaCha* ctx, const byte* key, word32 keySz)
{
#if !defined(USE_ARM_CHACHA_SPEEDUP)
#if !defined(USE_ARM_CHACHA_SPEEDUP) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
const word32* constants;
const byte* k;
#ifdef XSTREAM_ALIGN
Expand All @@ -167,7 +199,15 @@ int wc_Chacha_SetKey(ChaCha* ctx, const byte* key, word32 keySz)
if (keySz != (CHACHA_MAX_KEY_SZ/2) && keySz != CHACHA_MAX_KEY_SZ)
return BAD_FUNC_ARG;

#if !defined(USE_ARM_CHACHA_SPEEDUP)
#ifdef WOLFSSL_ARM_CHACHA_NEON_FALLBACK
if (chacha_use_neon())
wc_chacha_setkey(ctx->X, key, keySz);
else
#elif defined(USE_ARM_CHACHA_SPEEDUP) && !defined(WOLFSSL_ARM_CHACHA_C_ONLY)
wc_chacha_setkey(ctx->X, key, keySz);
#endif
#if !defined(USE_ARM_CHACHA_SPEEDUP) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
{
#ifdef XSTREAM_ALIGN
if ((wc_ptr_t)key % 4) {
WOLFSSL_MSG("wc_ChachaSetKey unaligned key");
Expand Down Expand Up @@ -211,16 +251,16 @@ int wc_Chacha_SetKey(ChaCha* ctx, const byte* key, word32 keySz)
ctx->X[ 1] = constants[1];
ctx->X[ 2] = constants[2];
ctx->X[ 3] = constants[3];
#else
wc_chacha_setkey(ctx->X, key, keySz);
}
#endif

ctx->left = 0; /* resets state */

return 0;
}

#if !defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP)
#if (!defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP)) || \
defined(WOLFSSL_ARM_CHACHA_NEED_C)
/**
* Converts word into bytes with rotations having been done.
*/
Expand Down Expand Up @@ -267,7 +307,8 @@ extern void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c,
#endif


#if !defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP)
#if (!defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP)) || \
defined(WOLFSSL_ARM_CHACHA_NEED_C)
/**
* Encrypt a stream of bytes
*/
Expand Down Expand Up @@ -366,23 +407,39 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
return 0;
}
#elif defined(USE_ARM_CHACHA_SPEEDUP)
/* Handle left over bytes from last block. */
if ((msglen > 0) && (ctx->left > 0)) {
byte* over = ((byte*)ctx->over) + CHACHA_CHUNK_BYTES - ctx->left;
word32 l = min(msglen, ctx->left);

wc_chacha_use_over(over, output, input, l);
#ifdef WOLFSSL_ARM_CHACHA_NEON_FALLBACK
if (chacha_use_neon())
#endif
#ifndef WOLFSSL_ARM_CHACHA_C_ONLY
{
/* Handle left over bytes from last block. */
if ((msglen > 0) && (ctx->left > 0)) {
byte* over = ((byte*)ctx->over) + CHACHA_CHUNK_BYTES - ctx->left;
word32 l = min(msglen, ctx->left);

wc_chacha_use_over(over, output, input, l);

ctx->left -= l;
input += l;
output += l;
msglen -= l;
}

ctx->left -= l;
input += l;
output += l;
msglen -= l;
if (msglen != 0) {
wc_chacha_crypt_bytes(ctx, output, input, msglen);
}
return 0;
}

if (msglen != 0) {
wc_chacha_crypt_bytes(ctx, output, input, msglen);
#endif
#ifdef WOLFSSL_ARM_CHACHA_NEED_C
#ifdef WOLFSSL_ARM_CHACHA_NEON_FALLBACK
else
#endif
{
wc_Chacha_encrypt_bytes(ctx, input, output, msglen);
return 0;
}
return 0;
#endif
#else
wc_Chacha_encrypt_bytes(ctx, input, output, msglen);
return 0;
Expand Down
91 changes: 80 additions & 11 deletions wolfcrypt/src/cpuid.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@
#define CPUID_AARCH64_FEAT_SHA3 ((word64)1 << 32)
#define CPUID_AARCH64_FEAT_SM3 ((word64)1 << 36)
#define CPUID_AARCH64_FEAT_SM4 ((word64)1 << 40)
#define CPUID_AARCH64_FEAT_ASMID ((word64)0xf << 20)

#ifdef WOLFSSL_AARCH64_PRIVILEGE_MODE
/* https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Registers
Expand All @@ -156,13 +157,27 @@
old_cpuid_flags = WC_CPUID_INITIALIZER;
word64 features;

#ifndef WOLFSSL_ARMASM_NO_NEON
__asm__ __volatile (
"mrs %[feat], ID_AA64PFR0_EL1\n"
: [feat] "=r" (features)
:
:
);

if ((features & CPUID_AARCH64_FEAT_ASMID) !=
CPUID_AARCH64_FEAT_ASMID)
new_cpuid_flags |= CPUID_ASIMD;
#endif

__asm__ __volatile (
"mrs %[feat], ID_AA64ISAR0_EL1\n"
: [feat] "=r" (features)
:
:
);

#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
if (features & CPUID_AARCH64_FEAT_AES)
new_cpuid_flags |= CPUID_AES;
if (features & CPUID_AARCH64_FEAT_AES_PMULL) {
Expand All @@ -171,16 +186,27 @@
}
if (features & CPUID_AARCH64_FEAT_SHA256)
new_cpuid_flags |= CPUID_SHA256;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
if (features & CPUID_AARCH64_FEAT_SHA256_512)
new_cpuid_flags |= CPUID_SHA256 | CPUID_SHA512;
#endif
#if !defined(WOLFSSL_AARCH64_NO_SQRDMLSH)
if (features & CPUID_AARCH64_FEAT_RDM)
new_cpuid_flags |= CPUID_RDM;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
if (features & CPUID_AARCH64_FEAT_SHA3)
new_cpuid_flags |= CPUID_SHA3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM3
if (features & CPUID_AARCH64_FEAT_SM3)
new_cpuid_flags |= CPUID_SM3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM4
if (features & CPUID_AARCH64_FEAT_SM4)
new_cpuid_flags |= CPUID_SM4;
#endif

(void)wolfSSL_Atomic_Uint_CompareExchange
(&cpuid_flags, &old_cpuid_flags, new_cpuid_flags);
Expand All @@ -200,6 +226,11 @@
old_cpuid_flags = WC_CPUID_INITIALIZER;
word64 hwcaps = getauxval(AT_HWCAP);

#ifndef WOLFSSL_ARMASM_NO_NEON
if (hwcaps & HWCAP_ASIMD)
new_cpuid_flags |= CPUID_ASIMD;
#endif

#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
if (hwcaps & HWCAP_AES)
new_cpuid_flags |= CPUID_AES;
Expand Down Expand Up @@ -247,12 +278,18 @@
old_cpuid_flags = WC_CPUID_INITIALIZER;
word64 features = android_getCpuFeatures();

#ifndef WOLFSSL_ARMASM_NO_NEON
/* All Android AArch64 chips support NEON. */
new_cpuid_flags |= CPUID_ASIMD;
#endif
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
if (features & ANDROID_CPU_ARM_FEATURE_AES)
new_cpuid_flags |= CPUID_AES;
if (features & ANDROID_CPU_ARM_FEATURE_PMULL)
new_cpuid_flags |= CPUID_PMULL;
if (features & ANDROID_CPU_ARM_FEATURE_SHA2)
new_cpuid_flags |= CPUID_SHA256;
#endif

(void)wolfSSL_Atomic_Uint_CompareExchange
(&cpuid_flags, &old_cpuid_flags, new_cpuid_flags);
Expand All @@ -279,18 +316,31 @@
if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
cpuid_flags_t new_cpuid_flags = 0,
old_cpuid_flags = WC_CPUID_INITIALIZER;

#ifndef WOLFSSL_ARMASM_NO_NEON
/* All Mac AArch64 chips support NEON. */
new_cpuid_flags |= CPUID_ASIMD;
#endif
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_AES") != 0)
new_cpuid_flags |= CPUID_AES;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_PMULL") != 0)
new_cpuid_flags |= CPUID_PMULL;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA256") != 0)
new_cpuid_flags |= CPUID_SHA256;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA512") != 0)
new_cpuid_flags |= CPUID_SHA512;
#endif
#if !defined(WOLFSSL_AARCH64_NO_SQRDMLSH)
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_RDM") != 0)
new_cpuid_flags |= CPUID_RDM;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA3") != 0)
new_cpuid_flags |= CPUID_SHA3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM3
new_cpuid_flags |= CPUID_SM3;
#endif
Expand All @@ -316,24 +366,40 @@

elf_aux_info(AT_HWCAP, &features, sizeof(features));

if (features & CPUID_AARCH64_FEAT_AES)
new_cpuid_flags |= CPUID_AES;
if (features & CPUID_AARCH64_FEAT_AES_PMULL) {
#ifndef WOLFSSL_ARMASM_NO_NEON
if (features & HWCAP_ASIMD)
new_cpuid_flags |= CPUID_ASIMD;
#endif

#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
if (features & HWCAP_AES)
new_cpuid_flags |= CPUID_AES;
if (features & HWCAP_PMULL)
new_cpuid_flags |= CPUID_PMULL;
}
if (features & CPUID_AARCH64_FEAT_SHA256)
if (features & HWCAP_SHA2)
new_cpuid_flags |= CPUID_SHA256;
if (features & CPUID_AARCH64_FEAT_SHA256_512)
new_cpuid_flags |= CPUID_SHA256 | CPUID_SHA512;
if (features & CPUID_AARCH64_FEAT_RDM)
#endif

#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
if (features & HWCAP_SHA512)
new_cpuid_flags |= CPUID_SHA512;
#endif
#if defined(HWCAP_ASIMDRDM) && !defined(WOLFSSL_AARCH64_NO_SQRDMLSH)
if (features & HWCAP_ASIMDRDM)
new_cpuid_flags |= CPUID_RDM;
if (features & CPUID_AARCH64_FEAT_SHA3)
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
if (features & HWCAP_SHA3)
new_cpuid_flags |= CPUID_SHA3;
if (features & CPUID_AARCH64_FEAT_SM3)
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM3
if (features & HWCAP_SM3)
new_cpuid_flags |= CPUID_SM3;
if (features & CPUID_AARCH64_FEAT_SM4)
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM4
if (features & HWCAP_SM4)
new_cpuid_flags |= CPUID_SM4;
#endif

(void)wolfSSL_Atomic_Uint_CompareExchange
(&cpuid_flags, &old_cpuid_flags, new_cpuid_flags);
Expand All @@ -345,6 +411,9 @@
if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
cpuid_flags_t new_cpuid_flags = 0,
old_cpuid_flags = WC_CPUID_INITIALIZER;
#ifndef WOLFSSL_ARMASM_NO_NEON
new_cpuid_flags |= CPUID_ASIMD;
#endif
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
new_cpuid_flags |= CPUID_AES;
new_cpuid_flags |= CPUID_PMULL;
Expand Down
Loading
Loading