diff options
Diffstat (limited to 'src/crypto/curve25519.c')
| -rw-r--r-- | src/crypto/curve25519.c | 81 |
1 files changed, 54 insertions, 27 deletions
diff --git a/src/crypto/curve25519.c b/src/crypto/curve25519.c index 032f275..f0e045e 100644 --- a/src/crypto/curve25519.c +++ b/src/crypto/curve25519.c @@ -24,7 +24,7 @@ static __always_inline void normalize_secret(u8 secret[CURVE25519_POINT_SIZE]) } static const u8 null_point[CURVE25519_POINT_SIZE] = { 0 }; -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) #include <asm/cpufeature.h> #include <asm/processor.h> #include <asm/fpu/api.h> @@ -110,6 +110,16 @@ static void curve25519_sandy2x_base(u8 pub[CURVE25519_POINT_SIZE], const u8 secr memzero_explicit(x_51, sizeof(x_51)); memzero_explicit(z_51, sizeof(z_51)); } +#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM) +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> +asmlinkage void curve25519_asm_neon(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE]); +static bool curve25519_use_neon __read_mostly = false; +void curve25519_fpu_init(void) +{ + curve25519_use_neon = elf_hwcap & HWCAP_NEON; +} #else void curve25519_fpu_init(void) { } #endif @@ -1326,23 +1336,32 @@ static void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *q) bool curve25519(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE]) { - limb bp[10], x[10], z[11], zmone[10]; - u8 e[32]; +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM) + if (curve25519_use_neon && may_use_simd()) { + kernel_neon_begin(); + curve25519_asm_neon(mypublic, secret, basepoint); + kernel_neon_end(); + } else +#endif + { + limb bp[10], x[10], z[11], zmone[10]; + u8 e[32]; - memcpy(e, secret, 32); - normalize_secret(e); + memcpy(e, secret, 32); + normalize_secret(e); - fexpand(bp, basepoint); - cmult(x, z, e, bp); - crecip(zmone, z); - fmul(z, x, zmone); - fcontract(mypublic, z); + fexpand(bp, basepoint); + cmult(x, z, e, bp); + crecip(zmone, z); + fmul(z, x, zmone); + fcontract(mypublic, z); - memzero_explicit(e, sizeof(e)); - memzero_explicit(bp, sizeof(bp)); - memzero_explicit(x, sizeof(x)); - memzero_explicit(z, sizeof(z)); - memzero_explicit(zmone, sizeof(zmone)); + memzero_explicit(e, sizeof(e)); + memzero_explicit(bp, sizeof(bp)); + memzero_explicit(x, sizeof(x)); + memzero_explicit(z, sizeof(z)); + memzero_explicit(zmone, sizeof(zmone)); + } return crypto_memneq(mypublic, null_point, CURVE25519_POINT_SIZE); } #else @@ -1491,21 +1510,29 @@ static void cmult(struct other_stack *s, limb *resultx, limb *resultz, const u8 bool curve25519(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE]) { - struct other_stack *s = kzalloc(sizeof(struct other_stack), GFP_KERNEL); - if (unlikely(!s)) - return false; - - memcpy(s->ee, secret, 32); - normalize_secret(s->ee); +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM) + if (curve25519_use_neon && may_use_simd()) { + kernel_neon_begin(); + curve25519_asm_neon(mypublic, secret, basepoint); + kernel_neon_end(); + } else +#endif + { + struct other_stack *s = kzalloc(sizeof(struct other_stack), GFP_KERNEL); + if (unlikely(!s)) + return false; - fexpand(s->bp, basepoint); - cmult(s, s->x, s->z, s->ee, s->bp); - crecip(s->zmone, s->z); - fmul(s->z, s->x, s->zmone); - fcontract(mypublic, s->z); + memcpy(s->ee, secret, 32); + normalize_secret(s->ee); - kzfree(s); + fexpand(s->bp, basepoint); + cmult(s, s->x, s->z, s->ee, s->bp); + crecip(s->zmone, s->z); + fmul(s->z, s->x, s->zmone); + fcontract(mypublic, s->z); + kzfree(s); + } return crypto_memneq(mypublic, null_point, CURVE25519_POINT_SIZE); } #endif |