diff options
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-arm-glue.h | 41 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-mips-glue.h | 2 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-x86_64-glue.h | 33 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305.c | 2 |
4 files changed, 49 insertions, 29 deletions
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h index 15ad53f..ca2ed5a 100644 --- a/src/crypto/zinc/poly1305/poly1305-arm-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h @@ -55,7 +55,7 @@ static void convert_to_base2_64(void *ctx) struct poly1305_arch_internal *state = ctx; u32 cy; - if (!state->is_base2_26) + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26) return; cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; @@ -87,18 +87,30 @@ static inline bool poly1305_init_arch(void *ctx, } static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, - const size_t len, const u32 padbit, + size_t len, const u32 padbit, simd_context_t *simd_context) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON)) { - if (poly1305_use_neon && simd_use(simd_context)) { - poly1305_blocks_neon(ctx, inp, len, padbit); - return true; - } + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { convert_to_base2_64(ctx); + poly1305_blocks_arm(ctx, inp, len, padbit); + return true; } - poly1305_blocks_arm(ctx, inp, len, padbit); + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + poly1305_blocks_neon(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } return true; } @@ -106,14 +118,11 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4], simd_context_t *simd_context) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON)) { - if (poly1305_use_neon && simd_use(simd_context)) { - poly1305_emit_neon(ctx, mac, nonce); - return true; - } + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { convert_to_base2_64(ctx); - } - - poly1305_emit_arm(ctx, mac, nonce); + poly1305_emit_arm(ctx, mac, nonce); + } else + poly1305_emit_neon(ctx, mac, nonce); return true; } diff --git a/src/crypto/zinc/poly1305/poly1305-mips-glue.h b/src/crypto/zinc/poly1305/poly1305-mips-glue.h index 4a0badb..f71c3f6 100644 --- a/src/crypto/zinc/poly1305/poly1305-mips-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-mips-glue.h @@ -20,7 +20,7 @@ static inline bool poly1305_init_arch(void *ctx, } static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, - const size_t len, const u32 padbit, + size_t len, const u32 padbit, simd_context_t *simd_context) { poly1305_blocks_mips(ctx, inp, len, padbit); diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h index 285cb31..b1248e8 100644 --- a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h @@ -93,11 +93,15 @@ static void convert_to_base2_64(void *ctx) } static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, - const size_t len, const u32 padbit, + size_t len, const u32 padbit, simd_context_t *simd_context) { struct poly1305_arch_internal *state = ctx; + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || !simd_use(simd_context)) { @@ -106,12 +110,21 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, return true; } - if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512) - poly1305_blocks_avx512(ctx, inp, len, padbit); - else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2) - poly1305_blocks_avx2(ctx, inp, len, padbit); - else - poly1305_blocks_avx(ctx, inp, len, padbit); + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } return true; } @@ -126,9 +139,7 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], !state->is_base2_26 || !simd_use(simd_context)) { convert_to_base2_64(ctx); poly1305_emit_x86_64(ctx, mac, nonce); - return true; - } - - poly1305_emit_avx(ctx, mac, nonce); + } else + poly1305_emit_avx(ctx, mac, nonce); return true; } diff --git a/src/crypto/zinc/poly1305/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c index d094ca7..ad758b5 100644 --- a/src/crypto/zinc/poly1305/poly1305.c +++ b/src/crypto/zinc/poly1305/poly1305.c @@ -28,7 +28,7 @@ static inline bool poly1305_init_arch(void *ctx, return false; } static inline bool poly1305_blocks_arch(void *ctx, const u8 *input, - const size_t len, const u32 padbit, + size_t len, const u32 padbit, simd_context_t *simd_context) { return false; |