diff options
| -rw-r--r-- | src/crypto/zinc/chacha20/chacha20-arm-glue.h | 31 | ||||
| -rw-r--r-- | src/crypto/zinc/chacha20/chacha20-mips-glue.h | 2 | ||||
| -rw-r--r-- | src/crypto/zinc/chacha20/chacha20-x86_64-glue.h | 40 | ||||
| -rw-r--r-- | src/crypto/zinc/chacha20/chacha20.c | 4 |
4 files changed, 54 insertions, 23 deletions
diff --git a/src/crypto/zinc/chacha20/chacha20-arm-glue.h b/src/crypto/zinc/chacha20/chacha20-arm-glue.h index b8d9e12..881e16c 100644 --- a/src/crypto/zinc/chacha20/chacha20-arm-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-arm-glue.h @@ -38,16 +38,33 @@ static void __init chacha20_fpu_init(void) } static inline bool chacha20_arch(struct chacha20_ctx *state, u8 *dst, - const u8 *src, const size_t len, + const u8 *src, size_t len, simd_context_t *simd_context) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon && - len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) - chacha20_neon(dst, src, len, state->key, state->counter); - else - chacha20_arm(dst, src, len, state->key, state->counter); + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + for (;;) { + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon && + len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + chacha20_neon(dst, src, bytes, state->key, state->counter); + state->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } else { + chacha20_arm(dst, src, len, state->key, state->counter); + state->counter[0] += (len + 63) / 64; + break; + } + } - state->counter[0] += (len + 63) / 64; return true; } diff --git a/src/crypto/zinc/chacha20/chacha20-mips-glue.h b/src/crypto/zinc/chacha20/chacha20-mips-glue.h index 13e9e8d..3904b34 100644 --- a/src/crypto/zinc/chacha20/chacha20-mips-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-mips-glue.h @@ -11,7 +11,7 @@ static void __init chacha20_fpu_init(void) } static inline bool chacha20_arch(struct chacha20_ctx *state, u8 *dst, - const u8 *src, const size_t len, + const u8 *src, size_t len, simd_context_t *simd_context) { chacha20_mips((u32 *)state, dst, src, len); diff --git a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h index 03075c9..2939f1e 100644 --- a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h @@ -51,26 +51,40 @@ static void __init chacha20_fpu_init(void) } static inline bool chacha20_arch(struct chacha20_ctx *state, u8 *dst, - const u8 *src, const size_t len, + const u8 *src, size_t len, simd_context_t *simd_context) { + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !chacha20_use_ssse3 || len <= CHACHA20_BLOCK_SIZE || !simd_use(simd_context)) return false; - if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 && - len >= CHACHA20_BLOCK_SIZE * 8) - chacha20_avx512(dst, src, len, state->key, state->counter); - else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl && - len >= CHACHA20_BLOCK_SIZE * 4) - chacha20_avx512vl(dst, src, len, state->key, state->counter); - else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 && - len >= CHACHA20_BLOCK_SIZE * 4) - chacha20_avx2(dst, src, len, state->key, state->counter); - else - chacha20_ssse3(dst, src, len, state->key, state->counter); + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 && + len >= CHACHA20_BLOCK_SIZE * 8) + chacha20_avx512(dst, src, bytes, state->key, state->counter); + else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx512vl(dst, src, bytes, state->key, state->counter); + else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx2(dst, src, bytes, state->key, state->counter); + else + chacha20_ssse3(dst, src, bytes, state->key, state->counter); + state->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } - state->counter[0] += (len + 63) / 64; return true; } diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c index e3fcbe8..c767e02 100644 --- a/src/crypto/zinc/chacha20/chacha20.c +++ b/src/crypto/zinc/chacha20/chacha20.c @@ -25,8 +25,8 @@ void __init chacha20_fpu_init(void) { } -static inline bool chacha20_arch(struct chacha20_ctx *state, u8 *out, - const u8 *in, const size_t len, +static inline bool chacha20_arch(struct chacha20_ctx *state, u8 *dst, + const u8 *src, size_t len, simd_context_t *simd_context) { return false; |