1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
/*
* Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*/
#include <asm/hwcap.h>
#include <asm/neon.h>
#if defined(CONFIG_ARM)
#include <asm/system_info.h>
#include <asm/cputype.h>
#endif
asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len,
const u32 key[8], const u32 counter[4]);
asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]);
asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len,
const u32 key[8], const u32 counter[4]);
static bool chacha20_use_neon __ro_after_init;
static void __init chacha20_fpu_init(void)
{
#if defined(CONFIG_ARM64)
chacha20_use_neon = elf_hwcap & HWCAP_ASIMD;
#elif defined(CONFIG_ARM)
switch (read_cpuid_part()) {
case ARM_CPU_PART_CORTEX_A7:
case ARM_CPU_PART_CORTEX_A5:
/* The Cortex-A7 and Cortex-A5 do not perform well with the NEON
* implementation but do incredibly with the scalar one and use
* less power.
*/
break;
default:
chacha20_use_neon = elf_hwcap & HWCAP_NEON;
}
#endif
}
static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
const u8 *src, size_t len,
simd_context_t *simd_context)
{
/* SIMD disables preemption, so relax after processing each page. */
BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
PAGE_SIZE % CHACHA20_BLOCK_SIZE);
for (;;) {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon &&
len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
const size_t bytes = min_t(size_t, len, PAGE_SIZE);
chacha20_neon(dst, src, bytes, ctx->key, ctx->counter);
ctx->counter[0] += (bytes + 63) / 64;
len -= bytes;
if (!len)
break;
dst += bytes;
src += bytes;
simd_relax(simd_context);
} else {
chacha20_arm(dst, src, len, ctx->key, ctx->counter);
ctx->counter[0] += (len + 63) / 64;
break;
}
}
return true;
}
static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
const u8 nonce[HCHACHA20_NONCE_SIZE],
const u8 key[HCHACHA20_KEY_SIZE],
simd_context_t *simd_context)
{
if (IS_ENABLED(CONFIG_ARM)) {
u32 x[] = { CHACHA20_CONSTANT_EXPA,
CHACHA20_CONSTANT_ND_3,
CHACHA20_CONSTANT_2_BY,
CHACHA20_CONSTANT_TE_K,
get_unaligned_le32(key + 0),
get_unaligned_le32(key + 4),
get_unaligned_le32(key + 8),
get_unaligned_le32(key + 12),
get_unaligned_le32(key + 16),
get_unaligned_le32(key + 20),
get_unaligned_le32(key + 24),
get_unaligned_le32(key + 28),
get_unaligned_le32(nonce + 0),
get_unaligned_le32(nonce + 4),
get_unaligned_le32(nonce + 8),
get_unaligned_le32(nonce + 12)
};
hchacha20_arm(x, derived_key);
return true;
}
return false;
}
|
