diff options
25 files changed, 172 insertions, 54 deletions
diff --git a/src/crypto/include/zinc/blake2s.h b/src/crypto/include/zinc/blake2s.h index 373c09e..9512815 100644 --- a/src/crypto/include/zinc/blake2s.h +++ b/src/crypto/include/zinc/blake2s.h @@ -92,10 +92,4 @@ static inline void blake2s(u8 *out, const u8 *in, const u8 *key, void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, const size_t inlen, const size_t keylen); -void blake2s_fpu_init(void); - -#ifdef DEBUG -bool blake2s_selftest(void); -#endif - #endif /* _ZINC_BLAKE2S_H */ diff --git a/src/crypto/include/zinc/chacha20.h b/src/crypto/include/zinc/chacha20.h index 8b3cc8c..afad694 100644 --- a/src/crypto/include/zinc/chacha20.h +++ b/src/crypto/include/zinc/chacha20.h @@ -25,8 +25,6 @@ struct chacha20_ctx { u32 counter[4]; } __aligned(32); -void chacha20_fpu_init(void); - static inline void chacha20_init(struct chacha20_ctx *state, const u8 key[CHACHA20_KEY_SIZE], const u64 nonce) diff --git a/src/crypto/include/zinc/chacha20poly1305.h b/src/crypto/include/zinc/chacha20poly1305.h index 03979b6..339980d 100644 --- a/src/crypto/include/zinc/chacha20poly1305.h +++ b/src/crypto/include/zinc/chacha20poly1305.h @@ -47,8 +47,4 @@ bool __must_check xchacha20poly1305_decrypt( const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCELEN], const u8 key[CHACHA20POLY1305_KEYLEN]); -#ifdef DEBUG -bool chacha20poly1305_selftest(void); -#endif - #endif /* _ZINC_CHACHA20POLY1305_H */ diff --git a/src/crypto/include/zinc/curve25519.h b/src/crypto/include/zinc/curve25519.h index 79bbf68..5da800f 100644 --- a/src/crypto/include/zinc/curve25519.h +++ b/src/crypto/include/zinc/curve25519.h @@ -19,10 +19,4 @@ void curve25519_generate_secret(u8 secret[CURVE25519_POINT_SIZE]); bool __must_check curve25519_generate_public( u8 pub[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE]); -void curve25519_fpu_init(void); - -#ifdef DEBUG -bool curve25519_selftest(void); -#endif - #endif /* _ZINC_CURVE25519_H */ diff --git a/src/crypto/include/zinc/poly1305.h b/src/crypto/include/zinc/poly1305.h index 1cdbadd..f6ea9dc 100644 --- a/src/crypto/include/zinc/poly1305.h +++ b/src/crypto/include/zinc/poly1305.h @@ -22,16 +22,10 @@ struct poly1305_ctx { size_t num; } __aligned(8); -void poly1305_fpu_init(void); - void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]); void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, simd_context_t *simd_context); void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_context_t *simd_context); -#ifdef DEBUG -bool poly1305_selftest(void); -#endif - #endif /* _ZINC_POLY1305_H */ diff --git a/src/crypto/zinc.h b/src/crypto/zinc.h new file mode 100644 index 0000000..9eab015 --- /dev/null +++ b/src/crypto/zinc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#ifndef _WG_ZINC_H +#define _WG_ZINC_H + +int chacha20_mod_init(void); +int poly1305_mod_init(void); +int chacha20poly1305_mod_init(void); +int blake2s_mod_init(void); +int curve25519_mod_init(void); + +#endif diff --git a/src/crypto/zinc/blake2s/blake2s-x86_64-glue.h b/src/crypto/zinc/blake2s/blake2s-x86_64-glue.h index e7cbef7..92798e2 100644 --- a/src/crypto/zinc/blake2s/blake2s-x86_64-glue.h +++ b/src/crypto/zinc/blake2s/blake2s-x86_64-glue.h @@ -22,7 +22,7 @@ asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, static bool blake2s_use_avx __ro_after_init; static bool blake2s_use_avx512 __ro_after_init; -void __init blake2s_fpu_init(void) +static void __init blake2s_fpu_init(void) { blake2s_use_avx = boot_cpu_has(X86_FEATURE_AVX) && diff --git a/src/crypto/zinc/blake2s/blake2s.c b/src/crypto/zinc/blake2s/blake2s.c index bedff1e..8cbaa6f 100644 --- a/src/crypto/zinc/blake2s/blake2s.c +++ b/src/crypto/zinc/blake2s/blake2s.c @@ -14,6 +14,8 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> #include <linux/bug.h> #include <asm/unaligned.h> @@ -114,9 +116,10 @@ EXPORT_SYMBOL(blake2s_init_key); #if defined(CONFIG_ZINC_ARCH_X86_64) #include "blake2s-x86_64-glue.h" #else -void __init blake2s_fpu_init(void) +static void __init blake2s_fpu_init(void) { } + static inline bool blake2s_arch(struct blake2s_state *state, const u8 *block, const size_t nblocks, const u32 inc) { @@ -274,3 +277,29 @@ void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, EXPORT_SYMBOL(blake2s_hmac); #include "../selftest/blake2s.h" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init blake2s_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + blake2s_fpu_init(); +#ifdef DEBUG + if (!blake2s_selftest()) + return -ENOTRECOVERABLE; +#endif + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); +#endif diff --git a/src/crypto/zinc/chacha20/chacha20-arm-glue.h b/src/crypto/zinc/chacha20/chacha20-arm-glue.h index 1de1a71..4d7eb8c 100644 --- a/src/crypto/zinc/chacha20/chacha20-arm-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-arm-glue.h @@ -17,7 +17,7 @@ asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len, static bool chacha20_use_neon __ro_after_init; -void __init chacha20_fpu_init(void) +static void __init chacha20_fpu_init(void) { #if defined(CONFIG_ARM64) chacha20_use_neon = elf_hwcap & HWCAP_ASIMD; diff --git a/src/crypto/zinc/chacha20/chacha20-mips-glue.h b/src/crypto/zinc/chacha20/chacha20-mips-glue.h index 4bdaea5..518deb5 100644 --- a/src/crypto/zinc/chacha20/chacha20-mips-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-mips-glue.h @@ -5,7 +5,7 @@ asmlinkage void chacha20_mips(u8 *out, const u8 *in, const size_t len, const u32 key[8], const u32 counter[4]); -void __init chacha20_fpu_init(void) +static void __init chacha20_fpu_init(void) { } diff --git a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h index e9a0d1e..7eacde3 100644 --- a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h +++ b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h @@ -30,7 +30,7 @@ static bool chacha20_use_avx2 __ro_after_init; static bool chacha20_use_avx512 __ro_after_init; static bool chacha20_use_avx512vl __ro_after_init; -void __init chacha20_fpu_init(void) +static void __init chacha20_fpu_init(void) { chacha20_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); chacha20_use_avx2 = diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c index a7156a9..2b3644f 100644 --- a/src/crypto/zinc/chacha20/chacha20.c +++ b/src/crypto/zinc/chacha20/chacha20.c @@ -10,6 +10,8 @@ #include <zinc/chacha20.h> #include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> #include <crypto/algapi.h> #if defined(CONFIG_ZINC_ARCH_X86_64) @@ -168,7 +170,26 @@ void hchacha20(u8 derived_key[CHACHA20_KEY_SIZE], if (!hchacha20_arch(derived_key, nonce, key, simd_context)) hchacha20_generic(derived_key, nonce, key); } -/* Deliberately not EXPORT_SYMBOL'd, since there are few reasons why somebody - * should be using this directly, rather than via xchacha20. Revisit only in - * the unlikely event that somebody has a good reason to export this. - */ +EXPORT_SYMBOL(hchacha20); + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + chacha20_fpu_init(); + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); +#endif diff --git a/src/crypto/zinc/chacha20poly1305.c b/src/crypto/zinc/chacha20poly1305.c index 7a8e03f..d5ebade 100644 --- a/src/crypto/zinc/chacha20poly1305.c +++ b/src/crypto/zinc/chacha20poly1305.c @@ -12,6 +12,8 @@ #include <zinc/poly1305.h> #include <asm/unaligned.h> #include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> #include <crypto/scatterwalk.h> static const u8 pad0[16] = { 0 }; @@ -334,3 +336,28 @@ bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, EXPORT_SYMBOL(xchacha20poly1305_decrypt); #include "selftest/chacha20poly1305.h" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ +#ifdef DEBUG + if (!chacha20poly1305_selftest()) + return -ENOTRECOVERABLE; +#endif + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); +#endif diff --git a/src/crypto/zinc/curve25519/curve25519-arm-glue.h b/src/crypto/zinc/curve25519/curve25519-arm-glue.h index 1b39920..a547732 100644 --- a/src/crypto/zinc/curve25519/curve25519-arm-glue.h +++ b/src/crypto/zinc/curve25519/curve25519-arm-glue.h @@ -16,7 +16,7 @@ asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_POINT_SIZE], static bool curve25519_use_neon __ro_after_init; -void __init curve25519_fpu_init(void) +static void __init curve25519_fpu_init(void) { curve25519_use_neon = elf_hwcap & HWCAP_NEON; } diff --git a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.h b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.h index 9159460..1b9b719 100644 --- a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.h +++ b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.h @@ -11,7 +11,7 @@ static bool curve25519_use_bmi2 __ro_after_init; static bool curve25519_use_adx __ro_after_init; -void __init curve25519_fpu_init(void) +static void __init curve25519_fpu_init(void) { curve25519_use_bmi2 = boot_cpu_has(X86_FEATURE_BMI2); curve25519_use_adx = boot_cpu_has(X86_FEATURE_BMI2) && diff --git a/src/crypto/zinc/curve25519/curve25519.c b/src/crypto/zinc/curve25519/curve25519.c index 7d19fb9..fca327f 100644 --- a/src/crypto/zinc/curve25519/curve25519.c +++ b/src/crypto/zinc/curve25519/curve25519.c @@ -15,6 +15,8 @@ #include <linux/version.h> #include <linux/string.h> #include <linux/random.h> +#include <linux/module.h> +#include <linux/init.h> #include <crypto/algapi.h> #if defined(CONFIG_ZINC_ARCH_X86_64) @@ -85,3 +87,29 @@ void curve25519_generate_secret(u8 secret[CURVE25519_POINT_SIZE]) EXPORT_SYMBOL(curve25519_generate_secret); #include "../selftest/curve25519.h" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init curve25519_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + curve25519_fpu_init(); +#ifdef DEBUG + if (!curve25519_selftest()) + return -ENOTRECOVERABLE; +#endif + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); +#endif diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h index 50fb519..dd3fa5a 100644 --- a/src/crypto/zinc/poly1305/poly1305-arm-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h @@ -20,7 +20,7 @@ asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); static bool poly1305_use_neon __ro_after_init; -void __init poly1305_fpu_init(void) +static void __init poly1305_fpu_init(void) { #if defined(CONFIG_ARM64) poly1305_use_neon = elf_hwcap & HWCAP_ASIMD; diff --git a/src/crypto/zinc/poly1305/poly1305-mips-glue.h b/src/crypto/zinc/poly1305/poly1305-mips-glue.h index 6af3f57..2932bb9 100644 --- a/src/crypto/zinc/poly1305/poly1305-mips-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-mips-glue.h @@ -7,7 +7,7 @@ asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, const u32 padbit); asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); -void __init poly1305_fpu_init(void) +static void __init poly1305_fpu_init(void) { } diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h index 63e2a5c..0ca3485 100644 --- a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h @@ -32,7 +32,7 @@ static bool poly1305_use_avx __ro_after_init; static bool poly1305_use_avx2 __ro_after_init; static bool poly1305_use_avx512 __ro_after_init; -void __init poly1305_fpu_init(void) +static void __init poly1305_fpu_init(void) { poly1305_use_avx = boot_cpu_has(X86_FEATURE_AVX) && diff --git a/src/crypto/zinc/poly1305/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c index aad6587..5377721 100644 --- a/src/crypto/zinc/poly1305/poly1305.c +++ b/src/crypto/zinc/poly1305/poly1305.c @@ -12,6 +12,8 @@ #include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/string.h> +#include <linux/module.h> +#include <linux/init.h> #if defined(CONFIG_ZINC_ARCH_X86_64) #include "poly1305-x86_64-glue.h" @@ -133,3 +135,29 @@ void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], EXPORT_SYMBOL(poly1305_final); #include "../selftest/poly1305.h" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + poly1305_fpu_init(); +#ifdef DEBUG + if (!poly1305_selftest()) + return -ENOTRECOVERABLE; +#endif + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Poly1305 one-time MAC"); +MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); +#endif diff --git a/src/crypto/zinc/selftest/blake2s.h b/src/crypto/zinc/selftest/blake2s.h index 61f09b8..67d3114 100644 --- a/src/crypto/zinc/selftest/blake2s.h +++ b/src/crypto/zinc/selftest/blake2s.h @@ -2058,7 +2058,7 @@ static const u8 blake2s_keyed_testvecs[][BLAKE2S_OUTBYTES] __initconst = { 0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd } }; -bool __init blake2s_selftest(void) +static bool __init blake2s_selftest(void) { u8 key[BLAKE2S_KEYBYTES]; u8 buf[ARRAY_SIZE(blake2s_testvecs)]; diff --git a/src/crypto/zinc/selftest/chacha20poly1305.h b/src/crypto/zinc/selftest/chacha20poly1305.h index 9aae3f5..aa375ae 100644 --- a/src/crypto/zinc/selftest/chacha20poly1305.h +++ b/src/crypto/zinc/selftest/chacha20poly1305.h @@ -7695,7 +7695,7 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) enum { MAXIMUM_TEST_BUFFER_LEN = 3000 }; -bool __init chacha20poly1305_selftest(void) +static bool __init chacha20poly1305_selftest(void) { size_t i; u8 computed_result[MAXIMUM_TEST_BUFFER_LEN], *heap_src, *heap_dst; diff --git a/src/crypto/zinc/selftest/curve25519.h b/src/crypto/zinc/selftest/curve25519.h index 2679120..b0cf55a 100644 --- a/src/crypto/zinc/selftest/curve25519.h +++ b/src/crypto/zinc/selftest/curve25519.h @@ -1279,7 +1279,7 @@ static const struct curve25519_test_vector curve25519_test_vectors[] __initconst } }; -bool __init curve25519_selftest(void) +static bool __init curve25519_selftest(void) { bool success = true, ret, ret2; size_t i = 0, j; diff --git a/src/crypto/zinc/selftest/poly1305.h b/src/crypto/zinc/selftest/poly1305.h index 02cd4ba..1439c98 100644 --- a/src/crypto/zinc/selftest/poly1305.h +++ b/src/crypto/zinc/selftest/poly1305.h @@ -818,7 +818,7 @@ static const struct poly1305_testvec poly1305_testvecs[] __initconst = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, } }; -bool __init poly1305_selftest(void) +static bool __init poly1305_selftest(void) { simd_context_t simd_context; bool success = true; @@ -10,12 +10,7 @@ #include "ratelimiter.h" #include "netlink.h" #include "uapi/wireguard.h" - -#include <zinc/chacha20poly1305.h> -#include <zinc/chacha20.h> -#include <zinc/poly1305.h> -#include <zinc/blake2s.h> -#include <zinc/curve25519.h> +#include "crypto/zinc.h" #include <linux/version.h> #include <linux/init.h> @@ -27,14 +22,13 @@ static int __init mod_init(void) { int ret; - chacha20_fpu_init(); - poly1305_fpu_init(); - blake2s_fpu_init(); - curve25519_fpu_init(); + if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) || + (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) || + (ret = curve25519_mod_init())) + return ret; + #ifdef DEBUG if (!allowedips_selftest() || !packet_counter_selftest() || - !curve25519_selftest() || !poly1305_selftest() || - !chacha20poly1305_selftest() || !blake2s_selftest() || !ratelimiter_selftest()) return -ENOTRECOVERABLE; #endif |