data: keep FPU on when possible

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

2 files changed, 41 insertions, 52 deletions

diff --git a/src/crypto/chacha20poly1305.c b/src/crypto/chacha20poly1305.c
index 944710c..cc48a9e 100644
--- a/src/crypto/chacha20poly1305.c
+++ b/src/crypto/chacha20poly1305.c
@@ -14,12 +14,6 @@
 #ifdef CONFIG_X86_64
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
-#include <asm/fpu/api.h>
-#include <asm/simd.h>
-#else
-#include <asm/i387.h>
-#endif
 #ifdef CONFIG_AS_SSSE3
 asmlinkage void chacha20_asm_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
 asmlinkage void chacha20_asm_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
@@ -532,13 +526,7 @@ bool chacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src, const size_t src
 	uint8_t block0[CHACHA20_BLOCK_SIZE] = { 0 };
 	__le64 len;
 	__le64 le_nonce = cpu_to_le64(nonce);
-	bool have_simd = false;
-
-#ifdef CONFIG_X86_64
-	have_simd = irq_fpu_usable();
-	if (have_simd)
-		kernel_fpu_begin();
-#endif
+	bool have_simd = chacha20poly1305_init_simd();
 
 	chacha20_keysetup(&chacha20_state, key, (uint8_t *)&le_nonce);
 
@@ -565,17 +553,15 @@ bool chacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src, const size_t src
 	memzero_explicit(&poly1305_state, sizeof(poly1305_state));
 	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
 
-#ifdef CONFIG_X86_64
-	if (have_simd)
-		kernel_fpu_end();
-#endif
+	chacha20poly1305_deinit_simd(have_simd);
 
 	return true;
 }
 
 bool chacha20poly1305_encrypt_sg(struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
 				 const uint8_t *ad, const size_t ad_len,
-				 const uint64_t nonce, const uint8_t key[static CHACHA20POLY1305_KEYLEN])
+				 const uint64_t nonce, const uint8_t key[static CHACHA20POLY1305_KEYLEN],
+				 bool have_simd)
 {
 	struct poly1305_ctx poly1305_state;
 	struct chacha20_ctx chacha20_state;
@@ -584,13 +570,6 @@ bool chacha20poly1305_encrypt_sg(struct scatterlist *dst, struct scatterlist *sr
 	uint8_t mac[POLY1305_MAC_SIZE];
 	__le64 len;
 	__le64 le_nonce = cpu_to_le64(nonce);
-	bool have_simd = false;
-
-#ifdef CONFIG_X86_64
-	have_simd = irq_fpu_usable();
-	if (have_simd)
-		kernel_fpu_begin();
-#endif
 
 	chacha20_keysetup(&chacha20_state, key, (uint8_t *)&le_nonce);
 
@@ -630,12 +609,6 @@ bool chacha20poly1305_encrypt_sg(struct scatterlist *dst, struct scatterlist *sr
 	memzero_explicit(&poly1305_state, sizeof(poly1305_state));
 	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
 	memzero_explicit(mac, sizeof(mac));
-
-#ifdef CONFIG_X86_64
-	if (have_simd)
-		kernel_fpu_end();
-#endif
-
 	return true;
 }
 
@@ -651,16 +624,12 @@ bool chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src
 	size_t dst_len;
 	__le64 len;
 	__le64 le_nonce = cpu_to_le64(nonce);
-	bool have_simd = false;
+	bool have_simd;
 
 	if (unlikely(src_len < POLY1305_MAC_SIZE))
 		return false;
 
-#ifdef CONFIG_X86_64
-	have_simd = irq_fpu_usable();
-	if (have_simd)
-		kernel_fpu_begin();
-#endif
+	have_simd = chacha20poly1305_init_simd();
 
 	chacha20_keysetup(&chacha20_state, key, (uint8_t *)&le_nonce);
 
@@ -690,10 +659,8 @@ bool chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src
 		chacha20_crypt(&chacha20_state, dst, src, dst_len, have_simd);
 
 	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
-#ifdef CONFIG_X86_64
-	if (have_simd)
-		kernel_fpu_end();
-#endif
+
+	chacha20poly1305_deinit_simd(have_simd);
 	return !ret;
 }
 
@@ -710,16 +677,12 @@ bool chacha20poly1305_decrypt_sg(struct scatterlist *dst, struct scatterlist *sr
 	size_t dst_len;
 	__le64 len;
 	__le64 le_nonce = cpu_to_le64(nonce);
-	bool have_simd = false;
+	bool have_simd;
 
 	if (unlikely(src_len < POLY1305_MAC_SIZE))
 		return false;
 
-#ifdef CONFIG_X86_64
-	have_simd = irq_fpu_usable();
-	if (have_simd)
-		kernel_fpu_begin();
-#endif
+	have_simd = chacha20poly1305_init_simd();
 
 	chacha20_keysetup(&chacha20_state, key, (uint8_t *)&le_nonce);
 
@@ -763,10 +726,7 @@ bool chacha20poly1305_decrypt_sg(struct scatterlist *dst, struct scatterlist *sr
 	memzero_explicit(read_mac, POLY1305_MAC_SIZE);
 	memzero_explicit(computed_mac, POLY1305_MAC_SIZE);
 	memzero_explicit(&chacha20_state, sizeof(chacha20_state));
-#ifdef CONFIG_X86_64
-	if (have_simd)
-		kernel_fpu_end();
-#endif
+	chacha20poly1305_deinit_simd(have_simd);
 	return !ret;
 }
 
diff --git a/src/crypto/chacha20poly1305.h b/src/crypto/chacha20poly1305.h
index e9ecaf8..57d3d23 100644
--- a/src/crypto/chacha20poly1305.h
+++ b/src/crypto/chacha20poly1305.h
@@ -20,7 +20,8 @@ bool chacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src, const size_t src
 
 bool chacha20poly1305_encrypt_sg(struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
 				 const uint8_t *ad, const size_t ad_len,
-				 const uint64_t nonce, const uint8_t key[static CHACHA20POLY1305_KEYLEN]);
+				 const uint64_t nonce, const uint8_t key[static CHACHA20POLY1305_KEYLEN],
+				 bool have_simd);
 
 bool chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
 			      const uint8_t *ad, const size_t ad_len,
@@ -30,6 +31,34 @@ bool chacha20poly1305_decrypt_sg(struct scatterlist *dst, struct scatterlist *sr
 				 const uint8_t *ad, const size_t ad_len,
 				 const uint64_t nonce, const uint8_t key[static CHACHA20POLY1305_KEYLEN]);
 
+#ifdef CONFIG_X86_64
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
+#include <asm/fpu/api.h>
+#include <asm/simd.h>
+#else
+#include <asm/i387.h>
+#endif
+#endif
+
+static inline bool chacha20poly1305_init_simd(void)
+{
+	bool have_simd = false;
+#ifdef CONFIG_X86_64
+	have_simd = irq_fpu_usable();
+	if (have_simd)
+		kernel_fpu_begin();
+#endif
+	return have_simd;
+}
+
+static inline void chacha20poly1305_deinit_simd(bool was_on)
+{
+#ifdef CONFIG_X86_64
+	if (was_on)
+		kernel_fpu_end();
+#endif
+}
+
 #ifdef DEBUG
 bool chacha20poly1305_selftest(void);
 #endif