curve25519: not all linkers support bmi2 and adx

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

2 files changed, 48 insertions, 6 deletions

diff --git a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c
index 19c86c6..d62bd37 100644
--- a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c
+++ b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c
@@ -15,8 +15,10 @@ static bool *const curve25519_nobs[] __initconst = {
 
 static void __init curve25519_fpu_init(void)
 {
-	curve25519_use_bmi2 = boot_cpu_has(X86_FEATURE_BMI2);
-	curve25519_use_adx = boot_cpu_has(X86_FEATURE_BMI2) &&
+	curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) &&
+			      boot_cpu_has(X86_FEATURE_BMI2);
+	curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) &&
+			     boot_cpu_has(X86_FEATURE_BMI2) &&
 			     boot_cpu_has(X86_FEATURE_ADX);
 }
 
@@ -24,10 +26,10 @@ static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
 				   const u8 secret[CURVE25519_KEY_SIZE],
 				   const u8 basepoint[CURVE25519_KEY_SIZE])
 {
-	if (curve25519_use_adx) {
+	if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) {
 		curve25519_adx(mypublic, secret, basepoint);
 		return true;
-	} else if (curve25519_use_bmi2) {
+	} else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) {
 		curve25519_bmi2(mypublic, secret, basepoint);
 		return true;
 	}
@@ -37,10 +39,10 @@ static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
 static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
 					const u8 secret[CURVE25519_KEY_SIZE])
 {
-	if (curve25519_use_adx) {
+	if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) {
 		curve25519_adx_base(pub, secret);
 		return true;
-	} else if (curve25519_use_bmi2) {
+	} else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) {
 		curve25519_bmi2_base(pub, secret);
 		return true;
 	}
diff --git a/src/crypto/zinc/curve25519/curve25519-x86_64.c b/src/crypto/zinc/curve25519/curve25519-x86_64.c
index 3d1806f..e8af3e2 100644
--- a/src/crypto/zinc/curve25519/curve25519-x86_64.c
+++ b/src/crypto/zinc/curve25519/curve25519-x86_64.c
@@ -582,6 +582,7 @@ __aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = {
 		  0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL
 };
 
+#ifdef CONFIG_AS_ADX
 /* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7]
  * a is two 256-bit integers: a0[0:3] and a1[4:7]
  * b is two 256-bit integers: b0[0:3] and b1[4:7]
@@ -736,7 +737,9 @@ static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a,
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11", "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a,
 				      const u64 *const b)
 {
@@ -885,7 +888,9 @@ static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a,
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11", "%r13", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1010,7 +1015,9 @@ static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11", "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1136,7 +1143,9 @@ static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
 		  "%r11", "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1201,7 +1210,9 @@ static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1264,7 +1275,9 @@ static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
 		  "%r11");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static void mul_256x256_integer_adx(u64 *const c, const u64 *const a,
 				    const u64 *const b)
 {
@@ -1354,7 +1367,9 @@ static void mul_256x256_integer_adx(u64 *const c, const u64 *const a,
 		: "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11",
 		  "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a,
 				     const u64 *const b)
 {
@@ -1433,7 +1448,9 @@ static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a,
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11", "%r13", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1499,7 +1516,9 @@ static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11", "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1566,7 +1585,9 @@ static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
 		  "%r11", "%r13", "%r14", "%r15");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1603,7 +1624,9 @@ static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9",
 		  "%r10", "%r11");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
 {
 	asm volatile(
@@ -1639,7 +1662,9 @@ static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
 		: "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10",
 		  "%r11");
 }
+#endif
 
+#ifdef CONFIG_AS_ADX
 static __always_inline void
 add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b)
 {
@@ -1671,7 +1696,9 @@ add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b)
 		: "r"(c), "r"(a), "r"(b)
 		: "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11");
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static __always_inline void
 add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b)
 {
@@ -1702,6 +1729,7 @@ add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b)
 		: "r"(c), "r"(a), "r"(b)
 		: "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11");
 }
+#endif
 
 static __always_inline void
 sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b)
@@ -1769,6 +1797,7 @@ mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a)
 		  "%r11");
 }
 
+#ifdef CONFIG_AS_ADX
 static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
 {
 	struct {
@@ -1815,7 +1844,9 @@ static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a)
 
 	memzero_explicit(&m, sizeof(m));
 }
+#endif
 
+#ifdef CONFIG_AS_BMI2
 static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
 {
 	struct {
@@ -1862,6 +1893,7 @@ static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a)
 
 	memzero_explicit(&m, sizeof(m));
 }
+#endif
 
 /* Given c, a 256-bit number, fred_eltfp25519_1w updates c
  * with a number such that 0 <= C < 2**255-19.
@@ -1939,6 +1971,7 @@ static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE],
 			   const u8 private_key[CURVE25519_KEY_SIZE],
 			   const u8 session_key[CURVE25519_KEY_SIZE])
 {
+#ifdef CONFIG_AS_ADX
 	struct {
 		u64 buffer[4 * NUM_WORDS_ELTFP25519];
 		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
@@ -2034,11 +2067,13 @@ static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE],
 	fred_eltfp25519_1w((u64 *)shared);
 
 	memzero_explicit(&m, sizeof(m));
+#endif
 }
 
 static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE],
 				const u8 private_key[CURVE25519_KEY_SIZE])
 {
+#ifdef CONFIG_AS_ADX
 	struct {
 		u64 buffer[4 * NUM_WORDS_ELTFP25519];
 		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
@@ -2128,12 +2163,14 @@ static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE],
 	fred_eltfp25519_1w((u64 *)session_key);
 
 	memzero_explicit(&m, sizeof(m));
+#endif
 }
 
 static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE],
 			    const u8 private_key[CURVE25519_KEY_SIZE],
 			    const u8 session_key[CURVE25519_KEY_SIZE])
 {
+#ifdef CONFIG_AS_BMI2
 	struct {
 		u64 buffer[4 * NUM_WORDS_ELTFP25519];
 		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
@@ -2229,11 +2266,13 @@ static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE],
 	fred_eltfp25519_1w((u64 *)shared);
 
 	memzero_explicit(&m, sizeof(m));
+#endif
 }
 
 static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE],
 				 const u8 private_key[CURVE25519_KEY_SIZE])
 {
+#ifdef CONFIG_AS_BMI2
 	struct {
 		u64 buffer[4 * NUM_WORDS_ELTFP25519];
 		u64 coordinates[4 * NUM_WORDS_ELTFP25519];
@@ -2323,4 +2362,5 @@ static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE],
 	fred_eltfp25519_1w((u64 *)session_key);
 
 	memzero_explicit(&m, sizeof(m));
+#endif
 }