curve25519: do dispatcher in C instead of asm, since shlx is haswell only

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

1 files changed, 73 insertions, 9 deletions

diff --git a/src/crypto/curve25519.c b/src/crypto/curve25519.c
index 539c31a..d9cdd8d 100644
--- a/src/crypto/curve25519.c
+++ b/src/crypto/curve25519.c
@@ -10,6 +10,13 @@
 #include <linux/random.h>
 #include <crypto/algapi.h>
 
+static __always_inline void normalize_secret(u8 secret[CURVE25519_POINT_SIZE])
+{
+	secret[0] &= 248;
+	secret[31] &= 127;
+	secret[31] |= 64;
+}
+
 #ifdef CONFIG_X86_64
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
@@ -20,19 +27,76 @@ void curve25519_fpu_init(void)
 {
 	curve25519_use_avx = boot_cpu_has(X86_FEATURE_AVX);
 }
-asmlinkage void curve25519_sandy2x(u8 *q, const u8 *n, const u8 *p);
-asmlinkage void curve25519_sandy2x_base(u8 *q, const u8 *n);
-#else
-void curve25519_fpu_init(void) { }
-#endif
 
-static __always_inline void normalize_secret(u8 secret[CURVE25519_POINT_SIZE])
+typedef u64 fe[10];
+typedef u64 fe51[5];
+asmlinkage void curve25519_sandy2x_ladder(fe *, const u8 *);
+asmlinkage void curve25519_sandy2x_ladder_base(fe *, const u8 *);
+asmlinkage void curve25519_sandy2x_fe_frombytes(fe, const u8 *);
+asmlinkage void curve25519_sandy2x_fe51_pack(u8 *, const fe51 *);
+asmlinkage void curve25519_sandy2x_fe51_mul(fe51 *, const fe51 *, const fe51 *);
+asmlinkage void curve25519_sandy2x_fe51_invert(fe51 *, const fe51 *);
+
+static void curve25519_sandy2x(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE])
 {
-	secret[0] &= 248;
-	secret[31] &= 127;
-	secret[31] |= 64;
+	u8 e[32];
+	fe var[3];
+	fe51 x_51, z_51;
+	memcpy(e, secret, 32);
+	normalize_secret(e);
+#define x1 var[0]
+#define x2 var[1]
+#define z2 var[2]
+	curve25519_sandy2x_fe_frombytes(x1, basepoint);
+	curve25519_sandy2x_ladder(var, e);
+	z_51[0] = (z2[1] << 26) + z2[0];
+	z_51[1] = (z2[3] << 26) + z2[2];
+	z_51[2] = (z2[5] << 26) + z2[4];
+	z_51[3] = (z2[7] << 26) + z2[6];
+	z_51[4] = (z2[9] << 26) + z2[8];
+	x_51[0] = (x2[1] << 26) + x2[0];
+	x_51[1] = (x2[3] << 26) + x2[2];
+	x_51[2] = (x2[5] << 26) + x2[4];
+	x_51[3] = (x2[7] << 26) + x2[6];
+	x_51[4] = (x2[9] << 26) + x2[8];
+#undef x1
+#undef x2
+#undef z2
+	curve25519_sandy2x_fe51_invert(&z_51, &z_51);
+	curve25519_sandy2x_fe51_mul(&x_51, &x_51, &z_51);
+	curve25519_sandy2x_fe51_pack(mypublic, &x_51);
 }
 
+static void curve25519_sandy2x_base(u8 pub[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE])
+{
+	u8 e[32];
+	fe var[3];
+	fe51 x_51, z_51;
+	memcpy(e, secret, 32);
+	normalize_secret(e);
+	curve25519_sandy2x_ladder_base(var, e);
+#define x2 var[0]
+#define z2 var[1]
+	z_51[0] = (z2[1] << 26) + z2[0];
+	z_51[1] = (z2[3] << 26) + z2[2];
+	z_51[2] = (z2[5] << 26) + z2[4];
+	z_51[3] = (z2[7] << 26) + z2[6];
+	z_51[4] = (z2[9] << 26) + z2[8];
+	x_51[0] = (x2[1] << 26) + x2[0];
+	x_51[1] = (x2[3] << 26) + x2[2];
+	x_51[2] = (x2[5] << 26) + x2[4];
+	x_51[3] = (x2[7] << 26) + x2[6];
+	x_51[4] = (x2[9] << 26) + x2[8];
+#undef x2
+#undef z2
+	curve25519_sandy2x_fe51_invert(&z_51, &z_51);
+	curve25519_sandy2x_fe51_mul(&x_51, &x_51, &z_51);
+	curve25519_sandy2x_fe51_pack(pub, &x_51);
+}
+#else
+void curve25519_fpu_init(void) { }
+#endif
+
 #ifdef __SIZEOF_INT128__
 typedef u64 limb;
 typedef limb felem[5];