/* SPDX-License-Identifier: OpenSSL OR (BSD-3-Clause OR GPL-2.0) * * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. */ #include <linux/linkage.h> .text #if defined(__thumb2__) .syntax unified .thumb #else .code 32 #endif .align 5 ENTRY(poly1305_init_arm) stmdb sp!,{r4-r11} eor r3,r3,r3 cmp r1,#0 str r3,[r0,#0] @ zero hash value str r3,[r0,#4] str r3,[r0,#8] str r3,[r0,#12] str r3,[r0,#16] str r3,[r0,#36] @ is_base2_26 add r0,r0,#20 #ifdef __thumb2__ it eq #endif moveq r0,#0 beq .Lno_key ldrb r4,[r1,#0] mov r10,#0x0fffffff ldrb r5,[r1,#1] and r3,r10,#-4 @ 0x0ffffffc ldrb r6,[r1,#2] ldrb r7,[r1,#3] orr r4,r4,r5,lsl#8 ldrb r5,[r1,#4] orr r4,r4,r6,lsl#16 ldrb r6,[r1,#5] orr r4,r4,r7,lsl#24 ldrb r7,[r1,#6] and r4,r4,r10 ldrb r8,[r1,#7] orr r5,r5,r6,lsl#8 ldrb r6,[r1,#8] orr r5,r5,r7,lsl#16 ldrb r7,[r1,#9] orr r5,r5,r8,lsl#24 ldrb r8,[r1,#10] and r5,r5,r3 ldrb r9,[r1,#11] orr r6,r6,r7,lsl#8 ldrb r7,[r1,#12] orr r6,r6,r8,lsl#16 ldrb r8,[r1,#13] orr r6,r6,r9,lsl#24 ldrb r9,[r1,#14] and r6,r6,r3 ldrb r10,[r1,#15] orr r7,r7,r8,lsl#8 str r4,[r0,#0] orr r7,r7,r9,lsl#16 str r5,[r0,#4] orr r7,r7,r10,lsl#24 str r6,[r0,#8] and r7,r7,r3 str r7,[r0,#12] .Lno_key: ldmia sp!,{r4-r11} #if __LINUX_ARM_ARCH__ >= 5 bx lr @ bx lr #else tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet .word 0xe12fff1e @ interoperable with Thumb ISA:-) #endif ENDPROC(poly1305_init_arm) .align 5 ENTRY(poly1305_blocks_arm) .Lpoly1305_blocks_arm: stmdb sp!,{r3-r11,lr} ands r2,r2,#-16 beq .Lno_data cmp r3,#0 add r2,r2,r1 @ end pointer sub sp,sp,#32 ldmia r0,{r4-r12} @ load context str r0,[sp,#12] @ offload stuff mov lr,r1 str r2,[sp,#16] str r10,[sp,#20] str r11,[sp,#24] str r12,[sp,#28] b .Loop .Loop: #if __LINUX_ARM_ARCH__ < 7 ldrb r0,[lr],#16 @ load input #ifdef __thumb2__ it hi #endif addhi r8,r8,#1 @ 1<<128 ldrb r1,[lr,#-15] ldrb r2,[lr,#-14] ldrb r3,[lr,#-13] orr r1,r0,r1,lsl#8 ldrb r0,[lr,#-12] orr r2,r1,r2,lsl#16 ldrb r1,[lr,#-11] orr r3,r2,r3,lsl#24 ldrb r2,[lr,#-10] adds r4,r4,r3 @ accumulate input ldrb r3,[lr,#-9] orr r1,r0,r1,lsl#8 ldrb r0,[lr,#-8] orr r2,r1,r2,lsl#16 ldrb r1,[lr,#-7] orr r3,r2,r3,lsl#24 ldrb r2,[lr,#-6] adcs r5,r5,r3 ldrb r3,[lr,#-5] orr r1,r0,r1,lsl#8 ldrb r0,[lr,#-4] orr r2,r1,r2,lsl#16 ldrb r1,[lr,#-3] orr r3,r2,r3,lsl#24 ldrb r2,[lr,#-2] adcs r6,r6,r3 ldrb r3,[lr,#-1] orr r1,r0,r1,lsl#8 str lr,[sp,#8] @ offload input pointer orr r2,r1,r2,lsl#16 add r10,r10,r10,lsr#2 orr r3,r2,r3,lsl#24 #else ldr r0,[lr],#16 @ load input #ifdef __thumb2__ it hi #endif addhi r8,r8,#1 @ padbit ldr r1,[lr,#-12] ldr r2,[lr,#-8] ldr r3,[lr,#-4] #ifdef __ARMEB__ rev r0,r0 rev r1,r1 rev r2,r2 rev r3,r3 #endif adds r4,r4,r0 @ accumulate input str lr,[sp,#8] @ offload input pointer adcs r5,r5,r1 add r10,r10,r10,lsr#2 adcs r6,r6,r2 #endif add r11,r11,r11,lsr#2 adcs r7,r7,r3 add r12,r12,r12,lsr#2 umull r2,r3,r5,r9 adc r8,r8,#0 umull r0,r1,r4,r9 umlal r2,r3,r8,r10 umlal r0,r1,r7,r10 ldr r10,[sp,#20] @ reload r10 umlal r2,r3,r6,r12 umlal r0,r1,r5,r12 umlal r2,r3,r7,r11 umlal r0,r1,r6,r11 umlal r2,r3,r4,r10 str r0,[sp,#0] @ future r4 mul r0,r11,r8 ldr r11,[sp,#24] @ reload r11 adds r2,r2,r1 @ d1+=d0>>32 eor r1,r1,r1 adc lr,r3,#0 @ future r6 str r2,[sp,#4] @ future r5 mul r2,r12,r8 eor r3,r3,r3 umlal r0,r1,r7,r12 ldr r12,[sp,#28] @ reload r12 umlal r2,r3,r7,r9 umlal r0,r1,r6,r9 umlal r2,r3,r6,r10 umlal r0,r1,r5,r10 umlal r2,r3,r5,r11 umlal r0,r1,r4,r11 umlal r2,r3,r4,r12 ldr r4,[sp,#0] mul r8,r9,r8 ldr r5,[sp,#4] adds r6,lr,r0 @ d2+=d1>>32 ldr lr,[sp,#8] @ reload input pointer adc r1,r1,#0 adds r7,r2,r1 @ d3+=d2>>32 ldr r0,[sp,#16] @ reload end pointer adc r3,r3,#0 add r8,r8,r3 @ h4+=d3>>32 and r1,r8,#-4 and r8,r8,#3 add r1,r1,r1,lsr#2 @ *=5 adds r4,r4,r1 adcs r5,r5,#0 adcs r6,r6,#0 adcs r7,r7,#0 adc r8,r8,#0 cmp r0,lr @ done yet? bhi .Loop ldr r0,[sp,#12] add sp,sp,#32 stmia r0,{r4-r8} @ store the result .Lno_data: #if __LINUX_ARM_ARCH__ >= 5 ldmia sp!,{r3-r11,pc} #else ldmia sp!,{r3-r11,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet .word 0xe12fff1e @ interoperable with Thumb ISA:-) #endif ENDPROC(poly1305_blocks_arm) .align 5 ENTRY(poly1305_emit_arm) stmdb sp!,{r4-r11} .Lpoly1305_emit_enter: ldmia r0,{r3-r7} adds r8,r3,#5 @ compare to modulus adcs r9,r4,#0 adcs r10,r5,#0 adcs r11,r6,#0 adc r7,r7,#0 tst r7,#4 @ did it carry/borrow? #ifdef __thumb2__ it ne #endif movne r3,r8 ldr r8,[r2,#0] #ifdef __thumb2__ it ne #endif movne r4,r9 ldr r9,[r2,#4] #ifdef __thumb2__ it ne #endif movne r5,r10 ldr r10,[r2,#8] #ifdef __thumb2__ it ne #endif movne r6,r11 ldr r11,[r2,#12] adds r3,r3,r8 adcs r4,r4,r9 adcs r5,r5,r10 adc r6,r6,r11 #if __LINUX_ARM_ARCH__ >= 7 #ifdef __ARMEB__ rev r3,r3 rev r4,r4 rev r5,r5 rev r6,r6 #endif str r3,[r1,#0] str r4,[r1,#4] str r5,[r1,#8] str r6,[r1,#12] #else strb r3,[r1,#0] mov r3,r3,lsr#8 strb r4,[r1,#4] mov r4,r4,lsr#8 strb r5,[r1,#8] mov r5,r5,lsr#8 strb r6,[r1,#12] mov r6,r6,lsr#8 strb r3,[r1,#1] mov r3,r3,lsr#8 strb r4,[r1,#5] mov r4,r4,lsr#8 strb r5,[r1,#9] mov r5,r5,lsr#8 strb r6,[r1,#13] mov r6,r6,lsr#8 strb r3,[r1,#2] mov r3,r3,lsr#8 strb r4,[r1,#6] mov r4,r4,lsr#8 strb r5,[r1,#10] mov r5,r5,lsr#8 strb r6,[r1,#14] mov r6,r6,lsr#8 strb r3,[r1,#3] strb r4,[r1,#7] strb r5,[r1,#11] strb r6,[r1,#15] #endif ldmia sp!,{r4-r11} #if __LINUX_ARM_ARCH__ >= 5 bx lr @ bx lr #else tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet .word 0xe12fff1e @ interoperable with Thumb ISA:-) #endif ENDPROC(poly1305_emit_arm) #if __LINUX_ARM_ARCH__ >= 7 .fpu neon .align 5 ENTRY(poly1305_init_neon) .Lpoly1305_init_neon: ldr r4,[r0,#20] @ load key base 2^32 ldr r5,[r0,#24] ldr r6,[r0,#28] ldr r7,[r0,#32] and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 mov r3,r4,lsr#26 mov r4,r5,lsr#20 orr r3,r3,r5,lsl#6 mov r5,r6,lsr#14 orr r4,r4,r6,lsl#12 mov r6,r7,lsr#8 orr r5,r5,r7,lsl#18 and r3,r3,#0x03ffffff and r4,r4,#0x03ffffff and r5,r5,#0x03ffffff vdup.32 d0,r2 @ r^1 in both lanes add r2,r3,r3,lsl#2 @ *5 vdup.32 d1,r3 add r3,r4,r4,lsl#2 vdup.32 d2,r2 vdup.32 d3,r4 add r4,r5,r5,lsl#2 vdup.32 d4,r3 vdup.32 d5,r5 add r5,r6,r6,lsl#2 vdup.32 d6,r4 vdup.32 d7,r6 vdup.32 d8,r5 mov r5,#2 @ counter .Lsquare_neon: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 vmull.u32 q5,d0,d0[1] vmull.u32 q6,d1,d0[1] vmull.u32 q7,d3,d0[1] vmull.u32 q8,d5,d0[1] vmull.u32 q9,d7,d0[1] vmlal.u32 q5,d7,d2[1] vmlal.u32 q6,d0,d1[1] vmlal.u32 q7,d1,d1[1] vmlal.u32 q8,d3,d1[1] vmlal.u32 q9,d5,d1[1] vmlal.u32 q5,d5,d4[1] vmlal.u32 q6,d7,d4[1] vmlal.u32 q8,d1,d3[1] vmlal.u32 q7,d0,d3[1] vmlal.u32 q9,d3,d3[1] vmlal.u32 q5,d3,d6[1] vmlal.u32 q8,d0,d5[1] vmlal.u32 q6,d5,d6[1] vmlal.u32 q7,d7,d6[1] vmlal.u32 q9,d1,d5[1] vmlal.u32 q8,d7,d8[1] vmlal.u32 q5,d1,d8[1] vmlal.u32 q6,d3,d8[1] vmlal.u32 q7,d5,d8[1] vmlal.u32 q9,d0,d7[1] @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein @ and P. Schwabe @ @ H0>>+H1>>+H2>>+H3>>+H4 @ H3>>+H4>>*5+H0>>+H1 @ @ Trivia. @ @ Result of multiplication of n-bit number by m-bit number is @ n+m bits wide. However! Even though 2^n is a n+1-bit number, @ m-bit number multiplied by 2^n is still n+m bits wide. @ @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2, @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit @ one is n+1 bits wide. @ @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4 @ can be 27. However! In cases when their width exceeds 26 bits @ they are limited by 2^26+2^6. This in turn means that *sum* @ of the products with these values can still be viewed as sum @ of 52-bit numbers as long as the amount of addends is not a @ power of 2. For example, @ @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4, @ @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than @ 8 * (2^52) or 2^55. However, the value is then multiplied by @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12), @ which is less than 32 * (2^52) or 2^57. And when processing @ data we are looking at triple as many addends... @ @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32 @ instruction accepts 2x32-bit input and writes 2x64-bit result. @ This means that result of reduction have to be compressed upon @ loop wrap-around. This can be done in the process of reduction @ to minimize amount of instructions [as well as amount of @ 128-bit instructions, which benefits low-end processors], but @ one has to watch for H2 (which is narrower than H0) and 5*H4 @ not being wider than 58 bits, so that result of right shift @ by 26 bits fits in 32 bits. This is also useful on x86, @ because it allows to use paddd in place for paddq, which @ benefits Atom, where paddq is ridiculously slow. vshr.u64 q15,q8,#26 vmovn.i64 d16,q8 vshr.u64 q4,q5,#26 vmovn.i64 d10,q5 vadd.i64 q9,q9,q15 @ h3 -> h4 vbic.i32 d16,#0xfc000000 @ &=0x03ffffff vadd.i64 q6,q6,q4 @ h0 -> h1 vbic.i32 d10,#0xfc000000 vshrn.u64 d30,q9,#26 vmovn.i64 d18,q9 vshr.u64 q4,q6,#26 vmovn.i64 d12,q6 vadd.i64 q7,q7,q4 @ h1 -> h2 vbic.i32 d18,#0xfc000000 vbic.i32 d12,#0xfc000000 vadd.i32 d10,d10,d30 vshl.u32 d30,d30,#2 vshrn.u64 d8,q7,#26 vmovn.i64 d14,q7 vadd.i32 d10,d10,d30 @ h4 -> h0 vadd.i32 d16,d16,d8 @ h2 -> h3 vbic.i32 d14,#0xfc000000 vshr.u32 d30,d10,#26 vbic.i32 d10,#0xfc000000 vshr.u32 d8,d16,#26 vbic.i32 d16,#0xfc000000 vadd.i32 d12,d12,d30 @ h0 -> h1 vadd.i32 d18,d18,d8 @ h3 -> h4 subs r5,r5,#1 beq .Lsquare_break_neon add r6,r0,#(48+0*9*4) add r7,r0,#(48+1*9*4) vtrn.32 d0,d10 @ r^2:r^1 vtrn.32 d3,d14 vtrn.32 d5,d16 vtrn.32 d1,d12 vtrn.32 d7,d18 vshl.u32 d4,d3,#2 @ *5 vshl.u32 d6,d5,#2 vshl.u32 d2,d1,#2 vshl.u32 d8,d7,#2 vadd.i32 d4,d4,d3 vadd.i32 d2,d2,d1 vadd.i32 d6,d6,d5 vadd.i32 d8,d8,d7 vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! vst1.32 {d8[0]},[r6,:32] vst1.32 {d8[1]},[r7,:32] b .Lsquare_neon .align 4 .Lsquare_break_neon: add r6,r0,#(48+2*4*9) add r7,r0,#(48+3*4*9) vmov d0,d10 @ r^4:r^3 vshl.u32 d2,d12,#2 @ *5 vmov d1,d12 vshl.u32 d4,d14,#2 vmov d3,d14 vshl.u32 d6,d16,#2 vmov d5,d16 vshl.u32 d8,d18,#2 vmov d7,d18 vadd.i32 d2,d2,d12 vadd.i32 d4,d4,d14 vadd.i32 d6,d6,d16 vadd.i32 d8,d8,d18 vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! vst1.32 {d8[0]},[r6] vst1.32 {d8[1]},[r7] bx lr @ bx lr ENDPROC(poly1305_init_neon) .align 5 ENTRY(poly1305_blocks_neon) ldr ip,[r0,#36] @ is_base2_26 ands r2,r2,#-16 beq .Lno_data_neon cmp r2,#64 bhs .Lenter_neon tst ip,ip @ is_base2_26? beq .Lpoly1305_blocks_arm .Lenter_neon: stmdb sp!,{r4-r7} vstmdb sp!,{d8-d15} @ ABI specification says so tst ip,ip @ is_base2_26? bne .Lbase2_26_neon stmdb sp!,{r1-r3,lr} bl .Lpoly1305_init_neon ldr r4,[r0,#0] @ load hash value base 2^32 ldr r5,[r0,#4] ldr r6,[r0,#8] ldr r7,[r0,#12] ldr ip,[r0,#16] and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 mov r3,r4,lsr#26 veor d10,d10,d10 mov r4,r5,lsr#20 orr r3,r3,r5,lsl#6 veor d12,d12,d12 mov r5,r6,lsr#14 orr r4,r4,r6,lsl#12 veor d14,d14,d14 mov r6,r7,lsr#8 orr r5,r5,r7,lsl#18 veor d16,d16,d16 and r3,r3,#0x03ffffff orr r6,r6,ip,lsl#24 veor d18,d18,d18 and r4,r4,#0x03ffffff mov r1,#1 and r5,r5,#0x03ffffff str r1,[r0,#36] @ is_base2_26 vmov.32 d10[0],r2 vmov.32 d12[0],r3 vmov.32 d14[0],r4 vmov.32 d16[0],r5 vmov.32 d18[0],r6 adr r5,.Lzeros ldmia sp!,{r1-r3,lr} b .Lbase2_32_neon .align 4 .Lbase2_26_neon: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ load hash value veor d10,d10,d10 veor d12,d12,d12 veor d14,d14,d14 veor d16,d16,d16 veor d18,d18,d18 vld4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]! adr r5,.Lzeros vld1.32 {d18[0]},[r0] sub r0,r0,#16 @ rewind .Lbase2_32_neon: add r4,r1,#32 mov r3,r3,lsl#24 tst r2,#31 beq .Leven vld4.32 {d20[0],d22[0],d24[0],d26[0]},[r1]! vmov.32 d28[0],r3 sub r2,r2,#16 add r4,r1,#32 #ifdef __ARMEB__ vrev32.8 q10,q10 vrev32.8 q13,q13 vrev32.8 q11,q11 vrev32.8 q12,q12 #endif vsri.u32 d28,d26,#8 @ base 2^32 -> base 2^26 vshl.u32 d26,d26,#18 vsri.u32 d26,d24,#14 vshl.u32 d24,d24,#12 vadd.i32 d29,d28,d18 @ add hash value and move to #hi vbic.i32 d26,#0xfc000000 vsri.u32 d24,d22,#20 vshl.u32 d22,d22,#6 vbic.i32 d24,#0xfc000000 vsri.u32 d22,d20,#26 vadd.i32 d27,d26,d16 vbic.i32 d20,#0xfc000000 vbic.i32 d22,#0xfc000000 vadd.i32 d25,d24,d14 vadd.i32 d21,d20,d10 vadd.i32 d23,d22,d12 mov r7,r5 add r6,r0,#48 cmp r2,r2 b .Long_tail .align 4 .Leven: subs r2,r2,#64 it lo movlo r4,r5 vmov.i32 q14,#1<<24 @ padbit, yes, always vld4.32 {d20,d22,d24,d26},[r1] @ inp[0:1] add r1,r1,#64 vld4.32 {d21,d23,d25,d27},[r4] @ inp[2:3] (or 0) add r4,r4,#64 itt hi addhi r7,r0,#(48+1*9*4) addhi r6,r0,#(48+3*9*4) #ifdef __ARMEB__ vrev32.8 q10,q10 vrev32.8 q13,q13 vrev32.8 q11,q11 vrev32.8 q12,q12 #endif vsri.u32 q14,q13,#8 @ base 2^32 -> base 2^26 vshl.u32 q13,q13,#18 vsri.u32 q13,q12,#14 vshl.u32 q12,q12,#12 vbic.i32 q13,#0xfc000000 vsri.u32 q12,q11,#20 vshl.u32 q11,q11,#6 vbic.i32 q12,#0xfc000000 vsri.u32 q11,q10,#26 vbic.i32 q10,#0xfc000000 vbic.i32 q11,#0xfc000000 bls .Lskip_loop vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^2 vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^4 vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! b .Loop_neon .align 5 .Loop_neon: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r @ ___________________/ @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r @ ___________________/ ____________________/ @ @ Note that we start with inp[2:3]*r^2. This is because it @ doesn't depend on reduction in previous iteration. @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ inp[2:3]*r^2 vadd.i32 d24,d24,d14 @ accumulate inp[0:1] vmull.u32 q7,d25,d0[1] vadd.i32 d20,d20,d10 vmull.u32 q5,d21,d0[1] vadd.i32 d26,d26,d16 vmull.u32 q8,d27,d0[1] vmlal.u32 q7,d23,d1[1] vadd.i32 d22,d22,d12 vmull.u32 q6,d23,d0[1] vadd.i32 d28,d28,d18 vmull.u32 q9,d29,d0[1] subs r2,r2,#64 vmlal.u32 q5,d29,d2[1] it lo movlo r4,r5 vmlal.u32 q8,d25,d1[1] vld1.32 d8[1],[r7,:32] vmlal.u32 q6,d21,d1[1] vmlal.u32 q9,d27,d1[1] vmlal.u32 q5,d27,d4[1] vmlal.u32 q8,d23,d3[1] vmlal.u32 q9,d25,d3[1] vmlal.u32 q6,d29,d4[1] vmlal.u32 q7,d21,d3[1] vmlal.u32 q8,d21,d5[1] vmlal.u32 q5,d25,d6[1] vmlal.u32 q9,d23,d5[1] vmlal.u32 q6,d27,d6[1] vmlal.u32 q7,d29,d6[1] vmlal.u32 q8,d29,d8[1] vmlal.u32 q5,d23,d8[1] vmlal.u32 q9,d21,d7[1] vmlal.u32 q6,d25,d8[1] vmlal.u32 q7,d27,d8[1] vld4.32 {d21,d23,d25,d27},[r4] @ inp[2:3] (or 0) add r4,r4,#64 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ (hash+inp[0:1])*r^4 and accumulate vmlal.u32 q8,d26,d0[0] vmlal.u32 q5,d20,d0[0] vmlal.u32 q9,d28,d0[0] vmlal.u32 q6,d22,d0[0] vmlal.u32 q7,d24,d0[0] vld1.32 d8[0],[r6,:32] vmlal.u32 q8,d24,d1[0] vmlal.u32 q5,d28,d2[0] vmlal.u32 q9,d26,d1[0] vmlal.u32 q6,d20,d1[0] vmlal.u32 q7,d22,d1[0] vmlal.u32 q8,d22,d3[0] vmlal.u32 q5,d26,d4[0] vmlal.u32 q9,d24,d3[0] vmlal.u32 q6,d28,d4[0] vmlal.u32 q7,d20,d3[0] vmlal.u32 q8,d20,d5[0] vmlal.u32 q5,d24,d6[0] vmlal.u32 q9,d22,d5[0] vmlal.u32 q6,d26,d6[0] vmlal.u32 q8,d28,d8[0] vmlal.u32 q7,d28,d6[0] vmlal.u32 q5,d22,d8[0] vmlal.u32 q9,d20,d7[0] vmov.i32 q14,#1<<24 @ padbit, yes, always vmlal.u32 q6,d24,d8[0] vmlal.u32 q7,d26,d8[0] vld4.32 {d20,d22,d24,d26},[r1] @ inp[0:1] add r1,r1,#64 #ifdef __ARMEB__ vrev32.8 q10,q10 vrev32.8 q11,q11 vrev32.8 q12,q12 vrev32.8 q13,q13 #endif @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ lazy reduction interleaved with base 2^32 -> base 2^26 of @ inp[0:3] previously loaded to q10-q13 and smashed to q10-q14. vshr.u64 q15,q8,#26 vmovn.i64 d16,q8 vshr.u64 q4,q5,#26 vmovn.i64 d10,q5 vadd.i64 q9,q9,q15 @ h3 -> h4 vbic.i32 d16,#0xfc000000 vsri.u32 q14,q13,#8 @ base 2^32 -> base 2^26 vadd.i64 q6,q6,q4 @ h0 -> h1 vshl.u32 q13,q13,#18 vbic.i32 d10,#0xfc000000 vshrn.u64 d30,q9,#26 vmovn.i64 d18,q9 vshr.u64 q4,q6,#26 vmovn.i64 d12,q6 vadd.i64 q7,q7,q4 @ h1 -> h2 vsri.u32 q13,q12,#14 vbic.i32 d18,#0xfc000000 vshl.u32 q12,q12,#12 vbic.i32 d12,#0xfc000000 vadd.i32 d10,d10,d30 vshl.u32 d30,d30,#2 vbic.i32 q13,#0xfc000000 vshrn.u64 d8,q7,#26 vmovn.i64 d14,q7 vaddl.u32 q5,d10,d30 @ h4 -> h0 [widen for a sec] vsri.u32 q12,q11,#20 vadd.i32 d16,d16,d8 @ h2 -> h3 vshl.u32 q11,q11,#6 vbic.i32 d14,#0xfc000000 vbic.i32 q12,#0xfc000000 vshrn.u64 d30,q5,#26 @ re-narrow vmovn.i64 d10,q5 vsri.u32 q11,q10,#26 vbic.i32 q10,#0xfc000000 vshr.u32 d8,d16,#26 vbic.i32 d16,#0xfc000000 vbic.i32 d10,#0xfc000000 vadd.i32 d12,d12,d30 @ h0 -> h1 vadd.i32 d18,d18,d8 @ h3 -> h4 vbic.i32 q11,#0xfc000000 bhi .Loop_neon .Lskip_loop: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 add r7,r0,#(48+0*9*4) add r6,r0,#(48+1*9*4) adds r2,r2,#32 it ne movne r2,#0 bne .Long_tail vadd.i32 d25,d24,d14 @ add hash value and move to #hi vadd.i32 d21,d20,d10 vadd.i32 d27,d26,d16 vadd.i32 d23,d22,d12 vadd.i32 d29,d28,d18 .Long_tail: vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^1 vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^2 vadd.i32 d24,d24,d14 @ can be redundant vmull.u32 q7,d25,d0 vadd.i32 d20,d20,d10 vmull.u32 q5,d21,d0 vadd.i32 d26,d26,d16 vmull.u32 q8,d27,d0 vadd.i32 d22,d22,d12 vmull.u32 q6,d23,d0 vadd.i32 d28,d28,d18 vmull.u32 q9,d29,d0 vmlal.u32 q5,d29,d2 vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! vmlal.u32 q8,d25,d1 vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! vmlal.u32 q6,d21,d1 vmlal.u32 q9,d27,d1 vmlal.u32 q7,d23,d1 vmlal.u32 q8,d23,d3 vld1.32 d8[1],[r7,:32] vmlal.u32 q5,d27,d4 vld1.32 d8[0],[r6,:32] vmlal.u32 q9,d25,d3 vmlal.u32 q6,d29,d4 vmlal.u32 q7,d21,d3 vmlal.u32 q8,d21,d5 it ne addne r7,r0,#(48+2*9*4) vmlal.u32 q5,d25,d6 it ne addne r6,r0,#(48+3*9*4) vmlal.u32 q9,d23,d5 vmlal.u32 q6,d27,d6 vmlal.u32 q7,d29,d6 vmlal.u32 q8,d29,d8 vorn q0,q0,q0 @ all-ones, can be redundant vmlal.u32 q5,d23,d8 vshr.u64 q0,q0,#38 vmlal.u32 q9,d21,d7 vmlal.u32 q6,d25,d8 vmlal.u32 q7,d27,d8 beq .Lshort_tail @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ (hash+inp[0:1])*r^4:r^3 and accumulate vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^3 vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^4 vmlal.u32 q7,d24,d0 vmlal.u32 q5,d20,d0 vmlal.u32 q8,d26,d0 vmlal.u32 q6,d22,d0 vmlal.u32 q9,d28,d0 vmlal.u32 q5,d28,d2 vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! vmlal.u32 q8,d24,d1 vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! vmlal.u32 q6,d20,d1 vmlal.u32 q9,d26,d1 vmlal.u32 q7,d22,d1 vmlal.u32 q8,d22,d3 vld1.32 d8[1],[r7,:32] vmlal.u32 q5,d26,d4 vld1.32 d8[0],[r6,:32] vmlal.u32 q9,d24,d3 vmlal.u32 q6,d28,d4 vmlal.u32 q7,d20,d3 vmlal.u32 q8,d20,d5 vmlal.u32 q5,d24,d6 vmlal.u32 q9,d22,d5 vmlal.u32 q6,d26,d6 vmlal.u32 q7,d28,d6 vmlal.u32 q8,d28,d8 vorn q0,q0,q0 @ all-ones vmlal.u32 q5,d22,d8 vshr.u64 q0,q0,#38 vmlal.u32 q9,d20,d7 vmlal.u32 q6,d24,d8 vmlal.u32 q7,d26,d8 .Lshort_tail: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ horizontal addition vadd.i64 d16,d16,d17 vadd.i64 d10,d10,d11 vadd.i64 d18,d18,d19 vadd.i64 d12,d12,d13 vadd.i64 d14,d14,d15 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ lazy reduction, but without narrowing vshr.u64 q15,q8,#26 vand.i64 q8,q8,q0 vshr.u64 q4,q5,#26 vand.i64 q5,q5,q0 vadd.i64 q9,q9,q15 @ h3 -> h4 vadd.i64 q6,q6,q4 @ h0 -> h1 vshr.u64 q15,q9,#26 vand.i64 q9,q9,q0 vshr.u64 q4,q6,#26 vand.i64 q6,q6,q0 vadd.i64 q7,q7,q4 @ h1 -> h2 vadd.i64 q5,q5,q15 vshl.u64 q15,q15,#2 vshr.u64 q4,q7,#26 vand.i64 q7,q7,q0 vadd.i64 q5,q5,q15 @ h4 -> h0 vadd.i64 q8,q8,q4 @ h2 -> h3 vshr.u64 q15,q5,#26 vand.i64 q5,q5,q0 vshr.u64 q4,q8,#26 vand.i64 q8,q8,q0 vadd.i64 q6,q6,q15 @ h0 -> h1 vadd.i64 q9,q9,q4 @ h3 -> h4 cmp r2,#0 bne .Leven @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ store hash value vst4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]! vst1.32 {d18[0]},[r0] vldmia sp!,{d8-d15} @ epilogue ldmia sp!,{r4-r7} .Lno_data_neon: bx lr @ bx lr ENDPROC(poly1305_blocks_neon) .align 5 ENTRY(poly1305_emit_neon) ldr ip,[r0,#36] @ is_base2_26 stmdb sp!,{r4-r11} tst ip,ip beq .Lpoly1305_emit_enter ldmia r0,{r3-r7} eor r8,r8,r8 adds r3,r3,r4,lsl#26 @ base 2^26 -> base 2^32 mov r4,r4,lsr#6 adcs r4,r4,r5,lsl#20 mov r5,r5,lsr#12 adcs r5,r5,r6,lsl#14 mov r6,r6,lsr#18 adcs r6,r6,r7,lsl#8 adc r7,r8,r7,lsr#24 @ can be partially reduced ... and r8,r7,#-4 @ ... so reduce and r7,r6,#3 add r8,r8,r8,lsr#2 @ *= 5 adds r3,r3,r8 adcs r4,r4,#0 adcs r5,r5,#0 adcs r6,r6,#0 adc r7,r7,#0 adds r8,r3,#5 @ compare to modulus adcs r9,r4,#0 adcs r10,r5,#0 adcs r11,r6,#0 adc r7,r7,#0 tst r7,#4 @ did it carry/borrow? it ne movne r3,r8 ldr r8,[r2,#0] it ne movne r4,r9 ldr r9,[r2,#4] it ne movne r5,r10 ldr r10,[r2,#8] it ne movne r6,r11 ldr r11,[r2,#12] adds r3,r3,r8 @ accumulate nonce adcs r4,r4,r9 adcs r5,r5,r10 adc r6,r6,r11 #ifdef __ARMEB__ rev r3,r3 rev r4,r4 rev r5,r5 rev r6,r6 #endif str r3,[r1,#0] @ store the result str r4,[r1,#4] str r5,[r1,#8] str r6,[r1,#12] ldmia sp!,{r4-r11} bx lr @ bx lr ENDPROC(poly1305_emit_neon) .align 5 .Lzeros: .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 #endif