/* SPDX-License-Identifier: OpenSSL OR (BSD-3-Clause OR GPL-2.0) * * Copyright (C) 2017 Samuel Neves . All Rights Reserved. * Copyright (C) 2015-2017 Jason A. Donenfeld . All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. */ #include .section .rodata.cst16.Lzero, "aM", @progbits, 16 .align 16 .Lzero: .long 0,0,0,0 .section .rodata.cst16.Lone, "aM", @progbits, 16 .align 16 .Lone: .long 1,0,0,0 .section .rodata.cst16.Linc, "aM", @progbits, 16 .align 16 .Linc: .long 0,1,2,3 .section .rodata.cst16.Lfour, "aM", @progbits, 16 .align 16 .Lfour: .long 4,4,4,4 .section .rodata.cst32.Lincy, "aM", @progbits, 32 .align 32 .Lincy: .long 0,2,4,6,1,3,5,7 .section .rodata.cst32.Leight, "aM", @progbits, 32 .align 32 .Leight: .long 8,8,8,8,8,8,8,8 .section .rodata.cst16.Lrot16, "aM", @progbits, 16 .align 16 .Lrot16: .byte 0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd .section .rodata.cst16.Lrot24, "aM", @progbits, 16 .align 16 .Lrot24: .byte 0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe .section .rodata.cst16.Lsigma, "aM", @progbits, 16 .align 16 .Lsigma: .byte 101,120,112,97,110,100,32,51,50,45,98,121,116,101,32,107,0 .section .rodata.cst64.Lzeroz, "aM", @progbits, 64 .align 64 .Lzeroz: .long 0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0 .section .rodata.cst64.Lfourz, "aM", @progbits, 64 .align 64 .Lfourz: .long 4,0,0,0, 4,0,0,0, 4,0,0,0, 4,0,0,0 .section .rodata.cst64.Lincz, "aM", @progbits, 64 .align 64 .Lincz: .long 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 .section .rodata.cst64.Lsixteen, "aM", @progbits, 64 .align 64 .Lsixteen: .long 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16 .text #ifdef CONFIG_AS_SSSE3 .align 32 ENTRY(hchacha20_ssse3) movdqa .Lsigma(%rip),%xmm0 movdqu (%rdx),%xmm1 movdqu 16(%rdx),%xmm2 movdqu (%rsi),%xmm3 movdqa .Lrot16(%rip),%xmm6 movdqa .Lrot24(%rip),%xmm7 movq $10,%r8 .align 32 .Loop_hssse3: paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm6,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $20,%xmm1 pslld $12,%xmm4 por %xmm4,%xmm1 paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm7,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $25,%xmm1 pslld $7,%xmm4 por %xmm4,%xmm1 pshufd $78,%xmm2,%xmm2 pshufd $57,%xmm1,%xmm1 pshufd $147,%xmm3,%xmm3 nop paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm6,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $20,%xmm1 pslld $12,%xmm4 por %xmm4,%xmm1 paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm7,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $25,%xmm1 pslld $7,%xmm4 por %xmm4,%xmm1 pshufd $78,%xmm2,%xmm2 pshufd $147,%xmm1,%xmm1 pshufd $57,%xmm3,%xmm3 decq %r8 jnz .Loop_hssse3 movdqu %xmm0,0(%rdi) movdqu %xmm3,16(%rdi) ret ENDPROC(hchacha20_ssse3) .align 32 ENTRY(chacha20_ssse3) .Lchacha20_ssse3: cmpq $0,%rdx je .Lssse3_epilogue leaq 8(%rsp),%r10 cmpq $128,%rdx ja .Lchacha20_4x .Ldo_sse3_after_all: subq $64+8,%rsp andq $-32,%rsp movdqa .Lsigma(%rip),%xmm0 movdqu (%rcx),%xmm1 movdqu 16(%rcx),%xmm2 movdqu (%r8),%xmm3 movdqa .Lrot16(%rip),%xmm6 movdqa .Lrot24(%rip),%xmm7 movdqa %xmm0,0(%rsp) movdqa %xmm1,16(%rsp) movdqa %xmm2,32(%rsp) movdqa %xmm3,48(%rsp) movq $10,%r8 jmp .Loop_ssse3 .align 32 .Loop_outer_ssse3: movdqa .Lone(%rip),%xmm3 movdqa 0(%rsp),%xmm0 movdqa 16(%rsp),%xmm1 movdqa 32(%rsp),%xmm2 paddd 48(%rsp),%xmm3 movq $10,%r8 movdqa %xmm3,48(%rsp) jmp .Loop_ssse3 .align 32 .Loop_ssse3: paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm6,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $20,%xmm1 pslld $12,%xmm4 por %xmm4,%xmm1 paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm7,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $25,%xmm1 pslld $7,%xmm4 por %xmm4,%xmm1 pshufd $78,%xmm2,%xmm2 pshufd $57,%xmm1,%xmm1 pshufd $147,%xmm3,%xmm3 nop paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm6,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $20,%xmm1 pslld $12,%xmm4 por %xmm4,%xmm1 paddd %xmm1,%xmm0 pxor %xmm0,%xmm3 pshufb %xmm7,%xmm3 paddd %xmm3,%xmm2 pxor %xmm2,%xmm1 movdqa %xmm1,%xmm4 psrld $25,%xmm1 pslld $7,%xmm4 por %xmm4,%xmm1 pshufd $78,%xmm2,%xmm2 pshufd $147,%xmm1,%xmm1 pshufd $57,%xmm3,%xmm3 decq %r8 jnz .Loop_ssse3 paddd 0(%rsp),%xmm0 paddd 16(%rsp),%xmm1 paddd 32(%rsp),%xmm2 paddd 48(%rsp),%xmm3 cmpq $64,%rdx jb .Ltail_ssse3 movdqu 0(%rsi),%xmm4 movdqu 16(%rsi),%xmm5 pxor %xmm4,%xmm0 movdqu 32(%rsi),%xmm4 pxor %xmm5,%xmm1 movdqu 48(%rsi),%xmm5 leaq 64(%rsi),%rsi pxor %xmm4,%xmm2 pxor %xmm5,%xmm3 movdqu %xmm0,0(%rdi) movdqu %xmm1,16(%rdi) movdqu %xmm2,32(%rdi) movdqu %xmm3,48(%rdi) leaq 64(%rdi),%rdi subq $64,%rdx jnz .Loop_outer_ssse3 jmp .Ldone_ssse3 .align 16 .Ltail_ssse3: movdqa %xmm0,0(%rsp) movdqa %xmm1,16(%rsp) movdqa %xmm2,32(%rsp) movdqa %xmm3,48(%rsp) xorq %r8,%r8 .Loop_tail_ssse3: movzbl (%rsi,%r8,1),%eax movzbl (%rsp,%r8,1),%ecx leaq 1(%r8),%r8 xorl %ecx,%eax movb %al,-1(%rdi,%r8,1) decq %rdx jnz .Loop_tail_ssse3 .Ldone_ssse3: leaq -8(%r10),%rsp .Lssse3_epilogue: ret .align 32 .Lchacha20_4x: leaq 8(%rsp),%r10 .Lproceed4x: subq $0x140+8,%rsp andq $-32,%rsp movdqa .Lsigma(%rip),%xmm11 movdqu (%rcx),%xmm15 movdqu 16(%rcx),%xmm7 movdqu (%r8),%xmm3 leaq 256(%rsp),%rcx leaq .Lrot16(%rip),%r9 leaq .Lrot24(%rip),%r11 pshufd $0x00,%xmm11,%xmm8 pshufd $0x55,%xmm11,%xmm9 movdqa %xmm8,64(%rsp) pshufd $0xaa,%xmm11,%xmm10 movdqa %xmm9,80(%rsp) pshufd $0xff,%xmm11,%xmm11 movdqa %xmm10,96(%rsp) movdqa %xmm11,112(%rsp) pshufd $0x00,%xmm15,%xmm12 pshufd $0x55,%xmm15,%xmm13 movdqa %xmm12,128-256(%rcx) pshufd $0xaa,%xmm15,%xmm14 movdqa %xmm13,144-256(%rcx) pshufd $0xff,%xmm15,%xmm15 movdqa %xmm14,160-256(%rcx) movdqa %xmm15,176-256(%rcx) pshufd $0x00,%xmm7,%xmm4 pshufd $0x55,%xmm7,%xmm5 movdqa %xmm4,192-256(%rcx) pshufd $0xaa,%xmm7,%xmm6 movdqa %xmm5,208-256(%rcx) pshufd $0xff,%xmm7,%xmm7 movdqa %xmm6,224-256(%rcx) movdqa %xmm7,240-256(%rcx) pshufd $0x00,%xmm3,%xmm0 pshufd $0x55,%xmm3,%xmm1 paddd .Linc(%rip),%xmm0 pshufd $0xaa,%xmm3,%xmm2 movdqa %xmm1,272-256(%rcx) pshufd $0xff,%xmm3,%xmm3 movdqa %xmm2,288-256(%rcx) movdqa %xmm3,304-256(%rcx) jmp .Loop_enter4x .align 32 .Loop_outer4x: movdqa 64(%rsp),%xmm8 movdqa 80(%rsp),%xmm9 movdqa 96(%rsp),%xmm10 movdqa 112(%rsp),%xmm11 movdqa 128-256(%rcx),%xmm12 movdqa 144-256(%rcx),%xmm13 movdqa 160-256(%rcx),%xmm14 movdqa 176-256(%rcx),%xmm15 movdqa 192-256(%rcx),%xmm4 movdqa 208-256(%rcx),%xmm5 movdqa 224-256(%rcx),%xmm6 movdqa 240-256(%rcx),%xmm7 movdqa 256-256(%rcx),%xmm0 movdqa 272-256(%rcx),%xmm1 movdqa 288-256(%rcx),%xmm2 movdqa 304-256(%rcx),%xmm3 paddd .Lfour(%rip),%xmm0 .Loop_enter4x: movdqa %xmm6,32(%rsp) movdqa %xmm7,48(%rsp) movdqa (%r9),%xmm7 movl $10,%eax movdqa %xmm0,256-256(%rcx) jmp .Loop4x .align 32 .Loop4x: paddd %xmm12,%xmm8 paddd %xmm13,%xmm9 pxor %xmm8,%xmm0 pxor %xmm9,%xmm1 pshufb %xmm7,%xmm0 pshufb %xmm7,%xmm1 paddd %xmm0,%xmm4 paddd %xmm1,%xmm5 pxor %xmm4,%xmm12 pxor %xmm5,%xmm13 movdqa %xmm12,%xmm6 pslld $12,%xmm12 psrld $20,%xmm6 movdqa %xmm13,%xmm7 pslld $12,%xmm13 por %xmm6,%xmm12 psrld $20,%xmm7 movdqa (%r11),%xmm6 por %xmm7,%xmm13 paddd %xmm12,%xmm8 paddd %xmm13,%xmm9 pxor %xmm8,%xmm0 pxor %xmm9,%xmm1 pshufb %xmm6,%xmm0 pshufb %xmm6,%xmm1 paddd %xmm0,%xmm4 paddd %xmm1,%xmm5 pxor %xmm4,%xmm12 pxor %xmm5,%xmm13 movdqa %xmm12,%xmm7 pslld $7,%xmm12 psrld $25,%xmm7 movdqa %xmm13,%xmm6 pslld $7,%xmm13 por %xmm7,%xmm12 psrld $25,%xmm6 movdqa (%r9),%xmm7 por %xmm6,%xmm13 movdqa %xmm4,0(%rsp) movdqa %xmm5,16(%rsp) movdqa 32(%rsp),%xmm4 movdqa 48(%rsp),%xmm5 paddd %xmm14,%xmm10 paddd %xmm15,%xmm11 pxor %xmm10,%xmm2 pxor %xmm11,%xmm3 pshufb %xmm7,%xmm2 pshufb %xmm7,%xmm3 paddd %xmm2,%xmm4 paddd %xmm3,%xmm5 pxor %xmm4,%xmm14 pxor %xmm5,%xmm15 movdqa %xmm14,%xmm6 pslld $12,%xmm14 psrld $20,%xmm6 movdqa %xmm15,%xmm7 pslld $12,%xmm15 por %xmm6,%xmm14 psrld $20,%xmm7 movdqa (%r11),%xmm6 por %xmm7,%xmm15 paddd %xmm14,%xmm10 paddd %xmm15,%xmm11 pxor %xmm10,%xmm2 pxor %xmm11,%xmm3 pshufb %xmm6,%xmm2 pshufb %xmm6,%xmm3 paddd %xmm2,%xmm4 paddd %xmm3,%xmm5 pxor %xmm4,%xmm14 pxor %xmm5,%xmm15 movdqa %xmm14,%xmm7 pslld $7,%xmm14 psrld $25,%xmm7 movdqa %xmm15,%xmm6 pslld $7,%xmm15 por %xmm7,%xmm14 psrld $25,%xmm6 movdqa (%r9),%xmm7 por %xmm6,%xmm15 paddd %xmm13,%xmm8 paddd %xmm14,%xmm9 pxor %xmm8,%xmm3 pxor %xmm9,%xmm0 pshufb %xmm7,%xmm3 pshufb %xmm7,%xmm0 paddd %xmm3,%xmm4 paddd %xmm0,%xmm5 pxor %xmm4,%xmm13 pxor %xmm5,%xmm14 movdqa %xmm13,%xmm6 pslld $12,%xmm13 psrld $20,%xmm6 movdqa %xmm14,%xmm7 pslld $12,%xmm14 por %xmm6,%xmm13 psrld $20,%xmm7 movdqa (%r11),%xmm6 por %xmm7,%xmm14 paddd %xmm13,%xmm8 paddd %xmm14,%xmm9 pxor %xmm8,%xmm3 pxor %xmm9,%xmm0 pshufb %xmm6,%xmm3 pshufb %xmm6,%xmm0 paddd %xmm3,%xmm4 paddd %xmm0,%xmm5 pxor %xmm4,%xmm13 pxor %xmm5,%xmm14 movdqa %xmm13,%xmm7 pslld $7,%xmm13 psrld $25,%xmm7 movdqa %xmm14,%xmm6 pslld $7,%xmm14 por %xmm7,%xmm13 psrld $25,%xmm6 movdqa (%r9),%xmm7 por %xmm6,%xmm14 movdqa %xmm4,32(%rsp) movdqa %xmm5,48(%rsp) movdqa 0(%rsp),%xmm4 movdqa 16(%rsp),%xmm5 paddd %xmm15,%xmm10 paddd %xmm12,%xmm11 pxor %xmm10,%xmm1 pxor %xmm11,%xmm2 pshufb %xmm7,%xmm1 pshufb %xmm7,%xmm2 paddd %xmm1,%xmm4 paddd %xmm2,%xmm5 pxor %xmm4,%xmm15 pxor %xmm5,%xmm12 movdqa %xmm15,%xmm6 pslld $12,%xmm15 psrld $20,%xmm6 movdqa %xmm12,%xmm7 pslld $12,%xmm12 por %xmm6,%xmm15 psrld $20,%xmm7 movdqa (%r11),%xmm6 por %xmm7,%xmm12 paddd %xmm15,%xmm10 paddd %xmm12,%xmm11 pxor %xmm10,%xmm1 pxor %xmm11,%xmm2 pshufb %xmm6,%xmm1 pshufb %xmm6,%xmm2 paddd %xmm1,%xmm4 paddd %xmm2,%xmm5 pxor %xmm4,%xmm15 pxor %xmm5,%xmm12 movdqa %xmm15,%xmm7 pslld $7,%xmm15 psrld $25,%xmm7 movdqa %xmm12,%xmm6 pslld $7,%xmm12 por %xmm7,%xmm15 psrld $25,%xmm6 movdqa (%r9),%xmm7 por %xmm6,%xmm12 decl %eax jnz .Loop4x paddd 64(%rsp),%xmm8 paddd 80(%rsp),%xmm9 paddd 96(%rsp),%xmm10 paddd 112(%rsp),%xmm11 movdqa %xmm8,%xmm6 punpckldq %xmm9,%xmm8 movdqa %xmm10,%xmm7 punpckldq %xmm11,%xmm10 punpckhdq %xmm9,%xmm6 punpckhdq %xmm11,%xmm7 movdqa %xmm8,%xmm9 punpcklqdq %xmm10,%xmm8 movdqa %xmm6,%xmm11 punpcklqdq %xmm7,%xmm6 punpckhqdq %xmm10,%xmm9 punpckhqdq %xmm7,%xmm11 paddd 128-256(%rcx),%xmm12 paddd 144-256(%rcx),%xmm13 paddd 160-256(%rcx),%xmm14 paddd 176-256(%rcx),%xmm15 movdqa %xmm8,0(%rsp) movdqa %xmm9,16(%rsp) movdqa 32(%rsp),%xmm8 movdqa 48(%rsp),%xmm9 movdqa %xmm12,%xmm10 punpckldq %xmm13,%xmm12 movdqa %xmm14,%xmm7 punpckldq %xmm15,%xmm14 punpckhdq %xmm13,%xmm10 punpckhdq %xmm15,%xmm7 movdqa %xmm12,%xmm13 punpcklqdq %xmm14,%xmm12 movdqa %xmm10,%xmm15 punpcklqdq %xmm7,%xmm10 punpckhqdq %xmm14,%xmm13 punpckhqdq %xmm7,%xmm15 paddd 192-256(%rcx),%xmm4 paddd 208-256(%rcx),%xmm5 paddd 224-256(%rcx),%xmm8 paddd 240-256(%rcx),%xmm9 movdqa %xmm6,32(%rsp) movdqa %xmm11,48(%rsp) movdqa %xmm4,%xmm14 punpckldq %xmm5,%xmm4 movdqa %xmm8,%xmm7 punpckldq %xmm9,%xmm8 punpckhdq %xmm5,%xmm14 punpckhdq %xmm9,%xmm7 movdqa %xmm4,%xmm5 punpcklqdq %xmm8,%xmm4 movdqa %xmm14,%xmm9 punpcklqdq %xmm7,%xmm14 punpckhqdq %xmm8,%xmm5 punpckhqdq %xmm7,%xmm9 paddd 256-256(%rcx),%xmm0 paddd 272-256(%rcx),%xmm1 paddd 288-256(%rcx),%xmm2 paddd 304-256(%rcx),%xmm3 movdqa %xmm0,%xmm8 punpckldq %xmm1,%xmm0 movdqa %xmm2,%xmm7 punpckldq %xmm3,%xmm2 punpckhdq %xmm1,%xmm8 punpckhdq %xmm3,%xmm7 movdqa %xmm0,%xmm1 punpcklqdq %xmm2,%xmm0 movdqa %xmm8,%xmm3 punpcklqdq %xmm7,%xmm8 punpckhqdq %xmm2,%xmm1 punpckhqdq %xmm7,%xmm3 cmpq $256,%rdx jb .Ltail4x movdqu 0(%rsi),%xmm6 movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 pxor 0(%rsp),%xmm6 pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 movdqu %xmm6,0(%rdi) movdqu 64(%rsi),%xmm6 movdqu %xmm11,16(%rdi) movdqu 80(%rsi),%xmm11 movdqu %xmm2,32(%rdi) movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 leaq 128(%rsi),%rsi pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 pxor %xmm1,%xmm7 movdqu %xmm6,64(%rdi) movdqu 0(%rsi),%xmm6 movdqu %xmm11,80(%rdi) movdqu 16(%rsi),%xmm11 movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) leaq 128(%rdi),%rdi movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 pxor %xmm14,%xmm2 pxor %xmm8,%xmm7 movdqu %xmm6,0(%rdi) movdqu 64(%rsi),%xmm6 movdqu %xmm11,16(%rdi) movdqu 80(%rsi),%xmm11 movdqu %xmm2,32(%rdi) movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 leaq 128(%rsi),%rsi pxor 48(%rsp),%xmm6 pxor %xmm15,%xmm11 pxor %xmm9,%xmm2 pxor %xmm3,%xmm7 movdqu %xmm6,64(%rdi) movdqu %xmm11,80(%rdi) movdqu %xmm2,96(%rdi) movdqu %xmm7,112(%rdi) leaq 128(%rdi),%rdi subq $256,%rdx jnz .Loop_outer4x jmp .Ldone4x .Ltail4x: cmpq $192,%rdx jae .L192_or_more4x cmpq $128,%rdx jae .L128_or_more4x cmpq $64,%rdx jae .L64_or_more4x xorq %r9,%r9 movdqa %xmm12,16(%rsp) movdqa %xmm4,32(%rsp) movdqa %xmm0,48(%rsp) jmp .Loop_tail4x .align 32 .L64_or_more4x: movdqu 0(%rsi),%xmm6 movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 pxor 0(%rsp),%xmm6 pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 movdqu %xmm6,0(%rdi) movdqu %xmm11,16(%rdi) movdqu %xmm2,32(%rdi) movdqu %xmm7,48(%rdi) je .Ldone4x movdqa 16(%rsp),%xmm6 leaq 64(%rsi),%rsi xorq %r9,%r9 movdqa %xmm6,0(%rsp) movdqa %xmm13,16(%rsp) leaq 64(%rdi),%rdi movdqa %xmm5,32(%rsp) subq $64,%rdx movdqa %xmm1,48(%rsp) jmp .Loop_tail4x .align 32 .L128_or_more4x: movdqu 0(%rsi),%xmm6 movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 pxor 0(%rsp),%xmm6 pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 movdqu %xmm6,0(%rdi) movdqu 64(%rsi),%xmm6 movdqu %xmm11,16(%rdi) movdqu 80(%rsi),%xmm11 movdqu %xmm2,32(%rdi) movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 pxor %xmm1,%xmm7 movdqu %xmm6,64(%rdi) movdqu %xmm11,80(%rdi) movdqu %xmm2,96(%rdi) movdqu %xmm7,112(%rdi) je .Ldone4x movdqa 32(%rsp),%xmm6 leaq 128(%rsi),%rsi xorq %r9,%r9 movdqa %xmm6,0(%rsp) movdqa %xmm10,16(%rsp) leaq 128(%rdi),%rdi movdqa %xmm14,32(%rsp) subq $128,%rdx movdqa %xmm8,48(%rsp) jmp .Loop_tail4x .align 32 .L192_or_more4x: movdqu 0(%rsi),%xmm6 movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 pxor 0(%rsp),%xmm6 pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 movdqu %xmm6,0(%rdi) movdqu 64(%rsi),%xmm6 movdqu %xmm11,16(%rdi) movdqu 80(%rsi),%xmm11 movdqu %xmm2,32(%rdi) movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 leaq 128(%rsi),%rsi pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 pxor %xmm1,%xmm7 movdqu %xmm6,64(%rdi) movdqu 0(%rsi),%xmm6 movdqu %xmm11,80(%rdi) movdqu 16(%rsi),%xmm11 movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) leaq 128(%rdi),%rdi movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 pxor %xmm14,%xmm2 pxor %xmm8,%xmm7 movdqu %xmm6,0(%rdi) movdqu %xmm11,16(%rdi) movdqu %xmm2,32(%rdi) movdqu %xmm7,48(%rdi) je .Ldone4x movdqa 48(%rsp),%xmm6 leaq 64(%rsi),%rsi xorq %r9,%r9 movdqa %xmm6,0(%rsp) movdqa %xmm15,16(%rsp) leaq 64(%rdi),%rdi movdqa %xmm9,32(%rsp) subq $192,%rdx movdqa %xmm3,48(%rsp) .Loop_tail4x: movzbl (%rsi,%r9,1),%eax movzbl (%rsp,%r9,1),%ecx leaq 1(%r9),%r9 xorl %ecx,%eax movb %al,-1(%rdi,%r9,1) decq %rdx jnz .Loop_tail4x .Ldone4x: leaq -8(%r10),%rsp .L4x_epilogue: ret ENDPROC(chacha20_ssse3) #endif /* CONFIG_AS_SSSE3 */ #ifdef CONFIG_AS_AVX2 .align 32 ENTRY(chacha20_avx2) .Lchacha20_avx2: cmpq $0,%rdx je .L8x_epilogue leaq 8(%rsp),%r10 subq $0x280+8,%rsp andq $-32,%rsp vzeroupper vbroadcasti128 .Lsigma(%rip),%ymm11 vbroadcasti128 (%rcx),%ymm3 vbroadcasti128 16(%rcx),%ymm15 vbroadcasti128 (%r8),%ymm7 leaq 256(%rsp),%rcx leaq 512(%rsp),%rax leaq .Lrot16(%rip),%r9 leaq .Lrot24(%rip),%r11 vpshufd $0x00,%ymm11,%ymm8 vpshufd $0x55,%ymm11,%ymm9 vmovdqa %ymm8,128-256(%rcx) vpshufd $0xaa,%ymm11,%ymm10 vmovdqa %ymm9,160-256(%rcx) vpshufd $0xff,%ymm11,%ymm11 vmovdqa %ymm10,192-256(%rcx) vmovdqa %ymm11,224-256(%rcx) vpshufd $0x00,%ymm3,%ymm0 vpshufd $0x55,%ymm3,%ymm1 vmovdqa %ymm0,256-256(%rcx) vpshufd $0xaa,%ymm3,%ymm2 vmovdqa %ymm1,288-256(%rcx) vpshufd $0xff,%ymm3,%ymm3 vmovdqa %ymm2,320-256(%rcx) vmovdqa %ymm3,352-256(%rcx) vpshufd $0x00,%ymm15,%ymm12 vpshufd $0x55,%ymm15,%ymm13 vmovdqa %ymm12,384-512(%rax) vpshufd $0xaa,%ymm15,%ymm14 vmovdqa %ymm13,416-512(%rax) vpshufd $0xff,%ymm15,%ymm15 vmovdqa %ymm14,448-512(%rax) vmovdqa %ymm15,480-512(%rax) vpshufd $0x00,%ymm7,%ymm4 vpshufd $0x55,%ymm7,%ymm5 vpaddd .Lincy(%rip),%ymm4,%ymm4 vpshufd $0xaa,%ymm7,%ymm6 vmovdqa %ymm5,544-512(%rax) vpshufd $0xff,%ymm7,%ymm7 vmovdqa %ymm6,576-512(%rax) vmovdqa %ymm7,608-512(%rax) jmp .Loop_enter8x .align 32 .Loop_outer8x: vmovdqa 128-256(%rcx),%ymm8 vmovdqa 160-256(%rcx),%ymm9 vmovdqa 192-256(%rcx),%ymm10 vmovdqa 224-256(%rcx),%ymm11 vmovdqa 256-256(%rcx),%ymm0 vmovdqa 288-256(%rcx),%ymm1 vmovdqa 320-256(%rcx),%ymm2 vmovdqa 352-256(%rcx),%ymm3 vmovdqa 384-512(%rax),%ymm12 vmovdqa 416-512(%rax),%ymm13 vmovdqa 448-512(%rax),%ymm14 vmovdqa 480-512(%rax),%ymm15 vmovdqa 512-512(%rax),%ymm4 vmovdqa 544-512(%rax),%ymm5 vmovdqa 576-512(%rax),%ymm6 vmovdqa 608-512(%rax),%ymm7 vpaddd .Leight(%rip),%ymm4,%ymm4 .Loop_enter8x: vmovdqa %ymm14,64(%rsp) vmovdqa %ymm15,96(%rsp) vbroadcasti128 (%r9),%ymm15 vmovdqa %ymm4,512-512(%rax) movl $10,%eax jmp .Loop8x .align 32 .Loop8x: vpaddd %ymm0,%ymm8,%ymm8 vpxor %ymm4,%ymm8,%ymm4 vpshufb %ymm15,%ymm4,%ymm4 vpaddd %ymm1,%ymm9,%ymm9 vpxor %ymm5,%ymm9,%ymm5 vpshufb %ymm15,%ymm5,%ymm5 vpaddd %ymm4,%ymm12,%ymm12 vpxor %ymm0,%ymm12,%ymm0 vpslld $12,%ymm0,%ymm14 vpsrld $20,%ymm0,%ymm0 vpor %ymm0,%ymm14,%ymm0 vbroadcasti128 (%r11),%ymm14 vpaddd %ymm5,%ymm13,%ymm13 vpxor %ymm1,%ymm13,%ymm1 vpslld $12,%ymm1,%ymm15 vpsrld $20,%ymm1,%ymm1 vpor %ymm1,%ymm15,%ymm1 vpaddd %ymm0,%ymm8,%ymm8 vpxor %ymm4,%ymm8,%ymm4 vpshufb %ymm14,%ymm4,%ymm4 vpaddd %ymm1,%ymm9,%ymm9 vpxor %ymm5,%ymm9,%ymm5 vpshufb %ymm14,%ymm5,%ymm5 vpaddd %ymm4,%ymm12,%ymm12 vpxor %ymm0,%ymm12,%ymm0 vpslld $7,%ymm0,%ymm15 vpsrld $25,%ymm0,%ymm0 vpor %ymm0,%ymm15,%ymm0 vbroadcasti128 (%r9),%ymm15 vpaddd %ymm5,%ymm13,%ymm13 vpxor %ymm1,%ymm13,%ymm1 vpslld $7,%ymm1,%ymm14 vpsrld $25,%ymm1,%ymm1 vpor %ymm1,%ymm14,%ymm1 vmovdqa %ymm12,0(%rsp) vmovdqa %ymm13,32(%rsp) vmovdqa 64(%rsp),%ymm12 vmovdqa 96(%rsp),%ymm13 vpaddd %ymm2,%ymm10,%ymm10 vpxor %ymm6,%ymm10,%ymm6 vpshufb %ymm15,%ymm6,%ymm6 vpaddd %ymm3,%ymm11,%ymm11 vpxor %ymm7,%ymm11,%ymm7 vpshufb %ymm15,%ymm7,%ymm7 vpaddd %ymm6,%ymm12,%ymm12 vpxor %ymm2,%ymm12,%ymm2 vpslld $12,%ymm2,%ymm14 vpsrld $20,%ymm2,%ymm2 vpor %ymm2,%ymm14,%ymm2 vbroadcasti128 (%r11),%ymm14 vpaddd %ymm7,%ymm13,%ymm13 vpxor %ymm3,%ymm13,%ymm3 vpslld $12,%ymm3,%ymm15 vpsrld $20,%ymm3,%ymm3 vpor %ymm3,%ymm15,%ymm3 vpaddd %ymm2,%ymm10,%ymm10 vpxor %ymm6,%ymm10,%ymm6 vpshufb %ymm14,%ymm6,%ymm6 vpaddd %ymm3,%ymm11,%ymm11 vpxor %ymm7,%ymm11,%ymm7 vpshufb %ymm14,%ymm7,%ymm7 vpaddd %ymm6,%ymm12,%ymm12 vpxor %ymm2,%ymm12,%ymm2 vpslld $7,%ymm2,%ymm15 vpsrld $25,%ymm2,%ymm2 vpor %ymm2,%ymm15,%ymm2 vbroadcasti128 (%r9),%ymm15 vpaddd %ymm7,%ymm13,%ymm13 vpxor %ymm3,%ymm13,%ymm3 vpslld $7,%ymm3,%ymm14 vpsrld $25,%ymm3,%ymm3 vpor %ymm3,%ymm14,%ymm3 vpaddd %ymm1,%ymm8,%ymm8 vpxor %ymm7,%ymm8,%ymm7 vpshufb %ymm15,%ymm7,%ymm7 vpaddd %ymm2,%ymm9,%ymm9 vpxor %ymm4,%ymm9,%ymm4 vpshufb %ymm15,%ymm4,%ymm4 vpaddd %ymm7,%ymm12,%ymm12 vpxor %ymm1,%ymm12,%ymm1 vpslld $12,%ymm1,%ymm14 vpsrld $20,%ymm1,%ymm1 vpor %ymm1,%ymm14,%ymm1 vbroadcasti128 (%r11),%ymm14 vpaddd %ymm4,%ymm13,%ymm13 vpxor %ymm2,%ymm13,%ymm2 vpslld $12,%ymm2,%ymm15 vpsrld $20,%ymm2,%ymm2 vpor %ymm2,%ymm15,%ymm2 vpaddd %ymm1,%ymm8,%ymm8 vpxor %ymm7,%ymm8,%ymm7 vpshufb %ymm14,%ymm7,%ymm7 vpaddd %ymm2,%ymm9,%ymm9 vpxor %ymm4,%ymm9,%ymm4 vpshufb %ymm14,%ymm4,%ymm4 vpaddd %ymm7,%ymm12,%ymm12 vpxor %ymm1,%ymm12,%ymm1 vpslld $7,%ymm1,%ymm15 vpsrld $25,%ymm1,%ymm1 vpor %ymm1,%ymm15,%ymm1 vbroadcasti128 (%r9),%ymm15 vpaddd %ymm4,%ymm13,%ymm13 vpxor %ymm2,%ymm13,%ymm2 vpslld $7,%ymm2,%ymm14 vpsrld $25,%ymm2,%ymm2 vpor %ymm2,%ymm14,%ymm2 vmovdqa %ymm12,64(%rsp) vmovdqa %ymm13,96(%rsp) vmovdqa 0(%rsp),%ymm12 vmovdqa 32(%rsp),%ymm13 vpaddd %ymm3,%ymm10,%ymm10 vpxor %ymm5,%ymm10,%ymm5 vpshufb %ymm15,%ymm5,%ymm5 vpaddd %ymm0,%ymm11,%ymm11 vpxor %ymm6,%ymm11,%ymm6 vpshufb %ymm15,%ymm6,%ymm6 vpaddd %ymm5,%ymm12,%ymm12 vpxor %ymm3,%ymm12,%ymm3 vpslld $12,%ymm3,%ymm14 vpsrld $20,%ymm3,%ymm3 vpor %ymm3,%ymm14,%ymm3 vbroadcasti128 (%r11),%ymm14 vpaddd %ymm6,%ymm13,%ymm13 vpxor %ymm0,%ymm13,%ymm0 vpslld $12,%ymm0,%ymm15 vpsrld $20,%ymm0,%ymm0 vpor %ymm0,%ymm15,%ymm0 vpaddd %ymm3,%ymm10,%ymm10 vpxor %ymm5,%ymm10,%ymm5 vpshufb %ymm14,%ymm5,%ymm5 vpaddd %ymm0,%ymm11,%ymm11 vpxor %ymm6,%ymm11,%ymm6 vpshufb %ymm14,%ymm6,%ymm6 vpaddd %ymm5,%ymm12,%ymm12 vpxor %ymm3,%ymm12,%ymm3 vpslld $7,%ymm3,%ymm15 vpsrld $25,%ymm3,%ymm3 vpor %ymm3,%ymm15,%ymm3 vbroadcasti128 (%r9),%ymm15 vpaddd %ymm6,%ymm13,%ymm13 vpxor %ymm0,%ymm13,%ymm0 vpslld $7,%ymm0,%ymm14 vpsrld $25,%ymm0,%ymm0 vpor %ymm0,%ymm14,%ymm0 decl %eax jnz .Loop8x leaq 512(%rsp),%rax vpaddd 128-256(%rcx),%ymm8,%ymm8 vpaddd 160-256(%rcx),%ymm9,%ymm9 vpaddd 192-256(%rcx),%ymm10,%ymm10 vpaddd 224-256(%rcx),%ymm11,%ymm11 vpunpckldq %ymm9,%ymm8,%ymm14 vpunpckldq %ymm11,%ymm10,%ymm15 vpunpckhdq %ymm9,%ymm8,%ymm8 vpunpckhdq %ymm11,%ymm10,%ymm10 vpunpcklqdq %ymm15,%ymm14,%ymm9 vpunpckhqdq %ymm15,%ymm14,%ymm14 vpunpcklqdq %ymm10,%ymm8,%ymm11 vpunpckhqdq %ymm10,%ymm8,%ymm8 vpaddd 256-256(%rcx),%ymm0,%ymm0 vpaddd 288-256(%rcx),%ymm1,%ymm1 vpaddd 320-256(%rcx),%ymm2,%ymm2 vpaddd 352-256(%rcx),%ymm3,%ymm3 vpunpckldq %ymm1,%ymm0,%ymm10 vpunpckldq %ymm3,%ymm2,%ymm15 vpunpckhdq %ymm1,%ymm0,%ymm0 vpunpckhdq %ymm3,%ymm2,%ymm2 vpunpcklqdq %ymm15,%ymm10,%ymm1 vpunpckhqdq %ymm15,%ymm10,%ymm10 vpunpcklqdq %ymm2,%ymm0,%ymm3 vpunpckhqdq %ymm2,%ymm0,%ymm0 vperm2i128 $0x20,%ymm1,%ymm9,%ymm15 vperm2i128 $0x31,%ymm1,%ymm9,%ymm1 vperm2i128 $0x20,%ymm10,%ymm14,%ymm9 vperm2i128 $0x31,%ymm10,%ymm14,%ymm10 vperm2i128 $0x20,%ymm3,%ymm11,%ymm14 vperm2i128 $0x31,%ymm3,%ymm11,%ymm3 vperm2i128 $0x20,%ymm0,%ymm8,%ymm11 vperm2i128 $0x31,%ymm0,%ymm8,%ymm0 vmovdqa %ymm15,0(%rsp) vmovdqa %ymm9,32(%rsp) vmovdqa 64(%rsp),%ymm15 vmovdqa 96(%rsp),%ymm9 vpaddd 384-512(%rax),%ymm12,%ymm12 vpaddd 416-512(%rax),%ymm13,%ymm13 vpaddd 448-512(%rax),%ymm15,%ymm15 vpaddd 480-512(%rax),%ymm9,%ymm9 vpunpckldq %ymm13,%ymm12,%ymm2 vpunpckldq %ymm9,%ymm15,%ymm8 vpunpckhdq %ymm13,%ymm12,%ymm12 vpunpckhdq %ymm9,%ymm15,%ymm15 vpunpcklqdq %ymm8,%ymm2,%ymm13 vpunpckhqdq %ymm8,%ymm2,%ymm2 vpunpcklqdq %ymm15,%ymm12,%ymm9 vpunpckhqdq %ymm15,%ymm12,%ymm12 vpaddd 512-512(%rax),%ymm4,%ymm4 vpaddd 544-512(%rax),%ymm5,%ymm5 vpaddd 576-512(%rax),%ymm6,%ymm6 vpaddd 608-512(%rax),%ymm7,%ymm7 vpunpckldq %ymm5,%ymm4,%ymm15 vpunpckldq %ymm7,%ymm6,%ymm8 vpunpckhdq %ymm5,%ymm4,%ymm4 vpunpckhdq %ymm7,%ymm6,%ymm6 vpunpcklqdq %ymm8,%ymm15,%ymm5 vpunpckhqdq %ymm8,%ymm15,%ymm15 vpunpcklqdq %ymm6,%ymm4,%ymm7 vpunpckhqdq %ymm6,%ymm4,%ymm4 vperm2i128 $0x20,%ymm5,%ymm13,%ymm8 vperm2i128 $0x31,%ymm5,%ymm13,%ymm5 vperm2i128 $0x20,%ymm15,%ymm2,%ymm13 vperm2i128 $0x31,%ymm15,%ymm2,%ymm15 vperm2i128 $0x20,%ymm7,%ymm9,%ymm2 vperm2i128 $0x31,%ymm7,%ymm9,%ymm7 vperm2i128 $0x20,%ymm4,%ymm12,%ymm9 vperm2i128 $0x31,%ymm4,%ymm12,%ymm4 vmovdqa 0(%rsp),%ymm6 vmovdqa 32(%rsp),%ymm12 cmpq $512,%rdx jb .Ltail8x vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 leaq 128(%rsi),%rsi vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) leaq 128(%rdi),%rdi vpxor 0(%rsi),%ymm12,%ymm12 vpxor 32(%rsi),%ymm13,%ymm13 vpxor 64(%rsi),%ymm10,%ymm10 vpxor 96(%rsi),%ymm15,%ymm15 leaq 128(%rsi),%rsi vmovdqu %ymm12,0(%rdi) vmovdqu %ymm13,32(%rdi) vmovdqu %ymm10,64(%rdi) vmovdqu %ymm15,96(%rdi) leaq 128(%rdi),%rdi vpxor 0(%rsi),%ymm14,%ymm14 vpxor 32(%rsi),%ymm2,%ymm2 vpxor 64(%rsi),%ymm3,%ymm3 vpxor 96(%rsi),%ymm7,%ymm7 leaq 128(%rsi),%rsi vmovdqu %ymm14,0(%rdi) vmovdqu %ymm2,32(%rdi) vmovdqu %ymm3,64(%rdi) vmovdqu %ymm7,96(%rdi) leaq 128(%rdi),%rdi vpxor 0(%rsi),%ymm11,%ymm11 vpxor 32(%rsi),%ymm9,%ymm9 vpxor 64(%rsi),%ymm0,%ymm0 vpxor 96(%rsi),%ymm4,%ymm4 leaq 128(%rsi),%rsi vmovdqu %ymm11,0(%rdi) vmovdqu %ymm9,32(%rdi) vmovdqu %ymm0,64(%rdi) vmovdqu %ymm4,96(%rdi) leaq 128(%rdi),%rdi subq $512,%rdx jnz .Loop_outer8x jmp .Ldone8x .Ltail8x: cmpq $448,%rdx jae .L448_or_more8x cmpq $384,%rdx jae .L384_or_more8x cmpq $320,%rdx jae .L320_or_more8x cmpq $256,%rdx jae .L256_or_more8x cmpq $192,%rdx jae .L192_or_more8x cmpq $128,%rdx jae .L128_or_more8x cmpq $64,%rdx jae .L64_or_more8x xorq %r9,%r9 vmovdqa %ymm6,0(%rsp) vmovdqa %ymm8,32(%rsp) jmp .Loop_tail8x .align 32 .L64_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) je .Ldone8x leaq 64(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm1,0(%rsp) leaq 64(%rdi),%rdi subq $64,%rdx vmovdqa %ymm5,32(%rsp) jmp .Loop_tail8x .align 32 .L128_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) je .Ldone8x leaq 128(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm12,0(%rsp) leaq 128(%rdi),%rdi subq $128,%rdx vmovdqa %ymm13,32(%rsp) jmp .Loop_tail8x .align 32 .L192_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vpxor 128(%rsi),%ymm12,%ymm12 vpxor 160(%rsi),%ymm13,%ymm13 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) vmovdqu %ymm12,128(%rdi) vmovdqu %ymm13,160(%rdi) je .Ldone8x leaq 192(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm10,0(%rsp) leaq 192(%rdi),%rdi subq $192,%rdx vmovdqa %ymm15,32(%rsp) jmp .Loop_tail8x .align 32 .L256_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vpxor 128(%rsi),%ymm12,%ymm12 vpxor 160(%rsi),%ymm13,%ymm13 vpxor 192(%rsi),%ymm10,%ymm10 vpxor 224(%rsi),%ymm15,%ymm15 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) vmovdqu %ymm12,128(%rdi) vmovdqu %ymm13,160(%rdi) vmovdqu %ymm10,192(%rdi) vmovdqu %ymm15,224(%rdi) je .Ldone8x leaq 256(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm14,0(%rsp) leaq 256(%rdi),%rdi subq $256,%rdx vmovdqa %ymm2,32(%rsp) jmp .Loop_tail8x .align 32 .L320_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vpxor 128(%rsi),%ymm12,%ymm12 vpxor 160(%rsi),%ymm13,%ymm13 vpxor 192(%rsi),%ymm10,%ymm10 vpxor 224(%rsi),%ymm15,%ymm15 vpxor 256(%rsi),%ymm14,%ymm14 vpxor 288(%rsi),%ymm2,%ymm2 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) vmovdqu %ymm12,128(%rdi) vmovdqu %ymm13,160(%rdi) vmovdqu %ymm10,192(%rdi) vmovdqu %ymm15,224(%rdi) vmovdqu %ymm14,256(%rdi) vmovdqu %ymm2,288(%rdi) je .Ldone8x leaq 320(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm3,0(%rsp) leaq 320(%rdi),%rdi subq $320,%rdx vmovdqa %ymm7,32(%rsp) jmp .Loop_tail8x .align 32 .L384_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vpxor 128(%rsi),%ymm12,%ymm12 vpxor 160(%rsi),%ymm13,%ymm13 vpxor 192(%rsi),%ymm10,%ymm10 vpxor 224(%rsi),%ymm15,%ymm15 vpxor 256(%rsi),%ymm14,%ymm14 vpxor 288(%rsi),%ymm2,%ymm2 vpxor 320(%rsi),%ymm3,%ymm3 vpxor 352(%rsi),%ymm7,%ymm7 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) vmovdqu %ymm12,128(%rdi) vmovdqu %ymm13,160(%rdi) vmovdqu %ymm10,192(%rdi) vmovdqu %ymm15,224(%rdi) vmovdqu %ymm14,256(%rdi) vmovdqu %ymm2,288(%rdi) vmovdqu %ymm3,320(%rdi) vmovdqu %ymm7,352(%rdi) je .Ldone8x leaq 384(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm11,0(%rsp) leaq 384(%rdi),%rdi subq $384,%rdx vmovdqa %ymm9,32(%rsp) jmp .Loop_tail8x .align 32 .L448_or_more8x: vpxor 0(%rsi),%ymm6,%ymm6 vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 vpxor 128(%rsi),%ymm12,%ymm12 vpxor 160(%rsi),%ymm13,%ymm13 vpxor 192(%rsi),%ymm10,%ymm10 vpxor 224(%rsi),%ymm15,%ymm15 vpxor 256(%rsi),%ymm14,%ymm14 vpxor 288(%rsi),%ymm2,%ymm2 vpxor 320(%rsi),%ymm3,%ymm3 vpxor 352(%rsi),%ymm7,%ymm7 vpxor 384(%rsi),%ymm11,%ymm11 vpxor 416(%rsi),%ymm9,%ymm9 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) vmovdqu %ymm12,128(%rdi) vmovdqu %ymm13,160(%rdi) vmovdqu %ymm10,192(%rdi) vmovdqu %ymm15,224(%rdi) vmovdqu %ymm14,256(%rdi) vmovdqu %ymm2,288(%rdi) vmovdqu %ymm3,320(%rdi) vmovdqu %ymm7,352(%rdi) vmovdqu %ymm11,384(%rdi) vmovdqu %ymm9,416(%rdi) je .Ldone8x leaq 448(%rsi),%rsi xorq %r9,%r9 vmovdqa %ymm0,0(%rsp) leaq 448(%rdi),%rdi subq $448,%rdx vmovdqa %ymm4,32(%rsp) .Loop_tail8x: movzbl (%rsi,%r9,1),%eax movzbl (%rsp,%r9,1),%ecx leaq 1(%r9),%r9 xorl %ecx,%eax movb %al,-1(%rdi,%r9,1) decq %rdx jnz .Loop_tail8x .Ldone8x: vzeroall leaq -8(%r10),%rsp .L8x_epilogue: ret ENDPROC(chacha20_avx2) #endif /* CONFIG_AS_AVX2 */ #ifdef CONFIG_AS_AVX512 .align 32 ENTRY(chacha20_avx512) .Lchacha20_avx512: cmpq $0,%rdx je .Lavx512_epilogue leaq 8(%rsp),%r10 cmpq $512,%rdx ja .Lchacha20_16x subq $64+8,%rsp andq $-64,%rsp vbroadcasti32x4 .Lsigma(%rip),%zmm0 vbroadcasti32x4 (%rcx),%zmm1 vbroadcasti32x4 16(%rcx),%zmm2 vbroadcasti32x4 (%r8),%zmm3 vmovdqa32 %zmm0,%zmm16 vmovdqa32 %zmm1,%zmm17 vmovdqa32 %zmm2,%zmm18 vpaddd .Lzeroz(%rip),%zmm3,%zmm3 vmovdqa32 .Lfourz(%rip),%zmm20 movq $10,%r8 vmovdqa32 %zmm3,%zmm19 jmp .Loop_avx512 .align 16 .Loop_outer_avx512: vmovdqa32 %zmm16,%zmm0 vmovdqa32 %zmm17,%zmm1 vmovdqa32 %zmm18,%zmm2 vpaddd %zmm20,%zmm19,%zmm3 movq $10,%r8 vmovdqa32 %zmm3,%zmm19 jmp .Loop_avx512 .align 32 .Loop_avx512: vpaddd %zmm1,%zmm0,%zmm0 vpxord %zmm0,%zmm3,%zmm3 vprold $16,%zmm3,%zmm3 vpaddd %zmm3,%zmm2,%zmm2 vpxord %zmm2,%zmm1,%zmm1 vprold $12,%zmm1,%zmm1 vpaddd %zmm1,%zmm0,%zmm0 vpxord %zmm0,%zmm3,%zmm3 vprold $8,%zmm3,%zmm3 vpaddd %zmm3,%zmm2,%zmm2 vpxord %zmm2,%zmm1,%zmm1 vprold $7,%zmm1,%zmm1 vpshufd $78,%zmm2,%zmm2 vpshufd $57,%zmm1,%zmm1 vpshufd $147,%zmm3,%zmm3 vpaddd %zmm1,%zmm0,%zmm0 vpxord %zmm0,%zmm3,%zmm3 vprold $16,%zmm3,%zmm3 vpaddd %zmm3,%zmm2,%zmm2 vpxord %zmm2,%zmm1,%zmm1 vprold $12,%zmm1,%zmm1 vpaddd %zmm1,%zmm0,%zmm0 vpxord %zmm0,%zmm3,%zmm3 vprold $8,%zmm3,%zmm3 vpaddd %zmm3,%zmm2,%zmm2 vpxord %zmm2,%zmm1,%zmm1 vprold $7,%zmm1,%zmm1 vpshufd $78,%zmm2,%zmm2 vpshufd $147,%zmm1,%zmm1 vpshufd $57,%zmm3,%zmm3 decq %r8 jnz .Loop_avx512 vpaddd %zmm16,%zmm0,%zmm0 vpaddd %zmm17,%zmm1,%zmm1 vpaddd %zmm18,%zmm2,%zmm2 vpaddd %zmm19,%zmm3,%zmm3 subq $64,%rdx jb .Ltail64_avx512 vpxor 0(%rsi),%xmm0,%xmm4 vpxor 16(%rsi),%xmm1,%xmm5 vpxor 32(%rsi),%xmm2,%xmm6 vpxor 48(%rsi),%xmm3,%xmm7 leaq 64(%rsi),%rsi vmovdqu %xmm4,0(%rdi) vmovdqu %xmm5,16(%rdi) vmovdqu %xmm6,32(%rdi) vmovdqu %xmm7,48(%rdi) leaq 64(%rdi),%rdi jz .Ldone_avx512 vextracti32x4 $1,%zmm0,%xmm4 vextracti32x4 $1,%zmm1,%xmm5 vextracti32x4 $1,%zmm2,%xmm6 vextracti32x4 $1,%zmm3,%xmm7 subq $64,%rdx jb .Ltail_avx512 vpxor 0(%rsi),%xmm4,%xmm4 vpxor 16(%rsi),%xmm5,%xmm5 vpxor 32(%rsi),%xmm6,%xmm6 vpxor 48(%rsi),%xmm7,%xmm7 leaq 64(%rsi),%rsi vmovdqu %xmm4,0(%rdi) vmovdqu %xmm5,16(%rdi) vmovdqu %xmm6,32(%rdi) vmovdqu %xmm7,48(%rdi) leaq 64(%rdi),%rdi jz .Ldone_avx512 vextracti32x4 $2,%zmm0,%xmm4 vextracti32x4 $2,%zmm1,%xmm5 vextracti32x4 $2,%zmm2,%xmm6 vextracti32x4 $2,%zmm3,%xmm7 subq $64,%rdx jb .Ltail_avx512 vpxor 0(%rsi),%xmm4,%xmm4 vpxor 16(%rsi),%xmm5,%xmm5 vpxor 32(%rsi),%xmm6,%xmm6 vpxor 48(%rsi),%xmm7,%xmm7 leaq 64(%rsi),%rsi vmovdqu %xmm4,0(%rdi) vmovdqu %xmm5,16(%rdi) vmovdqu %xmm6,32(%rdi) vmovdqu %xmm7,48(%rdi) leaq 64(%rdi),%rdi jz .Ldone_avx512 vextracti32x4 $3,%zmm0,%xmm4 vextracti32x4 $3,%zmm1,%xmm5 vextracti32x4 $3,%zmm2,%xmm6 vextracti32x4 $3,%zmm3,%xmm7 subq $64,%rdx jb .Ltail_avx512 vpxor 0(%rsi),%xmm4,%xmm4 vpxor 16(%rsi),%xmm5,%xmm5 vpxor 32(%rsi),%xmm6,%xmm6 vpxor 48(%rsi),%xmm7,%xmm7 leaq 64(%rsi),%rsi vmovdqu %xmm4,0(%rdi) vmovdqu %xmm5,16(%rdi) vmovdqu %xmm6,32(%rdi) vmovdqu %xmm7,48(%rdi) leaq 64(%rdi),%rdi jnz .Loop_outer_avx512 jmp .Ldone_avx512 .align 16 .Ltail64_avx512: vmovdqa %xmm0,0(%rsp) vmovdqa %xmm1,16(%rsp) vmovdqa %xmm2,32(%rsp) vmovdqa %xmm3,48(%rsp) addq $64,%rdx jmp .Loop_tail_avx512 .align 16 .Ltail_avx512: vmovdqa %xmm4,0(%rsp) vmovdqa %xmm5,16(%rsp) vmovdqa %xmm6,32(%rsp) vmovdqa %xmm7,48(%rsp) addq $64,%rdx .Loop_tail_avx512: movzbl (%rsi,%r8,1),%eax movzbl (%rsp,%r8,1),%ecx leaq 1(%r8),%r8 xorl %ecx,%eax movb %al,-1(%rdi,%r8,1) decq %rdx jnz .Loop_tail_avx512 vmovdqa32 %zmm16,0(%rsp) .Ldone_avx512: vzeroall leaq -8(%r10),%rsp .Lavx512_epilogue: ret .align 32 .Lchacha20_16x: leaq 8(%rsp),%r10 subq $64+8,%rsp andq $-64,%rsp vzeroupper leaq .Lsigma(%rip),%r9 vbroadcasti32x4 (%r9),%zmm3 vbroadcasti32x4 (%rcx),%zmm7 vbroadcasti32x4 16(%rcx),%zmm11 vbroadcasti32x4 (%r8),%zmm15 vpshufd $0x00,%zmm3,%zmm0 vpshufd $0x55,%zmm3,%zmm1 vpshufd $0xaa,%zmm3,%zmm2 vpshufd $0xff,%zmm3,%zmm3 vmovdqa64 %zmm0,%zmm16 vmovdqa64 %zmm1,%zmm17 vmovdqa64 %zmm2,%zmm18 vmovdqa64 %zmm3,%zmm19 vpshufd $0x00,%zmm7,%zmm4 vpshufd $0x55,%zmm7,%zmm5 vpshufd $0xaa,%zmm7,%zmm6 vpshufd $0xff,%zmm7,%zmm7 vmovdqa64 %zmm4,%zmm20 vmovdqa64 %zmm5,%zmm21 vmovdqa64 %zmm6,%zmm22 vmovdqa64 %zmm7,%zmm23 vpshufd $0x00,%zmm11,%zmm8 vpshufd $0x55,%zmm11,%zmm9 vpshufd $0xaa,%zmm11,%zmm10 vpshufd $0xff,%zmm11,%zmm11 vmovdqa64 %zmm8,%zmm24 vmovdqa64 %zmm9,%zmm25 vmovdqa64 %zmm10,%zmm26 vmovdqa64 %zmm11,%zmm27 vpshufd $0x00,%zmm15,%zmm12 vpshufd $0x55,%zmm15,%zmm13 vpshufd $0xaa,%zmm15,%zmm14 vpshufd $0xff,%zmm15,%zmm15 vpaddd .Lincz(%rip),%zmm12,%zmm12 vmovdqa64 %zmm12,%zmm28 vmovdqa64 %zmm13,%zmm29 vmovdqa64 %zmm14,%zmm30 vmovdqa64 %zmm15,%zmm31 movl $10,%eax jmp .Loop16x .align 32 .Loop_outer16x: vpbroadcastd 0(%r9),%zmm0 vpbroadcastd 4(%r9),%zmm1 vpbroadcastd 8(%r9),%zmm2 vpbroadcastd 12(%r9),%zmm3 vpaddd .Lsixteen(%rip),%zmm28,%zmm28 vmovdqa64 %zmm20,%zmm4 vmovdqa64 %zmm21,%zmm5 vmovdqa64 %zmm22,%zmm6 vmovdqa64 %zmm23,%zmm7 vmovdqa64 %zmm24,%zmm8 vmovdqa64 %zmm25,%zmm9 vmovdqa64 %zmm26,%zmm10 vmovdqa64 %zmm27,%zmm11 vmovdqa64 %zmm28,%zmm12 vmovdqa64 %zmm29,%zmm13 vmovdqa64 %zmm30,%zmm14 vmovdqa64 %zmm31,%zmm15 vmovdqa64 %zmm0,%zmm16 vmovdqa64 %zmm1,%zmm17 vmovdqa64 %zmm2,%zmm18 vmovdqa64 %zmm3,%zmm19 movl $10,%eax jmp .Loop16x .align 32 .Loop16x: vpaddd %zmm4,%zmm0,%zmm0 vpaddd %zmm5,%zmm1,%zmm1 vpaddd %zmm6,%zmm2,%zmm2 vpaddd %zmm7,%zmm3,%zmm3 vpxord %zmm0,%zmm12,%zmm12 vpxord %zmm1,%zmm13,%zmm13 vpxord %zmm2,%zmm14,%zmm14 vpxord %zmm3,%zmm15,%zmm15 vprold $16,%zmm12,%zmm12 vprold $16,%zmm13,%zmm13 vprold $16,%zmm14,%zmm14 vprold $16,%zmm15,%zmm15 vpaddd %zmm12,%zmm8,%zmm8 vpaddd %zmm13,%zmm9,%zmm9 vpaddd %zmm14,%zmm10,%zmm10 vpaddd %zmm15,%zmm11,%zmm11 vpxord %zmm8,%zmm4,%zmm4 vpxord %zmm9,%zmm5,%zmm5 vpxord %zmm10,%zmm6,%zmm6 vpxord %zmm11,%zmm7,%zmm7 vprold $12,%zmm4,%zmm4 vprold $12,%zmm5,%zmm5 vprold $12,%zmm6,%zmm6 vprold $12,%zmm7,%zmm7 vpaddd %zmm4,%zmm0,%zmm0 vpaddd %zmm5,%zmm1,%zmm1 vpaddd %zmm6,%zmm2,%zmm2 vpaddd %zmm7,%zmm3,%zmm3 vpxord %zmm0,%zmm12,%zmm12 vpxord %zmm1,%zmm13,%zmm13 vpxord %zmm2,%zmm14,%zmm14 vpxord %zmm3,%zmm15,%zmm15 vprold $8,%zmm12,%zmm12 vprold $8,%zmm13,%zmm13 vprold $8,%zmm14,%zmm14 vprold $8,%zmm15,%zmm15 vpaddd %zmm12,%zmm8,%zmm8 vpaddd %zmm13,%zmm9,%zmm9 vpaddd %zmm14,%zmm10,%zmm10 vpaddd %zmm15,%zmm11,%zmm11 vpxord %zmm8,%zmm4,%zmm4 vpxord %zmm9,%zmm5,%zmm5 vpxord %zmm10,%zmm6,%zmm6 vpxord %zmm11,%zmm7,%zmm7 vprold $7,%zmm4,%zmm4 vprold $7,%zmm5,%zmm5 vprold $7,%zmm6,%zmm6 vprold $7,%zmm7,%zmm7 vpaddd %zmm5,%zmm0,%zmm0 vpaddd %zmm6,%zmm1,%zmm1 vpaddd %zmm7,%zmm2,%zmm2 vpaddd %zmm4,%zmm3,%zmm3 vpxord %zmm0,%zmm15,%zmm15 vpxord %zmm1,%zmm12,%zmm12 vpxord %zmm2,%zmm13,%zmm13 vpxord %zmm3,%zmm14,%zmm14 vprold $16,%zmm15,%zmm15 vprold $16,%zmm12,%zmm12 vprold $16,%zmm13,%zmm13 vprold $16,%zmm14,%zmm14 vpaddd %zmm15,%zmm10,%zmm10 vpaddd %zmm12,%zmm11,%zmm11 vpaddd %zmm13,%zmm8,%zmm8 vpaddd %zmm14,%zmm9,%zmm9 vpxord %zmm10,%zmm5,%zmm5 vpxord %zmm11,%zmm6,%zmm6 vpxord %zmm8,%zmm7,%zmm7 vpxord %zmm9,%zmm4,%zmm4 vprold $12,%zmm5,%zmm5 vprold $12,%zmm6,%zmm6 vprold $12,%zmm7,%zmm7 vprold $12,%zmm4,%zmm4 vpaddd %zmm5,%zmm0,%zmm0 vpaddd %zmm6,%zmm1,%zmm1 vpaddd %zmm7,%zmm2,%zmm2 vpaddd %zmm4,%zmm3,%zmm3 vpxord %zmm0,%zmm15,%zmm15 vpxord %zmm1,%zmm12,%zmm12 vpxord %zmm2,%zmm13,%zmm13 vpxord %zmm3,%zmm14,%zmm14 vprold $8,%zmm15,%zmm15 vprold $8,%zmm12,%zmm12 vprold $8,%zmm13,%zmm13 vprold $8,%zmm14,%zmm14 vpaddd %zmm15,%zmm10,%zmm10 vpaddd %zmm12,%zmm11,%zmm11 vpaddd %zmm13,%zmm8,%zmm8 vpaddd %zmm14,%zmm9,%zmm9 vpxord %zmm10,%zmm5,%zmm5 vpxord %zmm11,%zmm6,%zmm6 vpxord %zmm8,%zmm7,%zmm7 vpxord %zmm9,%zmm4,%zmm4 vprold $7,%zmm5,%zmm5 vprold $7,%zmm6,%zmm6 vprold $7,%zmm7,%zmm7 vprold $7,%zmm4,%zmm4 decl %eax jnz .Loop16x vpaddd %zmm16,%zmm0,%zmm0 vpaddd %zmm17,%zmm1,%zmm1 vpaddd %zmm18,%zmm2,%zmm2 vpaddd %zmm19,%zmm3,%zmm3 vpunpckldq %zmm1,%zmm0,%zmm18 vpunpckldq %zmm3,%zmm2,%zmm19 vpunpckhdq %zmm1,%zmm0,%zmm0 vpunpckhdq %zmm3,%zmm2,%zmm2 vpunpcklqdq %zmm19,%zmm18,%zmm1 vpunpckhqdq %zmm19,%zmm18,%zmm18 vpunpcklqdq %zmm2,%zmm0,%zmm3 vpunpckhqdq %zmm2,%zmm0,%zmm0 vpaddd %zmm20,%zmm4,%zmm4 vpaddd %zmm21,%zmm5,%zmm5 vpaddd %zmm22,%zmm6,%zmm6 vpaddd %zmm23,%zmm7,%zmm7 vpunpckldq %zmm5,%zmm4,%zmm2 vpunpckldq %zmm7,%zmm6,%zmm19 vpunpckhdq %zmm5,%zmm4,%zmm4 vpunpckhdq %zmm7,%zmm6,%zmm6 vpunpcklqdq %zmm19,%zmm2,%zmm5 vpunpckhqdq %zmm19,%zmm2,%zmm2 vpunpcklqdq %zmm6,%zmm4,%zmm7 vpunpckhqdq %zmm6,%zmm4,%zmm4 vshufi32x4 $0x44,%zmm5,%zmm1,%zmm19 vshufi32x4 $0xee,%zmm5,%zmm1,%zmm5 vshufi32x4 $0x44,%zmm2,%zmm18,%zmm1 vshufi32x4 $0xee,%zmm2,%zmm18,%zmm2 vshufi32x4 $0x44,%zmm7,%zmm3,%zmm18 vshufi32x4 $0xee,%zmm7,%zmm3,%zmm7 vshufi32x4 $0x44,%zmm4,%zmm0,%zmm3 vshufi32x4 $0xee,%zmm4,%zmm0,%zmm4 vpaddd %zmm24,%zmm8,%zmm8 vpaddd %zmm25,%zmm9,%zmm9 vpaddd %zmm26,%zmm10,%zmm10 vpaddd %zmm27,%zmm11,%zmm11 vpunpckldq %zmm9,%zmm8,%zmm6 vpunpckldq %zmm11,%zmm10,%zmm0 vpunpckhdq %zmm9,%zmm8,%zmm8 vpunpckhdq %zmm11,%zmm10,%zmm10 vpunpcklqdq %zmm0,%zmm6,%zmm9 vpunpckhqdq %zmm0,%zmm6,%zmm6 vpunpcklqdq %zmm10,%zmm8,%zmm11 vpunpckhqdq %zmm10,%zmm8,%zmm8 vpaddd %zmm28,%zmm12,%zmm12 vpaddd %zmm29,%zmm13,%zmm13 vpaddd %zmm30,%zmm14,%zmm14 vpaddd %zmm31,%zmm15,%zmm15 vpunpckldq %zmm13,%zmm12,%zmm10 vpunpckldq %zmm15,%zmm14,%zmm0 vpunpckhdq %zmm13,%zmm12,%zmm12 vpunpckhdq %zmm15,%zmm14,%zmm14 vpunpcklqdq %zmm0,%zmm10,%zmm13 vpunpckhqdq %zmm0,%zmm10,%zmm10 vpunpcklqdq %zmm14,%zmm12,%zmm15 vpunpckhqdq %zmm14,%zmm12,%zmm12 vshufi32x4 $0x44,%zmm13,%zmm9,%zmm0 vshufi32x4 $0xee,%zmm13,%zmm9,%zmm13 vshufi32x4 $0x44,%zmm10,%zmm6,%zmm9 vshufi32x4 $0xee,%zmm10,%zmm6,%zmm10 vshufi32x4 $0x44,%zmm15,%zmm11,%zmm6 vshufi32x4 $0xee,%zmm15,%zmm11,%zmm15 vshufi32x4 $0x44,%zmm12,%zmm8,%zmm11 vshufi32x4 $0xee,%zmm12,%zmm8,%zmm12 vshufi32x4 $0x88,%zmm0,%zmm19,%zmm16 vshufi32x4 $0xdd,%zmm0,%zmm19,%zmm19 vshufi32x4 $0x88,%zmm13,%zmm5,%zmm0 vshufi32x4 $0xdd,%zmm13,%zmm5,%zmm13 vshufi32x4 $0x88,%zmm9,%zmm1,%zmm17 vshufi32x4 $0xdd,%zmm9,%zmm1,%zmm1 vshufi32x4 $0x88,%zmm10,%zmm2,%zmm9 vshufi32x4 $0xdd,%zmm10,%zmm2,%zmm10 vshufi32x4 $0x88,%zmm6,%zmm18,%zmm14 vshufi32x4 $0xdd,%zmm6,%zmm18,%zmm18 vshufi32x4 $0x88,%zmm15,%zmm7,%zmm6 vshufi32x4 $0xdd,%zmm15,%zmm7,%zmm15 vshufi32x4 $0x88,%zmm11,%zmm3,%zmm8 vshufi32x4 $0xdd,%zmm11,%zmm3,%zmm3 vshufi32x4 $0x88,%zmm12,%zmm4,%zmm11 vshufi32x4 $0xdd,%zmm12,%zmm4,%zmm12 cmpq $1024,%rdx jb .Ltail16x vpxord 0(%rsi),%zmm16,%zmm16 vpxord 64(%rsi),%zmm17,%zmm17 vpxord 128(%rsi),%zmm14,%zmm14 vpxord 192(%rsi),%zmm8,%zmm8 vmovdqu32 %zmm16,0(%rdi) vmovdqu32 %zmm17,64(%rdi) vmovdqu32 %zmm14,128(%rdi) vmovdqu32 %zmm8,192(%rdi) vpxord 256(%rsi),%zmm19,%zmm19 vpxord 320(%rsi),%zmm1,%zmm1 vpxord 384(%rsi),%zmm18,%zmm18 vpxord 448(%rsi),%zmm3,%zmm3 vmovdqu32 %zmm19,256(%rdi) vmovdqu32 %zmm1,320(%rdi) vmovdqu32 %zmm18,384(%rdi) vmovdqu32 %zmm3,448(%rdi) vpxord 512(%rsi),%zmm0,%zmm0 vpxord 576(%rsi),%zmm9,%zmm9 vpxord 640(%rsi),%zmm6,%zmm6 vpxord 704(%rsi),%zmm11,%zmm11 vmovdqu32 %zmm0,512(%rdi) vmovdqu32 %zmm9,576(%rdi) vmovdqu32 %zmm6,640(%rdi) vmovdqu32 %zmm11,704(%rdi) vpxord 768(%rsi),%zmm13,%zmm13 vpxord 832(%rsi),%zmm10,%zmm10 vpxord 896(%rsi),%zmm15,%zmm15 vpxord 960(%rsi),%zmm12,%zmm12 leaq 1024(%rsi),%rsi vmovdqu32 %zmm13,768(%rdi) vmovdqu32 %zmm10,832(%rdi) vmovdqu32 %zmm15,896(%rdi) vmovdqu32 %zmm12,960(%rdi) leaq 1024(%rdi),%rdi subq $1024,%rdx jnz .Loop_outer16x jmp .Ldone16x .align 32 .Ltail16x: xorq %r9,%r9 subq %rsi,%rdi cmpq $64,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm16,%zmm16 vmovdqu32 %zmm16,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm17,%zmm16 leaq 64(%rsi),%rsi cmpq $128,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm17,%zmm17 vmovdqu32 %zmm17,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm14,%zmm16 leaq 64(%rsi),%rsi cmpq $192,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm14,%zmm14 vmovdqu32 %zmm14,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm8,%zmm16 leaq 64(%rsi),%rsi cmpq $256,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm8,%zmm8 vmovdqu32 %zmm8,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm19,%zmm16 leaq 64(%rsi),%rsi cmpq $320,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm19,%zmm19 vmovdqu32 %zmm19,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm1,%zmm16 leaq 64(%rsi),%rsi cmpq $384,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm1,%zmm1 vmovdqu32 %zmm1,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm18,%zmm16 leaq 64(%rsi),%rsi cmpq $448,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm18,%zmm18 vmovdqu32 %zmm18,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm3,%zmm16 leaq 64(%rsi),%rsi cmpq $512,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm3,%zmm3 vmovdqu32 %zmm3,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm0,%zmm16 leaq 64(%rsi),%rsi cmpq $576,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm0,%zmm0 vmovdqu32 %zmm0,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm9,%zmm16 leaq 64(%rsi),%rsi cmpq $640,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm9,%zmm9 vmovdqu32 %zmm9,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm6,%zmm16 leaq 64(%rsi),%rsi cmpq $704,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm6,%zmm6 vmovdqu32 %zmm6,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm11,%zmm16 leaq 64(%rsi),%rsi cmpq $768,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm11,%zmm11 vmovdqu32 %zmm11,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm13,%zmm16 leaq 64(%rsi),%rsi cmpq $832,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm13,%zmm13 vmovdqu32 %zmm13,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm10,%zmm16 leaq 64(%rsi),%rsi cmpq $896,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm10,%zmm10 vmovdqu32 %zmm10,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm15,%zmm16 leaq 64(%rsi),%rsi cmpq $960,%rdx jb .Less_than_64_16x vpxord (%rsi),%zmm15,%zmm15 vmovdqu32 %zmm15,(%rdi,%rsi,1) je .Ldone16x vmovdqa32 %zmm12,%zmm16 leaq 64(%rsi),%rsi .Less_than_64_16x: vmovdqa32 %zmm16,0(%rsp) leaq (%rdi,%rsi,1),%rdi andq $63,%rdx .Loop_tail16x: movzbl (%rsi,%r9,1),%eax movzbl (%rsp,%r9,1),%ecx leaq 1(%r9),%r9 xorl %ecx,%eax movb %al,-1(%rdi,%r9,1) decq %rdx jnz .Loop_tail16x vpxord %zmm16,%zmm16,%zmm16 vmovdqa32 %zmm16,0(%rsp) .Ldone16x: vzeroall leaq -8(%r10),%rsp .L16x_epilogue: ret ENDPROC(chacha20_avx512) #endif /* CONFIG_AS_AVX512 */