Changeset 99371 in vbox

Timestamp:

Apr 11, 2023 10:16:56 AM (20 months ago)

Author:

vboxsync

Message:

openssl-3.1.0: After generating headers and asm (kmk recreate-headers recreate-headers)

Location:

trunk/src/libs/openssl-3.1.0

Files:

• crypto/bn/Makefile.kmk (modified) (4 diffs)
• crypto/genasm-nasm/aesni-gcm-x86_64.S (modified) (1 diff)
• crypto/genasm-nasm/aesni-mb-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/aesni-sha1-x86_64.S (modified) (8 diffs)
• crypto/genasm-nasm/aesni-sha256-x86_64.S (modified) (2 diffs)
• crypto/genasm-nasm/chacha-x86_64.S (modified) (6 diffs)
• crypto/genasm-nasm/ecp_nistz256-x86_64.S (modified) (24 diffs)
• crypto/genasm-nasm/ghash-x86_64.S (modified) (3 diffs)
• crypto/genasm-nasm/poly1305-x86_64.S (modified) (4 diffs)
• crypto/genasm-nasm/rsaz-avx2.S (modified) (1 diff)
• crypto/genasm-nasm/rsaz-x86_64.S (modified) (12 diffs)
• crypto/genasm-nasm/sha1-mb-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/sha1-x86_64.S (modified) (4 diffs)
• crypto/genasm-nasm/sha256-mb-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/sha256-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/sha512-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/x25519-x86_64.S (modified) (5 diffs)
• crypto/genasm-nasm/x86_64-mont.S (modified) (7 diffs)
• crypto/genasm-nasm/x86_64-mont5.S (modified) (6 diffs)
• gen-includes/openssl/asn1.h (modified) (2 diffs)
• gen-includes/openssl/cmp.h (modified) (3 diffs)
• gen-includes/openssl/conf.h (modified) (2 diffs)
• gen-includes/openssl/crypto.h (modified) (2 diffs)
• gen-includes/openssl/lhash.h (modified) (6 diffs)
• gen-includes/openssl/pkcs12.h (modified) (2 diffs)
• gen-includes/openssl/pkcs7.h (modified) (2 diffs)
• gen-includes/openssl/ssl.h (modified) (1 diff)
• gen-includes/openssl/x509.h (modified) (1 diff)
• gen-includes/openssl/x509v3.h (modified) (3 diffs)

trunk/src/libs/openssl-3.1.0/crypto/bn/Makefile.kmk

@@ -94 +94 @@
                 $(VBOX_PATH_CRYPTO_BN)/asm/x86-gf2m.pl \
                 $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx2.pl \
-                $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx512.pl \
                 $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-x86_64.pl \
                 $(VBOX_PATH_CRYPTO_BN)/asm/x86_64-gf2m.pl \
@@ -104 +103 @@
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86-gf2m.pl elf -fPIC $(VBOX_PATH_CRYPTO)/genasm-elf/x86-gf2m.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx2.pl elf $(VBOX_PATH_CRYPTO)/genasm-elf/rsaz-avx2.S
-        perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx512.pl elf $(VBOX_PATH_CRYPTO)/genasm-elf/rsaz-avx512.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-x86_64.pl elf $(VBOX_PATH_CRYPTO)/genasm-elf/rsaz-x86_64.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86_64-gf2m.pl elf $(VBOX_PATH_CRYPTO)/genasm-elf/x86_64-gf2m.S
@@ -114 +112 @@
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86-gf2m.pl macosx $(VBOX_PATH_CRYPTO)/genasm-macosx/x86-gf2m.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx2.pl macosx $(VBOX_PATH_CRYPTO)/genasm-macosx/rsaz-avx2.S
-        perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx512.pl macosx $(VBOX_PATH_CRYPTO)/genasm-macosx/rsaz-avx512.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-x86_64.pl macosx $(VBOX_PATH_CRYPTO)/genasm-macosx/rsaz-x86_64.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86_64-gf2m.pl macosx $(VBOX_PATH_CRYPTO)/genasm-macosx/x86_64-gf2m.S
@@ -124 +121 @@
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86-gf2m.pl win32n $(VBOX_PATH_CRYPTO)/genasm-nasm/x86-gf2m.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx2.pl nasm $(VBOX_PATH_CRYPTO)/genasm-nasm/rsaz-avx2.S
-        perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-avx512.pl nasm $(VBOX_PATH_CRYPTO)/genasm-nasm/rsaz-avx512.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/rsaz-x86_64.pl nasm $(VBOX_PATH_CRYPTO)/genasm-nasm/rsaz-x86_64.S
         perl $(VBOX_PATH_CRYPTO_BN)/asm/x86_64-gf2m.pl nasm $(VBOX_PATH_CRYPTO)/genasm-nasm/x86_64-gf2m.S

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/aesni-gcm-x86_64.S

@@ -6 +6 @@
 
 
+
+ALIGN   32
+_aesni_ctr32_ghash_6x:
+
+        vmovdqu xmm2,XMMWORD[32+r11]
+        sub     rdx,6
+        vpxor   xmm4,xmm4,xmm4
+        vmovdqu xmm15,XMMWORD[((0-128))+rcx]
+        vpaddb  xmm10,xmm1,xmm2
+        vpaddb  xmm11,xmm10,xmm2
+        vpaddb  xmm12,xmm11,xmm2
+        vpaddb  xmm13,xmm12,xmm2
+        vpaddb  xmm14,xmm13,xmm2
+        vpxor   xmm9,xmm1,xmm15
+        vmovdqu XMMWORD[(16+8)+rsp],xmm4
+        jmp     NEAR $L$oop6x
+
+ALIGN   32
+$L$oop6x:
+        add     ebx,100663296
+        jc      NEAR $L$handle_ctr32
+        vmovdqu xmm3,XMMWORD[((0-32))+r9]
+        vpaddb  xmm1,xmm14,xmm2
+        vpxor   xmm10,xmm10,xmm15
+        vpxor   xmm11,xmm11,xmm15
+
+$L$resume_ctr32:
+        vmovdqu XMMWORD[r8],xmm1
+        vpclmulqdq      xmm5,xmm7,xmm3,0x10
+        vpxor   xmm12,xmm12,xmm15
+        vmovups xmm2,XMMWORD[((16-128))+rcx]
+        vpclmulqdq      xmm6,xmm7,xmm3,0x01
+        xor     r12,r12
+        cmp     r15,r14
+
+        vaesenc xmm9,xmm9,xmm2
+        vmovdqu xmm0,XMMWORD[((48+8))+rsp]
+        vpxor   xmm13,xmm13,xmm15
+        vpclmulqdq      xmm1,xmm7,xmm3,0x00
+        vaesenc xmm10,xmm10,xmm2
+        vpxor   xmm14,xmm14,xmm15
+        setnc   r12b
+        vpclmulqdq      xmm7,xmm7,xmm3,0x11
+        vaesenc xmm11,xmm11,xmm2
+        vmovdqu xmm3,XMMWORD[((16-32))+r9]
+        neg     r12
+        vaesenc xmm12,xmm12,xmm2
+        vpxor   xmm6,xmm6,xmm5
+        vpclmulqdq      xmm5,xmm0,xmm3,0x00
+        vpxor   xmm8,xmm8,xmm4
+        vaesenc xmm13,xmm13,xmm2
+        vpxor   xmm4,xmm1,xmm5
+        and     r12,0x60
+        vmovups xmm15,XMMWORD[((32-128))+rcx]
+        vpclmulqdq      xmm1,xmm0,xmm3,0x10
+        vaesenc xmm14,xmm14,xmm2
+
+        vpclmulqdq      xmm2,xmm0,xmm3,0x01
+        lea     r14,[r12*1+r14]
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm8,xmm8,XMMWORD[((16+8))+rsp]
+        vpclmulqdq      xmm3,xmm0,xmm3,0x11
+        vmovdqu xmm0,XMMWORD[((64+8))+rsp]
+        vaesenc xmm10,xmm10,xmm15
+        movbe   r13,QWORD[88+r14]
+        vaesenc xmm11,xmm11,xmm15
+        movbe   r12,QWORD[80+r14]
+        vaesenc xmm12,xmm12,xmm15
+        mov     QWORD[((32+8))+rsp],r13
+        vaesenc xmm13,xmm13,xmm15
+        mov     QWORD[((40+8))+rsp],r12
+        vmovdqu xmm5,XMMWORD[((48-32))+r9]
+        vaesenc xmm14,xmm14,xmm15
+
+        vmovups xmm15,XMMWORD[((48-128))+rcx]
+        vpxor   xmm6,xmm6,xmm1
+        vpclmulqdq      xmm1,xmm0,xmm5,0x00
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm6,xmm6,xmm2
+        vpclmulqdq      xmm2,xmm0,xmm5,0x10
+        vaesenc xmm10,xmm10,xmm15
+        vpxor   xmm7,xmm7,xmm3
+        vpclmulqdq      xmm3,xmm0,xmm5,0x01
+        vaesenc xmm11,xmm11,xmm15
+        vpclmulqdq      xmm5,xmm0,xmm5,0x11
+        vmovdqu xmm0,XMMWORD[((80+8))+rsp]
+        vaesenc xmm12,xmm12,xmm15
+        vaesenc xmm13,xmm13,xmm15
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqu xmm1,XMMWORD[((64-32))+r9]
+        vaesenc xmm14,xmm14,xmm15
+
+        vmovups xmm15,XMMWORD[((64-128))+rcx]
+        vpxor   xmm6,xmm6,xmm2
+        vpclmulqdq      xmm2,xmm0,xmm1,0x00
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm6,xmm6,xmm3
+        vpclmulqdq      xmm3,xmm0,xmm1,0x10
+        vaesenc xmm10,xmm10,xmm15
+        movbe   r13,QWORD[72+r14]
+        vpxor   xmm7,xmm7,xmm5
+        vpclmulqdq      xmm5,xmm0,xmm1,0x01
+        vaesenc xmm11,xmm11,xmm15
+        movbe   r12,QWORD[64+r14]
+        vpclmulqdq      xmm1,xmm0,xmm1,0x11
+        vmovdqu xmm0,XMMWORD[((96+8))+rsp]
+        vaesenc xmm12,xmm12,xmm15
+        mov     QWORD[((48+8))+rsp],r13
+        vaesenc xmm13,xmm13,xmm15
+        mov     QWORD[((56+8))+rsp],r12
+        vpxor   xmm4,xmm4,xmm2
+        vmovdqu xmm2,XMMWORD[((96-32))+r9]
+        vaesenc xmm14,xmm14,xmm15
+
+        vmovups xmm15,XMMWORD[((80-128))+rcx]
+        vpxor   xmm6,xmm6,xmm3
+        vpclmulqdq      xmm3,xmm0,xmm2,0x00
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm6,xmm6,xmm5
+        vpclmulqdq      xmm5,xmm0,xmm2,0x10
+        vaesenc xmm10,xmm10,xmm15
+        movbe   r13,QWORD[56+r14]
+        vpxor   xmm7,xmm7,xmm1
+        vpclmulqdq      xmm1,xmm0,xmm2,0x01
+        vpxor   xmm8,xmm8,XMMWORD[((112+8))+rsp]
+        vaesenc xmm11,xmm11,xmm15
+        movbe   r12,QWORD[48+r14]
+        vpclmulqdq      xmm2,xmm0,xmm2,0x11
+        vaesenc xmm12,xmm12,xmm15
+        mov     QWORD[((64+8))+rsp],r13
+        vaesenc xmm13,xmm13,xmm15
+        mov     QWORD[((72+8))+rsp],r12
+        vpxor   xmm4,xmm4,xmm3
+        vmovdqu xmm3,XMMWORD[((112-32))+r9]
+        vaesenc xmm14,xmm14,xmm15
+
+        vmovups xmm15,XMMWORD[((96-128))+rcx]
+        vpxor   xmm6,xmm6,xmm5
+        vpclmulqdq      xmm5,xmm8,xmm3,0x10
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm6,xmm6,xmm1
+        vpclmulqdq      xmm1,xmm8,xmm3,0x01
+        vaesenc xmm10,xmm10,xmm15
+        movbe   r13,QWORD[40+r14]
+        vpxor   xmm7,xmm7,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm3,0x00
+        vaesenc xmm11,xmm11,xmm15
+        movbe   r12,QWORD[32+r14]
+        vpclmulqdq      xmm8,xmm8,xmm3,0x11
+        vaesenc xmm12,xmm12,xmm15
+        mov     QWORD[((80+8))+rsp],r13
+        vaesenc xmm13,xmm13,xmm15
+        mov     QWORD[((88+8))+rsp],r12
+        vpxor   xmm6,xmm6,xmm5
+        vaesenc xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm6,xmm1
+
+        vmovups xmm15,XMMWORD[((112-128))+rcx]
+        vpslldq xmm5,xmm6,8
+        vpxor   xmm4,xmm4,xmm2
+        vmovdqu xmm3,XMMWORD[16+r11]
+
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm7,xmm7,xmm8
+        vaesenc xmm10,xmm10,xmm15
+        vpxor   xmm4,xmm4,xmm5
+        movbe   r13,QWORD[24+r14]
+        vaesenc xmm11,xmm11,xmm15
+        movbe   r12,QWORD[16+r14]
+        vpalignr        xmm0,xmm4,xmm4,8
+        vpclmulqdq      xmm4,xmm4,xmm3,0x10
+        mov     QWORD[((96+8))+rsp],r13
+        vaesenc xmm12,xmm12,xmm15
+        mov     QWORD[((104+8))+rsp],r12
+        vaesenc xmm13,xmm13,xmm15
+        vmovups xmm1,XMMWORD[((128-128))+rcx]
+        vaesenc xmm14,xmm14,xmm15
+
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm15,XMMWORD[((144-128))+rcx]
+        vaesenc xmm10,xmm10,xmm1
+        vpsrldq xmm6,xmm6,8
+        vaesenc xmm11,xmm11,xmm1
+        vpxor   xmm7,xmm7,xmm6
+        vaesenc xmm12,xmm12,xmm1
+        vpxor   xmm4,xmm4,xmm0
+        movbe   r13,QWORD[8+r14]
+        vaesenc xmm13,xmm13,xmm1
+        movbe   r12,QWORD[r14]
+        vaesenc xmm14,xmm14,xmm1
+        vmovups xmm1,XMMWORD[((160-128))+rcx]
+        cmp     ebp,11
+        jb      NEAR $L$enc_tail
+
+        vaesenc xmm9,xmm9,xmm15
+        vaesenc xmm10,xmm10,xmm15
+        vaesenc xmm11,xmm11,xmm15
+        vaesenc xmm12,xmm12,xmm15
+        vaesenc xmm13,xmm13,xmm15
+        vaesenc xmm14,xmm14,xmm15
+
+        vaesenc xmm9,xmm9,xmm1
+        vaesenc xmm10,xmm10,xmm1
+        vaesenc xmm11,xmm11,xmm1
+        vaesenc xmm12,xmm12,xmm1
+        vaesenc xmm13,xmm13,xmm1
+        vmovups xmm15,XMMWORD[((176-128))+rcx]
+        vaesenc xmm14,xmm14,xmm1
+        vmovups xmm1,XMMWORD[((192-128))+rcx]
+        je      NEAR $L$enc_tail
+
+        vaesenc xmm9,xmm9,xmm15
+        vaesenc xmm10,xmm10,xmm15
+        vaesenc xmm11,xmm11,xmm15
+        vaesenc xmm12,xmm12,xmm15
+        vaesenc xmm13,xmm13,xmm15
+        vaesenc xmm14,xmm14,xmm15
+
+        vaesenc xmm9,xmm9,xmm1
+        vaesenc xmm10,xmm10,xmm1
+        vaesenc xmm11,xmm11,xmm1
+        vaesenc xmm12,xmm12,xmm1
+        vaesenc xmm13,xmm13,xmm1
+        vmovups xmm15,XMMWORD[((208-128))+rcx]
+        vaesenc xmm14,xmm14,xmm1
+        vmovups xmm1,XMMWORD[((224-128))+rcx]
+        jmp     NEAR $L$enc_tail
+
+ALIGN   32
+$L$handle_ctr32:
+        vmovdqu xmm0,XMMWORD[r11]
+        vpshufb xmm6,xmm1,xmm0
+        vmovdqu xmm5,XMMWORD[48+r11]
+        vpaddd  xmm10,xmm6,XMMWORD[64+r11]
+        vpaddd  xmm11,xmm6,xmm5
+        vmovdqu xmm3,XMMWORD[((0-32))+r9]
+        vpaddd  xmm12,xmm10,xmm5
+        vpshufb xmm10,xmm10,xmm0
+        vpaddd  xmm13,xmm11,xmm5
+        vpshufb xmm11,xmm11,xmm0
+        vpxor   xmm10,xmm10,xmm15
+        vpaddd  xmm14,xmm12,xmm5
+        vpshufb xmm12,xmm12,xmm0
+        vpxor   xmm11,xmm11,xmm15
+        vpaddd  xmm1,xmm13,xmm5
+        vpshufb xmm13,xmm13,xmm0
+        vpshufb xmm14,xmm14,xmm0
+        vpshufb xmm1,xmm1,xmm0
+        jmp     NEAR $L$resume_ctr32
+
+ALIGN   32
+$L$enc_tail:
+        vaesenc xmm9,xmm9,xmm15
+        vmovdqu XMMWORD[(16+8)+rsp],xmm7
+        vpalignr        xmm8,xmm4,xmm4,8
+        vaesenc xmm10,xmm10,xmm15
+        vpclmulqdq      xmm4,xmm4,xmm3,0x10
+        vpxor   xmm2,xmm1,XMMWORD[rdi]
+        vaesenc xmm11,xmm11,xmm15
+        vpxor   xmm0,xmm1,XMMWORD[16+rdi]
+        vaesenc xmm12,xmm12,xmm15
+        vpxor   xmm5,xmm1,XMMWORD[32+rdi]
+        vaesenc xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm1,XMMWORD[48+rdi]
+        vaesenc xmm14,xmm14,xmm15
+        vpxor   xmm7,xmm1,XMMWORD[64+rdi]
+        vpxor   xmm3,xmm1,XMMWORD[80+rdi]
+        vmovdqu xmm1,XMMWORD[r8]
+
+        vaesenclast     xmm9,xmm9,xmm2
+        vmovdqu xmm2,XMMWORD[32+r11]
+        vaesenclast     xmm10,xmm10,xmm0
+        vpaddb  xmm0,xmm1,xmm2
+        mov     QWORD[((112+8))+rsp],r13
+        lea     rdi,[96+rdi]
+        vaesenclast     xmm11,xmm11,xmm5
+        vpaddb  xmm5,xmm0,xmm2
+        mov     QWORD[((120+8))+rsp],r12
+        lea     rsi,[96+rsi]
+        vmovdqu xmm15,XMMWORD[((0-128))+rcx]
+        vaesenclast     xmm12,xmm12,xmm6
+        vpaddb  xmm6,xmm5,xmm2
+        vaesenclast     xmm13,xmm13,xmm7
+        vpaddb  xmm7,xmm6,xmm2
+        vaesenclast     xmm14,xmm14,xmm3
+        vpaddb  xmm3,xmm7,xmm2
+
+        add     r10,0x60
+        sub     rdx,0x6
+        jc      NEAR $L$6x_done
+
+        vmovups XMMWORD[(-96)+rsi],xmm9
+        vpxor   xmm9,xmm1,xmm15
+        vmovups XMMWORD[(-80)+rsi],xmm10
+        vmovdqa xmm10,xmm0
+        vmovups XMMWORD[(-64)+rsi],xmm11
+        vmovdqa xmm11,xmm5
+        vmovups XMMWORD[(-48)+rsi],xmm12
+        vmovdqa xmm12,xmm6
+        vmovups XMMWORD[(-32)+rsi],xmm13
+        vmovdqa xmm13,xmm7
+        vmovups XMMWORD[(-16)+rsi],xmm14
+        vmovdqa xmm14,xmm3
+        vmovdqu xmm7,XMMWORD[((32+8))+rsp]
+        jmp     NEAR $L$oop6x
+
+$L$6x_done:
+        vpxor   xmm8,xmm8,XMMWORD[((16+8))+rsp]
+        vpxor   xmm8,xmm8,xmm4
+
+        DB      0F3h,0C3h               ;repret
+
+
+global  aesni_gcm_decrypt
+
+ALIGN   32
+aesni_gcm_decrypt:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_gcm_decrypt:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        xor     r10,r10
+        cmp     rdx,0x60
+        jb      NEAR $L$gcm_dec_abort
+
+        lea     rax,[rsp]
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[(-216)+rax],xmm6
+        movaps  XMMWORD[(-200)+rax],xmm7
+        movaps  XMMWORD[(-184)+rax],xmm8
+        movaps  XMMWORD[(-168)+rax],xmm9
+        movaps  XMMWORD[(-152)+rax],xmm10
+        movaps  XMMWORD[(-136)+rax],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+$L$gcm_dec_body:
+        vzeroupper
+
+        vmovdqu xmm1,XMMWORD[r8]
+        add     rsp,-128
+        mov     ebx,DWORD[12+r8]
+        lea     r11,[$L$bswap_mask]
+        lea     r14,[((-128))+rcx]
+        mov     r15,0xf80
+        vmovdqu xmm8,XMMWORD[r9]
+        and     rsp,-128
+        vmovdqu xmm0,XMMWORD[r11]
+        lea     rcx,[128+rcx]
+        lea     r9,[((32+32))+r9]
+        mov     ebp,DWORD[((240-128))+rcx]
+        vpshufb xmm8,xmm8,xmm0
+
+        and     r14,r15
+        and     r15,rsp
+        sub     r15,r14
+        jc      NEAR $L$dec_no_key_aliasing
+        cmp     r15,768
+        jnc     NEAR $L$dec_no_key_aliasing
+        sub     rsp,r15
+$L$dec_no_key_aliasing:
+
+        vmovdqu xmm7,XMMWORD[80+rdi]
+        lea     r14,[rdi]
+        vmovdqu xmm4,XMMWORD[64+rdi]
+        lea     r15,[((-192))+rdx*1+rdi]
+        vmovdqu xmm5,XMMWORD[48+rdi]
+        shr     rdx,4
+        xor     r10,r10
+        vmovdqu xmm6,XMMWORD[32+rdi]
+        vpshufb xmm7,xmm7,xmm0
+        vmovdqu xmm2,XMMWORD[16+rdi]
+        vpshufb xmm4,xmm4,xmm0
+        vmovdqu xmm3,XMMWORD[rdi]
+        vpshufb xmm5,xmm5,xmm0
+        vmovdqu XMMWORD[48+rsp],xmm4
+        vpshufb xmm6,xmm6,xmm0
+        vmovdqu XMMWORD[64+rsp],xmm5
+        vpshufb xmm2,xmm2,xmm0
+        vmovdqu XMMWORD[80+rsp],xmm6
+        vpshufb xmm3,xmm3,xmm0
+        vmovdqu XMMWORD[96+rsp],xmm2
+        vmovdqu XMMWORD[112+rsp],xmm3
+
+        call    _aesni_ctr32_ghash_6x
+
+        vmovups XMMWORD[(-96)+rsi],xmm9
+        vmovups XMMWORD[(-80)+rsi],xmm10
+        vmovups XMMWORD[(-64)+rsi],xmm11
+        vmovups XMMWORD[(-48)+rsi],xmm12
+        vmovups XMMWORD[(-32)+rsi],xmm13
+        vmovups XMMWORD[(-16)+rsi],xmm14
+
+        vpshufb xmm8,xmm8,XMMWORD[r11]
+        vmovdqu XMMWORD[(-64)+r9],xmm8
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$gcm_dec_abort:
+        mov     rax,r10
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_gcm_decrypt:
+
+ALIGN   32
+_aesni_ctr32_6x:
+
+        vmovdqu xmm4,XMMWORD[((0-128))+rcx]
+        vmovdqu xmm2,XMMWORD[32+r11]
+        lea     r13,[((-1))+rbp]
+        vmovups xmm15,XMMWORD[((16-128))+rcx]
+        lea     r12,[((32-128))+rcx]
+        vpxor   xmm9,xmm1,xmm4
+        add     ebx,100663296
+        jc      NEAR $L$handle_ctr32_2
+        vpaddb  xmm10,xmm1,xmm2
+        vpaddb  xmm11,xmm10,xmm2
+        vpxor   xmm10,xmm10,xmm4
+        vpaddb  xmm12,xmm11,xmm2
+        vpxor   xmm11,xmm11,xmm4
+        vpaddb  xmm13,xmm12,xmm2
+        vpxor   xmm12,xmm12,xmm4
+        vpaddb  xmm14,xmm13,xmm2
+        vpxor   xmm13,xmm13,xmm4
+        vpaddb  xmm1,xmm14,xmm2
+        vpxor   xmm14,xmm14,xmm4
+        jmp     NEAR $L$oop_ctr32
+
+ALIGN   16
+$L$oop_ctr32:
+        vaesenc xmm9,xmm9,xmm15
+        vaesenc xmm10,xmm10,xmm15
+        vaesenc xmm11,xmm11,xmm15
+        vaesenc xmm12,xmm12,xmm15
+        vaesenc xmm13,xmm13,xmm15
+        vaesenc xmm14,xmm14,xmm15
+        vmovups xmm15,XMMWORD[r12]
+        lea     r12,[16+r12]
+        dec     r13d
+        jnz     NEAR $L$oop_ctr32
+
+        vmovdqu xmm3,XMMWORD[r12]
+        vaesenc xmm9,xmm9,xmm15
+        vpxor   xmm4,xmm3,XMMWORD[rdi]
+        vaesenc xmm10,xmm10,xmm15
+        vpxor   xmm5,xmm3,XMMWORD[16+rdi]
+        vaesenc xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm3,XMMWORD[32+rdi]
+        vaesenc xmm12,xmm12,xmm15
+        vpxor   xmm8,xmm3,XMMWORD[48+rdi]
+        vaesenc xmm13,xmm13,xmm15
+        vpxor   xmm2,xmm3,XMMWORD[64+rdi]
+        vaesenc xmm14,xmm14,xmm15
+        vpxor   xmm3,xmm3,XMMWORD[80+rdi]
+        lea     rdi,[96+rdi]
+
+        vaesenclast     xmm9,xmm9,xmm4
+        vaesenclast     xmm10,xmm10,xmm5
+        vaesenclast     xmm11,xmm11,xmm6
+        vaesenclast     xmm12,xmm12,xmm8
+        vaesenclast     xmm13,xmm13,xmm2
+        vaesenclast     xmm14,xmm14,xmm3
+        vmovups XMMWORD[rsi],xmm9
+        vmovups XMMWORD[16+rsi],xmm10
+        vmovups XMMWORD[32+rsi],xmm11
+        vmovups XMMWORD[48+rsi],xmm12
+        vmovups XMMWORD[64+rsi],xmm13
+        vmovups XMMWORD[80+rsi],xmm14
+        lea     rsi,[96+rsi]
+
+        DB      0F3h,0C3h               ;repret
+ALIGN   32
+$L$handle_ctr32_2:
+        vpshufb xmm6,xmm1,xmm0
+        vmovdqu xmm5,XMMWORD[48+r11]
+        vpaddd  xmm10,xmm6,XMMWORD[64+r11]
+        vpaddd  xmm11,xmm6,xmm5
+        vpaddd  xmm12,xmm10,xmm5
+        vpshufb xmm10,xmm10,xmm0
+        vpaddd  xmm13,xmm11,xmm5
+        vpshufb xmm11,xmm11,xmm0
+        vpxor   xmm10,xmm10,xmm4
+        vpaddd  xmm14,xmm12,xmm5
+        vpshufb xmm12,xmm12,xmm0
+        vpxor   xmm11,xmm11,xmm4
+        vpaddd  xmm1,xmm13,xmm5
+        vpshufb xmm13,xmm13,xmm0
+        vpxor   xmm12,xmm12,xmm4
+        vpshufb xmm14,xmm14,xmm0
+        vpxor   xmm13,xmm13,xmm4
+        vpshufb xmm1,xmm1,xmm0
+        vpxor   xmm14,xmm14,xmm4
+        jmp     NEAR $L$oop_ctr32
+
+
+
 global  aesni_gcm_encrypt
 
+ALIGN   32
 aesni_gcm_encrypt:
-
-        xor     eax,eax
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_gcm_encrypt:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        xor     r10,r10
+        cmp     rdx,0x60*3
+        jb      NEAR $L$gcm_enc_abort
+
+        lea     rax,[rsp]
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[(-216)+rax],xmm6
+        movaps  XMMWORD[(-200)+rax],xmm7
+        movaps  XMMWORD[(-184)+rax],xmm8
+        movaps  XMMWORD[(-168)+rax],xmm9
+        movaps  XMMWORD[(-152)+rax],xmm10
+        movaps  XMMWORD[(-136)+rax],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+$L$gcm_enc_body:
+        vzeroupper
+
+        vmovdqu xmm1,XMMWORD[r8]
+        add     rsp,-128
+        mov     ebx,DWORD[12+r8]
+        lea     r11,[$L$bswap_mask]
+        lea     r14,[((-128))+rcx]
+        mov     r15,0xf80
+        lea     rcx,[128+rcx]
+        vmovdqu xmm0,XMMWORD[r11]
+        and     rsp,-128
+        mov     ebp,DWORD[((240-128))+rcx]
+
+        and     r14,r15
+        and     r15,rsp
+        sub     r15,r14
+        jc      NEAR $L$enc_no_key_aliasing
+        cmp     r15,768
+        jnc     NEAR $L$enc_no_key_aliasing
+        sub     rsp,r15
+$L$enc_no_key_aliasing:
+
+        lea     r14,[rsi]
+        lea     r15,[((-192))+rdx*1+rsi]
+        shr     rdx,4
+
+        call    _aesni_ctr32_6x
+        vpshufb xmm8,xmm9,xmm0
+        vpshufb xmm2,xmm10,xmm0
+        vmovdqu XMMWORD[112+rsp],xmm8
+        vpshufb xmm4,xmm11,xmm0
+        vmovdqu XMMWORD[96+rsp],xmm2
+        vpshufb xmm5,xmm12,xmm0
+        vmovdqu XMMWORD[80+rsp],xmm4
+        vpshufb xmm6,xmm13,xmm0
+        vmovdqu XMMWORD[64+rsp],xmm5
+        vpshufb xmm7,xmm14,xmm0
+        vmovdqu XMMWORD[48+rsp],xmm6
+
+        call    _aesni_ctr32_6x
+
+        vmovdqu xmm8,XMMWORD[r9]
+        lea     r9,[((32+32))+r9]
+        sub     rdx,12
+        mov     r10,0x60*2
+        vpshufb xmm8,xmm8,xmm0
+
+        call    _aesni_ctr32_ghash_6x
+        vmovdqu xmm7,XMMWORD[32+rsp]
+        vmovdqu xmm0,XMMWORD[r11]
+        vmovdqu xmm3,XMMWORD[((0-32))+r9]
+        vpunpckhqdq     xmm1,xmm7,xmm7
+        vmovdqu xmm15,XMMWORD[((32-32))+r9]
+        vmovups XMMWORD[(-96)+rsi],xmm9
+        vpshufb xmm9,xmm9,xmm0
+        vpxor   xmm1,xmm1,xmm7
+        vmovups XMMWORD[(-80)+rsi],xmm10
+        vpshufb xmm10,xmm10,xmm0
+        vmovups XMMWORD[(-64)+rsi],xmm11
+        vpshufb xmm11,xmm11,xmm0
+        vmovups XMMWORD[(-48)+rsi],xmm12
+        vpshufb xmm12,xmm12,xmm0
+        vmovups XMMWORD[(-32)+rsi],xmm13
+        vpshufb xmm13,xmm13,xmm0
+        vmovups XMMWORD[(-16)+rsi],xmm14
+        vpshufb xmm14,xmm14,xmm0
+        vmovdqu XMMWORD[16+rsp],xmm9
+        vmovdqu xmm6,XMMWORD[48+rsp]
+        vmovdqu xmm0,XMMWORD[((16-32))+r9]
+        vpunpckhqdq     xmm2,xmm6,xmm6
+        vpclmulqdq      xmm5,xmm7,xmm3,0x00
+        vpxor   xmm2,xmm2,xmm6
+        vpclmulqdq      xmm7,xmm7,xmm3,0x11
+        vpclmulqdq      xmm1,xmm1,xmm15,0x00
+
+        vmovdqu xmm9,XMMWORD[64+rsp]
+        vpclmulqdq      xmm4,xmm6,xmm0,0x00
+        vmovdqu xmm3,XMMWORD[((48-32))+r9]
+        vpxor   xmm4,xmm4,xmm5
+        vpunpckhqdq     xmm5,xmm9,xmm9
+        vpclmulqdq      xmm6,xmm6,xmm0,0x11
+        vpxor   xmm5,xmm5,xmm9
+        vpxor   xmm6,xmm6,xmm7
+        vpclmulqdq      xmm2,xmm2,xmm15,0x10
+        vmovdqu xmm15,XMMWORD[((80-32))+r9]
+        vpxor   xmm2,xmm2,xmm1
+
+        vmovdqu xmm1,XMMWORD[80+rsp]
+        vpclmulqdq      xmm7,xmm9,xmm3,0x00
+        vmovdqu xmm0,XMMWORD[((64-32))+r9]
+        vpxor   xmm7,xmm7,xmm4
+        vpunpckhqdq     xmm4,xmm1,xmm1
+        vpclmulqdq      xmm9,xmm9,xmm3,0x11
+        vpxor   xmm4,xmm4,xmm1
+        vpxor   xmm9,xmm9,xmm6
+        vpclmulqdq      xmm5,xmm5,xmm15,0x00
+        vpxor   xmm5,xmm5,xmm2
+
+        vmovdqu xmm2,XMMWORD[96+rsp]
+        vpclmulqdq      xmm6,xmm1,xmm0,0x00
+        vmovdqu xmm3,XMMWORD[((96-32))+r9]
+        vpxor   xmm6,xmm6,xmm7
+        vpunpckhqdq     xmm7,xmm2,xmm2
+        vpclmulqdq      xmm1,xmm1,xmm0,0x11
+        vpxor   xmm7,xmm7,xmm2
+        vpxor   xmm1,xmm1,xmm9
+        vpclmulqdq      xmm4,xmm4,xmm15,0x10
+        vmovdqu xmm15,XMMWORD[((128-32))+r9]
+        vpxor   xmm4,xmm4,xmm5
+
+        vpxor   xmm8,xmm8,XMMWORD[112+rsp]
+        vpclmulqdq      xmm5,xmm2,xmm3,0x00
+        vmovdqu xmm0,XMMWORD[((112-32))+r9]
+        vpunpckhqdq     xmm9,xmm8,xmm8
+        vpxor   xmm5,xmm5,xmm6
+        vpclmulqdq      xmm2,xmm2,xmm3,0x11
+        vpxor   xmm9,xmm9,xmm8
+        vpxor   xmm2,xmm2,xmm1
+        vpclmulqdq      xmm7,xmm7,xmm15,0x00
+        vpxor   xmm4,xmm7,xmm4
+
+        vpclmulqdq      xmm6,xmm8,xmm0,0x00
+        vmovdqu xmm3,XMMWORD[((0-32))+r9]
+        vpunpckhqdq     xmm1,xmm14,xmm14
+        vpclmulqdq      xmm8,xmm8,xmm0,0x11
+        vpxor   xmm1,xmm1,xmm14
+        vpxor   xmm5,xmm6,xmm5
+        vpclmulqdq      xmm9,xmm9,xmm15,0x10
+        vmovdqu xmm15,XMMWORD[((32-32))+r9]
+        vpxor   xmm7,xmm8,xmm2
+        vpxor   xmm6,xmm9,xmm4
+
+        vmovdqu xmm0,XMMWORD[((16-32))+r9]
+        vpxor   xmm9,xmm7,xmm5
+        vpclmulqdq      xmm4,xmm14,xmm3,0x00
+        vpxor   xmm6,xmm6,xmm9
+        vpunpckhqdq     xmm2,xmm13,xmm13
+        vpclmulqdq      xmm14,xmm14,xmm3,0x11
+        vpxor   xmm2,xmm2,xmm13
+        vpslldq xmm9,xmm6,8
+        vpclmulqdq      xmm1,xmm1,xmm15,0x00
+        vpxor   xmm8,xmm5,xmm9
+        vpsrldq xmm6,xmm6,8
+        vpxor   xmm7,xmm7,xmm6
+
+        vpclmulqdq      xmm5,xmm13,xmm0,0x00
+        vmovdqu xmm3,XMMWORD[((48-32))+r9]
+        vpxor   xmm5,xmm5,xmm4
+        vpunpckhqdq     xmm9,xmm12,xmm12
+        vpclmulqdq      xmm13,xmm13,xmm0,0x11
+        vpxor   xmm9,xmm9,xmm12
+        vpxor   xmm13,xmm13,xmm14
+        vpalignr        xmm14,xmm8,xmm8,8
+        vpclmulqdq      xmm2,xmm2,xmm15,0x10
+        vmovdqu xmm15,XMMWORD[((80-32))+r9]
+        vpxor   xmm2,xmm2,xmm1
+
+        vpclmulqdq      xmm4,xmm12,xmm3,0x00
+        vmovdqu xmm0,XMMWORD[((64-32))+r9]
+        vpxor   xmm4,xmm4,xmm5
+        vpunpckhqdq     xmm1,xmm11,xmm11
+        vpclmulqdq      xmm12,xmm12,xmm3,0x11
+        vpxor   xmm1,xmm1,xmm11
+        vpxor   xmm12,xmm12,xmm13
+        vxorps  xmm7,xmm7,XMMWORD[16+rsp]
+        vpclmulqdq      xmm9,xmm9,xmm15,0x00
+        vpxor   xmm9,xmm9,xmm2
+
+        vpclmulqdq      xmm8,xmm8,XMMWORD[16+r11],0x10
+        vxorps  xmm8,xmm8,xmm14
+
+        vpclmulqdq      xmm5,xmm11,xmm0,0x00
+        vmovdqu xmm3,XMMWORD[((96-32))+r9]
+        vpxor   xmm5,xmm5,xmm4
+        vpunpckhqdq     xmm2,xmm10,xmm10
+        vpclmulqdq      xmm11,xmm11,xmm0,0x11
+        vpxor   xmm2,xmm2,xmm10
+        vpalignr        xmm14,xmm8,xmm8,8
+        vpxor   xmm11,xmm11,xmm12
+        vpclmulqdq      xmm1,xmm1,xmm15,0x10
+        vmovdqu xmm15,XMMWORD[((128-32))+r9]
+        vpxor   xmm1,xmm1,xmm9
+
+        vxorps  xmm14,xmm14,xmm7
+        vpclmulqdq      xmm8,xmm8,XMMWORD[16+r11],0x10
+        vxorps  xmm8,xmm8,xmm14
+
+        vpclmulqdq      xmm4,xmm10,xmm3,0x00
+        vmovdqu xmm0,XMMWORD[((112-32))+r9]
+        vpxor   xmm4,xmm4,xmm5
+        vpunpckhqdq     xmm9,xmm8,xmm8
+        vpclmulqdq      xmm10,xmm10,xmm3,0x11
+        vpxor   xmm9,xmm9,xmm8
+        vpxor   xmm10,xmm10,xmm11
+        vpclmulqdq      xmm2,xmm2,xmm15,0x00
+        vpxor   xmm2,xmm2,xmm1
+
+        vpclmulqdq      xmm5,xmm8,xmm0,0x00
+        vpclmulqdq      xmm7,xmm8,xmm0,0x11
+        vpxor   xmm5,xmm5,xmm4
+        vpclmulqdq      xmm6,xmm9,xmm15,0x10
+        vpxor   xmm7,xmm7,xmm10
+        vpxor   xmm6,xmm6,xmm2
+
+        vpxor   xmm4,xmm7,xmm5
+        vpxor   xmm6,xmm6,xmm4
+        vpslldq xmm1,xmm6,8
+        vmovdqu xmm3,XMMWORD[16+r11]
+        vpsrldq xmm6,xmm6,8
+        vpxor   xmm8,xmm5,xmm1
+        vpxor   xmm7,xmm7,xmm6
+
+        vpalignr        xmm2,xmm8,xmm8,8
+        vpclmulqdq      xmm8,xmm8,xmm3,0x10
+        vpxor   xmm8,xmm8,xmm2
+
+        vpalignr        xmm2,xmm8,xmm8,8
+        vpclmulqdq      xmm8,xmm8,xmm3,0x10
+        vpxor   xmm2,xmm2,xmm7
+        vpxor   xmm8,xmm8,xmm2
+        vpshufb xmm8,xmm8,XMMWORD[r11]
+        vmovdqu XMMWORD[(-64)+r9],xmm8
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$gcm_enc_abort:
+        mov     rax,r10
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
         DB      0F3h,0C3h               ;repret
 
-
-
-global  aesni_gcm_decrypt
-
-aesni_gcm_decrypt:
-
-        xor     eax,eax
+$L$SEH_end_aesni_gcm_encrypt:
+ALIGN   64
+$L$bswap_mask:
+DB      15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
+$L$poly:
+DB      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
+$L$one_msb:
+DB      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
+$L$two_lsb:
+DB      2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+$L$one_lsb:
+DB      1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+DB      65,69,83,45,78,73,32,71,67,77,32,109,111,100,117,108
+DB      101,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82
+DB      89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112
+DB      114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
+ALIGN   64
+EXTERN  __imp_RtlVirtualUnwind
+
+ALIGN   16
+gcm_se_handler:
+        push    rsi
+        push    rdi
+        push    rbx
+        push    rbp
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        pushfq
+        sub     rsp,64
+
+        mov     rax,QWORD[120+r8]
+        mov     rbx,QWORD[248+r8]
+
+        mov     rsi,QWORD[8+r9]
+        mov     r11,QWORD[56+r9]
+
+        mov     r10d,DWORD[r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jb      NEAR $L$common_seh_tail
+
+        mov     rax,QWORD[152+r8]
+
+        mov     r10d,DWORD[4+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jae     NEAR $L$common_seh_tail
+
+        mov     rax,QWORD[120+r8]
+
+        mov     r15,QWORD[((-48))+rax]
+        mov     r14,QWORD[((-40))+rax]
+        mov     r13,QWORD[((-32))+rax]
+        mov     r12,QWORD[((-24))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+        mov     rbx,QWORD[((-8))+rax]
+        mov     QWORD[240+r8],r15
+        mov     QWORD[232+r8],r14
+        mov     QWORD[224+r8],r13
+        mov     QWORD[216+r8],r12
+        mov     QWORD[160+r8],rbp
+        mov     QWORD[144+r8],rbx
+
+        lea     rsi,[((-216))+rax]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+
+$L$common_seh_tail:
+        mov     rdi,QWORD[8+rax]
+        mov     rsi,QWORD[16+rax]
+        mov     QWORD[152+r8],rax
+        mov     QWORD[168+r8],rsi
+        mov     QWORD[176+r8],rdi
+
+        mov     rdi,QWORD[40+r9]
+        mov     rsi,r8
+        mov     ecx,154
+        DD      0xa548f3fc
+
+        mov     rsi,r9
+        xor     rcx,rcx
+        mov     rdx,QWORD[8+rsi]
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[16+rsi]
+        mov     r10,QWORD[40+rsi]
+        lea     r11,[56+rsi]
+        lea     r12,[24+rsi]
+        mov     QWORD[32+rsp],r10
+        mov     QWORD[40+rsp],r11
+        mov     QWORD[48+rsp],r12
+        mov     QWORD[56+rsp],rcx
+        call    QWORD[__imp_RtlVirtualUnwind]
+
+        mov     eax,1
+        add     rsp,64
+        popfq
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     rbp
+        pop     rbx
+        pop     rdi
+        pop     rsi
         DB      0F3h,0C3h               ;repret
 
 
+section .pdata rdata align=4
+ALIGN   4
+        DD      $L$SEH_begin_aesni_gcm_decrypt wrt ..imagebase
+        DD      $L$SEH_end_aesni_gcm_decrypt wrt ..imagebase
+        DD      $L$SEH_gcm_dec_info wrt ..imagebase
+
+        DD      $L$SEH_begin_aesni_gcm_encrypt wrt ..imagebase
+        DD      $L$SEH_end_aesni_gcm_encrypt wrt ..imagebase
+        DD      $L$SEH_gcm_enc_info wrt ..imagebase
+section .xdata rdata align=8
+ALIGN   8
+$L$SEH_gcm_dec_info:
+DB      9,0,0,0
+        DD      gcm_se_handler wrt ..imagebase
+        DD      $L$gcm_dec_body wrt ..imagebase,$L$gcm_dec_abort wrt ..imagebase
+$L$SEH_gcm_enc_info:
+DB      9,0,0,0
+        DD      gcm_se_handler wrt ..imagebase
+        DD      $L$gcm_enc_body wrt ..imagebase,$L$gcm_enc_abort wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/aesni-mb-x86_64.S

@@ -22 +22 @@
 
 
+        cmp     edx,2
+        jb      NEAR $L$enc_non_avx
+        mov     ecx,DWORD[((OPENSSL_ia32cap_P+4))]
+        test    ecx,268435456
+        jnz     NEAR _avx_cbc_enc_shortcut
+        jmp     NEAR $L$enc_non_avx
+ALIGN   16
+$L$enc_non_avx:
         mov     rax,rsp
 
@@ -337 +345 @@
 
 
+        cmp     edx,2
+        jb      NEAR $L$dec_non_avx
+        mov     ecx,DWORD[((OPENSSL_ia32cap_P+4))]
+        test    ecx,268435456
+        jnz     NEAR _avx_cbc_dec_shortcut
+        jmp     NEAR $L$dec_non_avx
+ALIGN   16
+$L$dec_non_avx:
         mov     rax,rsp
 
@@ -627 +643 @@
 
 $L$SEH_end_aesni_multi_cbc_decrypt:
+
+ALIGN   32
+aesni_multi_cbc_encrypt_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_multi_cbc_encrypt_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx_cbc_enc_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[64+rsp],xmm10
+        movaps  XMMWORD[80+rsp],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+
+
+
+
+
+
+
+
+        sub     rsp,192
+        and     rsp,-128
+        mov     QWORD[16+rsp],rax
+
+
+$L$enc8x_body:
+        vzeroupper
+        vmovdqu xmm15,XMMWORD[rsi]
+        lea     rsi,[120+rsi]
+        lea     rdi,[160+rdi]
+        shr     edx,1
+
+$L$enc8x_loop_grande:
+
+        xor     edx,edx
+
+        mov     ecx,DWORD[((-144))+rdi]
+
+        mov     r8,QWORD[((-160))+rdi]
+        cmp     ecx,edx
+
+        mov     rbx,QWORD[((-152))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm2,XMMWORD[((-136))+rdi]
+        mov     DWORD[32+rsp],ecx
+        cmovle  r8,rsp
+        sub     rbx,r8
+        mov     QWORD[64+rsp],rbx
+
+        mov     ecx,DWORD[((-104))+rdi]
+
+        mov     r9,QWORD[((-120))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-112))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm3,XMMWORD[((-96))+rdi]
+        mov     DWORD[36+rsp],ecx
+        cmovle  r9,rsp
+        sub     rbp,r9
+        mov     QWORD[72+rsp],rbp
+
+        mov     ecx,DWORD[((-64))+rdi]
+
+        mov     r10,QWORD[((-80))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-72))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm4,XMMWORD[((-56))+rdi]
+        mov     DWORD[40+rsp],ecx
+        cmovle  r10,rsp
+        sub     rbp,r10
+        mov     QWORD[80+rsp],rbp
+
+        mov     ecx,DWORD[((-24))+rdi]
+
+        mov     r11,QWORD[((-40))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-32))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm5,XMMWORD[((-16))+rdi]
+        mov     DWORD[44+rsp],ecx
+        cmovle  r11,rsp
+        sub     rbp,r11
+        mov     QWORD[88+rsp],rbp
+
+        mov     ecx,DWORD[16+rdi]
+
+        mov     r12,QWORD[rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[8+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm6,XMMWORD[24+rdi]
+        mov     DWORD[48+rsp],ecx
+        cmovle  r12,rsp
+        sub     rbp,r12
+        mov     QWORD[96+rsp],rbp
+
+        mov     ecx,DWORD[56+rdi]
+
+        mov     r13,QWORD[40+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[48+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm7,XMMWORD[64+rdi]
+        mov     DWORD[52+rsp],ecx
+        cmovle  r13,rsp
+        sub     rbp,r13
+        mov     QWORD[104+rsp],rbp
+
+        mov     ecx,DWORD[96+rdi]
+
+        mov     r14,QWORD[80+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[88+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm8,XMMWORD[104+rdi]
+        mov     DWORD[56+rsp],ecx
+        cmovle  r14,rsp
+        sub     rbp,r14
+        mov     QWORD[112+rsp],rbp
+
+        mov     ecx,DWORD[136+rdi]
+
+        mov     r15,QWORD[120+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[128+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm9,XMMWORD[144+rdi]
+        mov     DWORD[60+rsp],ecx
+        cmovle  r15,rsp
+        sub     rbp,r15
+        mov     QWORD[120+rsp],rbp
+        test    edx,edx
+        jz      NEAR $L$enc8x_done
+
+        vmovups xmm1,XMMWORD[((16-120))+rsi]
+        vmovups xmm0,XMMWORD[((32-120))+rsi]
+        mov     eax,DWORD[((240-120))+rsi]
+
+        vpxor   xmm10,xmm15,XMMWORD[r8]
+        lea     rbp,[128+rsp]
+        vpxor   xmm11,xmm15,XMMWORD[r9]
+        vpxor   xmm12,xmm15,XMMWORD[r10]
+        vpxor   xmm13,xmm15,XMMWORD[r11]
+        vpxor   xmm2,xmm2,xmm10
+        vpxor   xmm10,xmm15,XMMWORD[r12]
+        vpxor   xmm3,xmm3,xmm11
+        vpxor   xmm11,xmm15,XMMWORD[r13]
+        vpxor   xmm4,xmm4,xmm12
+        vpxor   xmm12,xmm15,XMMWORD[r14]
+        vpxor   xmm5,xmm5,xmm13
+        vpxor   xmm13,xmm15,XMMWORD[r15]
+        vpxor   xmm6,xmm6,xmm10
+        mov     ecx,1
+        vpxor   xmm7,xmm7,xmm11
+        vpxor   xmm8,xmm8,xmm12
+        vpxor   xmm9,xmm9,xmm13
+        jmp     NEAR $L$oop_enc8x
+
+ALIGN   32
+$L$oop_enc8x:
+        vaesenc xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+0))+rsp]
+        vaesenc xmm3,xmm3,xmm1
+        prefetcht0      [31+r8]
+        vaesenc xmm4,xmm4,xmm1
+        vaesenc xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r8]
+        cmovge  r8,rsp
+        vaesenc xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm1
+        sub     rbx,r8
+        vaesenc xmm8,xmm8,xmm1
+        vpxor   xmm10,xmm15,XMMWORD[16+r8]
+        mov     QWORD[((64+0))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-72))+rsi]
+        lea     r8,[16+rbx*1+r8]
+        vmovdqu XMMWORD[rbp],xmm10
+        vaesenc xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+4))+rsp]
+        mov     rbx,QWORD[((64+8))+rsp]
+        vaesenc xmm3,xmm3,xmm0
+        prefetcht0      [31+r9]
+        vaesenc xmm4,xmm4,xmm0
+        vaesenc xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r9]
+        cmovge  r9,rsp
+        vaesenc xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm0
+        sub     rbx,r9
+        vaesenc xmm8,xmm8,xmm0
+        vpxor   xmm11,xmm15,XMMWORD[16+r9]
+        mov     QWORD[((64+8))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((-56))+rsi]
+        lea     r9,[16+rbx*1+r9]
+        vmovdqu XMMWORD[16+rbp],xmm11
+        vaesenc xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+8))+rsp]
+        mov     rbx,QWORD[((64+16))+rsp]
+        vaesenc xmm3,xmm3,xmm1
+        prefetcht0      [31+r10]
+        vaesenc xmm4,xmm4,xmm1
+        prefetcht0      [15+r8]
+        vaesenc xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r10]
+        cmovge  r10,rsp
+        vaesenc xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm1
+        sub     rbx,r10
+        vaesenc xmm8,xmm8,xmm1
+        vpxor   xmm12,xmm15,XMMWORD[16+r10]
+        mov     QWORD[((64+16))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-40))+rsi]
+        lea     r10,[16+rbx*1+r10]
+        vmovdqu XMMWORD[32+rbp],xmm12
+        vaesenc xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+12))+rsp]
+        mov     rbx,QWORD[((64+24))+rsp]
+        vaesenc xmm3,xmm3,xmm0
+        prefetcht0      [31+r11]
+        vaesenc xmm4,xmm4,xmm0
+        prefetcht0      [15+r9]
+        vaesenc xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r11]
+        cmovge  r11,rsp
+        vaesenc xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm0
+        sub     rbx,r11
+        vaesenc xmm8,xmm8,xmm0
+        vpxor   xmm13,xmm15,XMMWORD[16+r11]
+        mov     QWORD[((64+24))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((-24))+rsi]
+        lea     r11,[16+rbx*1+r11]
+        vmovdqu XMMWORD[48+rbp],xmm13
+        vaesenc xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+16))+rsp]
+        mov     rbx,QWORD[((64+32))+rsp]
+        vaesenc xmm3,xmm3,xmm1
+        prefetcht0      [31+r12]
+        vaesenc xmm4,xmm4,xmm1
+        prefetcht0      [15+r10]
+        vaesenc xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r12]
+        cmovge  r12,rsp
+        vaesenc xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm1
+        sub     rbx,r12
+        vaesenc xmm8,xmm8,xmm1
+        vpxor   xmm10,xmm15,XMMWORD[16+r12]
+        mov     QWORD[((64+32))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-8))+rsi]
+        lea     r12,[16+rbx*1+r12]
+        vaesenc xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+20))+rsp]
+        mov     rbx,QWORD[((64+40))+rsp]
+        vaesenc xmm3,xmm3,xmm0
+        prefetcht0      [31+r13]
+        vaesenc xmm4,xmm4,xmm0
+        prefetcht0      [15+r11]
+        vaesenc xmm5,xmm5,xmm0
+        lea     rbx,[r13*1+rbx]
+        cmovge  r13,rsp
+        vaesenc xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm0
+        sub     rbx,r13
+        vaesenc xmm8,xmm8,xmm0
+        vpxor   xmm11,xmm15,XMMWORD[16+r13]
+        mov     QWORD[((64+40))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[8+rsi]
+        lea     r13,[16+rbx*1+r13]
+        vaesenc xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+24))+rsp]
+        mov     rbx,QWORD[((64+48))+rsp]
+        vaesenc xmm3,xmm3,xmm1
+        prefetcht0      [31+r14]
+        vaesenc xmm4,xmm4,xmm1
+        prefetcht0      [15+r12]
+        vaesenc xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r14]
+        cmovge  r14,rsp
+        vaesenc xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm1
+        sub     rbx,r14
+        vaesenc xmm8,xmm8,xmm1
+        vpxor   xmm12,xmm15,XMMWORD[16+r14]
+        mov     QWORD[((64+48))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[24+rsi]
+        lea     r14,[16+rbx*1+r14]
+        vaesenc xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+28))+rsp]
+        mov     rbx,QWORD[((64+56))+rsp]
+        vaesenc xmm3,xmm3,xmm0
+        prefetcht0      [31+r15]
+        vaesenc xmm4,xmm4,xmm0
+        prefetcht0      [15+r13]
+        vaesenc xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r15]
+        cmovge  r15,rsp
+        vaesenc xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesenc xmm7,xmm7,xmm0
+        sub     rbx,r15
+        vaesenc xmm8,xmm8,xmm0
+        vpxor   xmm13,xmm15,XMMWORD[16+r15]
+        mov     QWORD[((64+56))+rsp],rbx
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[40+rsi]
+        lea     r15,[16+rbx*1+r15]
+        vmovdqu xmm14,XMMWORD[32+rsp]
+        prefetcht0      [15+r14]
+        prefetcht0      [15+r15]
+        cmp     eax,11
+        jb      NEAR $L$enc8x_tail
+
+        vaesenc xmm2,xmm2,xmm1
+        vaesenc xmm3,xmm3,xmm1
+        vaesenc xmm4,xmm4,xmm1
+        vaesenc xmm5,xmm5,xmm1
+        vaesenc xmm6,xmm6,xmm1
+        vaesenc xmm7,xmm7,xmm1
+        vaesenc xmm8,xmm8,xmm1
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((176-120))+rsi]
+
+        vaesenc xmm2,xmm2,xmm0
+        vaesenc xmm3,xmm3,xmm0
+        vaesenc xmm4,xmm4,xmm0
+        vaesenc xmm5,xmm5,xmm0
+        vaesenc xmm6,xmm6,xmm0
+        vaesenc xmm7,xmm7,xmm0
+        vaesenc xmm8,xmm8,xmm0
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((192-120))+rsi]
+        je      NEAR $L$enc8x_tail
+
+        vaesenc xmm2,xmm2,xmm1
+        vaesenc xmm3,xmm3,xmm1
+        vaesenc xmm4,xmm4,xmm1
+        vaesenc xmm5,xmm5,xmm1
+        vaesenc xmm6,xmm6,xmm1
+        vaesenc xmm7,xmm7,xmm1
+        vaesenc xmm8,xmm8,xmm1
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((208-120))+rsi]
+
+        vaesenc xmm2,xmm2,xmm0
+        vaesenc xmm3,xmm3,xmm0
+        vaesenc xmm4,xmm4,xmm0
+        vaesenc xmm5,xmm5,xmm0
+        vaesenc xmm6,xmm6,xmm0
+        vaesenc xmm7,xmm7,xmm0
+        vaesenc xmm8,xmm8,xmm0
+        vaesenc xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((224-120))+rsi]
+
+$L$enc8x_tail:
+        vaesenc xmm2,xmm2,xmm1
+        vpxor   xmm15,xmm15,xmm15
+        vaesenc xmm3,xmm3,xmm1
+        vaesenc xmm4,xmm4,xmm1
+        vpcmpgtd        xmm15,xmm14,xmm15
+        vaesenc xmm5,xmm5,xmm1
+        vaesenc xmm6,xmm6,xmm1
+        vpaddd  xmm15,xmm15,xmm14
+        vmovdqu xmm14,XMMWORD[48+rsp]
+        vaesenc xmm7,xmm7,xmm1
+        mov     rbx,QWORD[64+rsp]
+        vaesenc xmm8,xmm8,xmm1
+        vaesenc xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((16-120))+rsi]
+
+        vaesenclast     xmm2,xmm2,xmm0
+        vmovdqa XMMWORD[32+rsp],xmm15
+        vpxor   xmm15,xmm15,xmm15
+        vaesenclast     xmm3,xmm3,xmm0
+        vaesenclast     xmm4,xmm4,xmm0
+        vpcmpgtd        xmm15,xmm14,xmm15
+        vaesenclast     xmm5,xmm5,xmm0
+        vaesenclast     xmm6,xmm6,xmm0
+        vpaddd  xmm14,xmm14,xmm15
+        vmovdqu xmm15,XMMWORD[((-120))+rsi]
+        vaesenclast     xmm7,xmm7,xmm0
+        vaesenclast     xmm8,xmm8,xmm0
+        vmovdqa XMMWORD[48+rsp],xmm14
+        vaesenclast     xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((32-120))+rsi]
+
+        vmovups XMMWORD[(-16)+r8],xmm2
+        sub     r8,rbx
+        vpxor   xmm2,xmm2,XMMWORD[rbp]
+        vmovups XMMWORD[(-16)+r9],xmm3
+        sub     r9,QWORD[72+rsp]
+        vpxor   xmm3,xmm3,XMMWORD[16+rbp]
+        vmovups XMMWORD[(-16)+r10],xmm4
+        sub     r10,QWORD[80+rsp]
+        vpxor   xmm4,xmm4,XMMWORD[32+rbp]
+        vmovups XMMWORD[(-16)+r11],xmm5
+        sub     r11,QWORD[88+rsp]
+        vpxor   xmm5,xmm5,XMMWORD[48+rbp]
+        vmovups XMMWORD[(-16)+r12],xmm6
+        sub     r12,QWORD[96+rsp]
+        vpxor   xmm6,xmm6,xmm10
+        vmovups XMMWORD[(-16)+r13],xmm7
+        sub     r13,QWORD[104+rsp]
+        vpxor   xmm7,xmm7,xmm11
+        vmovups XMMWORD[(-16)+r14],xmm8
+        sub     r14,QWORD[112+rsp]
+        vpxor   xmm8,xmm8,xmm12
+        vmovups XMMWORD[(-16)+r15],xmm9
+        sub     r15,QWORD[120+rsp]
+        vpxor   xmm9,xmm9,xmm13
+
+        dec     edx
+        jnz     NEAR $L$oop_enc8x
+
+        mov     rax,QWORD[16+rsp]
+
+
+
+
+
+
+$L$enc8x_done:
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$enc8x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_multi_cbc_encrypt_avx:
+
+
+ALIGN   32
+aesni_multi_cbc_decrypt_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_multi_cbc_decrypt_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx_cbc_dec_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[64+rsp],xmm10
+        movaps  XMMWORD[80+rsp],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+
+
+
+
+
+
+
+
+
+        sub     rsp,256
+        and     rsp,-256
+        sub     rsp,192
+        mov     QWORD[16+rsp],rax
+
+
+$L$dec8x_body:
+        vzeroupper
+        vmovdqu xmm15,XMMWORD[rsi]
+        lea     rsi,[120+rsi]
+        lea     rdi,[160+rdi]
+        shr     edx,1
+
+$L$dec8x_loop_grande:
+
+        xor     edx,edx
+
+        mov     ecx,DWORD[((-144))+rdi]
+
+        mov     r8,QWORD[((-160))+rdi]
+        cmp     ecx,edx
+
+        mov     rbx,QWORD[((-152))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm2,XMMWORD[((-136))+rdi]
+        mov     DWORD[32+rsp],ecx
+        cmovle  r8,rsp
+        sub     rbx,r8
+        mov     QWORD[64+rsp],rbx
+        vmovdqu XMMWORD[192+rsp],xmm2
+
+        mov     ecx,DWORD[((-104))+rdi]
+
+        mov     r9,QWORD[((-120))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-112))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm3,XMMWORD[((-96))+rdi]
+        mov     DWORD[36+rsp],ecx
+        cmovle  r9,rsp
+        sub     rbp,r9
+        mov     QWORD[72+rsp],rbp
+        vmovdqu XMMWORD[208+rsp],xmm3
+
+        mov     ecx,DWORD[((-64))+rdi]
+
+        mov     r10,QWORD[((-80))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-72))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm4,XMMWORD[((-56))+rdi]
+        mov     DWORD[40+rsp],ecx
+        cmovle  r10,rsp
+        sub     rbp,r10
+        mov     QWORD[80+rsp],rbp
+        vmovdqu XMMWORD[224+rsp],xmm4
+
+        mov     ecx,DWORD[((-24))+rdi]
+
+        mov     r11,QWORD[((-40))+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[((-32))+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm5,XMMWORD[((-16))+rdi]
+        mov     DWORD[44+rsp],ecx
+        cmovle  r11,rsp
+        sub     rbp,r11
+        mov     QWORD[88+rsp],rbp
+        vmovdqu XMMWORD[240+rsp],xmm5
+
+        mov     ecx,DWORD[16+rdi]
+
+        mov     r12,QWORD[rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[8+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm6,XMMWORD[24+rdi]
+        mov     DWORD[48+rsp],ecx
+        cmovle  r12,rsp
+        sub     rbp,r12
+        mov     QWORD[96+rsp],rbp
+        vmovdqu XMMWORD[256+rsp],xmm6
+
+        mov     ecx,DWORD[56+rdi]
+
+        mov     r13,QWORD[40+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[48+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm7,XMMWORD[64+rdi]
+        mov     DWORD[52+rsp],ecx
+        cmovle  r13,rsp
+        sub     rbp,r13
+        mov     QWORD[104+rsp],rbp
+        vmovdqu XMMWORD[272+rsp],xmm7
+
+        mov     ecx,DWORD[96+rdi]
+
+        mov     r14,QWORD[80+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[88+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm8,XMMWORD[104+rdi]
+        mov     DWORD[56+rsp],ecx
+        cmovle  r14,rsp
+        sub     rbp,r14
+        mov     QWORD[112+rsp],rbp
+        vmovdqu XMMWORD[288+rsp],xmm8
+
+        mov     ecx,DWORD[136+rdi]
+
+        mov     r15,QWORD[120+rdi]
+        cmp     ecx,edx
+
+        mov     rbp,QWORD[128+rdi]
+        cmovg   edx,ecx
+        test    ecx,ecx
+
+        vmovdqu xmm9,XMMWORD[144+rdi]
+        mov     DWORD[60+rsp],ecx
+        cmovle  r15,rsp
+        sub     rbp,r15
+        mov     QWORD[120+rsp],rbp
+        vmovdqu XMMWORD[304+rsp],xmm9
+        test    edx,edx
+        jz      NEAR $L$dec8x_done
+
+        vmovups xmm1,XMMWORD[((16-120))+rsi]
+        vmovups xmm0,XMMWORD[((32-120))+rsi]
+        mov     eax,DWORD[((240-120))+rsi]
+        lea     rbp,[((192+128))+rsp]
+
+        vmovdqu xmm2,XMMWORD[r8]
+        vmovdqu xmm3,XMMWORD[r9]
+        vmovdqu xmm4,XMMWORD[r10]
+        vmovdqu xmm5,XMMWORD[r11]
+        vmovdqu xmm6,XMMWORD[r12]
+        vmovdqu xmm7,XMMWORD[r13]
+        vmovdqu xmm8,XMMWORD[r14]
+        vmovdqu xmm9,XMMWORD[r15]
+        vmovdqu XMMWORD[rbp],xmm2
+        vpxor   xmm2,xmm2,xmm15
+        vmovdqu XMMWORD[16+rbp],xmm3
+        vpxor   xmm3,xmm3,xmm15
+        vmovdqu XMMWORD[32+rbp],xmm4
+        vpxor   xmm4,xmm4,xmm15
+        vmovdqu XMMWORD[48+rbp],xmm5
+        vpxor   xmm5,xmm5,xmm15
+        vmovdqu XMMWORD[64+rbp],xmm6
+        vpxor   xmm6,xmm6,xmm15
+        vmovdqu XMMWORD[80+rbp],xmm7
+        vpxor   xmm7,xmm7,xmm15
+        vmovdqu XMMWORD[96+rbp],xmm8
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu XMMWORD[112+rbp],xmm9
+        vpxor   xmm9,xmm9,xmm15
+        xor     rbp,0x80
+        mov     ecx,1
+        jmp     NEAR $L$oop_dec8x
+
+ALIGN   32
+$L$oop_dec8x:
+        vaesdec xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+0))+rsp]
+        vaesdec xmm3,xmm3,xmm1
+        prefetcht0      [31+r8]
+        vaesdec xmm4,xmm4,xmm1
+        vaesdec xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r8]
+        cmovge  r8,rsp
+        vaesdec xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm1
+        sub     rbx,r8
+        vaesdec xmm8,xmm8,xmm1
+        vmovdqu xmm10,XMMWORD[16+r8]
+        mov     QWORD[((64+0))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-72))+rsi]
+        lea     r8,[16+rbx*1+r8]
+        vmovdqu XMMWORD[128+rsp],xmm10
+        vaesdec xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+4))+rsp]
+        mov     rbx,QWORD[((64+8))+rsp]
+        vaesdec xmm3,xmm3,xmm0
+        prefetcht0      [31+r9]
+        vaesdec xmm4,xmm4,xmm0
+        vaesdec xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r9]
+        cmovge  r9,rsp
+        vaesdec xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm0
+        sub     rbx,r9
+        vaesdec xmm8,xmm8,xmm0
+        vmovdqu xmm11,XMMWORD[16+r9]
+        mov     QWORD[((64+8))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((-56))+rsi]
+        lea     r9,[16+rbx*1+r9]
+        vmovdqu XMMWORD[144+rsp],xmm11
+        vaesdec xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+8))+rsp]
+        mov     rbx,QWORD[((64+16))+rsp]
+        vaesdec xmm3,xmm3,xmm1
+        prefetcht0      [31+r10]
+        vaesdec xmm4,xmm4,xmm1
+        prefetcht0      [15+r8]
+        vaesdec xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r10]
+        cmovge  r10,rsp
+        vaesdec xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm1
+        sub     rbx,r10
+        vaesdec xmm8,xmm8,xmm1
+        vmovdqu xmm12,XMMWORD[16+r10]
+        mov     QWORD[((64+16))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-40))+rsi]
+        lea     r10,[16+rbx*1+r10]
+        vmovdqu XMMWORD[160+rsp],xmm12
+        vaesdec xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+12))+rsp]
+        mov     rbx,QWORD[((64+24))+rsp]
+        vaesdec xmm3,xmm3,xmm0
+        prefetcht0      [31+r11]
+        vaesdec xmm4,xmm4,xmm0
+        prefetcht0      [15+r9]
+        vaesdec xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r11]
+        cmovge  r11,rsp
+        vaesdec xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm0
+        sub     rbx,r11
+        vaesdec xmm8,xmm8,xmm0
+        vmovdqu xmm13,XMMWORD[16+r11]
+        mov     QWORD[((64+24))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((-24))+rsi]
+        lea     r11,[16+rbx*1+r11]
+        vmovdqu XMMWORD[176+rsp],xmm13
+        vaesdec xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+16))+rsp]
+        mov     rbx,QWORD[((64+32))+rsp]
+        vaesdec xmm3,xmm3,xmm1
+        prefetcht0      [31+r12]
+        vaesdec xmm4,xmm4,xmm1
+        prefetcht0      [15+r10]
+        vaesdec xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r12]
+        cmovge  r12,rsp
+        vaesdec xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm1
+        sub     rbx,r12
+        vaesdec xmm8,xmm8,xmm1
+        vmovdqu xmm10,XMMWORD[16+r12]
+        mov     QWORD[((64+32))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((-8))+rsi]
+        lea     r12,[16+rbx*1+r12]
+        vaesdec xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+20))+rsp]
+        mov     rbx,QWORD[((64+40))+rsp]
+        vaesdec xmm3,xmm3,xmm0
+        prefetcht0      [31+r13]
+        vaesdec xmm4,xmm4,xmm0
+        prefetcht0      [15+r11]
+        vaesdec xmm5,xmm5,xmm0
+        lea     rbx,[r13*1+rbx]
+        cmovge  r13,rsp
+        vaesdec xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm0
+        sub     rbx,r13
+        vaesdec xmm8,xmm8,xmm0
+        vmovdqu xmm11,XMMWORD[16+r13]
+        mov     QWORD[((64+40))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[8+rsi]
+        lea     r13,[16+rbx*1+r13]
+        vaesdec xmm2,xmm2,xmm1
+        cmp     ecx,DWORD[((32+24))+rsp]
+        mov     rbx,QWORD[((64+48))+rsp]
+        vaesdec xmm3,xmm3,xmm1
+        prefetcht0      [31+r14]
+        vaesdec xmm4,xmm4,xmm1
+        prefetcht0      [15+r12]
+        vaesdec xmm5,xmm5,xmm1
+        lea     rbx,[rbx*1+r14]
+        cmovge  r14,rsp
+        vaesdec xmm6,xmm6,xmm1
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm1
+        sub     rbx,r14
+        vaesdec xmm8,xmm8,xmm1
+        vmovdqu xmm12,XMMWORD[16+r14]
+        mov     QWORD[((64+48))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[24+rsi]
+        lea     r14,[16+rbx*1+r14]
+        vaesdec xmm2,xmm2,xmm0
+        cmp     ecx,DWORD[((32+28))+rsp]
+        mov     rbx,QWORD[((64+56))+rsp]
+        vaesdec xmm3,xmm3,xmm0
+        prefetcht0      [31+r15]
+        vaesdec xmm4,xmm4,xmm0
+        prefetcht0      [15+r13]
+        vaesdec xmm5,xmm5,xmm0
+        lea     rbx,[rbx*1+r15]
+        cmovge  r15,rsp
+        vaesdec xmm6,xmm6,xmm0
+        cmovg   rbx,rsp
+        vaesdec xmm7,xmm7,xmm0
+        sub     rbx,r15
+        vaesdec xmm8,xmm8,xmm0
+        vmovdqu xmm13,XMMWORD[16+r15]
+        mov     QWORD[((64+56))+rsp],rbx
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[40+rsi]
+        lea     r15,[16+rbx*1+r15]
+        vmovdqu xmm14,XMMWORD[32+rsp]
+        prefetcht0      [15+r14]
+        prefetcht0      [15+r15]
+        cmp     eax,11
+        jb      NEAR $L$dec8x_tail
+
+        vaesdec xmm2,xmm2,xmm1
+        vaesdec xmm3,xmm3,xmm1
+        vaesdec xmm4,xmm4,xmm1
+        vaesdec xmm5,xmm5,xmm1
+        vaesdec xmm6,xmm6,xmm1
+        vaesdec xmm7,xmm7,xmm1
+        vaesdec xmm8,xmm8,xmm1
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((176-120))+rsi]
+
+        vaesdec xmm2,xmm2,xmm0
+        vaesdec xmm3,xmm3,xmm0
+        vaesdec xmm4,xmm4,xmm0
+        vaesdec xmm5,xmm5,xmm0
+        vaesdec xmm6,xmm6,xmm0
+        vaesdec xmm7,xmm7,xmm0
+        vaesdec xmm8,xmm8,xmm0
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((192-120))+rsi]
+        je      NEAR $L$dec8x_tail
+
+        vaesdec xmm2,xmm2,xmm1
+        vaesdec xmm3,xmm3,xmm1
+        vaesdec xmm4,xmm4,xmm1
+        vaesdec xmm5,xmm5,xmm1
+        vaesdec xmm6,xmm6,xmm1
+        vaesdec xmm7,xmm7,xmm1
+        vaesdec xmm8,xmm8,xmm1
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((208-120))+rsi]
+
+        vaesdec xmm2,xmm2,xmm0
+        vaesdec xmm3,xmm3,xmm0
+        vaesdec xmm4,xmm4,xmm0
+        vaesdec xmm5,xmm5,xmm0
+        vaesdec xmm6,xmm6,xmm0
+        vaesdec xmm7,xmm7,xmm0
+        vaesdec xmm8,xmm8,xmm0
+        vaesdec xmm9,xmm9,xmm0
+        vmovups xmm0,XMMWORD[((224-120))+rsi]
+
+$L$dec8x_tail:
+        vaesdec xmm2,xmm2,xmm1
+        vpxor   xmm15,xmm15,xmm15
+        vaesdec xmm3,xmm3,xmm1
+        vaesdec xmm4,xmm4,xmm1
+        vpcmpgtd        xmm15,xmm14,xmm15
+        vaesdec xmm5,xmm5,xmm1
+        vaesdec xmm6,xmm6,xmm1
+        vpaddd  xmm15,xmm15,xmm14
+        vmovdqu xmm14,XMMWORD[48+rsp]
+        vaesdec xmm7,xmm7,xmm1
+        mov     rbx,QWORD[64+rsp]
+        vaesdec xmm8,xmm8,xmm1
+        vaesdec xmm9,xmm9,xmm1
+        vmovups xmm1,XMMWORD[((16-120))+rsi]
+
+        vaesdeclast     xmm2,xmm2,xmm0
+        vmovdqa XMMWORD[32+rsp],xmm15
+        vpxor   xmm15,xmm15,xmm15
+        vaesdeclast     xmm3,xmm3,xmm0
+        vpxor   xmm2,xmm2,XMMWORD[rbp]
+        vaesdeclast     xmm4,xmm4,xmm0
+        vpxor   xmm3,xmm3,XMMWORD[16+rbp]
+        vpcmpgtd        xmm15,xmm14,xmm15
+        vaesdeclast     xmm5,xmm5,xmm0
+        vpxor   xmm4,xmm4,XMMWORD[32+rbp]
+        vaesdeclast     xmm6,xmm6,xmm0
+        vpxor   xmm5,xmm5,XMMWORD[48+rbp]
+        vpaddd  xmm14,xmm14,xmm15
+        vmovdqu xmm15,XMMWORD[((-120))+rsi]
+        vaesdeclast     xmm7,xmm7,xmm0
+        vpxor   xmm6,xmm6,XMMWORD[64+rbp]
+        vaesdeclast     xmm8,xmm8,xmm0
+        vpxor   xmm7,xmm7,XMMWORD[80+rbp]
+        vmovdqa XMMWORD[48+rsp],xmm14
+        vaesdeclast     xmm9,xmm9,xmm0
+        vpxor   xmm8,xmm8,XMMWORD[96+rbp]
+        vmovups xmm0,XMMWORD[((32-120))+rsi]
+
+        vmovups XMMWORD[(-16)+r8],xmm2
+        sub     r8,rbx
+        vmovdqu xmm2,XMMWORD[((128+0))+rsp]
+        vpxor   xmm9,xmm9,XMMWORD[112+rbp]
+        vmovups XMMWORD[(-16)+r9],xmm3
+        sub     r9,QWORD[72+rsp]
+        vmovdqu XMMWORD[rbp],xmm2
+        vpxor   xmm2,xmm2,xmm15
+        vmovdqu xmm3,XMMWORD[((128+16))+rsp]
+        vmovups XMMWORD[(-16)+r10],xmm4
+        sub     r10,QWORD[80+rsp]
+        vmovdqu XMMWORD[16+rbp],xmm3
+        vpxor   xmm3,xmm3,xmm15
+        vmovdqu xmm4,XMMWORD[((128+32))+rsp]
+        vmovups XMMWORD[(-16)+r11],xmm5
+        sub     r11,QWORD[88+rsp]
+        vmovdqu XMMWORD[32+rbp],xmm4
+        vpxor   xmm4,xmm4,xmm15
+        vmovdqu xmm5,XMMWORD[((128+48))+rsp]
+        vmovups XMMWORD[(-16)+r12],xmm6
+        sub     r12,QWORD[96+rsp]
+        vmovdqu XMMWORD[48+rbp],xmm5
+        vpxor   xmm5,xmm5,xmm15
+        vmovdqu XMMWORD[64+rbp],xmm10
+        vpxor   xmm6,xmm15,xmm10
+        vmovups XMMWORD[(-16)+r13],xmm7
+        sub     r13,QWORD[104+rsp]
+        vmovdqu XMMWORD[80+rbp],xmm11
+        vpxor   xmm7,xmm15,xmm11
+        vmovups XMMWORD[(-16)+r14],xmm8
+        sub     r14,QWORD[112+rsp]
+        vmovdqu XMMWORD[96+rbp],xmm12
+        vpxor   xmm8,xmm15,xmm12
+        vmovups XMMWORD[(-16)+r15],xmm9
+        sub     r15,QWORD[120+rsp]
+        vmovdqu XMMWORD[112+rbp],xmm13
+        vpxor   xmm9,xmm15,xmm13
+
+        xor     rbp,128
+        dec     edx
+        jnz     NEAR $L$oop_dec8x
+
+        mov     rax,QWORD[16+rsp]
+
+
+
+
+
+
+$L$dec8x_done:
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$dec8x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_multi_cbc_decrypt_avx:
 EXTERN  __imp_RtlVirtualUnwind
 
@@ -728 +1821 @@
         DD      $L$SEH_end_aesni_multi_cbc_decrypt wrt ..imagebase
         DD      $L$SEH_info_aesni_multi_cbc_decrypt wrt ..imagebase
+        DD      $L$SEH_begin_aesni_multi_cbc_encrypt_avx wrt ..imagebase
+        DD      $L$SEH_end_aesni_multi_cbc_encrypt_avx wrt ..imagebase
+        DD      $L$SEH_info_aesni_multi_cbc_encrypt_avx wrt ..imagebase
+        DD      $L$SEH_begin_aesni_multi_cbc_decrypt_avx wrt ..imagebase
+        DD      $L$SEH_end_aesni_multi_cbc_decrypt_avx wrt ..imagebase
+        DD      $L$SEH_info_aesni_multi_cbc_decrypt_avx wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -738 +1837 @@
         DD      se_handler wrt ..imagebase
         DD      $L$dec4x_body wrt ..imagebase,$L$dec4x_epilogue wrt ..imagebase
+$L$SEH_info_aesni_multi_cbc_encrypt_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$enc8x_body wrt ..imagebase,$L$enc8x_epilogue wrt ..imagebase
+$L$SEH_info_aesni_multi_cbc_decrypt_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$dec8x_body wrt ..imagebase,$L$dec8x_epilogue wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/aesni-sha1-x86_64.S

@@ -17 +17 @@
         bt      r11,61
         jc      NEAR aesni_cbc_sha1_enc_shaext
+        and     r11d,268435456
+        and     r10d,1073741824
+        or      r10d,r11d
+        cmp     r10d,1342177280
+        je      NEAR aesni_cbc_sha1_enc_avx
         jmp     NEAR aesni_cbc_sha1_enc_ssse3
         DB      0F3h,0C3h               ;repret
@@ -1427 +1432 @@
 
 $L$SEH_end_aesni_cbc_sha1_enc_ssse3:
+
+ALIGN   32
+aesni_cbc_sha1_enc_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_cbc_sha1_enc_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        mov     r10,QWORD[56+rsp]
+
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-264))+rsp]
+
+
+
+        movaps  XMMWORD[(96+0)+rsp],xmm6
+        movaps  XMMWORD[(96+16)+rsp],xmm7
+        movaps  XMMWORD[(96+32)+rsp],xmm8
+        movaps  XMMWORD[(96+48)+rsp],xmm9
+        movaps  XMMWORD[(96+64)+rsp],xmm10
+        movaps  XMMWORD[(96+80)+rsp],xmm11
+        movaps  XMMWORD[(96+96)+rsp],xmm12
+        movaps  XMMWORD[(96+112)+rsp],xmm13
+        movaps  XMMWORD[(96+128)+rsp],xmm14
+        movaps  XMMWORD[(96+144)+rsp],xmm15
+$L$prologue_avx:
+        vzeroall
+        mov     r12,rdi
+        mov     r13,rsi
+        mov     r14,rdx
+        lea     r15,[112+rcx]
+        vmovdqu xmm12,XMMWORD[r8]
+        mov     QWORD[88+rsp],r8
+        shl     r14,6
+        sub     r13,r12
+        mov     r8d,DWORD[((240-112))+r15]
+        add     r14,r10
+
+        lea     r11,[K_XX_XX]
+        mov     eax,DWORD[r9]
+        mov     ebx,DWORD[4+r9]
+        mov     ecx,DWORD[8+r9]
+        mov     edx,DWORD[12+r9]
+        mov     esi,ebx
+        mov     ebp,DWORD[16+r9]
+        mov     edi,ecx
+        xor     edi,edx
+        and     esi,edi
+
+        vmovdqa xmm6,XMMWORD[64+r11]
+        vmovdqa xmm10,XMMWORD[r11]
+        vmovdqu xmm0,XMMWORD[r10]
+        vmovdqu xmm1,XMMWORD[16+r10]
+        vmovdqu xmm2,XMMWORD[32+r10]
+        vmovdqu xmm3,XMMWORD[48+r10]
+        vpshufb xmm0,xmm0,xmm6
+        add     r10,64
+        vpshufb xmm1,xmm1,xmm6
+        vpshufb xmm2,xmm2,xmm6
+        vpshufb xmm3,xmm3,xmm6
+        vpaddd  xmm4,xmm0,xmm10
+        vpaddd  xmm5,xmm1,xmm10
+        vpaddd  xmm6,xmm2,xmm10
+        vmovdqa XMMWORD[rsp],xmm4
+        vmovdqa XMMWORD[16+rsp],xmm5
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        jmp     NEAR $L$oop_avx
+ALIGN   32
+$L$oop_avx:
+        shrd    ebx,ebx,2
+        vmovdqu xmm13,XMMWORD[r12]
+        vpxor   xmm13,xmm13,xmm15
+        vpxor   xmm12,xmm12,xmm13
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-80))+r15]
+        xor     esi,edx
+        vpalignr        xmm4,xmm1,xmm0,8
+        mov     edi,eax
+        add     ebp,DWORD[rsp]
+        vpaddd  xmm9,xmm10,xmm3
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vpsrldq xmm8,xmm3,4
+        add     ebp,esi
+        and     edi,ebx
+        vpxor   xmm4,xmm4,xmm0
+        xor     ebx,ecx
+        add     ebp,eax
+        vpxor   xmm8,xmm8,xmm2
+        shrd    eax,eax,7
+        xor     edi,ecx
+        mov     esi,ebp
+        add     edx,DWORD[4+rsp]
+        vpxor   xmm4,xmm4,xmm8
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vmovdqa XMMWORD[48+rsp],xmm9
+        add     edx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-64))+r15]
+        and     esi,eax
+        vpsrld  xmm8,xmm4,31
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     esi,ebx
+        vpslldq xmm9,xmm4,12
+        vpaddd  xmm4,xmm4,xmm4
+        mov     edi,edx
+        add     ecx,DWORD[8+rsp]
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpor    xmm4,xmm4,xmm8
+        vpsrld  xmm8,xmm9,30
+        add     ecx,esi
+        and     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        vpslld  xmm9,xmm9,2
+        vpxor   xmm4,xmm4,xmm8
+        shrd    edx,edx,7
+        xor     edi,eax
+        mov     esi,ecx
+        add     ebx,DWORD[12+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-48))+r15]
+        vpxor   xmm4,xmm4,xmm9
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        and     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        shrd    ecx,ecx,7
+        xor     esi,ebp
+        vpalignr        xmm5,xmm2,xmm1,8
+        mov     edi,ebx
+        add     eax,DWORD[16+rsp]
+        vpaddd  xmm9,xmm10,xmm4
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vpsrldq xmm8,xmm4,4
+        add     eax,esi
+        and     edi,ecx
+        vpxor   xmm5,xmm5,xmm1
+        xor     ecx,edx
+        add     eax,ebx
+        vpxor   xmm8,xmm8,xmm3
+        shrd    ebx,ebx,7
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-32))+r15]
+        xor     edi,edx
+        mov     esi,eax
+        add     ebp,DWORD[20+rsp]
+        vpxor   xmm5,xmm5,xmm8
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vmovdqa XMMWORD[rsp],xmm9
+        add     ebp,edi
+        and     esi,ebx
+        vpsrld  xmm8,xmm5,31
+        xor     ebx,ecx
+        add     ebp,eax
+        shrd    eax,eax,7
+        xor     esi,ecx
+        vpslldq xmm9,xmm5,12
+        vpaddd  xmm5,xmm5,xmm5
+        mov     edi,ebp
+        add     edx,DWORD[24+rsp]
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vpor    xmm5,xmm5,xmm8
+        vpsrld  xmm8,xmm9,30
+        add     edx,esi
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-16))+r15]
+        and     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        vpslld  xmm9,xmm9,2
+        vpxor   xmm5,xmm5,xmm8
+        shrd    ebp,ebp,7
+        xor     edi,ebx
+        mov     esi,edx
+        add     ecx,DWORD[28+rsp]
+        vpxor   xmm5,xmm5,xmm9
+        xor     ebp,eax
+        shld    edx,edx,5
+        vmovdqa xmm10,XMMWORD[16+r11]
+        add     ecx,edi
+        and     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        shrd    edx,edx,7
+        xor     esi,eax
+        vpalignr        xmm6,xmm3,xmm2,8
+        mov     edi,ecx
+        add     ebx,DWORD[32+rsp]
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[r15]
+        vpaddd  xmm9,xmm10,xmm5
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        vpsrldq xmm8,xmm5,4
+        add     ebx,esi
+        and     edi,edx
+        vpxor   xmm6,xmm6,xmm2
+        xor     edx,ebp
+        add     ebx,ecx
+        vpxor   xmm8,xmm8,xmm4
+        shrd    ecx,ecx,7
+        xor     edi,ebp
+        mov     esi,ebx
+        add     eax,DWORD[36+rsp]
+        vpxor   xmm6,xmm6,xmm8
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vmovdqa XMMWORD[16+rsp],xmm9
+        add     eax,edi
+        and     esi,ecx
+        vpsrld  xmm8,xmm6,31
+        xor     ecx,edx
+        add     eax,ebx
+        shrd    ebx,ebx,7
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[16+r15]
+        xor     esi,edx
+        vpslldq xmm9,xmm6,12
+        vpaddd  xmm6,xmm6,xmm6
+        mov     edi,eax
+        add     ebp,DWORD[40+rsp]
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vpor    xmm6,xmm6,xmm8
+        vpsrld  xmm8,xmm9,30
+        add     ebp,esi
+        and     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpslld  xmm9,xmm9,2
+        vpxor   xmm6,xmm6,xmm8
+        shrd    eax,eax,7
+        xor     edi,ecx
+        mov     esi,ebp
+        add     edx,DWORD[44+rsp]
+        vpxor   xmm6,xmm6,xmm9
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[32+r15]
+        and     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     esi,ebx
+        vpalignr        xmm7,xmm4,xmm3,8
+        mov     edi,edx
+        add     ecx,DWORD[48+rsp]
+        vpaddd  xmm9,xmm10,xmm6
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpsrldq xmm8,xmm6,4
+        add     ecx,esi
+        and     edi,ebp
+        vpxor   xmm7,xmm7,xmm3
+        xor     ebp,eax
+        add     ecx,edx
+        vpxor   xmm8,xmm8,xmm5
+        shrd    edx,edx,7
+        xor     edi,eax
+        mov     esi,ecx
+        add     ebx,DWORD[52+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[48+r15]
+        vpxor   xmm7,xmm7,xmm8
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     ebx,edi
+        and     esi,edx
+        vpsrld  xmm8,xmm7,31
+        xor     edx,ebp
+        add     ebx,ecx
+        shrd    ecx,ecx,7
+        xor     esi,ebp
+        vpslldq xmm9,xmm7,12
+        vpaddd  xmm7,xmm7,xmm7
+        mov     edi,ebx
+        add     eax,DWORD[56+rsp]
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vpor    xmm7,xmm7,xmm8
+        vpsrld  xmm8,xmm9,30
+        add     eax,esi
+        and     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        vpslld  xmm9,xmm9,2
+        vpxor   xmm7,xmm7,xmm8
+        shrd    ebx,ebx,7
+        cmp     r8d,11
+        jb      NEAR $L$vaesenclast6
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[64+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[80+r15]
+        je      NEAR $L$vaesenclast6
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[96+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[112+r15]
+$L$vaesenclast6:
+        vaesenclast     xmm12,xmm12,xmm15
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        xor     edi,edx
+        mov     esi,eax
+        add     ebp,DWORD[60+rsp]
+        vpxor   xmm7,xmm7,xmm9
+        xor     ebx,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        and     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpalignr        xmm8,xmm7,xmm6,8
+        vpxor   xmm0,xmm0,xmm4
+        shrd    eax,eax,7
+        xor     esi,ecx
+        mov     edi,ebp
+        add     edx,DWORD[rsp]
+        vpxor   xmm0,xmm0,xmm1
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vpaddd  xmm9,xmm10,xmm7
+        add     edx,esi
+        vmovdqu xmm13,XMMWORD[16+r12]
+        vpxor   xmm13,xmm13,xmm15
+        vmovups XMMWORD[r13*1+r12],xmm12
+        vpxor   xmm12,xmm12,xmm13
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-80))+r15]
+        and     edi,eax
+        vpxor   xmm0,xmm0,xmm8
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     edi,ebx
+        vpsrld  xmm8,xmm0,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        mov     esi,edx
+        add     ecx,DWORD[4+rsp]
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpslld  xmm0,xmm0,2
+        add     ecx,edi
+        and     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        shrd    edx,edx,7
+        xor     esi,eax
+        mov     edi,ecx
+        add     ebx,DWORD[8+rsp]
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-64))+r15]
+        vpor    xmm0,xmm0,xmm8
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        add     ebx,esi
+        and     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[12+rsp]
+        xor     edi,ebp
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpalignr        xmm8,xmm0,xmm7,8
+        vpxor   xmm1,xmm1,xmm5
+        add     ebp,DWORD[16+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-48))+r15]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        vpxor   xmm1,xmm1,xmm2
+        add     ebp,esi
+        xor     edi,ecx
+        vpaddd  xmm9,xmm10,xmm0
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpxor   xmm1,xmm1,xmm8
+        add     edx,DWORD[20+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        vpsrld  xmm8,xmm1,30
+        vmovdqa XMMWORD[rsp],xmm9
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpslld  xmm1,xmm1,2
+        add     ecx,DWORD[24+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-32))+r15]
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpor    xmm1,xmm1,xmm8
+        add     ebx,DWORD[28+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpalignr        xmm8,xmm1,xmm0,8
+        vpxor   xmm2,xmm2,xmm6
+        add     eax,DWORD[32+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        vpxor   xmm2,xmm2,xmm3
+        add     eax,esi
+        xor     edi,edx
+        vpaddd  xmm9,xmm10,xmm1
+        vmovdqa xmm10,XMMWORD[32+r11]
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpxor   xmm2,xmm2,xmm8
+        add     ebp,DWORD[36+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-16))+r15]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        vpsrld  xmm8,xmm2,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpslld  xmm2,xmm2,2
+        add     edx,DWORD[40+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpor    xmm2,xmm2,xmm8
+        add     ecx,DWORD[44+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[r15]
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpalignr        xmm8,xmm2,xmm1,8
+        vpxor   xmm3,xmm3,xmm7
+        add     ebx,DWORD[48+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        vpxor   xmm3,xmm3,xmm4
+        add     ebx,esi
+        xor     edi,ebp
+        vpaddd  xmm9,xmm10,xmm2
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpxor   xmm3,xmm3,xmm8
+        add     eax,DWORD[52+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        vpsrld  xmm8,xmm3,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpslld  xmm3,xmm3,2
+        add     ebp,DWORD[56+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[16+r15]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpor    xmm3,xmm3,xmm8
+        add     edx,DWORD[60+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpalignr        xmm8,xmm3,xmm2,8
+        vpxor   xmm4,xmm4,xmm0
+        add     ecx,DWORD[rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        vpxor   xmm4,xmm4,xmm5
+        add     ecx,esi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[32+r15]
+        xor     edi,eax
+        vpaddd  xmm9,xmm10,xmm3
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpxor   xmm4,xmm4,xmm8
+        add     ebx,DWORD[4+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        vpsrld  xmm8,xmm4,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpslld  xmm4,xmm4,2
+        add     eax,DWORD[8+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpor    xmm4,xmm4,xmm8
+        add     ebp,DWORD[12+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[48+r15]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpalignr        xmm8,xmm4,xmm3,8
+        vpxor   xmm5,xmm5,xmm1
+        add     edx,DWORD[16+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        vpxor   xmm5,xmm5,xmm6
+        add     edx,esi
+        xor     edi,ebx
+        vpaddd  xmm9,xmm10,xmm4
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpxor   xmm5,xmm5,xmm8
+        add     ecx,DWORD[20+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        vpsrld  xmm8,xmm5,30
+        vmovdqa XMMWORD[rsp],xmm9
+        add     ecx,edi
+        cmp     r8d,11
+        jb      NEAR $L$vaesenclast7
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[64+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[80+r15]
+        je      NEAR $L$vaesenclast7
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[96+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[112+r15]
+$L$vaesenclast7:
+        vaesenclast     xmm12,xmm12,xmm15
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpslld  xmm5,xmm5,2
+        add     ebx,DWORD[24+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpor    xmm5,xmm5,xmm8
+        add     eax,DWORD[28+rsp]
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        vpalignr        xmm8,xmm5,xmm4,8
+        vpxor   xmm6,xmm6,xmm2
+        add     ebp,DWORD[32+rsp]
+        vmovdqu xmm13,XMMWORD[32+r12]
+        vpxor   xmm13,xmm13,xmm15
+        vmovups XMMWORD[16+r12*1+r13],xmm12
+        vpxor   xmm12,xmm12,xmm13
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-80))+r15]
+        and     esi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        vpxor   xmm6,xmm6,xmm7
+        mov     edi,eax
+        xor     esi,ecx
+        vpaddd  xmm9,xmm10,xmm5
+        shld    eax,eax,5
+        add     ebp,esi
+        vpxor   xmm6,xmm6,xmm8
+        xor     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[36+rsp]
+        vpsrld  xmm8,xmm6,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        and     edi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     esi,ebp
+        vpslld  xmm6,xmm6,2
+        xor     edi,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-64))+r15]
+        xor     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[40+rsp]
+        and     esi,eax
+        vpor    xmm6,xmm6,xmm8
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     edi,edx
+        xor     esi,eax
+        shld    edx,edx,5
+        add     ecx,esi
+        xor     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[44+rsp]
+        and     edi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-48))+r15]
+        mov     esi,ecx
+        xor     edi,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        vpalignr        xmm8,xmm6,xmm5,8
+        vpxor   xmm7,xmm7,xmm3
+        add     eax,DWORD[48+rsp]
+        and     esi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        vpxor   xmm7,xmm7,xmm0
+        mov     edi,ebx
+        xor     esi,edx
+        vpaddd  xmm9,xmm10,xmm6
+        vmovdqa xmm10,XMMWORD[48+r11]
+        shld    ebx,ebx,5
+        add     eax,esi
+        vpxor   xmm7,xmm7,xmm8
+        xor     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[52+rsp]
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-32))+r15]
+        vpsrld  xmm8,xmm7,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        and     edi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     esi,eax
+        vpslld  xmm7,xmm7,2
+        xor     edi,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[56+rsp]
+        and     esi,ebx
+        vpor    xmm7,xmm7,xmm8
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     edi,ebp
+        xor     esi,ebx
+        shld    ebp,ebp,5
+        add     edx,esi
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-16))+r15]
+        xor     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[60+rsp]
+        and     edi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     esi,edx
+        xor     edi,eax
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        vpalignr        xmm8,xmm7,xmm6,8
+        vpxor   xmm0,xmm0,xmm4
+        add     ebx,DWORD[rsp]
+        and     esi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[r15]
+        vpxor   xmm0,xmm0,xmm1
+        mov     edi,ecx
+        xor     esi,ebp
+        vpaddd  xmm9,xmm10,xmm7
+        shld    ecx,ecx,5
+        add     ebx,esi
+        vpxor   xmm0,xmm0,xmm8
+        xor     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[4+rsp]
+        vpsrld  xmm8,xmm0,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        and     edi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        vpslld  xmm0,xmm0,2
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[8+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[16+r15]
+        and     esi,ecx
+        vpor    xmm0,xmm0,xmm8
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     edi,eax
+        xor     esi,ecx
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[12+rsp]
+        and     edi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     esi,ebp
+        xor     edi,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[32+r15]
+        xor     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        vpalignr        xmm8,xmm0,xmm7,8
+        vpxor   xmm1,xmm1,xmm5
+        add     ecx,DWORD[16+rsp]
+        and     esi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        vpxor   xmm1,xmm1,xmm2
+        mov     edi,edx
+        xor     esi,eax
+        vpaddd  xmm9,xmm10,xmm0
+        shld    edx,edx,5
+        add     ecx,esi
+        vpxor   xmm1,xmm1,xmm8
+        xor     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[20+rsp]
+        vpsrld  xmm8,xmm1,30
+        vmovdqa XMMWORD[rsp],xmm9
+        and     edi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[48+r15]
+        mov     esi,ecx
+        vpslld  xmm1,xmm1,2
+        xor     edi,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[24+rsp]
+        and     esi,edx
+        vpor    xmm1,xmm1,xmm8
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     edi,ebx
+        xor     esi,edx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[28+rsp]
+        cmp     r8d,11
+        jb      NEAR $L$vaesenclast8
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[64+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[80+r15]
+        je      NEAR $L$vaesenclast8
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[96+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[112+r15]
+$L$vaesenclast8:
+        vaesenclast     xmm12,xmm12,xmm15
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        and     edi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     esi,eax
+        xor     edi,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpalignr        xmm8,xmm1,xmm0,8
+        vpxor   xmm2,xmm2,xmm6
+        add     edx,DWORD[32+rsp]
+        and     esi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        vpxor   xmm2,xmm2,xmm3
+        mov     edi,ebp
+        xor     esi,ebx
+        vpaddd  xmm9,xmm10,xmm1
+        shld    ebp,ebp,5
+        add     edx,esi
+        vmovdqu xmm13,XMMWORD[48+r12]
+        vpxor   xmm13,xmm13,xmm15
+        vmovups XMMWORD[32+r12*1+r13],xmm12
+        vpxor   xmm12,xmm12,xmm13
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-80))+r15]
+        vpxor   xmm2,xmm2,xmm8
+        xor     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[36+rsp]
+        vpsrld  xmm8,xmm2,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        and     edi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     esi,edx
+        vpslld  xmm2,xmm2,2
+        xor     edi,eax
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[40+rsp]
+        and     esi,ebp
+        vpor    xmm2,xmm2,xmm8
+        xor     ebp,eax
+        shrd    edx,edx,7
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-64))+r15]
+        mov     edi,ecx
+        xor     esi,ebp
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[44+rsp]
+        and     edi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        add     eax,ebx
+        vpalignr        xmm8,xmm2,xmm1,8
+        vpxor   xmm3,xmm3,xmm7
+        add     ebp,DWORD[48+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-48))+r15]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        vpxor   xmm3,xmm3,xmm4
+        add     ebp,esi
+        xor     edi,ecx
+        vpaddd  xmm9,xmm10,xmm2
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpxor   xmm3,xmm3,xmm8
+        add     edx,DWORD[52+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        vpsrld  xmm8,xmm3,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpslld  xmm3,xmm3,2
+        add     ecx,DWORD[56+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[((-32))+r15]
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpor    xmm3,xmm3,xmm8
+        add     ebx,DWORD[60+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[rsp]
+        vpaddd  xmm9,xmm10,xmm3
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        vmovdqa XMMWORD[48+rsp],xmm9
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[4+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[((-16))+r15]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[8+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[12+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[r15]
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        cmp     r10,r14
+        je      NEAR $L$done_avx
+        vmovdqa xmm9,XMMWORD[64+r11]
+        vmovdqa xmm10,XMMWORD[r11]
+        vmovdqu xmm0,XMMWORD[r10]
+        vmovdqu xmm1,XMMWORD[16+r10]
+        vmovdqu xmm2,XMMWORD[32+r10]
+        vmovdqu xmm3,XMMWORD[48+r10]
+        vpshufb xmm0,xmm0,xmm9
+        add     r10,64
+        add     ebx,DWORD[16+rsp]
+        xor     esi,ebp
+        vpshufb xmm1,xmm1,xmm9
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        vpaddd  xmm8,xmm0,xmm10
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vmovdqa XMMWORD[rsp],xmm8
+        add     eax,DWORD[20+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[24+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[16+r15]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[28+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[32+rsp]
+        xor     esi,eax
+        vpshufb xmm2,xmm2,xmm9
+        mov     edi,edx
+        shld    edx,edx,5
+        vpaddd  xmm8,xmm1,xmm10
+        add     ecx,esi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[32+r15]
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vmovdqa XMMWORD[16+rsp],xmm8
+        add     ebx,DWORD[36+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[40+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[44+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[48+r15]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[48+rsp]
+        xor     esi,ebx
+        vpshufb xmm3,xmm3,xmm9
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        vpaddd  xmm8,xmm2,xmm10
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vmovdqa XMMWORD[32+rsp],xmm8
+        add     ecx,DWORD[52+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        cmp     r8d,11
+        jb      NEAR $L$vaesenclast9
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[64+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[80+r15]
+        je      NEAR $L$vaesenclast9
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[96+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[112+r15]
+$L$vaesenclast9:
+        vaesenclast     xmm12,xmm12,xmm15
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[56+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[60+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vmovups XMMWORD[48+r12*1+r13],xmm12
+        lea     r12,[64+r12]
+
+        add     eax,DWORD[r9]
+        add     esi,DWORD[4+r9]
+        add     ecx,DWORD[8+r9]
+        add     edx,DWORD[12+r9]
+        mov     DWORD[r9],eax
+        add     ebp,DWORD[16+r9]
+        mov     DWORD[4+r9],esi
+        mov     ebx,esi
+        mov     DWORD[8+r9],ecx
+        mov     edi,ecx
+        mov     DWORD[12+r9],edx
+        xor     edi,edx
+        mov     DWORD[16+r9],ebp
+        and     esi,edi
+        jmp     NEAR $L$oop_avx
+
+$L$done_avx:
+        add     ebx,DWORD[16+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[20+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[24+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[16+r15]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[28+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[32+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[32+r15]
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[36+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[40+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[44+rsp]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[48+r15]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[48+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[52+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        cmp     r8d,11
+        jb      NEAR $L$vaesenclast10
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[64+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[80+r15]
+        je      NEAR $L$vaesenclast10
+        vaesenc xmm12,xmm12,xmm15
+        vmovups xmm14,XMMWORD[96+r15]
+        vaesenc xmm12,xmm12,xmm14
+        vmovups xmm15,XMMWORD[112+r15]
+$L$vaesenclast10:
+        vaesenclast     xmm12,xmm12,xmm15
+        vmovups xmm15,XMMWORD[((-112))+r15]
+        vmovups xmm14,XMMWORD[((16-112))+r15]
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[56+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[60+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vmovups XMMWORD[48+r12*1+r13],xmm12
+        mov     r8,QWORD[88+rsp]
+
+        add     eax,DWORD[r9]
+        add     esi,DWORD[4+r9]
+        add     ecx,DWORD[8+r9]
+        mov     DWORD[r9],eax
+        add     edx,DWORD[12+r9]
+        mov     DWORD[4+r9],esi
+        add     ebp,DWORD[16+r9]
+        mov     DWORD[8+r9],ecx
+        mov     DWORD[12+r9],edx
+        mov     DWORD[16+r9],ebp
+        vmovups XMMWORD[r8],xmm12
+        vzeroall
+        movaps  xmm6,XMMWORD[((96+0))+rsp]
+        movaps  xmm7,XMMWORD[((96+16))+rsp]
+        movaps  xmm8,XMMWORD[((96+32))+rsp]
+        movaps  xmm9,XMMWORD[((96+48))+rsp]
+        movaps  xmm10,XMMWORD[((96+64))+rsp]
+        movaps  xmm11,XMMWORD[((96+80))+rsp]
+        movaps  xmm12,XMMWORD[((96+96))+rsp]
+        movaps  xmm13,XMMWORD[((96+112))+rsp]
+        movaps  xmm14,XMMWORD[((96+128))+rsp]
+        movaps  xmm15,XMMWORD[((96+144))+rsp]
+        lea     rsi,[264+rsp]
+
+        mov     r15,QWORD[rsi]
+
+        mov     r14,QWORD[8+rsi]
+
+        mov     r13,QWORD[16+rsi]
+
+        mov     r12,QWORD[24+rsi]
+
+        mov     rbp,QWORD[32+rsi]
+
+        mov     rbx,QWORD[40+rsi]
+
+        lea     rsp,[48+rsi]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_cbc_sha1_enc_avx:
 ALIGN   64
 K_XX_XX:
@@ -1546 +2901 @@
 DB      15,56,201,243
         cmp     r11d,11
-        jb      NEAR $L$aesenclast6
+        jb      NEAR $L$aesenclast11
         movups  xmm0,XMMWORD[64+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[80+rcx]
 DB      102,15,56,220,208
-        je      NEAR $L$aesenclast6
+        je      NEAR $L$aesenclast11
         movups  xmm0,XMMWORD[96+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[112+rcx]
 DB      102,15,56,220,208
-$L$aesenclast6:
+$L$aesenclast11:
 DB      102,15,56,221,209
         movups  xmm0,XMMWORD[((16-112))+rcx]
@@ -1612 +2967 @@
 DB      15,56,201,220
         cmp     r11d,11
-        jb      NEAR $L$aesenclast7
+        jb      NEAR $L$aesenclast12
         movups  xmm0,XMMWORD[64+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[80+rcx]
 DB      102,15,56,220,208
-        je      NEAR $L$aesenclast7
+        je      NEAR $L$aesenclast12
         movups  xmm0,XMMWORD[96+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[112+rcx]
 DB      102,15,56,220,208
-$L$aesenclast7:
+$L$aesenclast12:
 DB      102,15,56,221,209
         movups  xmm0,XMMWORD[((16-112))+rcx]
@@ -1678 +3033 @@
 DB      15,56,201,229
         cmp     r11d,11
-        jb      NEAR $L$aesenclast8
+        jb      NEAR $L$aesenclast13
         movups  xmm0,XMMWORD[64+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[80+rcx]
 DB      102,15,56,220,208
-        je      NEAR $L$aesenclast8
+        je      NEAR $L$aesenclast13
         movups  xmm0,XMMWORD[96+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[112+rcx]
 DB      102,15,56,220,208
-$L$aesenclast8:
+$L$aesenclast13:
 DB      102,15,56,221,209
         movups  xmm0,XMMWORD[((16-112))+rcx]
@@ -1742 +3097 @@
 DB      102,15,56,220,208
         cmp     r11d,11
-        jb      NEAR $L$aesenclast9
+        jb      NEAR $L$aesenclast14
         movups  xmm0,XMMWORD[64+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[80+rcx]
 DB      102,15,56,220,208
-        je      NEAR $L$aesenclast9
+        je      NEAR $L$aesenclast14
         movups  xmm0,XMMWORD[96+rcx]
 DB      102,15,56,220,209
         movups  xmm1,XMMWORD[112+rcx]
 DB      102,15,56,220,208
-$L$aesenclast9:
+$L$aesenclast14:
 DB      102,15,56,221,209
         movups  xmm0,XMMWORD[((16-112))+rcx]
@@ -1892 +3247 @@
         DD      $L$SEH_end_aesni_cbc_sha1_enc_ssse3 wrt ..imagebase
         DD      $L$SEH_info_aesni_cbc_sha1_enc_ssse3 wrt ..imagebase
+        DD      $L$SEH_begin_aesni_cbc_sha1_enc_avx wrt ..imagebase
+        DD      $L$SEH_end_aesni_cbc_sha1_enc_avx wrt ..imagebase
+        DD      $L$SEH_info_aesni_cbc_sha1_enc_avx wrt ..imagebase
         DD      $L$SEH_begin_aesni_cbc_sha1_enc_shaext wrt ..imagebase
         DD      $L$SEH_end_aesni_cbc_sha1_enc_shaext wrt ..imagebase
@@ -1901 +3259 @@
         DD      ssse3_handler wrt ..imagebase
         DD      $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase
+$L$SEH_info_aesni_cbc_sha1_enc_avx:
+DB      9,0,0,0
+        DD      ssse3_handler wrt ..imagebase
+        DD      $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
 $L$SEH_info_aesni_cbc_sha1_enc_shaext:
 DB      9,0,0,0

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/aesni-sha256-x86_64.S

@@ -12 +12 @@
 aesni_cbc_sha256_enc:
 
+        lea     r11,[OPENSSL_ia32cap_P]
+        mov     eax,1
+        cmp     rcx,0
+        je      NEAR $L$probe
+        mov     eax,DWORD[r11]
+        mov     r10,QWORD[4+r11]
+        bt      r10,61
+        jc      NEAR aesni_cbc_sha256_enc_shaext
+        mov     r11,r10
+        shr     r11,32
+
+        test    r10d,2048
+        jnz     NEAR aesni_cbc_sha256_enc_xop
+        and     r11d,296
+        cmp     r11d,296
+        je      NEAR aesni_cbc_sha256_enc_avx2
+        and     r10d,268435456
+        jnz     NEAR aesni_cbc_sha256_enc_avx
+        ud2
         xor     eax,eax
         cmp     rcx,0
@@ -67 +86 @@
 DB      46,111,114,103,62,0
 ALIGN   64
+
+ALIGN   64
+aesni_cbc_sha256_enc_xop:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_cbc_sha256_enc_xop:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+$L$xop_shortcut:
+        mov     r10,QWORD[56+rsp]
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,288
+        and     rsp,-64
+
+        shl     rdx,6
+        sub     rsi,rdi
+        sub     r10,rdi
+        add     rdx,rdi
+
+
+        mov     QWORD[((64+8))+rsp],rsi
+        mov     QWORD[((64+16))+rsp],rdx
+
+        mov     QWORD[((64+32))+rsp],r8
+        mov     QWORD[((64+40))+rsp],r9
+        mov     QWORD[((64+48))+rsp],r10
+        mov     QWORD[120+rsp],rax
+
+        movaps  XMMWORD[128+rsp],xmm6
+        movaps  XMMWORD[144+rsp],xmm7
+        movaps  XMMWORD[160+rsp],xmm8
+        movaps  XMMWORD[176+rsp],xmm9
+        movaps  XMMWORD[192+rsp],xmm10
+        movaps  XMMWORD[208+rsp],xmm11
+        movaps  XMMWORD[224+rsp],xmm12
+        movaps  XMMWORD[240+rsp],xmm13
+        movaps  XMMWORD[256+rsp],xmm14
+        movaps  XMMWORD[272+rsp],xmm15
+$L$prologue_xop:
+        vzeroall
+
+        mov     r12,rdi
+        lea     rdi,[128+rcx]
+        lea     r13,[((K256+544))]
+        mov     r14d,DWORD[((240-128))+rdi]
+        mov     r15,r9
+        mov     rsi,r10
+        vmovdqu xmm8,XMMWORD[r8]
+        sub     r14,9
+
+        mov     eax,DWORD[r15]
+        mov     ebx,DWORD[4+r15]
+        mov     ecx,DWORD[8+r15]
+        mov     edx,DWORD[12+r15]
+        mov     r8d,DWORD[16+r15]
+        mov     r9d,DWORD[20+r15]
+        mov     r10d,DWORD[24+r15]
+        mov     r11d,DWORD[28+r15]
+
+        vmovdqa xmm14,XMMWORD[r14*8+r13]
+        vmovdqa xmm13,XMMWORD[16+r14*8+r13]
+        vmovdqa xmm12,XMMWORD[32+r14*8+r13]
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        jmp     NEAR $L$loop_xop
+ALIGN   16
+$L$loop_xop:
+        vmovdqa xmm7,XMMWORD[((K256+512))]
+        vmovdqu xmm0,XMMWORD[r12*1+rsi]
+        vmovdqu xmm1,XMMWORD[16+r12*1+rsi]
+        vmovdqu xmm2,XMMWORD[32+r12*1+rsi]
+        vmovdqu xmm3,XMMWORD[48+r12*1+rsi]
+        vpshufb xmm0,xmm0,xmm7
+        lea     rbp,[K256]
+        vpshufb xmm1,xmm1,xmm7
+        vpshufb xmm2,xmm2,xmm7
+        vpaddd  xmm4,xmm0,XMMWORD[rbp]
+        vpshufb xmm3,xmm3,xmm7
+        vpaddd  xmm5,xmm1,XMMWORD[32+rbp]
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        vpaddd  xmm7,xmm3,XMMWORD[96+rbp]
+        vmovdqa XMMWORD[rsp],xmm4
+        mov     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm5
+        mov     esi,ebx
+        vmovdqa XMMWORD[32+rsp],xmm6
+        xor     esi,ecx
+        vmovdqa XMMWORD[48+rsp],xmm7
+        mov     r13d,r8d
+        jmp     NEAR $L$xop_00_47
+
+ALIGN   16
+$L$xop_00_47:
+        sub     rbp,-16*2*4
+        vmovdqu xmm9,XMMWORD[r12]
+        mov     QWORD[((64+0))+rsp],r12
+        vpalignr        xmm4,xmm1,xmm0,4
+        ror     r13d,14
+        mov     eax,r14d
+        vpalignr        xmm7,xmm3,xmm2,4
+        mov     r12d,r9d
+        xor     r13d,r8d
+DB      143,232,120,194,236,14
+        ror     r14d,9
+        xor     r12d,r10d
+        vpsrld  xmm4,xmm4,3
+        ror     r13d,5
+        xor     r14d,eax
+        vpaddd  xmm0,xmm0,xmm7
+        and     r12d,r8d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+DB      143,232,120,194,245,11
+        ror     r14d,11
+        xor     r12d,r10d
+        vpxor   xmm4,xmm4,xmm5
+        xor     r15d,ebx
+        ror     r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+DB      143,232,120,194,251,13
+        xor     r14d,eax
+        add     r11d,r13d
+        vpxor   xmm4,xmm4,xmm6
+        xor     esi,ebx
+        add     edx,r11d
+        vpsrld  xmm6,xmm3,10
+        ror     r14d,2
+        add     r11d,esi
+        vpaddd  xmm0,xmm0,xmm4
+        mov     r13d,edx
+        add     r14d,r11d
+DB      143,232,120,194,239,2
+        ror     r13d,14
+        mov     r11d,r14d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r12d,r8d
+        xor     r13d,edx
+        ror     r14d,9
+        xor     r12d,r9d
+        vpxor   xmm7,xmm7,xmm5
+        ror     r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vpxor   xmm9,xmm9,xmm8
+        xor     r13d,edx
+        vpsrldq xmm7,xmm7,8
+        add     r10d,DWORD[4+rsp]
+        mov     esi,r11d
+        ror     r14d,11
+        xor     r12d,r9d
+        vpaddd  xmm0,xmm0,xmm7
+        xor     esi,eax
+        ror     r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+DB      143,232,120,194,248,13
+        xor     r14d,r11d
+        add     r10d,r13d
+        vpsrld  xmm6,xmm0,10
+        xor     r15d,eax
+        add     ecx,r10d
+DB      143,232,120,194,239,2
+        ror     r14d,2
+        add     r10d,r15d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r13d,ecx
+        add     r14d,r10d
+        ror     r13d,14
+        mov     r10d,r14d
+        vpxor   xmm7,xmm7,xmm5
+        mov     r12d,edx
+        xor     r13d,ecx
+        ror     r14d,9
+        xor     r12d,r8d
+        vpslldq xmm7,xmm7,8
+        ror     r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r13d,ecx
+        vpaddd  xmm0,xmm0,xmm7
+        add     r9d,DWORD[8+rsp]
+        mov     r15d,r10d
+        ror     r14d,11
+        xor     r12d,r8d
+        vpaddd  xmm6,xmm0,XMMWORD[rbp]
+        xor     r15d,r11d
+        ror     r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        ror     r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        ror     r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        ror     r14d,9
+        xor     r12d,edx
+        ror     r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     esi,r9d
+        ror     r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        ror     r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        ror     r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[rsp],xmm6
+        vpalignr        xmm4,xmm2,xmm1,4
+        ror     r13d,14
+        mov     r8d,r14d
+        vpalignr        xmm7,xmm0,xmm3,4
+        mov     r12d,ebx
+        xor     r13d,eax
+DB      143,232,120,194,236,14
+        ror     r14d,9
+        xor     r12d,ecx
+        vpsrld  xmm4,xmm4,3
+        ror     r13d,5
+        xor     r14d,r8d
+        vpaddd  xmm1,xmm1,xmm7
+        and     r12d,eax
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+DB      143,232,120,194,245,11
+        ror     r14d,11
+        xor     r12d,ecx
+        vpxor   xmm4,xmm4,xmm5
+        xor     r15d,r9d
+        ror     r13d,6
+        add     edx,r12d
+        and     esi,r15d
+DB      143,232,120,194,248,13
+        xor     r14d,r8d
+        add     edx,r13d
+        vpxor   xmm4,xmm4,xmm6
+        xor     esi,r9d
+        add     r11d,edx
+        vpsrld  xmm6,xmm0,10
+        ror     r14d,2
+        add     edx,esi
+        vpaddd  xmm1,xmm1,xmm4
+        mov     r13d,r11d
+        add     r14d,edx
+DB      143,232,120,194,239,2
+        ror     r13d,14
+        mov     edx,r14d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r12d,eax
+        xor     r13d,r11d
+        ror     r14d,9
+        xor     r12d,ebx
+        vpxor   xmm7,xmm7,xmm5
+        ror     r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r13d,r11d
+        vpsrldq xmm7,xmm7,8
+        add     ecx,DWORD[20+rsp]
+        mov     esi,edx
+        ror     r14d,11
+        xor     r12d,ebx
+        vpaddd  xmm1,xmm1,xmm7
+        xor     esi,r8d
+        ror     r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+DB      143,232,120,194,249,13
+        xor     r14d,edx
+        add     ecx,r13d
+        vpsrld  xmm6,xmm1,10
+        xor     r15d,r8d
+        add     r10d,ecx
+DB      143,232,120,194,239,2
+        ror     r14d,2
+        add     ecx,r15d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r13d,r10d
+        add     r14d,ecx
+        ror     r13d,14
+        mov     ecx,r14d
+        vpxor   xmm7,xmm7,xmm5
+        mov     r12d,r11d
+        xor     r13d,r10d
+        ror     r14d,9
+        xor     r12d,eax
+        vpslldq xmm7,xmm7,8
+        ror     r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r13d,r10d
+        vpaddd  xmm1,xmm1,xmm7
+        add     ebx,DWORD[24+rsp]
+        mov     r15d,ecx
+        ror     r14d,11
+        xor     r12d,eax
+        vpaddd  xmm6,xmm1,XMMWORD[32+rbp]
+        xor     r15d,edx
+        ror     r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        ror     r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        ror     r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        ror     r14d,9
+        xor     r12d,r11d
+        ror     r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     esi,ebx
+        ror     r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        ror     r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        ror     r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm6
+        vpalignr        xmm4,xmm3,xmm2,4
+        ror     r13d,14
+        mov     eax,r14d
+        vpalignr        xmm7,xmm1,xmm0,4
+        mov     r12d,r9d
+        xor     r13d,r8d
+DB      143,232,120,194,236,14
+        ror     r14d,9
+        xor     r12d,r10d
+        vpsrld  xmm4,xmm4,3
+        ror     r13d,5
+        xor     r14d,eax
+        vpaddd  xmm2,xmm2,xmm7
+        and     r12d,r8d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+DB      143,232,120,194,245,11
+        ror     r14d,11
+        xor     r12d,r10d
+        vpxor   xmm4,xmm4,xmm5
+        xor     r15d,ebx
+        ror     r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+DB      143,232,120,194,249,13
+        xor     r14d,eax
+        add     r11d,r13d
+        vpxor   xmm4,xmm4,xmm6
+        xor     esi,ebx
+        add     edx,r11d
+        vpsrld  xmm6,xmm1,10
+        ror     r14d,2
+        add     r11d,esi
+        vpaddd  xmm2,xmm2,xmm4
+        mov     r13d,edx
+        add     r14d,r11d
+DB      143,232,120,194,239,2
+        ror     r13d,14
+        mov     r11d,r14d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r12d,r8d
+        xor     r13d,edx
+        ror     r14d,9
+        xor     r12d,r9d
+        vpxor   xmm7,xmm7,xmm5
+        ror     r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r13d,edx
+        vpsrldq xmm7,xmm7,8
+        add     r10d,DWORD[36+rsp]
+        mov     esi,r11d
+        ror     r14d,11
+        xor     r12d,r9d
+        vpaddd  xmm2,xmm2,xmm7
+        xor     esi,eax
+        ror     r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+DB      143,232,120,194,250,13
+        xor     r14d,r11d
+        add     r10d,r13d
+        vpsrld  xmm6,xmm2,10
+        xor     r15d,eax
+        add     ecx,r10d
+DB      143,232,120,194,239,2
+        ror     r14d,2
+        add     r10d,r15d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r13d,ecx
+        add     r14d,r10d
+        ror     r13d,14
+        mov     r10d,r14d
+        vpxor   xmm7,xmm7,xmm5
+        mov     r12d,edx
+        xor     r13d,ecx
+        ror     r14d,9
+        xor     r12d,r8d
+        vpslldq xmm7,xmm7,8
+        ror     r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r13d,ecx
+        vpaddd  xmm2,xmm2,xmm7
+        add     r9d,DWORD[40+rsp]
+        mov     r15d,r10d
+        ror     r14d,11
+        xor     r12d,r8d
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        xor     r15d,r11d
+        ror     r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        ror     r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        ror     r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        ror     r14d,9
+        xor     r12d,edx
+        ror     r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     esi,r9d
+        ror     r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        ror     r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        ror     r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vpalignr        xmm4,xmm0,xmm3,4
+        ror     r13d,14
+        mov     r8d,r14d
+        vpalignr        xmm7,xmm2,xmm1,4
+        mov     r12d,ebx
+        xor     r13d,eax
+DB      143,232,120,194,236,14
+        ror     r14d,9
+        xor     r12d,ecx
+        vpsrld  xmm4,xmm4,3
+        ror     r13d,5
+        xor     r14d,r8d
+        vpaddd  xmm3,xmm3,xmm7
+        and     r12d,eax
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+DB      143,232,120,194,245,11
+        ror     r14d,11
+        xor     r12d,ecx
+        vpxor   xmm4,xmm4,xmm5
+        xor     r15d,r9d
+        ror     r13d,6
+        add     edx,r12d
+        and     esi,r15d
+DB      143,232,120,194,250,13
+        xor     r14d,r8d
+        add     edx,r13d
+        vpxor   xmm4,xmm4,xmm6
+        xor     esi,r9d
+        add     r11d,edx
+        vpsrld  xmm6,xmm2,10
+        ror     r14d,2
+        add     edx,esi
+        vpaddd  xmm3,xmm3,xmm4
+        mov     r13d,r11d
+        add     r14d,edx
+DB      143,232,120,194,239,2
+        ror     r13d,14
+        mov     edx,r14d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r12d,eax
+        xor     r13d,r11d
+        ror     r14d,9
+        xor     r12d,ebx
+        vpxor   xmm7,xmm7,xmm5
+        ror     r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r13d,r11d
+        vpsrldq xmm7,xmm7,8
+        add     ecx,DWORD[52+rsp]
+        mov     esi,edx
+        ror     r14d,11
+        xor     r12d,ebx
+        vpaddd  xmm3,xmm3,xmm7
+        xor     esi,r8d
+        ror     r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+DB      143,232,120,194,251,13
+        xor     r14d,edx
+        add     ecx,r13d
+        vpsrld  xmm6,xmm3,10
+        xor     r15d,r8d
+        add     r10d,ecx
+DB      143,232,120,194,239,2
+        ror     r14d,2
+        add     ecx,r15d
+        vpxor   xmm7,xmm7,xmm6
+        mov     r13d,r10d
+        add     r14d,ecx
+        ror     r13d,14
+        mov     ecx,r14d
+        vpxor   xmm7,xmm7,xmm5
+        mov     r12d,r11d
+        xor     r13d,r10d
+        ror     r14d,9
+        xor     r12d,eax
+        vpslldq xmm7,xmm7,8
+        ror     r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r13d,r10d
+        vpaddd  xmm3,xmm3,xmm7
+        add     ebx,DWORD[56+rsp]
+        mov     r15d,ecx
+        ror     r14d,11
+        xor     r12d,eax
+        vpaddd  xmm6,xmm3,XMMWORD[96+rbp]
+        xor     r15d,edx
+        ror     r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        ror     r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        ror     r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        ror     r14d,9
+        xor     r12d,r11d
+        ror     r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     esi,ebx
+        ror     r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        ror     r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        ror     r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[48+rsp],xmm6
+        mov     r12,QWORD[((64+0))+rsp]
+        vpand   xmm11,xmm11,xmm14
+        mov     r15,QWORD[((64+8))+rsp]
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r12*1+r15],xmm8
+        lea     r12,[16+r12]
+        cmp     BYTE[131+rbp],0
+        jne     NEAR $L$xop_00_47
+        vmovdqu xmm9,XMMWORD[r12]
+        mov     QWORD[((64+0))+rsp],r12
+        ror     r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        xor     r13d,r8d
+        ror     r14d,9
+        xor     r12d,r10d
+        ror     r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+        ror     r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        ror     r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        add     edx,r11d
+        ror     r14d,2
+        add     r11d,esi
+        mov     r13d,edx
+        add     r14d,r11d
+        ror     r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        ror     r14d,9
+        xor     r12d,r9d
+        ror     r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vpxor   xmm9,xmm9,xmm8
+        xor     r13d,edx
+        add     r10d,DWORD[4+rsp]
+        mov     esi,r11d
+        ror     r14d,11
+        xor     r12d,r9d
+        xor     esi,eax
+        ror     r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        ror     r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        ror     r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        xor     r13d,ecx
+        ror     r14d,9
+        xor     r12d,r8d
+        ror     r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[8+rsp]
+        mov     r15d,r10d
+        ror     r14d,11
+        xor     r12d,r8d
+        xor     r15d,r11d
+        ror     r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        ror     r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        ror     r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        ror     r14d,9
+        xor     r12d,edx
+        ror     r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     esi,r9d
+        ror     r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        ror     r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        ror     r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        ror     r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        xor     r13d,eax
+        ror     r14d,9
+        xor     r12d,ecx
+        ror     r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+        ror     r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        ror     r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        add     r11d,edx
+        ror     r14d,2
+        add     edx,esi
+        mov     r13d,r11d
+        add     r14d,edx
+        ror     r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        ror     r14d,9
+        xor     r12d,ebx
+        ror     r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r13d,r11d
+        add     ecx,DWORD[20+rsp]
+        mov     esi,edx
+        ror     r14d,11
+        xor     r12d,ebx
+        xor     esi,r8d
+        ror     r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        ror     r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        ror     r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        xor     r13d,r10d
+        ror     r14d,9
+        xor     r12d,eax
+        ror     r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[24+rsp]
+        mov     r15d,ecx
+        ror     r14d,11
+        xor     r12d,eax
+        xor     r15d,edx
+        ror     r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        ror     r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        ror     r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        ror     r14d,9
+        xor     r12d,r11d
+        ror     r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     esi,ebx
+        ror     r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        ror     r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        ror     r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        ror     r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        xor     r13d,r8d
+        ror     r14d,9
+        xor     r12d,r10d
+        ror     r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+        ror     r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        ror     r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        add     edx,r11d
+        ror     r14d,2
+        add     r11d,esi
+        mov     r13d,edx
+        add     r14d,r11d
+        ror     r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        ror     r14d,9
+        xor     r12d,r9d
+        ror     r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r13d,edx
+        add     r10d,DWORD[36+rsp]
+        mov     esi,r11d
+        ror     r14d,11
+        xor     r12d,r9d
+        xor     esi,eax
+        ror     r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        ror     r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        ror     r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        xor     r13d,ecx
+        ror     r14d,9
+        xor     r12d,r8d
+        ror     r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[40+rsp]
+        mov     r15d,r10d
+        ror     r14d,11
+        xor     r12d,r8d
+        xor     r15d,r11d
+        ror     r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        ror     r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        ror     r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        ror     r14d,9
+        xor     r12d,edx
+        ror     r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     esi,r9d
+        ror     r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        ror     r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        ror     r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        ror     r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        xor     r13d,eax
+        ror     r14d,9
+        xor     r12d,ecx
+        ror     r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+        ror     r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        ror     r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        add     r11d,edx
+        ror     r14d,2
+        add     edx,esi
+        mov     r13d,r11d
+        add     r14d,edx
+        ror     r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        ror     r14d,9
+        xor     r12d,ebx
+        ror     r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r13d,r11d
+        add     ecx,DWORD[52+rsp]
+        mov     esi,edx
+        ror     r14d,11
+        xor     r12d,ebx
+        xor     esi,r8d
+        ror     r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        ror     r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        ror     r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        xor     r13d,r10d
+        ror     r14d,9
+        xor     r12d,eax
+        ror     r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[56+rsp]
+        mov     r15d,ecx
+        ror     r14d,11
+        xor     r12d,eax
+        xor     r15d,edx
+        ror     r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        ror     r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        ror     r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        ror     r14d,9
+        xor     r12d,r11d
+        ror     r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     esi,ebx
+        ror     r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        ror     r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        ror     r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        mov     r12,QWORD[((64+0))+rsp]
+        mov     r13,QWORD[((64+8))+rsp]
+        mov     r15,QWORD[((64+40))+rsp]
+        mov     rsi,QWORD[((64+48))+rsp]
+
+        vpand   xmm11,xmm11,xmm14
+        mov     eax,r14d
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r13*1+r12],xmm8
+        lea     r12,[16+r12]
+
+        add     eax,DWORD[r15]
+        add     ebx,DWORD[4+r15]
+        add     ecx,DWORD[8+r15]
+        add     edx,DWORD[12+r15]
+        add     r8d,DWORD[16+r15]
+        add     r9d,DWORD[20+r15]
+        add     r10d,DWORD[24+r15]
+        add     r11d,DWORD[28+r15]
+
+        cmp     r12,QWORD[((64+16))+rsp]
+
+        mov     DWORD[r15],eax
+        mov     DWORD[4+r15],ebx
+        mov     DWORD[8+r15],ecx
+        mov     DWORD[12+r15],edx
+        mov     DWORD[16+r15],r8d
+        mov     DWORD[20+r15],r9d
+        mov     DWORD[24+r15],r10d
+        mov     DWORD[28+r15],r11d
+
+        jb      NEAR $L$loop_xop
+
+        mov     r8,QWORD[((64+32))+rsp]
+        mov     rsi,QWORD[120+rsp]
+
+        vmovdqu XMMWORD[r8],xmm8
+        vzeroall
+        movaps  xmm6,XMMWORD[128+rsp]
+        movaps  xmm7,XMMWORD[144+rsp]
+        movaps  xmm8,XMMWORD[160+rsp]
+        movaps  xmm9,XMMWORD[176+rsp]
+        movaps  xmm10,XMMWORD[192+rsp]
+        movaps  xmm11,XMMWORD[208+rsp]
+        movaps  xmm12,XMMWORD[224+rsp]
+        movaps  xmm13,XMMWORD[240+rsp]
+        movaps  xmm14,XMMWORD[256+rsp]
+        movaps  xmm15,XMMWORD[272+rsp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_xop:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_cbc_sha256_enc_xop:
+
+ALIGN   64
+aesni_cbc_sha256_enc_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_cbc_sha256_enc_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+$L$avx_shortcut:
+        mov     r10,QWORD[56+rsp]
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,288
+        and     rsp,-64
+
+        shl     rdx,6
+        sub     rsi,rdi
+        sub     r10,rdi
+        add     rdx,rdi
+
+
+        mov     QWORD[((64+8))+rsp],rsi
+        mov     QWORD[((64+16))+rsp],rdx
+
+        mov     QWORD[((64+32))+rsp],r8
+        mov     QWORD[((64+40))+rsp],r9
+        mov     QWORD[((64+48))+rsp],r10
+        mov     QWORD[120+rsp],rax
+
+        movaps  XMMWORD[128+rsp],xmm6
+        movaps  XMMWORD[144+rsp],xmm7
+        movaps  XMMWORD[160+rsp],xmm8
+        movaps  XMMWORD[176+rsp],xmm9
+        movaps  XMMWORD[192+rsp],xmm10
+        movaps  XMMWORD[208+rsp],xmm11
+        movaps  XMMWORD[224+rsp],xmm12
+        movaps  XMMWORD[240+rsp],xmm13
+        movaps  XMMWORD[256+rsp],xmm14
+        movaps  XMMWORD[272+rsp],xmm15
+$L$prologue_avx:
+        vzeroall
+
+        mov     r12,rdi
+        lea     rdi,[128+rcx]
+        lea     r13,[((K256+544))]
+        mov     r14d,DWORD[((240-128))+rdi]
+        mov     r15,r9
+        mov     rsi,r10
+        vmovdqu xmm8,XMMWORD[r8]
+        sub     r14,9
+
+        mov     eax,DWORD[r15]
+        mov     ebx,DWORD[4+r15]
+        mov     ecx,DWORD[8+r15]
+        mov     edx,DWORD[12+r15]
+        mov     r8d,DWORD[16+r15]
+        mov     r9d,DWORD[20+r15]
+        mov     r10d,DWORD[24+r15]
+        mov     r11d,DWORD[28+r15]
+
+        vmovdqa xmm14,XMMWORD[r14*8+r13]
+        vmovdqa xmm13,XMMWORD[16+r14*8+r13]
+        vmovdqa xmm12,XMMWORD[32+r14*8+r13]
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        jmp     NEAR $L$loop_avx
+ALIGN   16
+$L$loop_avx:
+        vmovdqa xmm7,XMMWORD[((K256+512))]
+        vmovdqu xmm0,XMMWORD[r12*1+rsi]
+        vmovdqu xmm1,XMMWORD[16+r12*1+rsi]
+        vmovdqu xmm2,XMMWORD[32+r12*1+rsi]
+        vmovdqu xmm3,XMMWORD[48+r12*1+rsi]
+        vpshufb xmm0,xmm0,xmm7
+        lea     rbp,[K256]
+        vpshufb xmm1,xmm1,xmm7
+        vpshufb xmm2,xmm2,xmm7
+        vpaddd  xmm4,xmm0,XMMWORD[rbp]
+        vpshufb xmm3,xmm3,xmm7
+        vpaddd  xmm5,xmm1,XMMWORD[32+rbp]
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        vpaddd  xmm7,xmm3,XMMWORD[96+rbp]
+        vmovdqa XMMWORD[rsp],xmm4
+        mov     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm5
+        mov     esi,ebx
+        vmovdqa XMMWORD[32+rsp],xmm6
+        xor     esi,ecx
+        vmovdqa XMMWORD[48+rsp],xmm7
+        mov     r13d,r8d
+        jmp     NEAR $L$avx_00_47
+
+ALIGN   16
+$L$avx_00_47:
+        sub     rbp,-16*2*4
+        vmovdqu xmm9,XMMWORD[r12]
+        mov     QWORD[((64+0))+rsp],r12
+        vpalignr        xmm4,xmm1,xmm0,4
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        vpalignr        xmm7,xmm3,xmm2,4
+        xor     r13d,r8d
+        shrd    r14d,r14d,9
+        xor     r12d,r10d
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpaddd  xmm0,xmm0,xmm7
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+        vpsrld  xmm7,xmm4,3
+        shrd    r14d,r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        vpslld  xmm5,xmm4,14
+        shrd    r13d,r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        vpshufd xmm7,xmm3,250
+        add     edx,r11d
+        shrd    r14d,r14d,2
+        add     r11d,esi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        vpslld  xmm5,xmm5,11
+        shrd    r14d,r14d,9
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,r11d
+        and     r12d,edx
+        vpxor   xmm9,xmm9,xmm8
+        xor     r13d,edx
+        vpsrld  xmm6,xmm7,10
+        add     r10d,DWORD[4+rsp]
+        mov     esi,r11d
+        shrd    r14d,r14d,11
+        vpxor   xmm4,xmm4,xmm5
+        xor     r12d,r9d
+        xor     esi,eax
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        vpaddd  xmm0,xmm0,xmm4
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r14d,r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,edx
+        xor     r13d,ecx
+        shrd    r14d,r14d,9
+        vpshufd xmm6,xmm6,132
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        vpsrldq xmm6,xmm6,8
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[8+rsp]
+        vpaddd  xmm0,xmm0,xmm6
+        mov     r15d,r10d
+        shrd    r14d,r14d,11
+        xor     r12d,r8d
+        vpshufd xmm7,xmm0,80
+        xor     r15d,r11d
+        shrd    r13d,r13d,6
+        add     r9d,r12d
+        vpsrld  xmm6,xmm7,10
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        vpsrlq  xmm7,xmm7,17
+        xor     esi,r11d
+        add     ebx,r9d
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        vpsrlq  xmm7,xmm7,2
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        vpxor   xmm6,xmm6,xmm7
+        xor     r13d,ebx
+        shrd    r14d,r14d,9
+        xor     r12d,edx
+        vpshufd xmm6,xmm6,232
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vpslldq xmm6,xmm6,8
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     esi,r9d
+        vpaddd  xmm0,xmm0,xmm6
+        shrd    r14d,r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        vpaddd  xmm6,xmm0,XMMWORD[rbp]
+        shrd    r13d,r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        shrd    r14d,r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[rsp],xmm6
+        vpalignr        xmm4,xmm2,xmm1,4
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        vpalignr        xmm7,xmm0,xmm3,4
+        xor     r13d,eax
+        shrd    r14d,r14d,9
+        xor     r12d,ecx
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpaddd  xmm1,xmm1,xmm7
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+        vpsrld  xmm7,xmm4,3
+        shrd    r14d,r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        vpslld  xmm5,xmm4,14
+        shrd    r13d,r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        vpshufd xmm7,xmm0,250
+        add     r11d,edx
+        shrd    r14d,r14d,2
+        add     edx,esi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        vpslld  xmm5,xmm5,11
+        shrd    r14d,r14d,9
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r13d,r11d
+        vpsrld  xmm6,xmm7,10
+        add     ecx,DWORD[20+rsp]
+        mov     esi,edx
+        shrd    r14d,r14d,11
+        vpxor   xmm4,xmm4,xmm5
+        xor     r12d,ebx
+        xor     esi,r8d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        vpaddd  xmm1,xmm1,xmm4
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r14d,r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,r11d
+        xor     r13d,r10d
+        shrd    r14d,r14d,9
+        vpshufd xmm6,xmm6,132
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        vpsrldq xmm6,xmm6,8
+        and     r12d,r10d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[24+rsp]
+        vpaddd  xmm1,xmm1,xmm6
+        mov     r15d,ecx
+        shrd    r14d,r14d,11
+        xor     r12d,eax
+        vpshufd xmm7,xmm1,80
+        xor     r15d,edx
+        shrd    r13d,r13d,6
+        add     ebx,r12d
+        vpsrld  xmm6,xmm7,10
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        vpsrlq  xmm7,xmm7,17
+        xor     esi,edx
+        add     r9d,ebx
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        vpsrlq  xmm7,xmm7,2
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        vpxor   xmm6,xmm6,xmm7
+        xor     r13d,r9d
+        shrd    r14d,r14d,9
+        xor     r12d,r11d
+        vpshufd xmm6,xmm6,232
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpslldq xmm6,xmm6,8
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     esi,ebx
+        vpaddd  xmm1,xmm1,xmm6
+        shrd    r14d,r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        vpaddd  xmm6,xmm1,XMMWORD[32+rbp]
+        shrd    r13d,r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        shrd    r14d,r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm6
+        vpalignr        xmm4,xmm3,xmm2,4
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        vpalignr        xmm7,xmm1,xmm0,4
+        xor     r13d,r8d
+        shrd    r14d,r14d,9
+        xor     r12d,r10d
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpaddd  xmm2,xmm2,xmm7
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+        vpsrld  xmm7,xmm4,3
+        shrd    r14d,r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        vpslld  xmm5,xmm4,14
+        shrd    r13d,r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        vpshufd xmm7,xmm1,250
+        add     edx,r11d
+        shrd    r14d,r14d,2
+        add     r11d,esi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        vpslld  xmm5,xmm5,11
+        shrd    r14d,r14d,9
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,r11d
+        and     r12d,edx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r13d,edx
+        vpsrld  xmm6,xmm7,10
+        add     r10d,DWORD[36+rsp]
+        mov     esi,r11d
+        shrd    r14d,r14d,11
+        vpxor   xmm4,xmm4,xmm5
+        xor     r12d,r9d
+        xor     esi,eax
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        vpaddd  xmm2,xmm2,xmm4
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r14d,r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,edx
+        xor     r13d,ecx
+        shrd    r14d,r14d,9
+        vpshufd xmm6,xmm6,132
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        vpsrldq xmm6,xmm6,8
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[40+rsp]
+        vpaddd  xmm2,xmm2,xmm6
+        mov     r15d,r10d
+        shrd    r14d,r14d,11
+        xor     r12d,r8d
+        vpshufd xmm7,xmm2,80
+        xor     r15d,r11d
+        shrd    r13d,r13d,6
+        add     r9d,r12d
+        vpsrld  xmm6,xmm7,10
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        vpsrlq  xmm7,xmm7,17
+        xor     esi,r11d
+        add     ebx,r9d
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        vpsrlq  xmm7,xmm7,2
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        vpxor   xmm6,xmm6,xmm7
+        xor     r13d,ebx
+        shrd    r14d,r14d,9
+        xor     r12d,edx
+        vpshufd xmm6,xmm6,232
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vpslldq xmm6,xmm6,8
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     esi,r9d
+        vpaddd  xmm2,xmm2,xmm6
+        shrd    r14d,r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        shrd    r13d,r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        shrd    r14d,r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vpalignr        xmm4,xmm0,xmm3,4
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        vpalignr        xmm7,xmm2,xmm1,4
+        xor     r13d,eax
+        shrd    r14d,r14d,9
+        xor     r12d,ecx
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpaddd  xmm3,xmm3,xmm7
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+        vpsrld  xmm7,xmm4,3
+        shrd    r14d,r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        vpslld  xmm5,xmm4,14
+        shrd    r13d,r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        vpshufd xmm7,xmm2,250
+        add     r11d,edx
+        shrd    r14d,r14d,2
+        add     edx,esi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        vpslld  xmm5,xmm5,11
+        shrd    r14d,r14d,9
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r13d,r11d
+        vpsrld  xmm6,xmm7,10
+        add     ecx,DWORD[52+rsp]
+        mov     esi,edx
+        shrd    r14d,r14d,11
+        vpxor   xmm4,xmm4,xmm5
+        xor     r12d,ebx
+        xor     esi,r8d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        vpaddd  xmm3,xmm3,xmm4
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r14d,r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,r11d
+        xor     r13d,r10d
+        shrd    r14d,r14d,9
+        vpshufd xmm6,xmm6,132
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        vpsrldq xmm6,xmm6,8
+        and     r12d,r10d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[56+rsp]
+        vpaddd  xmm3,xmm3,xmm6
+        mov     r15d,ecx
+        shrd    r14d,r14d,11
+        xor     r12d,eax
+        vpshufd xmm7,xmm3,80
+        xor     r15d,edx
+        shrd    r13d,r13d,6
+        add     ebx,r12d
+        vpsrld  xmm6,xmm7,10
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        vpsrlq  xmm7,xmm7,17
+        xor     esi,edx
+        add     r9d,ebx
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        vpsrlq  xmm7,xmm7,2
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        vpxor   xmm6,xmm6,xmm7
+        xor     r13d,r9d
+        shrd    r14d,r14d,9
+        xor     r12d,r11d
+        vpshufd xmm6,xmm6,232
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpslldq xmm6,xmm6,8
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     esi,ebx
+        vpaddd  xmm3,xmm3,xmm6
+        shrd    r14d,r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        vpaddd  xmm6,xmm3,XMMWORD[96+rbp]
+        shrd    r13d,r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        shrd    r14d,r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[48+rsp],xmm6
+        mov     r12,QWORD[((64+0))+rsp]
+        vpand   xmm11,xmm11,xmm14
+        mov     r15,QWORD[((64+8))+rsp]
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r12*1+r15],xmm8
+        lea     r12,[16+r12]
+        cmp     BYTE[131+rbp],0
+        jne     NEAR $L$avx_00_47
+        vmovdqu xmm9,XMMWORD[r12]
+        mov     QWORD[((64+0))+rsp],r12
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        xor     r13d,r8d
+        shrd    r14d,r14d,9
+        xor     r12d,r10d
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+        shrd    r14d,r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        shrd    r13d,r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        add     edx,r11d
+        shrd    r14d,r14d,2
+        add     r11d,esi
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        shrd    r14d,r14d,9
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vpxor   xmm9,xmm9,xmm8
+        xor     r13d,edx
+        add     r10d,DWORD[4+rsp]
+        mov     esi,r11d
+        shrd    r14d,r14d,11
+        xor     r12d,r9d
+        xor     esi,eax
+        shrd    r13d,r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        shrd    r14d,r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        xor     r13d,ecx
+        shrd    r14d,r14d,9
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[8+rsp]
+        mov     r15d,r10d
+        shrd    r14d,r14d,11
+        xor     r12d,r8d
+        xor     r15d,r11d
+        shrd    r13d,r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        shrd    r14d,r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        shrd    r14d,r14d,9
+        xor     r12d,edx
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     esi,r9d
+        shrd    r14d,r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        shrd    r13d,r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        shrd    r14d,r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        xor     r13d,eax
+        shrd    r14d,r14d,9
+        xor     r12d,ecx
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+        shrd    r14d,r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        shrd    r13d,r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        add     r11d,edx
+        shrd    r14d,r14d,2
+        add     edx,esi
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        shrd    r14d,r14d,9
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r13d,r11d
+        add     ecx,DWORD[20+rsp]
+        mov     esi,edx
+        shrd    r14d,r14d,11
+        xor     r12d,ebx
+        xor     esi,r8d
+        shrd    r13d,r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        shrd    r14d,r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        xor     r13d,r10d
+        shrd    r14d,r14d,9
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[24+rsp]
+        mov     r15d,ecx
+        shrd    r14d,r14d,11
+        xor     r12d,eax
+        xor     r15d,edx
+        shrd    r13d,r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        shrd    r14d,r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        shrd    r14d,r14d,9
+        xor     r12d,r11d
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     esi,ebx
+        shrd    r14d,r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        shrd    r13d,r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        shrd    r14d,r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        xor     r13d,r8d
+        shrd    r14d,r14d,9
+        xor     r12d,r10d
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+        shrd    r14d,r14d,11
+        xor     r12d,r10d
+        xor     r15d,ebx
+        shrd    r13d,r13d,6
+        add     r11d,r12d
+        and     esi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     esi,ebx
+        add     edx,r11d
+        shrd    r14d,r14d,2
+        add     r11d,esi
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        xor     r13d,edx
+        shrd    r14d,r14d,9
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r13d,edx
+        add     r10d,DWORD[36+rsp]
+        mov     esi,r11d
+        shrd    r14d,r14d,11
+        xor     r12d,r9d
+        xor     esi,eax
+        shrd    r13d,r13d,6
+        add     r10d,r12d
+        and     r15d,esi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        add     ecx,r10d
+        shrd    r14d,r14d,2
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        xor     r13d,ecx
+        shrd    r14d,r14d,9
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r13d,ecx
+        add     r9d,DWORD[40+rsp]
+        mov     r15d,r10d
+        shrd    r14d,r14d,11
+        xor     r12d,r8d
+        xor     r15d,r11d
+        shrd    r13d,r13d,6
+        add     r9d,r12d
+        and     esi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     esi,r11d
+        add     ebx,r9d
+        shrd    r14d,r14d,2
+        add     r9d,esi
+        mov     r13d,ebx
+        add     r14d,r9d
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        xor     r13d,ebx
+        shrd    r14d,r14d,9
+        xor     r12d,edx
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     esi,r9d
+        shrd    r14d,r14d,11
+        xor     r12d,edx
+        xor     esi,r10d
+        shrd    r13d,r13d,6
+        add     r8d,r12d
+        and     r15d,esi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        add     eax,r8d
+        shrd    r14d,r14d,2
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        xor     r13d,eax
+        shrd    r14d,r14d,9
+        xor     r12d,ecx
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+        shrd    r14d,r14d,11
+        xor     r12d,ecx
+        xor     r15d,r9d
+        shrd    r13d,r13d,6
+        add     edx,r12d
+        and     esi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     esi,r9d
+        add     r11d,edx
+        shrd    r14d,r14d,2
+        add     edx,esi
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        xor     r13d,r11d
+        shrd    r14d,r14d,9
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r13d,r11d
+        add     ecx,DWORD[52+rsp]
+        mov     esi,edx
+        shrd    r14d,r14d,11
+        xor     r12d,ebx
+        xor     esi,r8d
+        shrd    r13d,r13d,6
+        add     ecx,r12d
+        and     r15d,esi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        add     r10d,ecx
+        shrd    r14d,r14d,2
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        xor     r13d,r10d
+        shrd    r14d,r14d,9
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r13d,r10d
+        add     ebx,DWORD[56+rsp]
+        mov     r15d,ecx
+        shrd    r14d,r14d,11
+        xor     r12d,eax
+        xor     r15d,edx
+        shrd    r13d,r13d,6
+        add     ebx,r12d
+        and     esi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     esi,edx
+        add     r9d,ebx
+        shrd    r14d,r14d,2
+        add     ebx,esi
+        mov     r13d,r9d
+        add     r14d,ebx
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        xor     r13d,r9d
+        shrd    r14d,r14d,9
+        xor     r12d,r11d
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     esi,ebx
+        shrd    r14d,r14d,11
+        xor     r12d,r11d
+        xor     esi,ecx
+        shrd    r13d,r13d,6
+        add     eax,r12d
+        and     r15d,esi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        add     r8d,eax
+        shrd    r14d,r14d,2
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        mov     r12,QWORD[((64+0))+rsp]
+        mov     r13,QWORD[((64+8))+rsp]
+        mov     r15,QWORD[((64+40))+rsp]
+        mov     rsi,QWORD[((64+48))+rsp]
+
+        vpand   xmm11,xmm11,xmm14
+        mov     eax,r14d
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r13*1+r12],xmm8
+        lea     r12,[16+r12]
+
+        add     eax,DWORD[r15]
+        add     ebx,DWORD[4+r15]
+        add     ecx,DWORD[8+r15]
+        add     edx,DWORD[12+r15]
+        add     r8d,DWORD[16+r15]
+        add     r9d,DWORD[20+r15]
+        add     r10d,DWORD[24+r15]
+        add     r11d,DWORD[28+r15]
+
+        cmp     r12,QWORD[((64+16))+rsp]
+
+        mov     DWORD[r15],eax
+        mov     DWORD[4+r15],ebx
+        mov     DWORD[8+r15],ecx
+        mov     DWORD[12+r15],edx
+        mov     DWORD[16+r15],r8d
+        mov     DWORD[20+r15],r9d
+        mov     DWORD[24+r15],r10d
+        mov     DWORD[28+r15],r11d
+        jb      NEAR $L$loop_avx
+
+        mov     r8,QWORD[((64+32))+rsp]
+        mov     rsi,QWORD[120+rsp]
+
+        vmovdqu XMMWORD[r8],xmm8
+        vzeroall
+        movaps  xmm6,XMMWORD[128+rsp]
+        movaps  xmm7,XMMWORD[144+rsp]
+        movaps  xmm8,XMMWORD[160+rsp]
+        movaps  xmm9,XMMWORD[176+rsp]
+        movaps  xmm10,XMMWORD[192+rsp]
+        movaps  xmm11,XMMWORD[208+rsp]
+        movaps  xmm12,XMMWORD[224+rsp]
+        movaps  xmm13,XMMWORD[240+rsp]
+        movaps  xmm14,XMMWORD[256+rsp]
+        movaps  xmm15,XMMWORD[272+rsp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_cbc_sha256_enc_avx:
+
+ALIGN   64
+aesni_cbc_sha256_enc_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_cbc_sha256_enc_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+$L$avx2_shortcut:
+        mov     r10,QWORD[56+rsp]
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,736
+        and     rsp,-256*4
+        add     rsp,448
+
+        shl     rdx,6
+        sub     rsi,rdi
+        sub     r10,rdi
+        add     rdx,rdi
+
+
+
+        mov     QWORD[((64+16))+rsp],rdx
+
+        mov     QWORD[((64+32))+rsp],r8
+        mov     QWORD[((64+40))+rsp],r9
+        mov     QWORD[((64+48))+rsp],r10
+        mov     QWORD[120+rsp],rax
+
+        movaps  XMMWORD[128+rsp],xmm6
+        movaps  XMMWORD[144+rsp],xmm7
+        movaps  XMMWORD[160+rsp],xmm8
+        movaps  XMMWORD[176+rsp],xmm9
+        movaps  XMMWORD[192+rsp],xmm10
+        movaps  XMMWORD[208+rsp],xmm11
+        movaps  XMMWORD[224+rsp],xmm12
+        movaps  XMMWORD[240+rsp],xmm13
+        movaps  XMMWORD[256+rsp],xmm14
+        movaps  XMMWORD[272+rsp],xmm15
+$L$prologue_avx2:
+        vzeroall
+
+        mov     r13,rdi
+        vpinsrq xmm15,xmm15,rsi,1
+        lea     rdi,[128+rcx]
+        lea     r12,[((K256+544))]
+        mov     r14d,DWORD[((240-128))+rdi]
+        mov     r15,r9
+        mov     rsi,r10
+        vmovdqu xmm8,XMMWORD[r8]
+        lea     r14,[((-9))+r14]
+
+        vmovdqa xmm14,XMMWORD[r14*8+r12]
+        vmovdqa xmm13,XMMWORD[16+r14*8+r12]
+        vmovdqa xmm12,XMMWORD[32+r14*8+r12]
+
+        sub     r13,-16*4
+        mov     eax,DWORD[r15]
+        lea     r12,[r13*1+rsi]
+        mov     ebx,DWORD[4+r15]
+        cmp     r13,rdx
+        mov     ecx,DWORD[8+r15]
+        cmove   r12,rsp
+        mov     edx,DWORD[12+r15]
+        mov     r8d,DWORD[16+r15]
+        mov     r9d,DWORD[20+r15]
+        mov     r10d,DWORD[24+r15]
+        mov     r11d,DWORD[28+r15]
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        jmp     NEAR $L$oop_avx2
+ALIGN   16
+$L$oop_avx2:
+        vmovdqa ymm7,YMMWORD[((K256+512))]
+        vmovdqu xmm0,XMMWORD[((-64+0))+r13*1+rsi]
+        vmovdqu xmm1,XMMWORD[((-64+16))+r13*1+rsi]
+        vmovdqu xmm2,XMMWORD[((-64+32))+r13*1+rsi]
+        vmovdqu xmm3,XMMWORD[((-64+48))+r13*1+rsi]
+
+        vinserti128     ymm0,ymm0,XMMWORD[r12],1
+        vinserti128     ymm1,ymm1,XMMWORD[16+r12],1
+        vpshufb ymm0,ymm0,ymm7
+        vinserti128     ymm2,ymm2,XMMWORD[32+r12],1
+        vpshufb ymm1,ymm1,ymm7
+        vinserti128     ymm3,ymm3,XMMWORD[48+r12],1
+
+        lea     rbp,[K256]
+        vpshufb ymm2,ymm2,ymm7
+        lea     r13,[((-64))+r13]
+        vpaddd  ymm4,ymm0,YMMWORD[rbp]
+        vpshufb ymm3,ymm3,ymm7
+        vpaddd  ymm5,ymm1,YMMWORD[32+rbp]
+        vpaddd  ymm6,ymm2,YMMWORD[64+rbp]
+        vpaddd  ymm7,ymm3,YMMWORD[96+rbp]
+        vmovdqa YMMWORD[rsp],ymm4
+        xor     r14d,r14d
+        vmovdqa YMMWORD[32+rsp],ymm5
+        lea     rsp,[((-64))+rsp]
+        mov     esi,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        xor     esi,ecx
+        vmovdqa YMMWORD[32+rsp],ymm7
+        mov     r12d,r9d
+        sub     rbp,-16*2*4
+        jmp     NEAR $L$avx2_00_47
+
+ALIGN   16
+$L$avx2_00_47:
+        vmovdqu xmm9,XMMWORD[r13]
+        vpinsrq xmm15,xmm15,r13,0
+        lea     rsp,[((-64))+rsp]
+        vpalignr        ymm4,ymm1,ymm0,4
+        add     r11d,DWORD[((0+128))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        vpalignr        ymm7,ymm3,ymm2,4
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        vpaddd  ymm0,ymm0,ymm7
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        vpxor   ymm4,ymm7,ymm6
+        and     esi,r15d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        vpshufd ymm7,ymm3,250
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        vpsrld  ymm6,ymm6,11
+        add     r10d,DWORD[((4+128))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,esi
+        vpxor   xmm9,xmm9,xmm8
+        xor     r14d,r12d
+        xor     r15d,eax
+        vpaddd  ymm0,ymm0,ymm4
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        vpxor   ymm6,ymm6,ymm7
+        add     r9d,DWORD[((8+128))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        vpshufd ymm6,ymm6,132
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        vpsrldq ymm6,ymm6,8
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        vpaddd  ymm0,ymm0,ymm6
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        vpshufd ymm7,ymm0,80
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        vpsrld  ymm6,ymm7,10
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        vpsrlq  ymm7,ymm7,17
+        add     r8d,DWORD[((12+128))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        vpxor   ymm6,ymm6,ymm7
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        vpsrlq  ymm7,ymm7,2
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        vpxor   ymm6,ymm6,ymm7
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        vpshufd ymm6,ymm6,232
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        vpslldq ymm6,ymm6,8
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        vpaddd  ymm0,ymm0,ymm6
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        vpaddd  ymm6,ymm0,YMMWORD[rbp]
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        vpalignr        ymm4,ymm2,ymm1,4
+        add     edx,DWORD[((32+128))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        vpalignr        ymm7,ymm0,ymm3,4
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        vpaddd  ymm1,ymm1,ymm7
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        vpxor   ymm4,ymm7,ymm6
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        vpshufd ymm7,ymm0,250
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        vpsrld  ymm6,ymm6,11
+        add     ecx,DWORD[((36+128))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        vpaddd  ymm1,ymm1,ymm4
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        vpxor   ymm6,ymm6,ymm7
+        add     ebx,DWORD[((40+128))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        vpshufd ymm6,ymm6,132
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        vpsrldq ymm6,ymm6,8
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        vpaddd  ymm1,ymm1,ymm6
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        vpshufd ymm7,ymm1,80
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        vpsrld  ymm6,ymm7,10
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        vpsrlq  ymm7,ymm7,17
+        add     eax,DWORD[((44+128))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        vpxor   ymm6,ymm6,ymm7
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        vpsrlq  ymm7,ymm7,2
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        vpxor   ymm6,ymm6,ymm7
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        vpshufd ymm6,ymm6,232
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        vpslldq ymm6,ymm6,8
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        vpaddd  ymm1,ymm1,ymm6
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        vpaddd  ymm6,ymm1,YMMWORD[32+rbp]
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vmovdqa YMMWORD[32+rsp],ymm6
+        lea     rsp,[((-64))+rsp]
+        vpalignr        ymm4,ymm3,ymm2,4
+        add     r11d,DWORD[((0+128))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        vpalignr        ymm7,ymm1,ymm0,4
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        vpaddd  ymm2,ymm2,ymm7
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        vpxor   ymm4,ymm7,ymm6
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        vpshufd ymm7,ymm1,250
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        vpsrld  ymm6,ymm6,11
+        add     r10d,DWORD[((4+128))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,eax
+        vpaddd  ymm2,ymm2,ymm4
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        vpxor   ymm6,ymm6,ymm7
+        add     r9d,DWORD[((8+128))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        vpshufd ymm6,ymm6,132
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        vpsrldq ymm6,ymm6,8
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        vpaddd  ymm2,ymm2,ymm6
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        vpshufd ymm7,ymm2,80
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        vpsrld  ymm6,ymm7,10
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        vpsrlq  ymm7,ymm7,17
+        add     r8d,DWORD[((12+128))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        vpxor   ymm6,ymm6,ymm7
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        vpsrlq  ymm7,ymm7,2
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        vpxor   ymm6,ymm6,ymm7
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        vpshufd ymm6,ymm6,232
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        vpslldq ymm6,ymm6,8
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        vpaddd  ymm2,ymm2,ymm6
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        vpaddd  ymm6,ymm2,YMMWORD[64+rbp]
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        vpalignr        ymm4,ymm0,ymm3,4
+        add     edx,DWORD[((32+128))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        vpalignr        ymm7,ymm2,ymm1,4
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        vpaddd  ymm3,ymm3,ymm7
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        vpxor   ymm4,ymm7,ymm6
+        and     esi,r15d
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        vpshufd ymm7,ymm2,250
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        vpsrld  ymm6,ymm6,11
+        add     ecx,DWORD[((36+128))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        vpaddd  ymm3,ymm3,ymm4
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        vpxor   ymm6,ymm6,ymm7
+        add     ebx,DWORD[((40+128))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        vpshufd ymm6,ymm6,132
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        vpsrldq ymm6,ymm6,8
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        vpaddd  ymm3,ymm3,ymm6
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        vpshufd ymm7,ymm3,80
+        and     esi,r15d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        vpsrld  ymm6,ymm7,10
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        vpsrlq  ymm7,ymm7,17
+        add     eax,DWORD[((44+128))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        vpxor   ymm6,ymm6,ymm7
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        vpsrlq  ymm7,ymm7,2
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        vpxor   ymm6,ymm6,ymm7
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        vpshufd ymm6,ymm6,232
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        vpslldq ymm6,ymm6,8
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        vpaddd  ymm3,ymm3,ymm6
+        and     r15d,esi
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        vpaddd  ymm6,ymm3,YMMWORD[96+rbp]
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vmovdqa YMMWORD[32+rsp],ymm6
+        vmovq   r13,xmm15
+        vpextrq r15,xmm15,1
+        vpand   xmm11,xmm11,xmm14
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r13*1+r15],xmm8
+        lea     r13,[16+r13]
+        lea     rbp,[128+rbp]
+        cmp     BYTE[3+rbp],0
+        jne     NEAR $L$avx2_00_47
+        vmovdqu xmm9,XMMWORD[r13]
+        vpinsrq xmm15,xmm15,r13,0
+        add     r11d,DWORD[((0+64))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     esi,r15d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[((4+64))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,esi
+        vpxor   xmm9,xmm9,xmm8
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[((8+64))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[((12+64))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[((32+64))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[((36+64))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[((40+64))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[((44+64))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        add     r11d,DWORD[rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[4+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[8+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[12+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[32+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     esi,r15d
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[36+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[40+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     esi,r15d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[44+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,esi
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vpextrq r12,xmm15,1
+        vmovq   r13,xmm15
+        mov     r15,QWORD[552+rsp]
+        add     eax,r14d
+        lea     rbp,[448+rsp]
+
+        vpand   xmm11,xmm11,xmm14
+        vpor    xmm8,xmm8,xmm11
+        vmovdqu XMMWORD[r13*1+r12],xmm8
+        lea     r13,[16+r13]
+
+        add     eax,DWORD[r15]
+        add     ebx,DWORD[4+r15]
+        add     ecx,DWORD[8+r15]
+        add     edx,DWORD[12+r15]
+        add     r8d,DWORD[16+r15]
+        add     r9d,DWORD[20+r15]
+        add     r10d,DWORD[24+r15]
+        add     r11d,DWORD[28+r15]
+
+        mov     DWORD[r15],eax
+        mov     DWORD[4+r15],ebx
+        mov     DWORD[8+r15],ecx
+        mov     DWORD[12+r15],edx
+        mov     DWORD[16+r15],r8d
+        mov     DWORD[20+r15],r9d
+        mov     DWORD[24+r15],r10d
+        mov     DWORD[28+r15],r11d
+
+        cmp     r13,QWORD[80+rbp]
+        je      NEAR $L$done_avx2
+
+        xor     r14d,r14d
+        mov     esi,ebx
+        mov     r12d,r9d
+        xor     esi,ecx
+        jmp     NEAR $L$ower_avx2
+ALIGN   16
+$L$ower_avx2:
+        vmovdqu xmm9,XMMWORD[r13]
+        vpinsrq xmm15,xmm15,r13,0
+        add     r11d,DWORD[((0+16))+rbp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     esi,r15d
+        vpxor   xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((16-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[((4+16))+rbp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,esi
+        vpxor   xmm9,xmm9,xmm8
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[((8+16))+rbp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((32-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[((12+16))+rbp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((48-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[((32+16))+rbp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[((36+16))+rbp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((80-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[((40+16))+rbp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((96-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[((44+16))+rbp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((112-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        lea     rbp,[((-64))+rbp]
+        add     r11d,DWORD[((0+16))+rbp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((128-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rsi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[((4+16))+rbp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    esi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,esi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     esi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     esi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,esi
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((144-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[((8+16))+rbp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     esi,r15d
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((160-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rsi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[((12+16))+rbp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    esi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,esi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     esi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     esi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((176-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[((32+16))+rbp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     esi,r15d
+        vpand   xmm8,xmm11,xmm12
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((192-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,r9d
+        xor     r14d,r13d
+        lea     edx,[rsi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[((36+16))+rbp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    esi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,esi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     esi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     esi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,esi
+        vaesenclast     xmm11,xmm9,xmm10
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((208-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[((40+16))+rbp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     esi,r15d
+        vpand   xmm11,xmm11,xmm13
+        vaesenc xmm9,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((224-128))+rdi]
+        xor     r14d,r12d
+        xor     esi,edx
+        xor     r14d,r13d
+        lea     ebx,[rsi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[((44+16))+rbp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    esi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,esi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     esi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     esi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,esi
+        vpor    xmm8,xmm8,xmm11
+        vaesenclast     xmm11,xmm9,xmm10
+        vmovdqu xmm10,XMMWORD[((0-128))+rdi]
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vmovq   r13,xmm15
+        vpextrq r15,xmm15,1
+        vpand   xmm11,xmm11,xmm14
+        vpor    xmm8,xmm8,xmm11
+        lea     rbp,[((-64))+rbp]
+        vmovdqu XMMWORD[r13*1+r15],xmm8
+        lea     r13,[16+r13]
+        cmp     rbp,rsp
+        jae     NEAR $L$ower_avx2
+
+        mov     r15,QWORD[552+rsp]
+        lea     r13,[64+r13]
+        mov     rsi,QWORD[560+rsp]
+        add     eax,r14d
+        lea     rsp,[448+rsp]
+
+        add     eax,DWORD[r15]
+        add     ebx,DWORD[4+r15]
+        add     ecx,DWORD[8+r15]
+        add     edx,DWORD[12+r15]
+        add     r8d,DWORD[16+r15]
+        add     r9d,DWORD[20+r15]
+        add     r10d,DWORD[24+r15]
+        lea     r12,[r13*1+rsi]
+        add     r11d,DWORD[28+r15]
+
+        cmp     r13,QWORD[((64+16))+rsp]
+
+        mov     DWORD[r15],eax
+        cmove   r12,rsp
+        mov     DWORD[4+r15],ebx
+        mov     DWORD[8+r15],ecx
+        mov     DWORD[12+r15],edx
+        mov     DWORD[16+r15],r8d
+        mov     DWORD[20+r15],r9d
+        mov     DWORD[24+r15],r10d
+        mov     DWORD[28+r15],r11d
+
+        jbe     NEAR $L$oop_avx2
+        lea     rbp,[rsp]
+
+
+
+
+$L$done_avx2:
+        mov     r8,QWORD[((64+32))+rbp]
+        mov     rsi,QWORD[((64+56))+rbp]
+
+        vmovdqu XMMWORD[r8],xmm8
+        vzeroall
+        movaps  xmm6,XMMWORD[128+rbp]
+        movaps  xmm7,XMMWORD[144+rbp]
+        movaps  xmm8,XMMWORD[160+rbp]
+        movaps  xmm9,XMMWORD[176+rbp]
+        movaps  xmm10,XMMWORD[192+rbp]
+        movaps  xmm11,XMMWORD[208+rbp]
+        movaps  xmm12,XMMWORD[224+rbp]
+        movaps  xmm13,XMMWORD[240+rbp]
+        movaps  xmm14,XMMWORD[256+rbp]
+        movaps  xmm15,XMMWORD[272+rbp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_cbc_sha256_enc_avx2:
+
+ALIGN   32
+aesni_cbc_sha256_enc_shaext:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_aesni_cbc_sha256_enc_shaext:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        mov     r10,QWORD[56+rsp]
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[(-8-160)+rax],xmm6
+        movaps  XMMWORD[(-8-144)+rax],xmm7
+        movaps  XMMWORD[(-8-128)+rax],xmm8
+        movaps  XMMWORD[(-8-112)+rax],xmm9
+        movaps  XMMWORD[(-8-96)+rax],xmm10
+        movaps  XMMWORD[(-8-80)+rax],xmm11
+        movaps  XMMWORD[(-8-64)+rax],xmm12
+        movaps  XMMWORD[(-8-48)+rax],xmm13
+        movaps  XMMWORD[(-8-32)+rax],xmm14
+        movaps  XMMWORD[(-8-16)+rax],xmm15
+$L$prologue_shaext:
+        lea     rax,[((K256+128))]
+        movdqu  xmm1,XMMWORD[r9]
+        movdqu  xmm2,XMMWORD[16+r9]
+        movdqa  xmm3,XMMWORD[((512-128))+rax]
+
+        mov     r11d,DWORD[240+rcx]
+        sub     rsi,rdi
+        movups  xmm15,XMMWORD[rcx]
+        movups  xmm6,XMMWORD[r8]
+        movups  xmm4,XMMWORD[16+rcx]
+        lea     rcx,[112+rcx]
+
+        pshufd  xmm0,xmm1,0x1b
+        pshufd  xmm1,xmm1,0xb1
+        pshufd  xmm2,xmm2,0x1b
+        movdqa  xmm7,xmm3
+DB      102,15,58,15,202,8
+        punpcklqdq      xmm2,xmm0
+
+        jmp     NEAR $L$oop_shaext
+
+ALIGN   16
+$L$oop_shaext:
+        movdqu  xmm10,XMMWORD[r10]
+        movdqu  xmm11,XMMWORD[16+r10]
+        movdqu  xmm12,XMMWORD[32+r10]
+DB      102,68,15,56,0,211
+        movdqu  xmm13,XMMWORD[48+r10]
+
+        movdqa  xmm0,XMMWORD[((0-128))+rax]
+        paddd   xmm0,xmm10
+DB      102,68,15,56,0,219
+        movdqa  xmm9,xmm2
+        movdqa  xmm8,xmm1
+        movups  xmm14,XMMWORD[rdi]
+        xorps   xmm14,xmm15
+        xorps   xmm6,xmm14
+        movups  xmm5,XMMWORD[((-80))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movups  xmm4,XMMWORD[((-64))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,202
+
+        movdqa  xmm0,XMMWORD[((32-128))+rax]
+        paddd   xmm0,xmm11
+DB      102,68,15,56,0,227
+        lea     r10,[64+r10]
+        movups  xmm5,XMMWORD[((-48))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movups  xmm4,XMMWORD[((-32))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,202
+
+        movdqa  xmm0,XMMWORD[((64-128))+rax]
+        paddd   xmm0,xmm12
+DB      102,68,15,56,0,235
+DB      69,15,56,204,211
+        movups  xmm5,XMMWORD[((-16))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm13
+DB      102,65,15,58,15,220,4
+        paddd   xmm10,xmm3
+        movups  xmm4,XMMWORD[rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,202
+
+        movdqa  xmm0,XMMWORD[((96-128))+rax]
+        paddd   xmm0,xmm13
+DB      69,15,56,205,213
+DB      69,15,56,204,220
+        movups  xmm5,XMMWORD[16+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movups  xmm4,XMMWORD[32+rcx]
+        aesenc  xmm6,xmm5
+        movdqa  xmm3,xmm10
+DB      102,65,15,58,15,221,4
+        paddd   xmm11,xmm3
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((128-128))+rax]
+        paddd   xmm0,xmm10
+DB      69,15,56,205,218
+DB      69,15,56,204,229
+        movups  xmm5,XMMWORD[48+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm11
+DB      102,65,15,58,15,218,4
+        paddd   xmm12,xmm3
+        cmp     r11d,11
+        jb      NEAR $L$aesenclast1
+        movups  xmm4,XMMWORD[64+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[80+rcx]
+        aesenc  xmm6,xmm4
+        je      NEAR $L$aesenclast1
+        movups  xmm4,XMMWORD[96+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[112+rcx]
+        aesenc  xmm6,xmm4
+$L$aesenclast1:
+        aesenclast      xmm6,xmm5
+        movups  xmm4,XMMWORD[((16-112))+rcx]
+        nop
+DB      15,56,203,202
+        movups  xmm14,XMMWORD[16+rdi]
+        xorps   xmm14,xmm15
+        movups  XMMWORD[rdi*1+rsi],xmm6
+        xorps   xmm6,xmm14
+        movups  xmm5,XMMWORD[((-80))+rcx]
+        aesenc  xmm6,xmm4
+        movdqa  xmm0,XMMWORD[((160-128))+rax]
+        paddd   xmm0,xmm11
+DB      69,15,56,205,227
+DB      69,15,56,204,234
+        movups  xmm4,XMMWORD[((-64))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm12
+DB      102,65,15,58,15,219,4
+        paddd   xmm13,xmm3
+        movups  xmm5,XMMWORD[((-48))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((192-128))+rax]
+        paddd   xmm0,xmm12
+DB      69,15,56,205,236
+DB      69,15,56,204,211
+        movups  xmm4,XMMWORD[((-32))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm13
+DB      102,65,15,58,15,220,4
+        paddd   xmm10,xmm3
+        movups  xmm5,XMMWORD[((-16))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((224-128))+rax]
+        paddd   xmm0,xmm13
+DB      69,15,56,205,213
+DB      69,15,56,204,220
+        movups  xmm4,XMMWORD[rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm10
+DB      102,65,15,58,15,221,4
+        paddd   xmm11,xmm3
+        movups  xmm5,XMMWORD[16+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((256-128))+rax]
+        paddd   xmm0,xmm10
+DB      69,15,56,205,218
+DB      69,15,56,204,229
+        movups  xmm4,XMMWORD[32+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm11
+DB      102,65,15,58,15,218,4
+        paddd   xmm12,xmm3
+        movups  xmm5,XMMWORD[48+rcx]
+        aesenc  xmm6,xmm4
+        cmp     r11d,11
+        jb      NEAR $L$aesenclast2
+        movups  xmm4,XMMWORD[64+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[80+rcx]
+        aesenc  xmm6,xmm4
+        je      NEAR $L$aesenclast2
+        movups  xmm4,XMMWORD[96+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[112+rcx]
+        aesenc  xmm6,xmm4
+$L$aesenclast2:
+        aesenclast      xmm6,xmm5
+        movups  xmm4,XMMWORD[((16-112))+rcx]
+        nop
+DB      15,56,203,202
+        movups  xmm14,XMMWORD[32+rdi]
+        xorps   xmm14,xmm15
+        movups  XMMWORD[16+rdi*1+rsi],xmm6
+        xorps   xmm6,xmm14
+        movups  xmm5,XMMWORD[((-80))+rcx]
+        aesenc  xmm6,xmm4
+        movdqa  xmm0,XMMWORD[((288-128))+rax]
+        paddd   xmm0,xmm11
+DB      69,15,56,205,227
+DB      69,15,56,204,234
+        movups  xmm4,XMMWORD[((-64))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm12
+DB      102,65,15,58,15,219,4
+        paddd   xmm13,xmm3
+        movups  xmm5,XMMWORD[((-48))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((320-128))+rax]
+        paddd   xmm0,xmm12
+DB      69,15,56,205,236
+DB      69,15,56,204,211
+        movups  xmm4,XMMWORD[((-32))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm13
+DB      102,65,15,58,15,220,4
+        paddd   xmm10,xmm3
+        movups  xmm5,XMMWORD[((-16))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((352-128))+rax]
+        paddd   xmm0,xmm13
+DB      69,15,56,205,213
+DB      69,15,56,204,220
+        movups  xmm4,XMMWORD[rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm10
+DB      102,65,15,58,15,221,4
+        paddd   xmm11,xmm3
+        movups  xmm5,XMMWORD[16+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((384-128))+rax]
+        paddd   xmm0,xmm10
+DB      69,15,56,205,218
+DB      69,15,56,204,229
+        movups  xmm4,XMMWORD[32+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm11
+DB      102,65,15,58,15,218,4
+        paddd   xmm12,xmm3
+        movups  xmm5,XMMWORD[48+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+        movdqa  xmm0,XMMWORD[((416-128))+rax]
+        paddd   xmm0,xmm11
+DB      69,15,56,205,227
+DB      69,15,56,204,234
+        cmp     r11d,11
+        jb      NEAR $L$aesenclast3
+        movups  xmm4,XMMWORD[64+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[80+rcx]
+        aesenc  xmm6,xmm4
+        je      NEAR $L$aesenclast3
+        movups  xmm4,XMMWORD[96+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[112+rcx]
+        aesenc  xmm6,xmm4
+$L$aesenclast3:
+        aesenclast      xmm6,xmm5
+        movups  xmm4,XMMWORD[((16-112))+rcx]
+        nop
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movdqa  xmm3,xmm12
+DB      102,65,15,58,15,219,4
+        paddd   xmm13,xmm3
+        movups  xmm14,XMMWORD[48+rdi]
+        xorps   xmm14,xmm15
+        movups  XMMWORD[32+rdi*1+rsi],xmm6
+        xorps   xmm6,xmm14
+        movups  xmm5,XMMWORD[((-80))+rcx]
+        aesenc  xmm6,xmm4
+        movups  xmm4,XMMWORD[((-64))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,202
+
+        movdqa  xmm0,XMMWORD[((448-128))+rax]
+        paddd   xmm0,xmm12
+DB      69,15,56,205,236
+        movdqa  xmm3,xmm7
+        movups  xmm5,XMMWORD[((-48))+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movups  xmm4,XMMWORD[((-32))+rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,202
+
+        movdqa  xmm0,XMMWORD[((480-128))+rax]
+        paddd   xmm0,xmm13
+        movups  xmm5,XMMWORD[((-16))+rcx]
+        aesenc  xmm6,xmm4
+        movups  xmm4,XMMWORD[rcx]
+        aesenc  xmm6,xmm5
+DB      15,56,203,209
+        pshufd  xmm0,xmm0,0x0e
+        movups  xmm5,XMMWORD[16+rcx]
+        aesenc  xmm6,xmm4
+DB      15,56,203,202
+
+        movups  xmm4,XMMWORD[32+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[48+rcx]
+        aesenc  xmm6,xmm4
+        cmp     r11d,11
+        jb      NEAR $L$aesenclast4
+        movups  xmm4,XMMWORD[64+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[80+rcx]
+        aesenc  xmm6,xmm4
+        je      NEAR $L$aesenclast4
+        movups  xmm4,XMMWORD[96+rcx]
+        aesenc  xmm6,xmm5
+        movups  xmm5,XMMWORD[112+rcx]
+        aesenc  xmm6,xmm4
+$L$aesenclast4:
+        aesenclast      xmm6,xmm5
+        movups  xmm4,XMMWORD[((16-112))+rcx]
+        nop
+
+        paddd   xmm2,xmm9
+        paddd   xmm1,xmm8
+
+        dec     rdx
+        movups  XMMWORD[48+rdi*1+rsi],xmm6
+        lea     rdi,[64+rdi]
+        jnz     NEAR $L$oop_shaext
+
+        pshufd  xmm2,xmm2,0xb1
+        pshufd  xmm3,xmm1,0x1b
+        pshufd  xmm1,xmm1,0xb1
+        punpckhqdq      xmm1,xmm2
+DB      102,15,58,15,211,8
+
+        movups  XMMWORD[r8],xmm6
+        movdqu  XMMWORD[r9],xmm1
+        movdqu  XMMWORD[16+r9],xmm2
+        movaps  xmm6,XMMWORD[rsp]
+        movaps  xmm7,XMMWORD[16+rsp]
+        movaps  xmm8,XMMWORD[32+rsp]
+        movaps  xmm9,XMMWORD[48+rsp]
+        movaps  xmm10,XMMWORD[64+rsp]
+        movaps  xmm11,XMMWORD[80+rsp]
+        movaps  xmm12,XMMWORD[96+rsp]
+        movaps  xmm13,XMMWORD[112+rsp]
+        movaps  xmm14,XMMWORD[128+rsp]
+        movaps  xmm15,XMMWORD[144+rsp]
+        lea     rsp,[((8+160))+rsp]
+$L$epilogue_shaext:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_aesni_cbc_sha256_enc_shaext:
+EXTERN  __imp_RtlVirtualUnwind
+
+ALIGN   16
+se_handler:
+        push    rsi
+        push    rdi
+        push    rbx
+        push    rbp
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        pushfq
+        sub     rsp,64
+
+        mov     rax,QWORD[120+r8]
+        mov     rbx,QWORD[248+r8]
+
+        mov     rsi,QWORD[8+r9]
+        mov     r11,QWORD[56+r9]
+
+        mov     r10d,DWORD[r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jb      NEAR $L$in_prologue
+
+        mov     rax,QWORD[152+r8]
+
+        mov     r10d,DWORD[4+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jae     NEAR $L$in_prologue
+        lea     r10,[aesni_cbc_sha256_enc_shaext]
+        cmp     rbx,r10
+        jb      NEAR $L$not_in_shaext
+
+        lea     rsi,[rax]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+        lea     rax,[168+rax]
+        jmp     NEAR $L$in_prologue
+$L$not_in_shaext:
+        lea     r10,[$L$avx2_shortcut]
+        cmp     rbx,r10
+        jb      NEAR $L$not_in_avx2
+
+        and     rax,-256*4
+        add     rax,448
+$L$not_in_avx2:
+        mov     rsi,rax
+        mov     rax,QWORD[((64+56))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+        mov     r12,QWORD[((-24))+rax]
+        mov     r13,QWORD[((-32))+rax]
+        mov     r14,QWORD[((-40))+rax]
+        mov     r15,QWORD[((-48))+rax]
+        mov     QWORD[144+r8],rbx
+        mov     QWORD[160+r8],rbp
+        mov     QWORD[216+r8],r12
+        mov     QWORD[224+r8],r13
+        mov     QWORD[232+r8],r14
+        mov     QWORD[240+r8],r15
+
+        lea     rsi,[((64+64))+rsi]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+
+$L$in_prologue:
+        mov     rdi,QWORD[8+rax]
+        mov     rsi,QWORD[16+rax]
+        mov     QWORD[152+r8],rax
+        mov     QWORD[168+r8],rsi
+        mov     QWORD[176+r8],rdi
+
+        mov     rdi,QWORD[40+r9]
+        mov     rsi,r8
+        mov     ecx,154
+        DD      0xa548f3fc
+
+        mov     rsi,r9
+        xor     rcx,rcx
+        mov     rdx,QWORD[8+rsi]
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[16+rsi]
+        mov     r10,QWORD[40+rsi]
+        lea     r11,[56+rsi]
+        lea     r12,[24+rsi]
+        mov     QWORD[32+rsp],r10
+        mov     QWORD[40+rsp],r11
+        mov     QWORD[48+rsp],r12
+        mov     QWORD[56+rsp],rcx
+        call    QWORD[__imp_RtlVirtualUnwind]
+
+        mov     eax,1
+        add     rsp,64
+        popfq
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     rbp
+        pop     rbx
+        pop     rdi
+        pop     rsi
+        DB      0F3h,0C3h               ;repret
+
+
+section .pdata rdata align=4
+        DD      $L$SEH_begin_aesni_cbc_sha256_enc_xop wrt ..imagebase
+        DD      $L$SEH_end_aesni_cbc_sha256_enc_xop wrt ..imagebase
+        DD      $L$SEH_info_aesni_cbc_sha256_enc_xop wrt ..imagebase
+
+        DD      $L$SEH_begin_aesni_cbc_sha256_enc_avx wrt ..imagebase
+        DD      $L$SEH_end_aesni_cbc_sha256_enc_avx wrt ..imagebase
+        DD      $L$SEH_info_aesni_cbc_sha256_enc_avx wrt ..imagebase
+        DD      $L$SEH_begin_aesni_cbc_sha256_enc_avx2 wrt ..imagebase
+        DD      $L$SEH_end_aesni_cbc_sha256_enc_avx2 wrt ..imagebase
+        DD      $L$SEH_info_aesni_cbc_sha256_enc_avx2 wrt ..imagebase
+        DD      $L$SEH_begin_aesni_cbc_sha256_enc_shaext wrt ..imagebase
+        DD      $L$SEH_end_aesni_cbc_sha256_enc_shaext wrt ..imagebase
+        DD      $L$SEH_info_aesni_cbc_sha256_enc_shaext wrt ..imagebase
+section .xdata rdata align=8
+ALIGN   8
+$L$SEH_info_aesni_cbc_sha256_enc_xop:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_xop wrt ..imagebase,$L$epilogue_xop wrt ..imagebase
+
+$L$SEH_info_aesni_cbc_sha256_enc_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_aesni_cbc_sha256_enc_avx2:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase
+$L$SEH_info_aesni_cbc_sha256_enc_shaext:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_shaext wrt ..imagebase,$L$epilogue_shaext wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/chacha-x86_64.S

@@ -62 +62 @@
         je      NEAR $L$no_data
         mov     r10,QWORD[((OPENSSL_ia32cap_P+4))]
+        bt      r10,48
+        jc      NEAR $L$ChaCha20_avx512
+        test    r10,r10
+        js      NEAR $L$ChaCha20_avx512vl
         test    r10d,512
         jnz     NEAR $L$ChaCha20_ssse3
@@ -357 +361 @@
         mov     r9,rsp
 
+        test    r10d,2048
+        jnz     NEAR $L$ChaCha20_4xop
         cmp     rdx,128
         je      NEAR $L$ChaCha20_128
@@ -696 +702 @@
 
         mov     r11,r10
+        shr     r10,32
+        test    r10,32
+        jnz     NEAR $L$ChaCha20_8x
         cmp     rdx,192
         ja      NEAR $L$proceed4x
@@ -1260 +1269 @@
 
 $L$SEH_end_ChaCha20_4x:
+
+ALIGN   32
+ChaCha20_4xop:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_4xop:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_4xop:
+        mov     r9,rsp
+
+        sub     rsp,0x140+168
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$4xop_body:
+        vzeroupper
+
+        vmovdqa xmm11,XMMWORD[$L$sigma]
+        vmovdqu xmm3,XMMWORD[rcx]
+        vmovdqu xmm15,XMMWORD[16+rcx]
+        vmovdqu xmm7,XMMWORD[r8]
+        lea     rcx,[256+rsp]
+
+        vpshufd xmm8,xmm11,0x00
+        vpshufd xmm9,xmm11,0x55
+        vmovdqa XMMWORD[64+rsp],xmm8
+        vpshufd xmm10,xmm11,0xaa
+        vmovdqa XMMWORD[80+rsp],xmm9
+        vpshufd xmm11,xmm11,0xff
+        vmovdqa XMMWORD[96+rsp],xmm10
+        vmovdqa XMMWORD[112+rsp],xmm11
+
+        vpshufd xmm0,xmm3,0x00
+        vpshufd xmm1,xmm3,0x55
+        vmovdqa XMMWORD[(128-256)+rcx],xmm0
+        vpshufd xmm2,xmm3,0xaa
+        vmovdqa XMMWORD[(144-256)+rcx],xmm1
+        vpshufd xmm3,xmm3,0xff
+        vmovdqa XMMWORD[(160-256)+rcx],xmm2
+        vmovdqa XMMWORD[(176-256)+rcx],xmm3
+
+        vpshufd xmm12,xmm15,0x00
+        vpshufd xmm13,xmm15,0x55
+        vmovdqa XMMWORD[(192-256)+rcx],xmm12
+        vpshufd xmm14,xmm15,0xaa
+        vmovdqa XMMWORD[(208-256)+rcx],xmm13
+        vpshufd xmm15,xmm15,0xff
+        vmovdqa XMMWORD[(224-256)+rcx],xmm14
+        vmovdqa XMMWORD[(240-256)+rcx],xmm15
+
+        vpshufd xmm4,xmm7,0x00
+        vpshufd xmm5,xmm7,0x55
+        vpaddd  xmm4,xmm4,XMMWORD[$L$inc]
+        vpshufd xmm6,xmm7,0xaa
+        vmovdqa XMMWORD[(272-256)+rcx],xmm5
+        vpshufd xmm7,xmm7,0xff
+        vmovdqa XMMWORD[(288-256)+rcx],xmm6
+        vmovdqa XMMWORD[(304-256)+rcx],xmm7
+
+        jmp     NEAR $L$oop_enter4xop
+
+ALIGN   32
+$L$oop_outer4xop:
+        vmovdqa xmm8,XMMWORD[64+rsp]
+        vmovdqa xmm9,XMMWORD[80+rsp]
+        vmovdqa xmm10,XMMWORD[96+rsp]
+        vmovdqa xmm11,XMMWORD[112+rsp]
+        vmovdqa xmm0,XMMWORD[((128-256))+rcx]
+        vmovdqa xmm1,XMMWORD[((144-256))+rcx]
+        vmovdqa xmm2,XMMWORD[((160-256))+rcx]
+        vmovdqa xmm3,XMMWORD[((176-256))+rcx]
+        vmovdqa xmm12,XMMWORD[((192-256))+rcx]
+        vmovdqa xmm13,XMMWORD[((208-256))+rcx]
+        vmovdqa xmm14,XMMWORD[((224-256))+rcx]
+        vmovdqa xmm15,XMMWORD[((240-256))+rcx]
+        vmovdqa xmm4,XMMWORD[((256-256))+rcx]
+        vmovdqa xmm5,XMMWORD[((272-256))+rcx]
+        vmovdqa xmm6,XMMWORD[((288-256))+rcx]
+        vmovdqa xmm7,XMMWORD[((304-256))+rcx]
+        vpaddd  xmm4,xmm4,XMMWORD[$L$four]
+
+$L$oop_enter4xop:
+        mov     eax,10
+        vmovdqa XMMWORD[(256-256)+rcx],xmm4
+        jmp     NEAR $L$oop4xop
+
+ALIGN   32
+$L$oop4xop:
+        vpaddd  xmm8,xmm8,xmm0
+        vpaddd  xmm9,xmm9,xmm1
+        vpaddd  xmm10,xmm10,xmm2
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm8,xmm4
+        vpxor   xmm5,xmm9,xmm5
+        vpxor   xmm6,xmm10,xmm6
+        vpxor   xmm7,xmm11,xmm7
+DB      143,232,120,194,228,16
+DB      143,232,120,194,237,16
+DB      143,232,120,194,246,16
+DB      143,232,120,194,255,16
+        vpaddd  xmm12,xmm12,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm14,xmm14,xmm6
+        vpaddd  xmm15,xmm15,xmm7
+        vpxor   xmm0,xmm12,xmm0
+        vpxor   xmm1,xmm13,xmm1
+        vpxor   xmm2,xmm2,xmm14
+        vpxor   xmm3,xmm3,xmm15
+DB      143,232,120,194,192,12
+DB      143,232,120,194,201,12
+DB      143,232,120,194,210,12
+DB      143,232,120,194,219,12
+        vpaddd  xmm8,xmm0,xmm8
+        vpaddd  xmm9,xmm1,xmm9
+        vpaddd  xmm10,xmm10,xmm2
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm8,xmm4
+        vpxor   xmm5,xmm9,xmm5
+        vpxor   xmm6,xmm10,xmm6
+        vpxor   xmm7,xmm11,xmm7
+DB      143,232,120,194,228,8
+DB      143,232,120,194,237,8
+DB      143,232,120,194,246,8
+DB      143,232,120,194,255,8
+        vpaddd  xmm12,xmm12,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm14,xmm14,xmm6
+        vpaddd  xmm15,xmm15,xmm7
+        vpxor   xmm0,xmm12,xmm0
+        vpxor   xmm1,xmm13,xmm1
+        vpxor   xmm2,xmm2,xmm14
+        vpxor   xmm3,xmm3,xmm15
+DB      143,232,120,194,192,7
+DB      143,232,120,194,201,7
+DB      143,232,120,194,210,7
+DB      143,232,120,194,219,7
+        vpaddd  xmm8,xmm8,xmm1
+        vpaddd  xmm9,xmm9,xmm2
+        vpaddd  xmm10,xmm10,xmm3
+        vpaddd  xmm11,xmm11,xmm0
+        vpxor   xmm7,xmm8,xmm7
+        vpxor   xmm4,xmm9,xmm4
+        vpxor   xmm5,xmm10,xmm5
+        vpxor   xmm6,xmm11,xmm6
+DB      143,232,120,194,255,16
+DB      143,232,120,194,228,16
+DB      143,232,120,194,237,16
+DB      143,232,120,194,246,16
+        vpaddd  xmm14,xmm14,xmm7
+        vpaddd  xmm15,xmm15,xmm4
+        vpaddd  xmm12,xmm12,xmm5
+        vpaddd  xmm13,xmm13,xmm6
+        vpxor   xmm1,xmm14,xmm1
+        vpxor   xmm2,xmm15,xmm2
+        vpxor   xmm3,xmm3,xmm12
+        vpxor   xmm0,xmm0,xmm13
+DB      143,232,120,194,201,12
+DB      143,232,120,194,210,12
+DB      143,232,120,194,219,12
+DB      143,232,120,194,192,12
+        vpaddd  xmm8,xmm1,xmm8
+        vpaddd  xmm9,xmm2,xmm9
+        vpaddd  xmm10,xmm10,xmm3
+        vpaddd  xmm11,xmm11,xmm0
+        vpxor   xmm7,xmm8,xmm7
+        vpxor   xmm4,xmm9,xmm4
+        vpxor   xmm5,xmm10,xmm5
+        vpxor   xmm6,xmm11,xmm6
+DB      143,232,120,194,255,8
+DB      143,232,120,194,228,8
+DB      143,232,120,194,237,8
+DB      143,232,120,194,246,8
+        vpaddd  xmm14,xmm14,xmm7
+        vpaddd  xmm15,xmm15,xmm4
+        vpaddd  xmm12,xmm12,xmm5
+        vpaddd  xmm13,xmm13,xmm6
+        vpxor   xmm1,xmm14,xmm1
+        vpxor   xmm2,xmm15,xmm2
+        vpxor   xmm3,xmm3,xmm12
+        vpxor   xmm0,xmm0,xmm13
+DB      143,232,120,194,201,7
+DB      143,232,120,194,210,7
+DB      143,232,120,194,219,7
+DB      143,232,120,194,192,7
+        dec     eax
+        jnz     NEAR $L$oop4xop
+
+        vpaddd  xmm8,xmm8,XMMWORD[64+rsp]
+        vpaddd  xmm9,xmm9,XMMWORD[80+rsp]
+        vpaddd  xmm10,xmm10,XMMWORD[96+rsp]
+        vpaddd  xmm11,xmm11,XMMWORD[112+rsp]
+
+        vmovdqa XMMWORD[32+rsp],xmm14
+        vmovdqa XMMWORD[48+rsp],xmm15
+
+        vpunpckldq      xmm14,xmm8,xmm9
+        vpunpckldq      xmm15,xmm10,xmm11
+        vpunpckhdq      xmm8,xmm8,xmm9
+        vpunpckhdq      xmm10,xmm10,xmm11
+        vpunpcklqdq     xmm9,xmm14,xmm15
+        vpunpckhqdq     xmm14,xmm14,xmm15
+        vpunpcklqdq     xmm11,xmm8,xmm10
+        vpunpckhqdq     xmm8,xmm8,xmm10
+        vpaddd  xmm0,xmm0,XMMWORD[((128-256))+rcx]
+        vpaddd  xmm1,xmm1,XMMWORD[((144-256))+rcx]
+        vpaddd  xmm2,xmm2,XMMWORD[((160-256))+rcx]
+        vpaddd  xmm3,xmm3,XMMWORD[((176-256))+rcx]
+
+        vmovdqa XMMWORD[rsp],xmm9
+        vmovdqa XMMWORD[16+rsp],xmm14
+        vmovdqa xmm9,XMMWORD[32+rsp]
+        vmovdqa xmm14,XMMWORD[48+rsp]
+
+        vpunpckldq      xmm10,xmm0,xmm1
+        vpunpckldq      xmm15,xmm2,xmm3
+        vpunpckhdq      xmm0,xmm0,xmm1
+        vpunpckhdq      xmm2,xmm2,xmm3
+        vpunpcklqdq     xmm1,xmm10,xmm15
+        vpunpckhqdq     xmm10,xmm10,xmm15
+        vpunpcklqdq     xmm3,xmm0,xmm2
+        vpunpckhqdq     xmm0,xmm0,xmm2
+        vpaddd  xmm12,xmm12,XMMWORD[((192-256))+rcx]
+        vpaddd  xmm13,xmm13,XMMWORD[((208-256))+rcx]
+        vpaddd  xmm9,xmm9,XMMWORD[((224-256))+rcx]
+        vpaddd  xmm14,xmm14,XMMWORD[((240-256))+rcx]
+
+        vpunpckldq      xmm2,xmm12,xmm13
+        vpunpckldq      xmm15,xmm9,xmm14
+        vpunpckhdq      xmm12,xmm12,xmm13
+        vpunpckhdq      xmm9,xmm9,xmm14
+        vpunpcklqdq     xmm13,xmm2,xmm15
+        vpunpckhqdq     xmm2,xmm2,xmm15
+        vpunpcklqdq     xmm14,xmm12,xmm9
+        vpunpckhqdq     xmm12,xmm12,xmm9
+        vpaddd  xmm4,xmm4,XMMWORD[((256-256))+rcx]
+        vpaddd  xmm5,xmm5,XMMWORD[((272-256))+rcx]
+        vpaddd  xmm6,xmm6,XMMWORD[((288-256))+rcx]
+        vpaddd  xmm7,xmm7,XMMWORD[((304-256))+rcx]
+
+        vpunpckldq      xmm9,xmm4,xmm5
+        vpunpckldq      xmm15,xmm6,xmm7
+        vpunpckhdq      xmm4,xmm4,xmm5
+        vpunpckhdq      xmm6,xmm6,xmm7
+        vpunpcklqdq     xmm5,xmm9,xmm15
+        vpunpckhqdq     xmm9,xmm9,xmm15
+        vpunpcklqdq     xmm7,xmm4,xmm6
+        vpunpckhqdq     xmm4,xmm4,xmm6
+        vmovdqa xmm6,XMMWORD[rsp]
+        vmovdqa xmm15,XMMWORD[16+rsp]
+
+        cmp     rdx,64*4
+        jb      NEAR $L$tail4xop
+
+        vpxor   xmm6,xmm6,XMMWORD[rsi]
+        vpxor   xmm1,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm13,xmm13,XMMWORD[32+rsi]
+        vpxor   xmm5,xmm5,XMMWORD[48+rsi]
+        vpxor   xmm15,xmm15,XMMWORD[64+rsi]
+        vpxor   xmm10,xmm10,XMMWORD[80+rsi]
+        vpxor   xmm2,xmm2,XMMWORD[96+rsi]
+        vpxor   xmm9,xmm9,XMMWORD[112+rsi]
+        lea     rsi,[128+rsi]
+        vpxor   xmm11,xmm11,XMMWORD[rsi]
+        vpxor   xmm3,xmm3,XMMWORD[16+rsi]
+        vpxor   xmm14,xmm14,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+        vpxor   xmm8,xmm8,XMMWORD[64+rsi]
+        vpxor   xmm0,xmm0,XMMWORD[80+rsi]
+        vpxor   xmm12,xmm12,XMMWORD[96+rsi]
+        vpxor   xmm4,xmm4,XMMWORD[112+rsi]
+        lea     rsi,[128+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm6
+        vmovdqu XMMWORD[16+rdi],xmm1
+        vmovdqu XMMWORD[32+rdi],xmm13
+        vmovdqu XMMWORD[48+rdi],xmm5
+        vmovdqu XMMWORD[64+rdi],xmm15
+        vmovdqu XMMWORD[80+rdi],xmm10
+        vmovdqu XMMWORD[96+rdi],xmm2
+        vmovdqu XMMWORD[112+rdi],xmm9
+        lea     rdi,[128+rdi]
+        vmovdqu XMMWORD[rdi],xmm11
+        vmovdqu XMMWORD[16+rdi],xmm3
+        vmovdqu XMMWORD[32+rdi],xmm14
+        vmovdqu XMMWORD[48+rdi],xmm7
+        vmovdqu XMMWORD[64+rdi],xmm8
+        vmovdqu XMMWORD[80+rdi],xmm0
+        vmovdqu XMMWORD[96+rdi],xmm12
+        vmovdqu XMMWORD[112+rdi],xmm4
+        lea     rdi,[128+rdi]
+
+        sub     rdx,64*4
+        jnz     NEAR $L$oop_outer4xop
+
+        jmp     NEAR $L$done4xop
+
+ALIGN   32
+$L$tail4xop:
+        cmp     rdx,192
+        jae     NEAR $L$192_or_more4xop
+        cmp     rdx,128
+        jae     NEAR $L$128_or_more4xop
+        cmp     rdx,64
+        jae     NEAR $L$64_or_more4xop
+
+        xor     r10,r10
+        vmovdqa XMMWORD[rsp],xmm6
+        vmovdqa XMMWORD[16+rsp],xmm1
+        vmovdqa XMMWORD[32+rsp],xmm13
+        vmovdqa XMMWORD[48+rsp],xmm5
+        jmp     NEAR $L$oop_tail4xop
+
+ALIGN   32
+$L$64_or_more4xop:
+        vpxor   xmm6,xmm6,XMMWORD[rsi]
+        vpxor   xmm1,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm13,xmm13,XMMWORD[32+rsi]
+        vpxor   xmm5,xmm5,XMMWORD[48+rsi]
+        vmovdqu XMMWORD[rdi],xmm6
+        vmovdqu XMMWORD[16+rdi],xmm1
+        vmovdqu XMMWORD[32+rdi],xmm13
+        vmovdqu XMMWORD[48+rdi],xmm5
+        je      NEAR $L$done4xop
+
+        lea     rsi,[64+rsi]
+        vmovdqa XMMWORD[rsp],xmm15
+        xor     r10,r10
+        vmovdqa XMMWORD[16+rsp],xmm10
+        lea     rdi,[64+rdi]
+        vmovdqa XMMWORD[32+rsp],xmm2
+        sub     rdx,64
+        vmovdqa XMMWORD[48+rsp],xmm9
+        jmp     NEAR $L$oop_tail4xop
+
+ALIGN   32
+$L$128_or_more4xop:
+        vpxor   xmm6,xmm6,XMMWORD[rsi]
+        vpxor   xmm1,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm13,xmm13,XMMWORD[32+rsi]
+        vpxor   xmm5,xmm5,XMMWORD[48+rsi]
+        vpxor   xmm15,xmm15,XMMWORD[64+rsi]
+        vpxor   xmm10,xmm10,XMMWORD[80+rsi]
+        vpxor   xmm2,xmm2,XMMWORD[96+rsi]
+        vpxor   xmm9,xmm9,XMMWORD[112+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm6
+        vmovdqu XMMWORD[16+rdi],xmm1
+        vmovdqu XMMWORD[32+rdi],xmm13
+        vmovdqu XMMWORD[48+rdi],xmm5
+        vmovdqu XMMWORD[64+rdi],xmm15
+        vmovdqu XMMWORD[80+rdi],xmm10
+        vmovdqu XMMWORD[96+rdi],xmm2
+        vmovdqu XMMWORD[112+rdi],xmm9
+        je      NEAR $L$done4xop
+
+        lea     rsi,[128+rsi]
+        vmovdqa XMMWORD[rsp],xmm11
+        xor     r10,r10
+        vmovdqa XMMWORD[16+rsp],xmm3
+        lea     rdi,[128+rdi]
+        vmovdqa XMMWORD[32+rsp],xmm14
+        sub     rdx,128
+        vmovdqa XMMWORD[48+rsp],xmm7
+        jmp     NEAR $L$oop_tail4xop
+
+ALIGN   32
+$L$192_or_more4xop:
+        vpxor   xmm6,xmm6,XMMWORD[rsi]
+        vpxor   xmm1,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm13,xmm13,XMMWORD[32+rsi]
+        vpxor   xmm5,xmm5,XMMWORD[48+rsi]
+        vpxor   xmm15,xmm15,XMMWORD[64+rsi]
+        vpxor   xmm10,xmm10,XMMWORD[80+rsi]
+        vpxor   xmm2,xmm2,XMMWORD[96+rsi]
+        vpxor   xmm9,xmm9,XMMWORD[112+rsi]
+        lea     rsi,[128+rsi]
+        vpxor   xmm11,xmm11,XMMWORD[rsi]
+        vpxor   xmm3,xmm3,XMMWORD[16+rsi]
+        vpxor   xmm14,xmm14,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm6
+        vmovdqu XMMWORD[16+rdi],xmm1
+        vmovdqu XMMWORD[32+rdi],xmm13
+        vmovdqu XMMWORD[48+rdi],xmm5
+        vmovdqu XMMWORD[64+rdi],xmm15
+        vmovdqu XMMWORD[80+rdi],xmm10
+        vmovdqu XMMWORD[96+rdi],xmm2
+        vmovdqu XMMWORD[112+rdi],xmm9
+        lea     rdi,[128+rdi]
+        vmovdqu XMMWORD[rdi],xmm11
+        vmovdqu XMMWORD[16+rdi],xmm3
+        vmovdqu XMMWORD[32+rdi],xmm14
+        vmovdqu XMMWORD[48+rdi],xmm7
+        je      NEAR $L$done4xop
+
+        lea     rsi,[64+rsi]
+        vmovdqa XMMWORD[rsp],xmm8
+        xor     r10,r10
+        vmovdqa XMMWORD[16+rsp],xmm0
+        lea     rdi,[64+rdi]
+        vmovdqa XMMWORD[32+rsp],xmm12
+        sub     rdx,192
+        vmovdqa XMMWORD[48+rsp],xmm4
+
+$L$oop_tail4xop:
+        movzx   eax,BYTE[r10*1+rsi]
+        movzx   ecx,BYTE[r10*1+rsp]
+        lea     r10,[1+r10]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r10*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail4xop
+
+$L$done4xop:
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$4xop_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_4xop:
+
+ALIGN   32
+ChaCha20_8x:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_8x:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_8x:
+        mov     r9,rsp
+
+        sub     rsp,0x280+168
+        and     rsp,-32
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$8x_body:
+        vzeroupper
+
+
+
+
+
+
+
+
+
+
+        vbroadcasti128  ymm11,XMMWORD[$L$sigma]
+        vbroadcasti128  ymm3,XMMWORD[rcx]
+        vbroadcasti128  ymm15,XMMWORD[16+rcx]
+        vbroadcasti128  ymm7,XMMWORD[r8]
+        lea     rcx,[256+rsp]
+        lea     rax,[512+rsp]
+        lea     r10,[$L$rot16]
+        lea     r11,[$L$rot24]
+
+        vpshufd ymm8,ymm11,0x00
+        vpshufd ymm9,ymm11,0x55
+        vmovdqa YMMWORD[(128-256)+rcx],ymm8
+        vpshufd ymm10,ymm11,0xaa
+        vmovdqa YMMWORD[(160-256)+rcx],ymm9
+        vpshufd ymm11,ymm11,0xff
+        vmovdqa YMMWORD[(192-256)+rcx],ymm10
+        vmovdqa YMMWORD[(224-256)+rcx],ymm11
+
+        vpshufd ymm0,ymm3,0x00
+        vpshufd ymm1,ymm3,0x55
+        vmovdqa YMMWORD[(256-256)+rcx],ymm0
+        vpshufd ymm2,ymm3,0xaa
+        vmovdqa YMMWORD[(288-256)+rcx],ymm1
+        vpshufd ymm3,ymm3,0xff
+        vmovdqa YMMWORD[(320-256)+rcx],ymm2
+        vmovdqa YMMWORD[(352-256)+rcx],ymm3
+
+        vpshufd ymm12,ymm15,0x00
+        vpshufd ymm13,ymm15,0x55
+        vmovdqa YMMWORD[(384-512)+rax],ymm12
+        vpshufd ymm14,ymm15,0xaa
+        vmovdqa YMMWORD[(416-512)+rax],ymm13
+        vpshufd ymm15,ymm15,0xff
+        vmovdqa YMMWORD[(448-512)+rax],ymm14
+        vmovdqa YMMWORD[(480-512)+rax],ymm15
+
+        vpshufd ymm4,ymm7,0x00
+        vpshufd ymm5,ymm7,0x55
+        vpaddd  ymm4,ymm4,YMMWORD[$L$incy]
+        vpshufd ymm6,ymm7,0xaa
+        vmovdqa YMMWORD[(544-512)+rax],ymm5
+        vpshufd ymm7,ymm7,0xff
+        vmovdqa YMMWORD[(576-512)+rax],ymm6
+        vmovdqa YMMWORD[(608-512)+rax],ymm7
+
+        jmp     NEAR $L$oop_enter8x
+
+ALIGN   32
+$L$oop_outer8x:
+        vmovdqa ymm8,YMMWORD[((128-256))+rcx]
+        vmovdqa ymm9,YMMWORD[((160-256))+rcx]
+        vmovdqa ymm10,YMMWORD[((192-256))+rcx]
+        vmovdqa ymm11,YMMWORD[((224-256))+rcx]
+        vmovdqa ymm0,YMMWORD[((256-256))+rcx]
+        vmovdqa ymm1,YMMWORD[((288-256))+rcx]
+        vmovdqa ymm2,YMMWORD[((320-256))+rcx]
+        vmovdqa ymm3,YMMWORD[((352-256))+rcx]
+        vmovdqa ymm12,YMMWORD[((384-512))+rax]
+        vmovdqa ymm13,YMMWORD[((416-512))+rax]
+        vmovdqa ymm14,YMMWORD[((448-512))+rax]
+        vmovdqa ymm15,YMMWORD[((480-512))+rax]
+        vmovdqa ymm4,YMMWORD[((512-512))+rax]
+        vmovdqa ymm5,YMMWORD[((544-512))+rax]
+        vmovdqa ymm6,YMMWORD[((576-512))+rax]
+        vmovdqa ymm7,YMMWORD[((608-512))+rax]
+        vpaddd  ymm4,ymm4,YMMWORD[$L$eight]
+
+$L$oop_enter8x:
+        vmovdqa YMMWORD[64+rsp],ymm14
+        vmovdqa YMMWORD[96+rsp],ymm15
+        vbroadcasti128  ymm15,XMMWORD[r10]
+        vmovdqa YMMWORD[(512-512)+rax],ymm4
+        mov     eax,10
+        jmp     NEAR $L$oop8x
+
+ALIGN   32
+$L$oop8x:
+        vpaddd  ymm8,ymm8,ymm0
+        vpxor   ymm4,ymm8,ymm4
+        vpshufb ymm4,ymm4,ymm15
+        vpaddd  ymm9,ymm9,ymm1
+        vpxor   ymm5,ymm9,ymm5
+        vpshufb ymm5,ymm5,ymm15
+        vpaddd  ymm12,ymm12,ymm4
+        vpxor   ymm0,ymm12,ymm0
+        vpslld  ymm14,ymm0,12
+        vpsrld  ymm0,ymm0,20
+        vpor    ymm0,ymm14,ymm0
+        vbroadcasti128  ymm14,XMMWORD[r11]
+        vpaddd  ymm13,ymm13,ymm5
+        vpxor   ymm1,ymm13,ymm1
+        vpslld  ymm15,ymm1,12
+        vpsrld  ymm1,ymm1,20
+        vpor    ymm1,ymm15,ymm1
+        vpaddd  ymm8,ymm8,ymm0
+        vpxor   ymm4,ymm8,ymm4
+        vpshufb ymm4,ymm4,ymm14
+        vpaddd  ymm9,ymm9,ymm1
+        vpxor   ymm5,ymm9,ymm5
+        vpshufb ymm5,ymm5,ymm14
+        vpaddd  ymm12,ymm12,ymm4
+        vpxor   ymm0,ymm12,ymm0
+        vpslld  ymm15,ymm0,7
+        vpsrld  ymm0,ymm0,25
+        vpor    ymm0,ymm15,ymm0
+        vbroadcasti128  ymm15,XMMWORD[r10]
+        vpaddd  ymm13,ymm13,ymm5
+        vpxor   ymm1,ymm13,ymm1
+        vpslld  ymm14,ymm1,7
+        vpsrld  ymm1,ymm1,25
+        vpor    ymm1,ymm14,ymm1
+        vmovdqa YMMWORD[rsp],ymm12
+        vmovdqa YMMWORD[32+rsp],ymm13
+        vmovdqa ymm12,YMMWORD[64+rsp]
+        vmovdqa ymm13,YMMWORD[96+rsp]
+        vpaddd  ymm10,ymm10,ymm2
+        vpxor   ymm6,ymm10,ymm6
+        vpshufb ymm6,ymm6,ymm15
+        vpaddd  ymm11,ymm11,ymm3
+        vpxor   ymm7,ymm11,ymm7
+        vpshufb ymm7,ymm7,ymm15
+        vpaddd  ymm12,ymm12,ymm6
+        vpxor   ymm2,ymm12,ymm2
+        vpslld  ymm14,ymm2,12
+        vpsrld  ymm2,ymm2,20
+        vpor    ymm2,ymm14,ymm2
+        vbroadcasti128  ymm14,XMMWORD[r11]
+        vpaddd  ymm13,ymm13,ymm7
+        vpxor   ymm3,ymm13,ymm3
+        vpslld  ymm15,ymm3,12
+        vpsrld  ymm3,ymm3,20
+        vpor    ymm3,ymm15,ymm3
+        vpaddd  ymm10,ymm10,ymm2
+        vpxor   ymm6,ymm10,ymm6
+        vpshufb ymm6,ymm6,ymm14
+        vpaddd  ymm11,ymm11,ymm3
+        vpxor   ymm7,ymm11,ymm7
+        vpshufb ymm7,ymm7,ymm14
+        vpaddd  ymm12,ymm12,ymm6
+        vpxor   ymm2,ymm12,ymm2
+        vpslld  ymm15,ymm2,7
+        vpsrld  ymm2,ymm2,25
+        vpor    ymm2,ymm15,ymm2
+        vbroadcasti128  ymm15,XMMWORD[r10]
+        vpaddd  ymm13,ymm13,ymm7
+        vpxor   ymm3,ymm13,ymm3
+        vpslld  ymm14,ymm3,7
+        vpsrld  ymm3,ymm3,25
+        vpor    ymm3,ymm14,ymm3
+        vpaddd  ymm8,ymm8,ymm1
+        vpxor   ymm7,ymm8,ymm7
+        vpshufb ymm7,ymm7,ymm15
+        vpaddd  ymm9,ymm9,ymm2
+        vpxor   ymm4,ymm9,ymm4
+        vpshufb ymm4,ymm4,ymm15
+        vpaddd  ymm12,ymm12,ymm7
+        vpxor   ymm1,ymm12,ymm1
+        vpslld  ymm14,ymm1,12
+        vpsrld  ymm1,ymm1,20
+        vpor    ymm1,ymm14,ymm1
+        vbroadcasti128  ymm14,XMMWORD[r11]
+        vpaddd  ymm13,ymm13,ymm4
+        vpxor   ymm2,ymm13,ymm2
+        vpslld  ymm15,ymm2,12
+        vpsrld  ymm2,ymm2,20
+        vpor    ymm2,ymm15,ymm2
+        vpaddd  ymm8,ymm8,ymm1
+        vpxor   ymm7,ymm8,ymm7
+        vpshufb ymm7,ymm7,ymm14
+        vpaddd  ymm9,ymm9,ymm2
+        vpxor   ymm4,ymm9,ymm4
+        vpshufb ymm4,ymm4,ymm14
+        vpaddd  ymm12,ymm12,ymm7
+        vpxor   ymm1,ymm12,ymm1
+        vpslld  ymm15,ymm1,7
+        vpsrld  ymm1,ymm1,25
+        vpor    ymm1,ymm15,ymm1
+        vbroadcasti128  ymm15,XMMWORD[r10]
+        vpaddd  ymm13,ymm13,ymm4
+        vpxor   ymm2,ymm13,ymm2
+        vpslld  ymm14,ymm2,7
+        vpsrld  ymm2,ymm2,25
+        vpor    ymm2,ymm14,ymm2
+        vmovdqa YMMWORD[64+rsp],ymm12
+        vmovdqa YMMWORD[96+rsp],ymm13
+        vmovdqa ymm12,YMMWORD[rsp]
+        vmovdqa ymm13,YMMWORD[32+rsp]
+        vpaddd  ymm10,ymm10,ymm3
+        vpxor   ymm5,ymm10,ymm5
+        vpshufb ymm5,ymm5,ymm15
+        vpaddd  ymm11,ymm11,ymm0
+        vpxor   ymm6,ymm11,ymm6
+        vpshufb ymm6,ymm6,ymm15
+        vpaddd  ymm12,ymm12,ymm5
+        vpxor   ymm3,ymm12,ymm3
+        vpslld  ymm14,ymm3,12
+        vpsrld  ymm3,ymm3,20
+        vpor    ymm3,ymm14,ymm3
+        vbroadcasti128  ymm14,XMMWORD[r11]
+        vpaddd  ymm13,ymm13,ymm6
+        vpxor   ymm0,ymm13,ymm0
+        vpslld  ymm15,ymm0,12
+        vpsrld  ymm0,ymm0,20
+        vpor    ymm0,ymm15,ymm0
+        vpaddd  ymm10,ymm10,ymm3
+        vpxor   ymm5,ymm10,ymm5
+        vpshufb ymm5,ymm5,ymm14
+        vpaddd  ymm11,ymm11,ymm0
+        vpxor   ymm6,ymm11,ymm6
+        vpshufb ymm6,ymm6,ymm14
+        vpaddd  ymm12,ymm12,ymm5
+        vpxor   ymm3,ymm12,ymm3
+        vpslld  ymm15,ymm3,7
+        vpsrld  ymm3,ymm3,25
+        vpor    ymm3,ymm15,ymm3
+        vbroadcasti128  ymm15,XMMWORD[r10]
+        vpaddd  ymm13,ymm13,ymm6
+        vpxor   ymm0,ymm13,ymm0
+        vpslld  ymm14,ymm0,7
+        vpsrld  ymm0,ymm0,25
+        vpor    ymm0,ymm14,ymm0
+        dec     eax
+        jnz     NEAR $L$oop8x
+
+        lea     rax,[512+rsp]
+        vpaddd  ymm8,ymm8,YMMWORD[((128-256))+rcx]
+        vpaddd  ymm9,ymm9,YMMWORD[((160-256))+rcx]
+        vpaddd  ymm10,ymm10,YMMWORD[((192-256))+rcx]
+        vpaddd  ymm11,ymm11,YMMWORD[((224-256))+rcx]
+
+        vpunpckldq      ymm14,ymm8,ymm9
+        vpunpckldq      ymm15,ymm10,ymm11
+        vpunpckhdq      ymm8,ymm8,ymm9
+        vpunpckhdq      ymm10,ymm10,ymm11
+        vpunpcklqdq     ymm9,ymm14,ymm15
+        vpunpckhqdq     ymm14,ymm14,ymm15
+        vpunpcklqdq     ymm11,ymm8,ymm10
+        vpunpckhqdq     ymm8,ymm8,ymm10
+        vpaddd  ymm0,ymm0,YMMWORD[((256-256))+rcx]
+        vpaddd  ymm1,ymm1,YMMWORD[((288-256))+rcx]
+        vpaddd  ymm2,ymm2,YMMWORD[((320-256))+rcx]
+        vpaddd  ymm3,ymm3,YMMWORD[((352-256))+rcx]
+
+        vpunpckldq      ymm10,ymm0,ymm1
+        vpunpckldq      ymm15,ymm2,ymm3
+        vpunpckhdq      ymm0,ymm0,ymm1
+        vpunpckhdq      ymm2,ymm2,ymm3
+        vpunpcklqdq     ymm1,ymm10,ymm15
+        vpunpckhqdq     ymm10,ymm10,ymm15
+        vpunpcklqdq     ymm3,ymm0,ymm2
+        vpunpckhqdq     ymm0,ymm0,ymm2
+        vperm2i128      ymm15,ymm9,ymm1,0x20
+        vperm2i128      ymm1,ymm9,ymm1,0x31
+        vperm2i128      ymm9,ymm14,ymm10,0x20
+        vperm2i128      ymm10,ymm14,ymm10,0x31
+        vperm2i128      ymm14,ymm11,ymm3,0x20
+        vperm2i128      ymm3,ymm11,ymm3,0x31
+        vperm2i128      ymm11,ymm8,ymm0,0x20
+        vperm2i128      ymm0,ymm8,ymm0,0x31
+        vmovdqa YMMWORD[rsp],ymm15
+        vmovdqa YMMWORD[32+rsp],ymm9
+        vmovdqa ymm15,YMMWORD[64+rsp]
+        vmovdqa ymm9,YMMWORD[96+rsp]
+
+        vpaddd  ymm12,ymm12,YMMWORD[((384-512))+rax]
+        vpaddd  ymm13,ymm13,YMMWORD[((416-512))+rax]
+        vpaddd  ymm15,ymm15,YMMWORD[((448-512))+rax]
+        vpaddd  ymm9,ymm9,YMMWORD[((480-512))+rax]
+
+        vpunpckldq      ymm2,ymm12,ymm13
+        vpunpckldq      ymm8,ymm15,ymm9
+        vpunpckhdq      ymm12,ymm12,ymm13
+        vpunpckhdq      ymm15,ymm15,ymm9
+        vpunpcklqdq     ymm13,ymm2,ymm8
+        vpunpckhqdq     ymm2,ymm2,ymm8
+        vpunpcklqdq     ymm9,ymm12,ymm15
+        vpunpckhqdq     ymm12,ymm12,ymm15
+        vpaddd  ymm4,ymm4,YMMWORD[((512-512))+rax]
+        vpaddd  ymm5,ymm5,YMMWORD[((544-512))+rax]
+        vpaddd  ymm6,ymm6,YMMWORD[((576-512))+rax]
+        vpaddd  ymm7,ymm7,YMMWORD[((608-512))+rax]
+
+        vpunpckldq      ymm15,ymm4,ymm5
+        vpunpckldq      ymm8,ymm6,ymm7
+        vpunpckhdq      ymm4,ymm4,ymm5
+        vpunpckhdq      ymm6,ymm6,ymm7
+        vpunpcklqdq     ymm5,ymm15,ymm8
+        vpunpckhqdq     ymm15,ymm15,ymm8
+        vpunpcklqdq     ymm7,ymm4,ymm6
+        vpunpckhqdq     ymm4,ymm4,ymm6
+        vperm2i128      ymm8,ymm13,ymm5,0x20
+        vperm2i128      ymm5,ymm13,ymm5,0x31
+        vperm2i128      ymm13,ymm2,ymm15,0x20
+        vperm2i128      ymm15,ymm2,ymm15,0x31
+        vperm2i128      ymm2,ymm9,ymm7,0x20
+        vperm2i128      ymm7,ymm9,ymm7,0x31
+        vperm2i128      ymm9,ymm12,ymm4,0x20
+        vperm2i128      ymm4,ymm12,ymm4,0x31
+        vmovdqa ymm6,YMMWORD[rsp]
+        vmovdqa ymm12,YMMWORD[32+rsp]
+
+        cmp     rdx,64*8
+        jb      NEAR $L$tail8x
+
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        lea     rsi,[128+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        lea     rdi,[128+rdi]
+
+        vpxor   ymm12,ymm12,YMMWORD[rsi]
+        vpxor   ymm13,ymm13,YMMWORD[32+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[64+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[96+rsi]
+        lea     rsi,[128+rsi]
+        vmovdqu YMMWORD[rdi],ymm12
+        vmovdqu YMMWORD[32+rdi],ymm13
+        vmovdqu YMMWORD[64+rdi],ymm10
+        vmovdqu YMMWORD[96+rdi],ymm15
+        lea     rdi,[128+rdi]
+
+        vpxor   ymm14,ymm14,YMMWORD[rsi]
+        vpxor   ymm2,ymm2,YMMWORD[32+rsi]
+        vpxor   ymm3,ymm3,YMMWORD[64+rsi]
+        vpxor   ymm7,ymm7,YMMWORD[96+rsi]
+        lea     rsi,[128+rsi]
+        vmovdqu YMMWORD[rdi],ymm14
+        vmovdqu YMMWORD[32+rdi],ymm2
+        vmovdqu YMMWORD[64+rdi],ymm3
+        vmovdqu YMMWORD[96+rdi],ymm7
+        lea     rdi,[128+rdi]
+
+        vpxor   ymm11,ymm11,YMMWORD[rsi]
+        vpxor   ymm9,ymm9,YMMWORD[32+rsi]
+        vpxor   ymm0,ymm0,YMMWORD[64+rsi]
+        vpxor   ymm4,ymm4,YMMWORD[96+rsi]
+        lea     rsi,[128+rsi]
+        vmovdqu YMMWORD[rdi],ymm11
+        vmovdqu YMMWORD[32+rdi],ymm9
+        vmovdqu YMMWORD[64+rdi],ymm0
+        vmovdqu YMMWORD[96+rdi],ymm4
+        lea     rdi,[128+rdi]
+
+        sub     rdx,64*8
+        jnz     NEAR $L$oop_outer8x
+
+        jmp     NEAR $L$done8x
+
+$L$tail8x:
+        cmp     rdx,448
+        jae     NEAR $L$448_or_more8x
+        cmp     rdx,384
+        jae     NEAR $L$384_or_more8x
+        cmp     rdx,320
+        jae     NEAR $L$320_or_more8x
+        cmp     rdx,256
+        jae     NEAR $L$256_or_more8x
+        cmp     rdx,192
+        jae     NEAR $L$192_or_more8x
+        cmp     rdx,128
+        jae     NEAR $L$128_or_more8x
+        cmp     rdx,64
+        jae     NEAR $L$64_or_more8x
+
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm6
+        vmovdqa YMMWORD[32+rsp],ymm8
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$64_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        je      NEAR $L$done8x
+
+        lea     rsi,[64+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm1
+        lea     rdi,[64+rdi]
+        sub     rdx,64
+        vmovdqa YMMWORD[32+rsp],ymm5
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$128_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        je      NEAR $L$done8x
+
+        lea     rsi,[128+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm12
+        lea     rdi,[128+rdi]
+        sub     rdx,128
+        vmovdqa YMMWORD[32+rsp],ymm13
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$192_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[128+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[160+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        vmovdqu YMMWORD[128+rdi],ymm12
+        vmovdqu YMMWORD[160+rdi],ymm13
+        je      NEAR $L$done8x
+
+        lea     rsi,[192+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm10
+        lea     rdi,[192+rdi]
+        sub     rdx,192
+        vmovdqa YMMWORD[32+rsp],ymm15
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$256_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[128+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[160+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[192+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[224+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        vmovdqu YMMWORD[128+rdi],ymm12
+        vmovdqu YMMWORD[160+rdi],ymm13
+        vmovdqu YMMWORD[192+rdi],ymm10
+        vmovdqu YMMWORD[224+rdi],ymm15
+        je      NEAR $L$done8x
+
+        lea     rsi,[256+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm14
+        lea     rdi,[256+rdi]
+        sub     rdx,256
+        vmovdqa YMMWORD[32+rsp],ymm2
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$320_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[128+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[160+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[192+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[224+rsi]
+        vpxor   ymm14,ymm14,YMMWORD[256+rsi]
+        vpxor   ymm2,ymm2,YMMWORD[288+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        vmovdqu YMMWORD[128+rdi],ymm12
+        vmovdqu YMMWORD[160+rdi],ymm13
+        vmovdqu YMMWORD[192+rdi],ymm10
+        vmovdqu YMMWORD[224+rdi],ymm15
+        vmovdqu YMMWORD[256+rdi],ymm14
+        vmovdqu YMMWORD[288+rdi],ymm2
+        je      NEAR $L$done8x
+
+        lea     rsi,[320+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm3
+        lea     rdi,[320+rdi]
+        sub     rdx,320
+        vmovdqa YMMWORD[32+rsp],ymm7
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$384_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[128+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[160+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[192+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[224+rsi]
+        vpxor   ymm14,ymm14,YMMWORD[256+rsi]
+        vpxor   ymm2,ymm2,YMMWORD[288+rsi]
+        vpxor   ymm3,ymm3,YMMWORD[320+rsi]
+        vpxor   ymm7,ymm7,YMMWORD[352+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        vmovdqu YMMWORD[128+rdi],ymm12
+        vmovdqu YMMWORD[160+rdi],ymm13
+        vmovdqu YMMWORD[192+rdi],ymm10
+        vmovdqu YMMWORD[224+rdi],ymm15
+        vmovdqu YMMWORD[256+rdi],ymm14
+        vmovdqu YMMWORD[288+rdi],ymm2
+        vmovdqu YMMWORD[320+rdi],ymm3
+        vmovdqu YMMWORD[352+rdi],ymm7
+        je      NEAR $L$done8x
+
+        lea     rsi,[384+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm11
+        lea     rdi,[384+rdi]
+        sub     rdx,384
+        vmovdqa YMMWORD[32+rsp],ymm9
+        jmp     NEAR $L$oop_tail8x
+
+ALIGN   32
+$L$448_or_more8x:
+        vpxor   ymm6,ymm6,YMMWORD[rsi]
+        vpxor   ymm8,ymm8,YMMWORD[32+rsi]
+        vpxor   ymm1,ymm1,YMMWORD[64+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[96+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[128+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[160+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[192+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[224+rsi]
+        vpxor   ymm14,ymm14,YMMWORD[256+rsi]
+        vpxor   ymm2,ymm2,YMMWORD[288+rsi]
+        vpxor   ymm3,ymm3,YMMWORD[320+rsi]
+        vpxor   ymm7,ymm7,YMMWORD[352+rsi]
+        vpxor   ymm11,ymm11,YMMWORD[384+rsi]
+        vpxor   ymm9,ymm9,YMMWORD[416+rsi]
+        vmovdqu YMMWORD[rdi],ymm6
+        vmovdqu YMMWORD[32+rdi],ymm8
+        vmovdqu YMMWORD[64+rdi],ymm1
+        vmovdqu YMMWORD[96+rdi],ymm5
+        vmovdqu YMMWORD[128+rdi],ymm12
+        vmovdqu YMMWORD[160+rdi],ymm13
+        vmovdqu YMMWORD[192+rdi],ymm10
+        vmovdqu YMMWORD[224+rdi],ymm15
+        vmovdqu YMMWORD[256+rdi],ymm14
+        vmovdqu YMMWORD[288+rdi],ymm2
+        vmovdqu YMMWORD[320+rdi],ymm3
+        vmovdqu YMMWORD[352+rdi],ymm7
+        vmovdqu YMMWORD[384+rdi],ymm11
+        vmovdqu YMMWORD[416+rdi],ymm9
+        je      NEAR $L$done8x
+
+        lea     rsi,[448+rsi]
+        xor     r10,r10
+        vmovdqa YMMWORD[rsp],ymm0
+        lea     rdi,[448+rdi]
+        sub     rdx,448
+        vmovdqa YMMWORD[32+rsp],ymm4
+
+$L$oop_tail8x:
+        movzx   eax,BYTE[r10*1+rsi]
+        movzx   ecx,BYTE[r10*1+rsp]
+        lea     r10,[1+r10]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r10*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail8x
+
+$L$done8x:
+        vzeroall
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$8x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_8x:
+
+ALIGN   32
+ChaCha20_avx512:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_avx512:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_avx512:
+        mov     r9,rsp
+
+        cmp     rdx,512
+        ja      NEAR $L$ChaCha20_16x
+
+        sub     rsp,64+168
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$avx512_body:
+        vbroadcasti32x4 zmm0,ZMMWORD[$L$sigma]
+        vbroadcasti32x4 zmm1,ZMMWORD[rcx]
+        vbroadcasti32x4 zmm2,ZMMWORD[16+rcx]
+        vbroadcasti32x4 zmm3,ZMMWORD[r8]
+
+        vmovdqa32       zmm16,zmm0
+        vmovdqa32       zmm17,zmm1
+        vmovdqa32       zmm18,zmm2
+        vpaddd  zmm3,zmm3,ZMMWORD[$L$zeroz]
+        vmovdqa32       zmm20,ZMMWORD[$L$fourz]
+        mov     r8,10
+        vmovdqa32       zmm19,zmm3
+        jmp     NEAR $L$oop_avx512
+
+ALIGN   16
+$L$oop_outer_avx512:
+        vmovdqa32       zmm0,zmm16
+        vmovdqa32       zmm1,zmm17
+        vmovdqa32       zmm2,zmm18
+        vpaddd  zmm3,zmm19,zmm20
+        mov     r8,10
+        vmovdqa32       zmm19,zmm3
+        jmp     NEAR $L$oop_avx512
+
+ALIGN   32
+$L$oop_avx512:
+        vpaddd  zmm0,zmm0,zmm1
+        vpxord  zmm3,zmm3,zmm0
+        vprold  zmm3,zmm3,16
+        vpaddd  zmm2,zmm2,zmm3
+        vpxord  zmm1,zmm1,zmm2
+        vprold  zmm1,zmm1,12
+        vpaddd  zmm0,zmm0,zmm1
+        vpxord  zmm3,zmm3,zmm0
+        vprold  zmm3,zmm3,8
+        vpaddd  zmm2,zmm2,zmm3
+        vpxord  zmm1,zmm1,zmm2
+        vprold  zmm1,zmm1,7
+        vpshufd zmm2,zmm2,78
+        vpshufd zmm1,zmm1,57
+        vpshufd zmm3,zmm3,147
+        vpaddd  zmm0,zmm0,zmm1
+        vpxord  zmm3,zmm3,zmm0
+        vprold  zmm3,zmm3,16
+        vpaddd  zmm2,zmm2,zmm3
+        vpxord  zmm1,zmm1,zmm2
+        vprold  zmm1,zmm1,12
+        vpaddd  zmm0,zmm0,zmm1
+        vpxord  zmm3,zmm3,zmm0
+        vprold  zmm3,zmm3,8
+        vpaddd  zmm2,zmm2,zmm3
+        vpxord  zmm1,zmm1,zmm2
+        vprold  zmm1,zmm1,7
+        vpshufd zmm2,zmm2,78
+        vpshufd zmm1,zmm1,147
+        vpshufd zmm3,zmm3,57
+        dec     r8
+        jnz     NEAR $L$oop_avx512
+        vpaddd  zmm0,zmm0,zmm16
+        vpaddd  zmm1,zmm1,zmm17
+        vpaddd  zmm2,zmm2,zmm18
+        vpaddd  zmm3,zmm3,zmm19
+
+        sub     rdx,64
+        jb      NEAR $L$tail64_avx512
+
+        vpxor   xmm4,xmm0,XMMWORD[rsi]
+        vpxor   xmm5,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm2,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm3,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        jz      NEAR $L$done_avx512
+
+        vextracti32x4   xmm4,zmm0,1
+        vextracti32x4   xmm5,zmm1,1
+        vextracti32x4   xmm6,zmm2,1
+        vextracti32x4   xmm7,zmm3,1
+
+        sub     rdx,64
+        jb      NEAR $L$tail_avx512
+
+        vpxor   xmm4,xmm4,XMMWORD[rsi]
+        vpxor   xmm5,xmm5,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm6,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        jz      NEAR $L$done_avx512
+
+        vextracti32x4   xmm4,zmm0,2
+        vextracti32x4   xmm5,zmm1,2
+        vextracti32x4   xmm6,zmm2,2
+        vextracti32x4   xmm7,zmm3,2
+
+        sub     rdx,64
+        jb      NEAR $L$tail_avx512
+
+        vpxor   xmm4,xmm4,XMMWORD[rsi]
+        vpxor   xmm5,xmm5,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm6,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        jz      NEAR $L$done_avx512
+
+        vextracti32x4   xmm4,zmm0,3
+        vextracti32x4   xmm5,zmm1,3
+        vextracti32x4   xmm6,zmm2,3
+        vextracti32x4   xmm7,zmm3,3
+
+        sub     rdx,64
+        jb      NEAR $L$tail_avx512
+
+        vpxor   xmm4,xmm4,XMMWORD[rsi]
+        vpxor   xmm5,xmm5,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm6,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        jnz     NEAR $L$oop_outer_avx512
+
+        jmp     NEAR $L$done_avx512
+
+ALIGN   16
+$L$tail64_avx512:
+        vmovdqa XMMWORD[rsp],xmm0
+        vmovdqa XMMWORD[16+rsp],xmm1
+        vmovdqa XMMWORD[32+rsp],xmm2
+        vmovdqa XMMWORD[48+rsp],xmm3
+        add     rdx,64
+        jmp     NEAR $L$oop_tail_avx512
+
+ALIGN   16
+$L$tail_avx512:
+        vmovdqa XMMWORD[rsp],xmm4
+        vmovdqa XMMWORD[16+rsp],xmm5
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vmovdqa XMMWORD[48+rsp],xmm7
+        add     rdx,64
+
+$L$oop_tail_avx512:
+        movzx   eax,BYTE[r8*1+rsi]
+        movzx   ecx,BYTE[r8*1+rsp]
+        lea     r8,[1+r8]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r8*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail_avx512
+
+        vmovdqu32       ZMMWORD[rsp],zmm16
+
+$L$done_avx512:
+        vzeroall
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$avx512_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_avx512:
+
+ALIGN   32
+ChaCha20_avx512vl:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_avx512vl:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_avx512vl:
+        mov     r9,rsp
+
+        cmp     rdx,128
+        ja      NEAR $L$ChaCha20_8xvl
+
+        sub     rsp,64+168
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$avx512vl_body:
+        vbroadcasti128  ymm0,XMMWORD[$L$sigma]
+        vbroadcasti128  ymm1,XMMWORD[rcx]
+        vbroadcasti128  ymm2,XMMWORD[16+rcx]
+        vbroadcasti128  ymm3,XMMWORD[r8]
+
+        vmovdqa32       ymm16,ymm0
+        vmovdqa32       ymm17,ymm1
+        vmovdqa32       ymm18,ymm2
+        vpaddd  ymm3,ymm3,YMMWORD[$L$zeroz]
+        vmovdqa32       ymm20,YMMWORD[$L$twoy]
+        mov     r8,10
+        vmovdqa32       ymm19,ymm3
+        jmp     NEAR $L$oop_avx512vl
+
+ALIGN   16
+$L$oop_outer_avx512vl:
+        vmovdqa32       ymm2,ymm18
+        vpaddd  ymm3,ymm19,ymm20
+        mov     r8,10
+        vmovdqa32       ymm19,ymm3
+        jmp     NEAR $L$oop_avx512vl
+
+ALIGN   32
+$L$oop_avx512vl:
+        vpaddd  ymm0,ymm0,ymm1
+        vpxor   ymm3,ymm3,ymm0
+        vprold  ymm3,ymm3,16
+        vpaddd  ymm2,ymm2,ymm3
+        vpxor   ymm1,ymm1,ymm2
+        vprold  ymm1,ymm1,12
+        vpaddd  ymm0,ymm0,ymm1
+        vpxor   ymm3,ymm3,ymm0
+        vprold  ymm3,ymm3,8
+        vpaddd  ymm2,ymm2,ymm3
+        vpxor   ymm1,ymm1,ymm2
+        vprold  ymm1,ymm1,7
+        vpshufd ymm2,ymm2,78
+        vpshufd ymm1,ymm1,57
+        vpshufd ymm3,ymm3,147
+        vpaddd  ymm0,ymm0,ymm1
+        vpxor   ymm3,ymm3,ymm0
+        vprold  ymm3,ymm3,16
+        vpaddd  ymm2,ymm2,ymm3
+        vpxor   ymm1,ymm1,ymm2
+        vprold  ymm1,ymm1,12
+        vpaddd  ymm0,ymm0,ymm1
+        vpxor   ymm3,ymm3,ymm0
+        vprold  ymm3,ymm3,8
+        vpaddd  ymm2,ymm2,ymm3
+        vpxor   ymm1,ymm1,ymm2
+        vprold  ymm1,ymm1,7
+        vpshufd ymm2,ymm2,78
+        vpshufd ymm1,ymm1,147
+        vpshufd ymm3,ymm3,57
+        dec     r8
+        jnz     NEAR $L$oop_avx512vl
+        vpaddd  ymm0,ymm0,ymm16
+        vpaddd  ymm1,ymm1,ymm17
+        vpaddd  ymm2,ymm2,ymm18
+        vpaddd  ymm3,ymm3,ymm19
+
+        sub     rdx,64
+        jb      NEAR $L$tail64_avx512vl
+
+        vpxor   xmm4,xmm0,XMMWORD[rsi]
+        vpxor   xmm5,xmm1,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm2,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm3,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        jz      NEAR $L$done_avx512vl
+
+        vextracti128    xmm4,ymm0,1
+        vextracti128    xmm5,ymm1,1
+        vextracti128    xmm6,ymm2,1
+        vextracti128    xmm7,ymm3,1
+
+        sub     rdx,64
+        jb      NEAR $L$tail_avx512vl
+
+        vpxor   xmm4,xmm4,XMMWORD[rsi]
+        vpxor   xmm5,xmm5,XMMWORD[16+rsi]
+        vpxor   xmm6,xmm6,XMMWORD[32+rsi]
+        vpxor   xmm7,xmm7,XMMWORD[48+rsi]
+        lea     rsi,[64+rsi]
+
+        vmovdqu XMMWORD[rdi],xmm4
+        vmovdqu XMMWORD[16+rdi],xmm5
+        vmovdqu XMMWORD[32+rdi],xmm6
+        vmovdqu XMMWORD[48+rdi],xmm7
+        lea     rdi,[64+rdi]
+
+        vmovdqa32       ymm0,ymm16
+        vmovdqa32       ymm1,ymm17
+        jnz     NEAR $L$oop_outer_avx512vl
+
+        jmp     NEAR $L$done_avx512vl
+
+ALIGN   16
+$L$tail64_avx512vl:
+        vmovdqa XMMWORD[rsp],xmm0
+        vmovdqa XMMWORD[16+rsp],xmm1
+        vmovdqa XMMWORD[32+rsp],xmm2
+        vmovdqa XMMWORD[48+rsp],xmm3
+        add     rdx,64
+        jmp     NEAR $L$oop_tail_avx512vl
+
+ALIGN   16
+$L$tail_avx512vl:
+        vmovdqa XMMWORD[rsp],xmm4
+        vmovdqa XMMWORD[16+rsp],xmm5
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vmovdqa XMMWORD[48+rsp],xmm7
+        add     rdx,64
+
+$L$oop_tail_avx512vl:
+        movzx   eax,BYTE[r8*1+rsi]
+        movzx   ecx,BYTE[r8*1+rsp]
+        lea     r8,[1+r8]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r8*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail_avx512vl
+
+        vmovdqu32       YMMWORD[rsp],ymm16
+        vmovdqu32       YMMWORD[32+rsp],ymm16
+
+$L$done_avx512vl:
+        vzeroall
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$avx512vl_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_avx512vl:
+
+ALIGN   32
+ChaCha20_16x:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_16x:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_16x:
+        mov     r9,rsp
+
+        sub     rsp,64+168
+        and     rsp,-64
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$16x_body:
+        vzeroupper
+
+        lea     r10,[$L$sigma]
+        vbroadcasti32x4 zmm3,ZMMWORD[r10]
+        vbroadcasti32x4 zmm7,ZMMWORD[rcx]
+        vbroadcasti32x4 zmm11,ZMMWORD[16+rcx]
+        vbroadcasti32x4 zmm15,ZMMWORD[r8]
+
+        vpshufd zmm0,zmm3,0x00
+        vpshufd zmm1,zmm3,0x55
+        vpshufd zmm2,zmm3,0xaa
+        vpshufd zmm3,zmm3,0xff
+        vmovdqa64       zmm16,zmm0
+        vmovdqa64       zmm17,zmm1
+        vmovdqa64       zmm18,zmm2
+        vmovdqa64       zmm19,zmm3
+
+        vpshufd zmm4,zmm7,0x00
+        vpshufd zmm5,zmm7,0x55
+        vpshufd zmm6,zmm7,0xaa
+        vpshufd zmm7,zmm7,0xff
+        vmovdqa64       zmm20,zmm4
+        vmovdqa64       zmm21,zmm5
+        vmovdqa64       zmm22,zmm6
+        vmovdqa64       zmm23,zmm7
+
+        vpshufd zmm8,zmm11,0x00
+        vpshufd zmm9,zmm11,0x55
+        vpshufd zmm10,zmm11,0xaa
+        vpshufd zmm11,zmm11,0xff
+        vmovdqa64       zmm24,zmm8
+        vmovdqa64       zmm25,zmm9
+        vmovdqa64       zmm26,zmm10
+        vmovdqa64       zmm27,zmm11
+
+        vpshufd zmm12,zmm15,0x00
+        vpshufd zmm13,zmm15,0x55
+        vpshufd zmm14,zmm15,0xaa
+        vpshufd zmm15,zmm15,0xff
+        vpaddd  zmm12,zmm12,ZMMWORD[$L$incz]
+        vmovdqa64       zmm28,zmm12
+        vmovdqa64       zmm29,zmm13
+        vmovdqa64       zmm30,zmm14
+        vmovdqa64       zmm31,zmm15
+
+        mov     eax,10
+        jmp     NEAR $L$oop16x
+
+ALIGN   32
+$L$oop_outer16x:
+        vpbroadcastd    zmm0,DWORD[r10]
+        vpbroadcastd    zmm1,DWORD[4+r10]
+        vpbroadcastd    zmm2,DWORD[8+r10]
+        vpbroadcastd    zmm3,DWORD[12+r10]
+        vpaddd  zmm28,zmm28,ZMMWORD[$L$sixteen]
+        vmovdqa64       zmm4,zmm20
+        vmovdqa64       zmm5,zmm21
+        vmovdqa64       zmm6,zmm22
+        vmovdqa64       zmm7,zmm23
+        vmovdqa64       zmm8,zmm24
+        vmovdqa64       zmm9,zmm25
+        vmovdqa64       zmm10,zmm26
+        vmovdqa64       zmm11,zmm27
+        vmovdqa64       zmm12,zmm28
+        vmovdqa64       zmm13,zmm29
+        vmovdqa64       zmm14,zmm30
+        vmovdqa64       zmm15,zmm31
+
+        vmovdqa64       zmm16,zmm0
+        vmovdqa64       zmm17,zmm1
+        vmovdqa64       zmm18,zmm2
+        vmovdqa64       zmm19,zmm3
+
+        mov     eax,10
+        jmp     NEAR $L$oop16x
+
+ALIGN   32
+$L$oop16x:
+        vpaddd  zmm0,zmm0,zmm4
+        vpaddd  zmm1,zmm1,zmm5
+        vpaddd  zmm2,zmm2,zmm6
+        vpaddd  zmm3,zmm3,zmm7
+        vpxord  zmm12,zmm12,zmm0
+        vpxord  zmm13,zmm13,zmm1
+        vpxord  zmm14,zmm14,zmm2
+        vpxord  zmm15,zmm15,zmm3
+        vprold  zmm12,zmm12,16
+        vprold  zmm13,zmm13,16
+        vprold  zmm14,zmm14,16
+        vprold  zmm15,zmm15,16
+        vpaddd  zmm8,zmm8,zmm12
+        vpaddd  zmm9,zmm9,zmm13
+        vpaddd  zmm10,zmm10,zmm14
+        vpaddd  zmm11,zmm11,zmm15
+        vpxord  zmm4,zmm4,zmm8
+        vpxord  zmm5,zmm5,zmm9
+        vpxord  zmm6,zmm6,zmm10
+        vpxord  zmm7,zmm7,zmm11
+        vprold  zmm4,zmm4,12
+        vprold  zmm5,zmm5,12
+        vprold  zmm6,zmm6,12
+        vprold  zmm7,zmm7,12
+        vpaddd  zmm0,zmm0,zmm4
+        vpaddd  zmm1,zmm1,zmm5
+        vpaddd  zmm2,zmm2,zmm6
+        vpaddd  zmm3,zmm3,zmm7
+        vpxord  zmm12,zmm12,zmm0
+        vpxord  zmm13,zmm13,zmm1
+        vpxord  zmm14,zmm14,zmm2
+        vpxord  zmm15,zmm15,zmm3
+        vprold  zmm12,zmm12,8
+        vprold  zmm13,zmm13,8
+        vprold  zmm14,zmm14,8
+        vprold  zmm15,zmm15,8
+        vpaddd  zmm8,zmm8,zmm12
+        vpaddd  zmm9,zmm9,zmm13
+        vpaddd  zmm10,zmm10,zmm14
+        vpaddd  zmm11,zmm11,zmm15
+        vpxord  zmm4,zmm4,zmm8
+        vpxord  zmm5,zmm5,zmm9
+        vpxord  zmm6,zmm6,zmm10
+        vpxord  zmm7,zmm7,zmm11
+        vprold  zmm4,zmm4,7
+        vprold  zmm5,zmm5,7
+        vprold  zmm6,zmm6,7
+        vprold  zmm7,zmm7,7
+        vpaddd  zmm0,zmm0,zmm5
+        vpaddd  zmm1,zmm1,zmm6
+        vpaddd  zmm2,zmm2,zmm7
+        vpaddd  zmm3,zmm3,zmm4
+        vpxord  zmm15,zmm15,zmm0
+        vpxord  zmm12,zmm12,zmm1
+        vpxord  zmm13,zmm13,zmm2
+        vpxord  zmm14,zmm14,zmm3
+        vprold  zmm15,zmm15,16
+        vprold  zmm12,zmm12,16
+        vprold  zmm13,zmm13,16
+        vprold  zmm14,zmm14,16
+        vpaddd  zmm10,zmm10,zmm15
+        vpaddd  zmm11,zmm11,zmm12
+        vpaddd  zmm8,zmm8,zmm13
+        vpaddd  zmm9,zmm9,zmm14
+        vpxord  zmm5,zmm5,zmm10
+        vpxord  zmm6,zmm6,zmm11
+        vpxord  zmm7,zmm7,zmm8
+        vpxord  zmm4,zmm4,zmm9
+        vprold  zmm5,zmm5,12
+        vprold  zmm6,zmm6,12
+        vprold  zmm7,zmm7,12
+        vprold  zmm4,zmm4,12
+        vpaddd  zmm0,zmm0,zmm5
+        vpaddd  zmm1,zmm1,zmm6
+        vpaddd  zmm2,zmm2,zmm7
+        vpaddd  zmm3,zmm3,zmm4
+        vpxord  zmm15,zmm15,zmm0
+        vpxord  zmm12,zmm12,zmm1
+        vpxord  zmm13,zmm13,zmm2
+        vpxord  zmm14,zmm14,zmm3
+        vprold  zmm15,zmm15,8
+        vprold  zmm12,zmm12,8
+        vprold  zmm13,zmm13,8
+        vprold  zmm14,zmm14,8
+        vpaddd  zmm10,zmm10,zmm15
+        vpaddd  zmm11,zmm11,zmm12
+        vpaddd  zmm8,zmm8,zmm13
+        vpaddd  zmm9,zmm9,zmm14
+        vpxord  zmm5,zmm5,zmm10
+        vpxord  zmm6,zmm6,zmm11
+        vpxord  zmm7,zmm7,zmm8
+        vpxord  zmm4,zmm4,zmm9
+        vprold  zmm5,zmm5,7
+        vprold  zmm6,zmm6,7
+        vprold  zmm7,zmm7,7
+        vprold  zmm4,zmm4,7
+        dec     eax
+        jnz     NEAR $L$oop16x
+
+        vpaddd  zmm0,zmm0,zmm16
+        vpaddd  zmm1,zmm1,zmm17
+        vpaddd  zmm2,zmm2,zmm18
+        vpaddd  zmm3,zmm3,zmm19
+
+        vpunpckldq      zmm18,zmm0,zmm1
+        vpunpckldq      zmm19,zmm2,zmm3
+        vpunpckhdq      zmm0,zmm0,zmm1
+        vpunpckhdq      zmm2,zmm2,zmm3
+        vpunpcklqdq     zmm1,zmm18,zmm19
+        vpunpckhqdq     zmm18,zmm18,zmm19
+        vpunpcklqdq     zmm3,zmm0,zmm2
+        vpunpckhqdq     zmm0,zmm0,zmm2
+        vpaddd  zmm4,zmm4,zmm20
+        vpaddd  zmm5,zmm5,zmm21
+        vpaddd  zmm6,zmm6,zmm22
+        vpaddd  zmm7,zmm7,zmm23
+
+        vpunpckldq      zmm2,zmm4,zmm5
+        vpunpckldq      zmm19,zmm6,zmm7
+        vpunpckhdq      zmm4,zmm4,zmm5
+        vpunpckhdq      zmm6,zmm6,zmm7
+        vpunpcklqdq     zmm5,zmm2,zmm19
+        vpunpckhqdq     zmm2,zmm2,zmm19
+        vpunpcklqdq     zmm7,zmm4,zmm6
+        vpunpckhqdq     zmm4,zmm4,zmm6
+        vshufi32x4      zmm19,zmm1,zmm5,0x44
+        vshufi32x4      zmm5,zmm1,zmm5,0xee
+        vshufi32x4      zmm1,zmm18,zmm2,0x44
+        vshufi32x4      zmm2,zmm18,zmm2,0xee
+        vshufi32x4      zmm18,zmm3,zmm7,0x44
+        vshufi32x4      zmm7,zmm3,zmm7,0xee
+        vshufi32x4      zmm3,zmm0,zmm4,0x44
+        vshufi32x4      zmm4,zmm0,zmm4,0xee
+        vpaddd  zmm8,zmm8,zmm24
+        vpaddd  zmm9,zmm9,zmm25
+        vpaddd  zmm10,zmm10,zmm26
+        vpaddd  zmm11,zmm11,zmm27
+
+        vpunpckldq      zmm6,zmm8,zmm9
+        vpunpckldq      zmm0,zmm10,zmm11
+        vpunpckhdq      zmm8,zmm8,zmm9
+        vpunpckhdq      zmm10,zmm10,zmm11
+        vpunpcklqdq     zmm9,zmm6,zmm0
+        vpunpckhqdq     zmm6,zmm6,zmm0
+        vpunpcklqdq     zmm11,zmm8,zmm10
+        vpunpckhqdq     zmm8,zmm8,zmm10
+        vpaddd  zmm12,zmm12,zmm28
+        vpaddd  zmm13,zmm13,zmm29
+        vpaddd  zmm14,zmm14,zmm30
+        vpaddd  zmm15,zmm15,zmm31
+
+        vpunpckldq      zmm10,zmm12,zmm13
+        vpunpckldq      zmm0,zmm14,zmm15
+        vpunpckhdq      zmm12,zmm12,zmm13
+        vpunpckhdq      zmm14,zmm14,zmm15
+        vpunpcklqdq     zmm13,zmm10,zmm0
+        vpunpckhqdq     zmm10,zmm10,zmm0
+        vpunpcklqdq     zmm15,zmm12,zmm14
+        vpunpckhqdq     zmm12,zmm12,zmm14
+        vshufi32x4      zmm0,zmm9,zmm13,0x44
+        vshufi32x4      zmm13,zmm9,zmm13,0xee
+        vshufi32x4      zmm9,zmm6,zmm10,0x44
+        vshufi32x4      zmm10,zmm6,zmm10,0xee
+        vshufi32x4      zmm6,zmm11,zmm15,0x44
+        vshufi32x4      zmm15,zmm11,zmm15,0xee
+        vshufi32x4      zmm11,zmm8,zmm12,0x44
+        vshufi32x4      zmm12,zmm8,zmm12,0xee
+        vshufi32x4      zmm16,zmm19,zmm0,0x88
+        vshufi32x4      zmm19,zmm19,zmm0,0xdd
+        vshufi32x4      zmm0,zmm5,zmm13,0x88
+        vshufi32x4      zmm13,zmm5,zmm13,0xdd
+        vshufi32x4      zmm17,zmm1,zmm9,0x88
+        vshufi32x4      zmm1,zmm1,zmm9,0xdd
+        vshufi32x4      zmm9,zmm2,zmm10,0x88
+        vshufi32x4      zmm10,zmm2,zmm10,0xdd
+        vshufi32x4      zmm14,zmm18,zmm6,0x88
+        vshufi32x4      zmm18,zmm18,zmm6,0xdd
+        vshufi32x4      zmm6,zmm7,zmm15,0x88
+        vshufi32x4      zmm15,zmm7,zmm15,0xdd
+        vshufi32x4      zmm8,zmm3,zmm11,0x88
+        vshufi32x4      zmm3,zmm3,zmm11,0xdd
+        vshufi32x4      zmm11,zmm4,zmm12,0x88
+        vshufi32x4      zmm12,zmm4,zmm12,0xdd
+        cmp     rdx,64*16
+        jb      NEAR $L$tail16x
+
+        vpxord  zmm16,zmm16,ZMMWORD[rsi]
+        vpxord  zmm17,zmm17,ZMMWORD[64+rsi]
+        vpxord  zmm14,zmm14,ZMMWORD[128+rsi]
+        vpxord  zmm8,zmm8,ZMMWORD[192+rsi]
+        vmovdqu32       ZMMWORD[rdi],zmm16
+        vmovdqu32       ZMMWORD[64+rdi],zmm17
+        vmovdqu32       ZMMWORD[128+rdi],zmm14
+        vmovdqu32       ZMMWORD[192+rdi],zmm8
+
+        vpxord  zmm19,zmm19,ZMMWORD[256+rsi]
+        vpxord  zmm1,zmm1,ZMMWORD[320+rsi]
+        vpxord  zmm18,zmm18,ZMMWORD[384+rsi]
+        vpxord  zmm3,zmm3,ZMMWORD[448+rsi]
+        vmovdqu32       ZMMWORD[256+rdi],zmm19
+        vmovdqu32       ZMMWORD[320+rdi],zmm1
+        vmovdqu32       ZMMWORD[384+rdi],zmm18
+        vmovdqu32       ZMMWORD[448+rdi],zmm3
+
+        vpxord  zmm0,zmm0,ZMMWORD[512+rsi]
+        vpxord  zmm9,zmm9,ZMMWORD[576+rsi]
+        vpxord  zmm6,zmm6,ZMMWORD[640+rsi]
+        vpxord  zmm11,zmm11,ZMMWORD[704+rsi]
+        vmovdqu32       ZMMWORD[512+rdi],zmm0
+        vmovdqu32       ZMMWORD[576+rdi],zmm9
+        vmovdqu32       ZMMWORD[640+rdi],zmm6
+        vmovdqu32       ZMMWORD[704+rdi],zmm11
+
+        vpxord  zmm13,zmm13,ZMMWORD[768+rsi]
+        vpxord  zmm10,zmm10,ZMMWORD[832+rsi]
+        vpxord  zmm15,zmm15,ZMMWORD[896+rsi]
+        vpxord  zmm12,zmm12,ZMMWORD[960+rsi]
+        lea     rsi,[1024+rsi]
+        vmovdqu32       ZMMWORD[768+rdi],zmm13
+        vmovdqu32       ZMMWORD[832+rdi],zmm10
+        vmovdqu32       ZMMWORD[896+rdi],zmm15
+        vmovdqu32       ZMMWORD[960+rdi],zmm12
+        lea     rdi,[1024+rdi]
+
+        sub     rdx,64*16
+        jnz     NEAR $L$oop_outer16x
+
+        jmp     NEAR $L$done16x
+
+ALIGN   32
+$L$tail16x:
+        xor     r10,r10
+        sub     rdi,rsi
+        cmp     rdx,64*1
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm16,zmm16,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm16
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm17
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*2
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm17,zmm17,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm17
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm14
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*3
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm14,zmm14,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm14
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm8
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*4
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm8,zmm8,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm8
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm19
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*5
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm19,zmm19,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm19
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm1
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*6
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm1,zmm1,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm1
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm18
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*7
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm18,zmm18,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm18
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm3
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*8
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm3,zmm3,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm3
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm0
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*9
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm0,zmm0,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm0
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm9
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*10
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm9,zmm9,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm9
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm6
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*11
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm6,zmm6,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm6
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm11
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*12
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm11,zmm11,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm11
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm13
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*13
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm13,zmm13,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm13
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm10
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*14
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm10,zmm10,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm10
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm15
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*15
+        jb      NEAR $L$ess_than_64_16x
+        vpxord  zmm15,zmm15,ZMMWORD[rsi]
+        vmovdqu32       ZMMWORD[rsi*1+rdi],zmm15
+        je      NEAR $L$done16x
+        vmovdqa32       zmm16,zmm12
+        lea     rsi,[64+rsi]
+
+$L$ess_than_64_16x:
+        vmovdqa32       ZMMWORD[rsp],zmm16
+        lea     rdi,[rsi*1+rdi]
+        and     rdx,63
+
+$L$oop_tail16x:
+        movzx   eax,BYTE[r10*1+rsi]
+        movzx   ecx,BYTE[r10*1+rsp]
+        lea     r10,[1+r10]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r10*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail16x
+
+        vpxord  zmm16,zmm16,zmm16
+        vmovdqa32       ZMMWORD[rsp],zmm16
+
+$L$done16x:
+        vzeroall
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$16x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_16x:
+
+ALIGN   32
+ChaCha20_8xvl:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ChaCha20_8xvl:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+$L$ChaCha20_8xvl:
+        mov     r9,rsp
+
+        sub     rsp,64+168
+        and     rsp,-64
+        movaps  XMMWORD[(-168)+r9],xmm6
+        movaps  XMMWORD[(-152)+r9],xmm7
+        movaps  XMMWORD[(-136)+r9],xmm8
+        movaps  XMMWORD[(-120)+r9],xmm9
+        movaps  XMMWORD[(-104)+r9],xmm10
+        movaps  XMMWORD[(-88)+r9],xmm11
+        movaps  XMMWORD[(-72)+r9],xmm12
+        movaps  XMMWORD[(-56)+r9],xmm13
+        movaps  XMMWORD[(-40)+r9],xmm14
+        movaps  XMMWORD[(-24)+r9],xmm15
+$L$8xvl_body:
+        vzeroupper
+
+        lea     r10,[$L$sigma]
+        vbroadcasti128  ymm3,XMMWORD[r10]
+        vbroadcasti128  ymm7,XMMWORD[rcx]
+        vbroadcasti128  ymm11,XMMWORD[16+rcx]
+        vbroadcasti128  ymm15,XMMWORD[r8]
+
+        vpshufd ymm0,ymm3,0x00
+        vpshufd ymm1,ymm3,0x55
+        vpshufd ymm2,ymm3,0xaa
+        vpshufd ymm3,ymm3,0xff
+        vmovdqa64       ymm16,ymm0
+        vmovdqa64       ymm17,ymm1
+        vmovdqa64       ymm18,ymm2
+        vmovdqa64       ymm19,ymm3
+
+        vpshufd ymm4,ymm7,0x00
+        vpshufd ymm5,ymm7,0x55
+        vpshufd ymm6,ymm7,0xaa
+        vpshufd ymm7,ymm7,0xff
+        vmovdqa64       ymm20,ymm4
+        vmovdqa64       ymm21,ymm5
+        vmovdqa64       ymm22,ymm6
+        vmovdqa64       ymm23,ymm7
+
+        vpshufd ymm8,ymm11,0x00
+        vpshufd ymm9,ymm11,0x55
+        vpshufd ymm10,ymm11,0xaa
+        vpshufd ymm11,ymm11,0xff
+        vmovdqa64       ymm24,ymm8
+        vmovdqa64       ymm25,ymm9
+        vmovdqa64       ymm26,ymm10
+        vmovdqa64       ymm27,ymm11
+
+        vpshufd ymm12,ymm15,0x00
+        vpshufd ymm13,ymm15,0x55
+        vpshufd ymm14,ymm15,0xaa
+        vpshufd ymm15,ymm15,0xff
+        vpaddd  ymm12,ymm12,YMMWORD[$L$incy]
+        vmovdqa64       ymm28,ymm12
+        vmovdqa64       ymm29,ymm13
+        vmovdqa64       ymm30,ymm14
+        vmovdqa64       ymm31,ymm15
+
+        mov     eax,10
+        jmp     NEAR $L$oop8xvl
+
+ALIGN   32
+$L$oop_outer8xvl:
+
+
+        vpbroadcastd    ymm2,DWORD[8+r10]
+        vpbroadcastd    ymm3,DWORD[12+r10]
+        vpaddd  ymm28,ymm28,YMMWORD[$L$eight]
+        vmovdqa64       ymm4,ymm20
+        vmovdqa64       ymm5,ymm21
+        vmovdqa64       ymm6,ymm22
+        vmovdqa64       ymm7,ymm23
+        vmovdqa64       ymm8,ymm24
+        vmovdqa64       ymm9,ymm25
+        vmovdqa64       ymm10,ymm26
+        vmovdqa64       ymm11,ymm27
+        vmovdqa64       ymm12,ymm28
+        vmovdqa64       ymm13,ymm29
+        vmovdqa64       ymm14,ymm30
+        vmovdqa64       ymm15,ymm31
+
+        vmovdqa64       ymm16,ymm0
+        vmovdqa64       ymm17,ymm1
+        vmovdqa64       ymm18,ymm2
+        vmovdqa64       ymm19,ymm3
+
+        mov     eax,10
+        jmp     NEAR $L$oop8xvl
+
+ALIGN   32
+$L$oop8xvl:
+        vpaddd  ymm0,ymm0,ymm4
+        vpaddd  ymm1,ymm1,ymm5
+        vpaddd  ymm2,ymm2,ymm6
+        vpaddd  ymm3,ymm3,ymm7
+        vpxor   ymm12,ymm12,ymm0
+        vpxor   ymm13,ymm13,ymm1
+        vpxor   ymm14,ymm14,ymm2
+        vpxor   ymm15,ymm15,ymm3
+        vprold  ymm12,ymm12,16
+        vprold  ymm13,ymm13,16
+        vprold  ymm14,ymm14,16
+        vprold  ymm15,ymm15,16
+        vpaddd  ymm8,ymm8,ymm12
+        vpaddd  ymm9,ymm9,ymm13
+        vpaddd  ymm10,ymm10,ymm14
+        vpaddd  ymm11,ymm11,ymm15
+        vpxor   ymm4,ymm4,ymm8
+        vpxor   ymm5,ymm5,ymm9
+        vpxor   ymm6,ymm6,ymm10
+        vpxor   ymm7,ymm7,ymm11
+        vprold  ymm4,ymm4,12
+        vprold  ymm5,ymm5,12
+        vprold  ymm6,ymm6,12
+        vprold  ymm7,ymm7,12
+        vpaddd  ymm0,ymm0,ymm4
+        vpaddd  ymm1,ymm1,ymm5
+        vpaddd  ymm2,ymm2,ymm6
+        vpaddd  ymm3,ymm3,ymm7
+        vpxor   ymm12,ymm12,ymm0
+        vpxor   ymm13,ymm13,ymm1
+        vpxor   ymm14,ymm14,ymm2
+        vpxor   ymm15,ymm15,ymm3
+        vprold  ymm12,ymm12,8
+        vprold  ymm13,ymm13,8
+        vprold  ymm14,ymm14,8
+        vprold  ymm15,ymm15,8
+        vpaddd  ymm8,ymm8,ymm12
+        vpaddd  ymm9,ymm9,ymm13
+        vpaddd  ymm10,ymm10,ymm14
+        vpaddd  ymm11,ymm11,ymm15
+        vpxor   ymm4,ymm4,ymm8
+        vpxor   ymm5,ymm5,ymm9
+        vpxor   ymm6,ymm6,ymm10
+        vpxor   ymm7,ymm7,ymm11
+        vprold  ymm4,ymm4,7
+        vprold  ymm5,ymm5,7
+        vprold  ymm6,ymm6,7
+        vprold  ymm7,ymm7,7
+        vpaddd  ymm0,ymm0,ymm5
+        vpaddd  ymm1,ymm1,ymm6
+        vpaddd  ymm2,ymm2,ymm7
+        vpaddd  ymm3,ymm3,ymm4
+        vpxor   ymm15,ymm15,ymm0
+        vpxor   ymm12,ymm12,ymm1
+        vpxor   ymm13,ymm13,ymm2
+        vpxor   ymm14,ymm14,ymm3
+        vprold  ymm15,ymm15,16
+        vprold  ymm12,ymm12,16
+        vprold  ymm13,ymm13,16
+        vprold  ymm14,ymm14,16
+        vpaddd  ymm10,ymm10,ymm15
+        vpaddd  ymm11,ymm11,ymm12
+        vpaddd  ymm8,ymm8,ymm13
+        vpaddd  ymm9,ymm9,ymm14
+        vpxor   ymm5,ymm5,ymm10
+        vpxor   ymm6,ymm6,ymm11
+        vpxor   ymm7,ymm7,ymm8
+        vpxor   ymm4,ymm4,ymm9
+        vprold  ymm5,ymm5,12
+        vprold  ymm6,ymm6,12
+        vprold  ymm7,ymm7,12
+        vprold  ymm4,ymm4,12
+        vpaddd  ymm0,ymm0,ymm5
+        vpaddd  ymm1,ymm1,ymm6
+        vpaddd  ymm2,ymm2,ymm7
+        vpaddd  ymm3,ymm3,ymm4
+        vpxor   ymm15,ymm15,ymm0
+        vpxor   ymm12,ymm12,ymm1
+        vpxor   ymm13,ymm13,ymm2
+        vpxor   ymm14,ymm14,ymm3
+        vprold  ymm15,ymm15,8
+        vprold  ymm12,ymm12,8
+        vprold  ymm13,ymm13,8
+        vprold  ymm14,ymm14,8
+        vpaddd  ymm10,ymm10,ymm15
+        vpaddd  ymm11,ymm11,ymm12
+        vpaddd  ymm8,ymm8,ymm13
+        vpaddd  ymm9,ymm9,ymm14
+        vpxor   ymm5,ymm5,ymm10
+        vpxor   ymm6,ymm6,ymm11
+        vpxor   ymm7,ymm7,ymm8
+        vpxor   ymm4,ymm4,ymm9
+        vprold  ymm5,ymm5,7
+        vprold  ymm6,ymm6,7
+        vprold  ymm7,ymm7,7
+        vprold  ymm4,ymm4,7
+        dec     eax
+        jnz     NEAR $L$oop8xvl
+
+        vpaddd  ymm0,ymm0,ymm16
+        vpaddd  ymm1,ymm1,ymm17
+        vpaddd  ymm2,ymm2,ymm18
+        vpaddd  ymm3,ymm3,ymm19
+
+        vpunpckldq      ymm18,ymm0,ymm1
+        vpunpckldq      ymm19,ymm2,ymm3
+        vpunpckhdq      ymm0,ymm0,ymm1
+        vpunpckhdq      ymm2,ymm2,ymm3
+        vpunpcklqdq     ymm1,ymm18,ymm19
+        vpunpckhqdq     ymm18,ymm18,ymm19
+        vpunpcklqdq     ymm3,ymm0,ymm2
+        vpunpckhqdq     ymm0,ymm0,ymm2
+        vpaddd  ymm4,ymm4,ymm20
+        vpaddd  ymm5,ymm5,ymm21
+        vpaddd  ymm6,ymm6,ymm22
+        vpaddd  ymm7,ymm7,ymm23
+
+        vpunpckldq      ymm2,ymm4,ymm5
+        vpunpckldq      ymm19,ymm6,ymm7
+        vpunpckhdq      ymm4,ymm4,ymm5
+        vpunpckhdq      ymm6,ymm6,ymm7
+        vpunpcklqdq     ymm5,ymm2,ymm19
+        vpunpckhqdq     ymm2,ymm2,ymm19
+        vpunpcklqdq     ymm7,ymm4,ymm6
+        vpunpckhqdq     ymm4,ymm4,ymm6
+        vshufi32x4      ymm19,ymm1,ymm5,0
+        vshufi32x4      ymm5,ymm1,ymm5,3
+        vshufi32x4      ymm1,ymm18,ymm2,0
+        vshufi32x4      ymm2,ymm18,ymm2,3
+        vshufi32x4      ymm18,ymm3,ymm7,0
+        vshufi32x4      ymm7,ymm3,ymm7,3
+        vshufi32x4      ymm3,ymm0,ymm4,0
+        vshufi32x4      ymm4,ymm0,ymm4,3
+        vpaddd  ymm8,ymm8,ymm24
+        vpaddd  ymm9,ymm9,ymm25
+        vpaddd  ymm10,ymm10,ymm26
+        vpaddd  ymm11,ymm11,ymm27
+
+        vpunpckldq      ymm6,ymm8,ymm9
+        vpunpckldq      ymm0,ymm10,ymm11
+        vpunpckhdq      ymm8,ymm8,ymm9
+        vpunpckhdq      ymm10,ymm10,ymm11
+        vpunpcklqdq     ymm9,ymm6,ymm0
+        vpunpckhqdq     ymm6,ymm6,ymm0
+        vpunpcklqdq     ymm11,ymm8,ymm10
+        vpunpckhqdq     ymm8,ymm8,ymm10
+        vpaddd  ymm12,ymm12,ymm28
+        vpaddd  ymm13,ymm13,ymm29
+        vpaddd  ymm14,ymm14,ymm30
+        vpaddd  ymm15,ymm15,ymm31
+
+        vpunpckldq      ymm10,ymm12,ymm13
+        vpunpckldq      ymm0,ymm14,ymm15
+        vpunpckhdq      ymm12,ymm12,ymm13
+        vpunpckhdq      ymm14,ymm14,ymm15
+        vpunpcklqdq     ymm13,ymm10,ymm0
+        vpunpckhqdq     ymm10,ymm10,ymm0
+        vpunpcklqdq     ymm15,ymm12,ymm14
+        vpunpckhqdq     ymm12,ymm12,ymm14
+        vperm2i128      ymm0,ymm9,ymm13,0x20
+        vperm2i128      ymm13,ymm9,ymm13,0x31
+        vperm2i128      ymm9,ymm6,ymm10,0x20
+        vperm2i128      ymm10,ymm6,ymm10,0x31
+        vperm2i128      ymm6,ymm11,ymm15,0x20
+        vperm2i128      ymm15,ymm11,ymm15,0x31
+        vperm2i128      ymm11,ymm8,ymm12,0x20
+        vperm2i128      ymm12,ymm8,ymm12,0x31
+        cmp     rdx,64*8
+        jb      NEAR $L$tail8xvl
+
+        mov     eax,0x80
+        vpxord  ymm19,ymm19,YMMWORD[rsi]
+        vpxor   ymm0,ymm0,YMMWORD[32+rsi]
+        vpxor   ymm5,ymm5,YMMWORD[64+rsi]
+        vpxor   ymm13,ymm13,YMMWORD[96+rsi]
+        lea     rsi,[rax*1+rsi]
+        vmovdqu32       YMMWORD[rdi],ymm19
+        vmovdqu YMMWORD[32+rdi],ymm0
+        vmovdqu YMMWORD[64+rdi],ymm5
+        vmovdqu YMMWORD[96+rdi],ymm13
+        lea     rdi,[rax*1+rdi]
+
+        vpxor   ymm1,ymm1,YMMWORD[rsi]
+        vpxor   ymm9,ymm9,YMMWORD[32+rsi]
+        vpxor   ymm2,ymm2,YMMWORD[64+rsi]
+        vpxor   ymm10,ymm10,YMMWORD[96+rsi]
+        lea     rsi,[rax*1+rsi]
+        vmovdqu YMMWORD[rdi],ymm1
+        vmovdqu YMMWORD[32+rdi],ymm9
+        vmovdqu YMMWORD[64+rdi],ymm2
+        vmovdqu YMMWORD[96+rdi],ymm10
+        lea     rdi,[rax*1+rdi]
+
+        vpxord  ymm18,ymm18,YMMWORD[rsi]
+        vpxor   ymm6,ymm6,YMMWORD[32+rsi]
+        vpxor   ymm7,ymm7,YMMWORD[64+rsi]
+        vpxor   ymm15,ymm15,YMMWORD[96+rsi]
+        lea     rsi,[rax*1+rsi]
+        vmovdqu32       YMMWORD[rdi],ymm18
+        vmovdqu YMMWORD[32+rdi],ymm6
+        vmovdqu YMMWORD[64+rdi],ymm7
+        vmovdqu YMMWORD[96+rdi],ymm15
+        lea     rdi,[rax*1+rdi]
+
+        vpxor   ymm3,ymm3,YMMWORD[rsi]
+        vpxor   ymm11,ymm11,YMMWORD[32+rsi]
+        vpxor   ymm4,ymm4,YMMWORD[64+rsi]
+        vpxor   ymm12,ymm12,YMMWORD[96+rsi]
+        lea     rsi,[rax*1+rsi]
+        vmovdqu YMMWORD[rdi],ymm3
+        vmovdqu YMMWORD[32+rdi],ymm11
+        vmovdqu YMMWORD[64+rdi],ymm4
+        vmovdqu YMMWORD[96+rdi],ymm12
+        lea     rdi,[rax*1+rdi]
+
+        vpbroadcastd    ymm0,DWORD[r10]
+        vpbroadcastd    ymm1,DWORD[4+r10]
+
+        sub     rdx,64*8
+        jnz     NEAR $L$oop_outer8xvl
+
+        jmp     NEAR $L$done8xvl
+
+ALIGN   32
+$L$tail8xvl:
+        vmovdqa64       ymm8,ymm19
+        xor     r10,r10
+        sub     rdi,rsi
+        cmp     rdx,64*1
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm8,ymm8,YMMWORD[rsi]
+        vpxor   ymm0,ymm0,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm8
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm0
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm5
+        vmovdqa ymm0,ymm13
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*2
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm5,ymm5,YMMWORD[rsi]
+        vpxor   ymm13,ymm13,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm5
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm13
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm1
+        vmovdqa ymm0,ymm9
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*3
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm1,ymm1,YMMWORD[rsi]
+        vpxor   ymm9,ymm9,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm1
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm9
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm2
+        vmovdqa ymm0,ymm10
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*4
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm2,ymm2,YMMWORD[rsi]
+        vpxor   ymm10,ymm10,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm2
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm10
+        je      NEAR $L$done8xvl
+        vmovdqa32       ymm8,ymm18
+        vmovdqa ymm0,ymm6
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*5
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxord  ymm18,ymm18,YMMWORD[rsi]
+        vpxor   ymm6,ymm6,YMMWORD[32+rsi]
+        vmovdqu32       YMMWORD[rsi*1+rdi],ymm18
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm6
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm7
+        vmovdqa ymm0,ymm15
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*6
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm7,ymm7,YMMWORD[rsi]
+        vpxor   ymm15,ymm15,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm7
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm15
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm3
+        vmovdqa ymm0,ymm11
+        lea     rsi,[64+rsi]
+
+        cmp     rdx,64*7
+        jb      NEAR $L$ess_than_64_8xvl
+        vpxor   ymm3,ymm3,YMMWORD[rsi]
+        vpxor   ymm11,ymm11,YMMWORD[32+rsi]
+        vmovdqu YMMWORD[rsi*1+rdi],ymm3
+        vmovdqu YMMWORD[32+rsi*1+rdi],ymm11
+        je      NEAR $L$done8xvl
+        vmovdqa ymm8,ymm4
+        vmovdqa ymm0,ymm12
+        lea     rsi,[64+rsi]
+
+$L$ess_than_64_8xvl:
+        vmovdqa YMMWORD[rsp],ymm8
+        vmovdqa YMMWORD[32+rsp],ymm0
+        lea     rdi,[rsi*1+rdi]
+        and     rdx,63
+
+$L$oop_tail8xvl:
+        movzx   eax,BYTE[r10*1+rsi]
+        movzx   ecx,BYTE[r10*1+rsp]
+        lea     r10,[1+r10]
+        xor     eax,ecx
+        mov     BYTE[((-1))+r10*1+rdi],al
+        dec     rdx
+        jnz     NEAR $L$oop_tail8xvl
+
+        vpxor   ymm8,ymm8,ymm8
+        vmovdqa YMMWORD[rsp],ymm8
+        vmovdqa YMMWORD[32+rsp],ymm8
+
+$L$done8xvl:
+        vzeroall
+        movaps  xmm6,XMMWORD[((-168))+r9]
+        movaps  xmm7,XMMWORD[((-152))+r9]
+        movaps  xmm8,XMMWORD[((-136))+r9]
+        movaps  xmm9,XMMWORD[((-120))+r9]
+        movaps  xmm10,XMMWORD[((-104))+r9]
+        movaps  xmm11,XMMWORD[((-88))+r9]
+        movaps  xmm12,XMMWORD[((-72))+r9]
+        movaps  xmm13,XMMWORD[((-56))+r9]
+        movaps  xmm14,XMMWORD[((-40))+r9]
+        movaps  xmm15,XMMWORD[((-24))+r9]
+        lea     rsp,[r9]
+
+$L$8xvl_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ChaCha20_8xvl:
 EXTERN  __imp_RtlVirtualUnwind
 
@@ -1406 +3871 @@
         DD      $L$SEH_end_ChaCha20_4x wrt ..imagebase
         DD      $L$SEH_info_ChaCha20_4x wrt ..imagebase
+        DD      $L$SEH_begin_ChaCha20_4xop wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_4xop wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_4xop wrt ..imagebase
+        DD      $L$SEH_begin_ChaCha20_8x wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_8x wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_8x wrt ..imagebase
+        DD      $L$SEH_begin_ChaCha20_avx512 wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_avx512 wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_avx512 wrt ..imagebase
+
+        DD      $L$SEH_begin_ChaCha20_avx512vl wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_avx512vl wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_avx512vl wrt ..imagebase
+
+        DD      $L$SEH_begin_ChaCha20_16x wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_16x wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_16x wrt ..imagebase
+
+        DD      $L$SEH_begin_ChaCha20_8xvl wrt ..imagebase
+        DD      $L$SEH_end_ChaCha20_8xvl wrt ..imagebase
+        DD      $L$SEH_info_ChaCha20_8xvl wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -1429 +3915 @@
         DD      $L$4x_body wrt ..imagebase,$L$4x_epilogue wrt ..imagebase
         DD      0xa0,0
+$L$SEH_info_ChaCha20_4xop:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$4xop_body wrt ..imagebase,$L$4xop_epilogue wrt ..imagebase
+        DD      0xa0,0
+$L$SEH_info_ChaCha20_8x:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$8x_body wrt ..imagebase,$L$8x_epilogue wrt ..imagebase
+        DD      0xa0,0
+$L$SEH_info_ChaCha20_avx512:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$avx512_body wrt ..imagebase,$L$avx512_epilogue wrt ..imagebase
+        DD      0x20,0
+
+$L$SEH_info_ChaCha20_avx512vl:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$avx512vl_body wrt ..imagebase,$L$avx512vl_epilogue wrt ..imagebase
+        DD      0x20,0
+
+$L$SEH_info_ChaCha20_16x:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$16x_body wrt ..imagebase,$L$16x_epilogue wrt ..imagebase
+        DD      0xa0,0
+
+$L$SEH_info_ChaCha20_8xvl:
+DB      9,0,0,0
+        DD      simd_handler wrt ..imagebase
+        DD      $L$8xvl_body wrt ..imagebase,$L$8xvl_epilogue wrt ..imagebase
+        DD      0xa0,0

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/ecp_nistz256-x86_64.S

@@ -2854 +2854 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     ecx,0x80100
+        je      NEAR $L$ecp_nistz256_ord_mul_montx
         push    rbp
 
@@ -3187 +3191 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     ecx,0x80100
+        je      NEAR $L$ecp_nistz256_ord_sqr_montx
         push    rbp
 
@@ -3471 +3479 @@
 
 
+ALIGN   32
+ecp_nistz256_ord_mul_montx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ecp_nistz256_ord_mul_montx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$ecp_nistz256_ord_mul_montx:
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$ord_mulx_body:
+
+        mov     rbx,rdx
+        mov     rdx,QWORD[rdx]
+        mov     r9,QWORD[rsi]
+        mov     r10,QWORD[8+rsi]
+        mov     r11,QWORD[16+rsi]
+        mov     r12,QWORD[24+rsi]
+        lea     rsi,[((-128))+rsi]
+        lea     r14,[(($L$ord-128))]
+        mov     r15,QWORD[$L$ordK]
+
+
+        mulx    r9,r8,r9
+        mulx    r10,rcx,r10
+        mulx    r11,rbp,r11
+        add     r9,rcx
+        mulx    r12,rcx,r12
+        mov     rdx,r8
+        mulx    rax,rdx,r15
+        adc     r10,rbp
+        adc     r11,rcx
+        adc     r12,0
+
+
+        xor     r13,r13
+        mulx    rbp,rcx,QWORD[((0+128))+r14]
+        adcx    r8,rcx
+        adox    r9,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+r14]
+        adcx    r9,rcx
+        adox    r10,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+r14]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+r14]
+        mov     rdx,QWORD[8+rbx]
+        adcx    r11,rcx
+        adox    r12,rbp
+        adcx    r12,r8
+        adox    r13,r8
+        adc     r13,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r9,rcx
+        adox    r10,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r9
+        mulx    rax,rdx,r15
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        adcx    r13,r8
+        adox    r8,r8
+        adc     r8,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+r14]
+        adcx    r9,rcx
+        adox    r10,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+r14]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+r14]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+r14]
+        mov     rdx,QWORD[16+rbx]
+        adcx    r12,rcx
+        adox    r13,rbp
+        adcx    r13,r9
+        adox    r8,r9
+        adc     r8,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r10
+        mulx    rax,rdx,r15
+        adcx    r13,rcx
+        adox    r8,rbp
+
+        adcx    r8,r9
+        adox    r9,r9
+        adc     r9,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+r14]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+r14]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+r14]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+r14]
+        mov     rdx,QWORD[24+rbx]
+        adcx    r13,rcx
+        adox    r8,rbp
+        adcx    r8,r10
+        adox    r9,r10
+        adc     r9,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r13,rcx
+        adox    r8,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r11
+        mulx    rax,rdx,r15
+        adcx    r8,rcx
+        adox    r9,rbp
+
+        adcx    r9,r10
+        adox    r10,r10
+        adc     r10,0
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+r14]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+r14]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+r14]
+        adcx    r13,rcx
+        adox    r8,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+r14]
+        lea     r14,[128+r14]
+        mov     rbx,r12
+        adcx    r8,rcx
+        adox    r9,rbp
+        mov     rdx,r13
+        adcx    r9,r11
+        adox    r10,r11
+        adc     r10,0
+
+
+
+        mov     rcx,r8
+        sub     r12,QWORD[r14]
+        sbb     r13,QWORD[8+r14]
+        sbb     r8,QWORD[16+r14]
+        mov     rbp,r9
+        sbb     r9,QWORD[24+r14]
+        sbb     r10,0
+
+        cmovc   r12,rbx
+        cmovc   r13,rdx
+        cmovc   r8,rcx
+        cmovc   r9,rbp
+
+        mov     QWORD[rdi],r12
+        mov     QWORD[8+rdi],r13
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbx,QWORD[32+rsp]
+
+        mov     rbp,QWORD[40+rsp]
+
+        lea     rsp,[48+rsp]
+
+$L$ord_mulx_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_ord_mul_montx:
+
+
+ALIGN   32
+ecp_nistz256_ord_sqr_montx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ecp_nistz256_ord_sqr_montx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$ecp_nistz256_ord_sqr_montx:
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$ord_sqrx_body:
+
+        mov     rbx,rdx
+        mov     rdx,QWORD[rsi]
+        mov     r14,QWORD[8+rsi]
+        mov     r15,QWORD[16+rsi]
+        mov     r8,QWORD[24+rsi]
+        lea     rsi,[$L$ord]
+        jmp     NEAR $L$oop_ord_sqrx
+
+ALIGN   32
+$L$oop_ord_sqrx:
+        mulx    r10,r9,r14
+        mulx    r11,rcx,r15
+        mov     rax,rdx
+DB      102,73,15,110,206
+        mulx    r12,rbp,r8
+        mov     rdx,r14
+        add     r10,rcx
+DB      102,73,15,110,215
+        adc     r11,rbp
+        adc     r12,0
+        xor     r13,r13
+
+        mulx    rbp,rcx,r15
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,r8
+        mov     rdx,r15
+        adcx    r12,rcx
+        adox    r13,rbp
+        adc     r13,0
+
+        mulx    r14,rcx,r8
+        mov     rdx,rax
+DB      102,73,15,110,216
+        xor     r15,r15
+        adcx    r9,r9
+        adox    r13,rcx
+        adcx    r10,r10
+        adox    r14,r15
+
+
+        mulx    rbp,r8,rdx
+DB      102,72,15,126,202
+        adcx    r11,r11
+        adox    r9,rbp
+        adcx    r12,r12
+        mulx    rax,rcx,rdx
+DB      102,72,15,126,210
+        adcx    r13,r13
+        adox    r10,rcx
+        adcx    r14,r14
+        mulx    rbp,rcx,rdx
+DB      0x67
+DB      102,72,15,126,218
+        adox    r11,rax
+        adcx    r15,r15
+        adox    r12,rcx
+        adox    r13,rbp
+        mulx    rax,rcx,rdx
+        adox    r14,rcx
+        adox    r15,rax
+
+
+        mov     rdx,r8
+        mulx    rcx,rdx,QWORD[32+rsi]
+
+        xor     rax,rax
+        mulx    rbp,rcx,QWORD[rsi]
+        adcx    r8,rcx
+        adox    r9,rbp
+        mulx    rbp,rcx,QWORD[8+rsi]
+        adcx    r9,rcx
+        adox    r10,rbp
+        mulx    rbp,rcx,QWORD[16+rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+        mulx    rbp,rcx,QWORD[24+rsi]
+        adcx    r11,rcx
+        adox    r8,rbp
+        adcx    r8,rax
+
+
+        mov     rdx,r9
+        mulx    rcx,rdx,QWORD[32+rsi]
+
+        mulx    rbp,rcx,QWORD[rsi]
+        adox    r9,rcx
+        adcx    r10,rbp
+        mulx    rbp,rcx,QWORD[8+rsi]
+        adox    r10,rcx
+        adcx    r11,rbp
+        mulx    rbp,rcx,QWORD[16+rsi]
+        adox    r11,rcx
+        adcx    r8,rbp
+        mulx    rbp,rcx,QWORD[24+rsi]
+        adox    r8,rcx
+        adcx    r9,rbp
+        adox    r9,rax
+
+
+        mov     rdx,r10
+        mulx    rcx,rdx,QWORD[32+rsi]
+
+        mulx    rbp,rcx,QWORD[rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+        mulx    rbp,rcx,QWORD[8+rsi]
+        adcx    r11,rcx
+        adox    r8,rbp
+        mulx    rbp,rcx,QWORD[16+rsi]
+        adcx    r8,rcx
+        adox    r9,rbp
+        mulx    rbp,rcx,QWORD[24+rsi]
+        adcx    r9,rcx
+        adox    r10,rbp
+        adcx    r10,rax
+
+
+        mov     rdx,r11
+        mulx    rcx,rdx,QWORD[32+rsi]
+
+        mulx    rbp,rcx,QWORD[rsi]
+        adox    r11,rcx
+        adcx    r8,rbp
+        mulx    rbp,rcx,QWORD[8+rsi]
+        adox    r8,rcx
+        adcx    r9,rbp
+        mulx    rbp,rcx,QWORD[16+rsi]
+        adox    r9,rcx
+        adcx    r10,rbp
+        mulx    rbp,rcx,QWORD[24+rsi]
+        adox    r10,rcx
+        adcx    r11,rbp
+        adox    r11,rax
+
+
+        add     r12,r8
+        adc     r9,r13
+        mov     rdx,r12
+        adc     r10,r14
+        adc     r11,r15
+        mov     r14,r9
+        adc     rax,0
+
+
+        sub     r12,QWORD[rsi]
+        mov     r15,r10
+        sbb     r9,QWORD[8+rsi]
+        sbb     r10,QWORD[16+rsi]
+        mov     r8,r11
+        sbb     r11,QWORD[24+rsi]
+        sbb     rax,0
+
+        cmovnc  rdx,r12
+        cmovnc  r14,r9
+        cmovnc  r15,r10
+        cmovnc  r8,r11
+
+        dec     rbx
+        jnz     NEAR $L$oop_ord_sqrx
+
+        mov     QWORD[rdi],rdx
+        mov     QWORD[8+rdi],r14
+        pxor    xmm1,xmm1
+        mov     QWORD[16+rdi],r15
+        pxor    xmm2,xmm2
+        mov     QWORD[24+rdi],r8
+        pxor    xmm3,xmm3
+
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbx,QWORD[32+rsp]
+
+        mov     rbp,QWORD[40+rsp]
+
+        lea     rsp,[48+rsp]
+
+$L$ord_sqrx_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_ord_sqr_montx:
+
+
 
 
@@ -3486 +3960 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
         lea     rdx,[$L$RR]
         jmp     NEAR $L$mul_mont
@@ -3511 +3987 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
 $L$mul_mont:
         push    rbp
@@ -3525 +4003 @@
 
 $L$mul_body:
+        cmp     ecx,0x80100
+        je      NEAR $L$mul_montx
         mov     rbx,rdx
         mov     rax,QWORD[rdx]
@@ -3533 +4013 @@
 
         call    __ecp_nistz256_mul_montq
+        jmp     NEAR $L$mul_mont_done
+
+ALIGN   32
+$L$mul_montx:
+        mov     rbx,rdx
+        mov     rdx,QWORD[rdx]
+        mov     r9,QWORD[rsi]
+        mov     r10,QWORD[8+rsi]
+        mov     r11,QWORD[16+rsi]
+        mov     r12,QWORD[24+rsi]
+        lea     rsi,[((-128))+rsi]
+
+        call    __ecp_nistz256_mul_montx
 $L$mul_mont_done:
         mov     r15,QWORD[rsp]
@@ -3793 +4286 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
         push    rbp
 
@@ -3806 +4301 @@
 
 $L$sqr_body:
+        cmp     ecx,0x80100
+        je      NEAR $L$sqr_montx
         mov     rax,QWORD[rsi]
         mov     r14,QWORD[8+rsi]
@@ -3812 +4309 @@
 
         call    __ecp_nistz256_sqr_montq
+        jmp     NEAR $L$sqr_mont_done
+
+ALIGN   32
+$L$sqr_montx:
+        mov     rdx,QWORD[rsi]
+        mov     r14,QWORD[8+rsi]
+        mov     r15,QWORD[16+rsi]
+        mov     r8,QWORD[24+rsi]
+        lea     rsi,[((-128))+rsi]
+
+        call    __ecp_nistz256_sqr_montx
 $L$sqr_mont_done:
         mov     r15,QWORD[rsp]
@@ -3991 +4499 @@
         mov     QWORD[8+rdi],r13
         cmovc   r15,rcx
+        mov     QWORD[16+rdi],r14
+        mov     QWORD[24+rdi],r15
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+ALIGN   32
+__ecp_nistz256_mul_montx:
+
+
+
+        mulx    r9,r8,r9
+        mulx    r10,rcx,r10
+        mov     r14,32
+        xor     r13,r13
+        mulx    r11,rbp,r11
+        mov     r15,QWORD[(($L$poly+24))]
+        adc     r9,rcx
+        mulx    r12,rcx,r12
+        mov     rdx,r8
+        adc     r10,rbp
+        shlx    rbp,r8,r14
+        adc     r11,rcx
+        shrx    rcx,r8,r14
+        adc     r12,0
+
+
+
+        add     r9,rbp
+        adc     r10,rcx
+
+        mulx    rbp,rcx,r15
+        mov     rdx,QWORD[8+rbx]
+        adc     r11,rcx
+        adc     r12,rbp
+        adc     r13,0
+        xor     r8,r8
+
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r9,rcx
+        adox    r10,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r9
+        adcx    r12,rcx
+        shlx    rcx,r9,r14
+        adox    r13,rbp
+        shrx    rbp,r9,r14
+
+        adcx    r13,r8
+        adox    r8,r8
+        adc     r8,0
+
+
+
+        add     r10,rcx
+        adc     r11,rbp
+
+        mulx    rbp,rcx,r15
+        mov     rdx,QWORD[16+rbx]
+        adc     r12,rcx
+        adc     r13,rbp
+        adc     r8,0
+        xor     r9,r9
+
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r10,rcx
+        adox    r11,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r10
+        adcx    r13,rcx
+        shlx    rcx,r10,r14
+        adox    r8,rbp
+        shrx    rbp,r10,r14
+
+        adcx    r8,r9
+        adox    r9,r9
+        adc     r9,0
+
+
+
+        add     r11,rcx
+        adc     r12,rbp
+
+        mulx    rbp,rcx,r15
+        mov     rdx,QWORD[24+rbx]
+        adc     r13,rcx
+        adc     r8,rbp
+        adc     r9,0
+        xor     r10,r10
+
+
+
+        mulx    rbp,rcx,QWORD[((0+128))+rsi]
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,QWORD[((8+128))+rsi]
+        adcx    r12,rcx
+        adox    r13,rbp
+
+        mulx    rbp,rcx,QWORD[((16+128))+rsi]
+        adcx    r13,rcx
+        adox    r8,rbp
+
+        mulx    rbp,rcx,QWORD[((24+128))+rsi]
+        mov     rdx,r11
+        adcx    r8,rcx
+        shlx    rcx,r11,r14
+        adox    r9,rbp
+        shrx    rbp,r11,r14
+
+        adcx    r9,r10
+        adox    r10,r10
+        adc     r10,0
+
+
+
+        add     r12,rcx
+        adc     r13,rbp
+
+        mulx    rbp,rcx,r15
+        mov     rbx,r12
+        mov     r14,QWORD[(($L$poly+8))]
+        adc     r8,rcx
+        mov     rdx,r13
+        adc     r9,rbp
+        adc     r10,0
+
+
+
+        xor     eax,eax
+        mov     rcx,r8
+        sbb     r12,-1
+        sbb     r13,r14
+        sbb     r8,0
+        mov     rbp,r9
+        sbb     r9,r15
+        sbb     r10,0
+
+        cmovc   r12,rbx
+        cmovc   r13,rdx
+        mov     QWORD[rdi],r12
+        cmovc   r8,rcx
+        mov     QWORD[8+rdi],r13
+        cmovc   r9,rbp
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+__ecp_nistz256_sqr_montx:
+
+        mulx    r10,r9,r14
+        mulx    r11,rcx,r15
+        xor     eax,eax
+        adc     r10,rcx
+        mulx    r12,rbp,r8
+        mov     rdx,r14
+        adc     r11,rbp
+        adc     r12,0
+        xor     r13,r13
+
+
+        mulx    rbp,rcx,r15
+        adcx    r11,rcx
+        adox    r12,rbp
+
+        mulx    rbp,rcx,r8
+        mov     rdx,r15
+        adcx    r12,rcx
+        adox    r13,rbp
+        adc     r13,0
+
+
+        mulx    r14,rcx,r8
+        mov     rdx,QWORD[((0+128))+rsi]
+        xor     r15,r15
+        adcx    r9,r9
+        adox    r13,rcx
+        adcx    r10,r10
+        adox    r14,r15
+
+        mulx    rbp,r8,rdx
+        mov     rdx,QWORD[((8+128))+rsi]
+        adcx    r11,r11
+        adox    r9,rbp
+        adcx    r12,r12
+        mulx    rax,rcx,rdx
+        mov     rdx,QWORD[((16+128))+rsi]
+        adcx    r13,r13
+        adox    r10,rcx
+        adcx    r14,r14
+DB      0x67
+        mulx    rbp,rcx,rdx
+        mov     rdx,QWORD[((24+128))+rsi]
+        adox    r11,rax
+        adcx    r15,r15
+        adox    r12,rcx
+        mov     rsi,32
+        adox    r13,rbp
+DB      0x67,0x67
+        mulx    rax,rcx,rdx
+        mov     rdx,QWORD[(($L$poly+24))]
+        adox    r14,rcx
+        shlx    rcx,r8,rsi
+        adox    r15,rax
+        shrx    rax,r8,rsi
+        mov     rbp,rdx
+
+
+        add     r9,rcx
+        adc     r10,rax
+
+        mulx    r8,rcx,r8
+        adc     r11,rcx
+        shlx    rcx,r9,rsi
+        adc     r8,0
+        shrx    rax,r9,rsi
+
+
+        add     r10,rcx
+        adc     r11,rax
+
+        mulx    r9,rcx,r9
+        adc     r8,rcx
+        shlx    rcx,r10,rsi
+        adc     r9,0
+        shrx    rax,r10,rsi
+
+
+        add     r11,rcx
+        adc     r8,rax
+
+        mulx    r10,rcx,r10
+        adc     r9,rcx
+        shlx    rcx,r11,rsi
+        adc     r10,0
+        shrx    rax,r11,rsi
+
+
+        add     r8,rcx
+        adc     r9,rax
+
+        mulx    r11,rcx,r11
+        adc     r10,rcx
+        adc     r11,0
+
+        xor     rdx,rdx
+        add     r12,r8
+        mov     rsi,QWORD[(($L$poly+8))]
+        adc     r13,r9
+        mov     r8,r12
+        adc     r14,r10
+        adc     r15,r11
+        mov     r9,r13
+        adc     rdx,0
+
+        sub     r12,-1
+        mov     r10,r14
+        sbb     r13,rsi
+        sbb     r14,0
+        mov     r11,r15
+        sbb     r15,rbp
+        sbb     rdx,0
+
+        cmovc   r12,r8
+        cmovc   r13,r9
+        mov     QWORD[rdi],r12
+        cmovc   r14,r10
+        mov     QWORD[8+rdi],r13
+        cmovc   r15,r11
         mov     QWORD[16+rdi],r14
         mov     QWORD[24+rdi],r15
@@ -4146 +4952 @@
 ecp_nistz256_gather_w5:
 
+        mov     eax,DWORD[((OPENSSL_ia32cap_P+8))]
+        test    eax,32
+        jnz     NEAR $L$avx2_gather_w5
         lea     rax,[((-136))+rsp]
 $L$SEH_begin_ecp_nistz256_gather_w5:
@@ -4253 +5062 @@
 ecp_nistz256_gather_w7:
 
+        mov     eax,DWORD[((OPENSSL_ia32cap_P+8))]
+        test    eax,32
+        jnz     NEAR $L$avx2_gather_w7
         lea     rax,[((-136))+rsp]
 $L$SEH_begin_ecp_nistz256_gather_w7:
@@ -4320 +5132 @@
 $L$SEH_end_ecp_nistz256_gather_w7:
 
+
+
+
+ALIGN   32
+ecp_nistz256_avx2_gather_w5:
+
+$L$avx2_gather_w5:
+        vzeroupper
+        lea     rax,[((-136))+rsp]
+        mov     r11,rsp
+$L$SEH_begin_ecp_nistz256_avx2_gather_w5:
+DB      0x48,0x8d,0x60,0xe0
+DB      0xc5,0xf8,0x29,0x70,0xe0
+DB      0xc5,0xf8,0x29,0x78,0xf0
+DB      0xc5,0x78,0x29,0x40,0x00
+DB      0xc5,0x78,0x29,0x48,0x10
+DB      0xc5,0x78,0x29,0x50,0x20
+DB      0xc5,0x78,0x29,0x58,0x30
+DB      0xc5,0x78,0x29,0x60,0x40
+DB      0xc5,0x78,0x29,0x68,0x50
+DB      0xc5,0x78,0x29,0x70,0x60
+DB      0xc5,0x78,0x29,0x78,0x70
+        vmovdqa ymm0,YMMWORD[$L$Two]
+
+        vpxor   ymm2,ymm2,ymm2
+        vpxor   ymm3,ymm3,ymm3
+        vpxor   ymm4,ymm4,ymm4
+
+        vmovdqa ymm5,YMMWORD[$L$One]
+        vmovdqa ymm10,YMMWORD[$L$Two]
+
+        vmovd   xmm1,r8d
+        vpermd  ymm1,ymm2,ymm1
+
+        mov     rax,8
+$L$select_loop_avx2_w5:
+
+        vmovdqa ymm6,YMMWORD[rdx]
+        vmovdqa ymm7,YMMWORD[32+rdx]
+        vmovdqa ymm8,YMMWORD[64+rdx]
+
+        vmovdqa ymm11,YMMWORD[96+rdx]
+        vmovdqa ymm12,YMMWORD[128+rdx]
+        vmovdqa ymm13,YMMWORD[160+rdx]
+
+        vpcmpeqd        ymm9,ymm5,ymm1
+        vpcmpeqd        ymm14,ymm10,ymm1
+
+        vpaddd  ymm5,ymm5,ymm0
+        vpaddd  ymm10,ymm10,ymm0
+        lea     rdx,[192+rdx]
+
+        vpand   ymm6,ymm6,ymm9
+        vpand   ymm7,ymm7,ymm9
+        vpand   ymm8,ymm8,ymm9
+        vpand   ymm11,ymm11,ymm14
+        vpand   ymm12,ymm12,ymm14
+        vpand   ymm13,ymm13,ymm14
+
+        vpxor   ymm2,ymm2,ymm6
+        vpxor   ymm3,ymm3,ymm7
+        vpxor   ymm4,ymm4,ymm8
+        vpxor   ymm2,ymm2,ymm11
+        vpxor   ymm3,ymm3,ymm12
+        vpxor   ymm4,ymm4,ymm13
+
+        dec     rax
+        jnz     NEAR $L$select_loop_avx2_w5
+
+        vmovdqu YMMWORD[rcx],ymm2
+        vmovdqu YMMWORD[32+rcx],ymm3
+        vmovdqu YMMWORD[64+rcx],ymm4
+        vzeroupper
+        movaps  xmm6,XMMWORD[rsp]
+        movaps  xmm7,XMMWORD[16+rsp]
+        movaps  xmm8,XMMWORD[32+rsp]
+        movaps  xmm9,XMMWORD[48+rsp]
+        movaps  xmm10,XMMWORD[64+rsp]
+        movaps  xmm11,XMMWORD[80+rsp]
+        movaps  xmm12,XMMWORD[96+rsp]
+        movaps  xmm13,XMMWORD[112+rsp]
+        movaps  xmm14,XMMWORD[128+rsp]
+        movaps  xmm15,XMMWORD[144+rsp]
+        lea     rsp,[r11]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_avx2_gather_w5:
+
+
+
+
 global  ecp_nistz256_avx2_gather_w7
 
 ALIGN   32
 ecp_nistz256_avx2_gather_w7:
-        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
-        mov     QWORD[16+rsp],rsi
-        mov     rax,rsp
+
+$L$avx2_gather_w7:
+        vzeroupper
+        mov     r11,rsp
+        lea     rax,[((-136))+rsp]
 $L$SEH_begin_ecp_nistz256_avx2_gather_w7:
-        mov     rdi,rcx
-        mov     rsi,rdx
-        mov     rdx,r8
-
-
-
-DB      0x0f,0x0b
-        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
-        mov     rsi,QWORD[16+rsp]
+DB      0x48,0x8d,0x60,0xe0
+DB      0xc5,0xf8,0x29,0x70,0xe0
+DB      0xc5,0xf8,0x29,0x78,0xf0
+DB      0xc5,0x78,0x29,0x40,0x00
+DB      0xc5,0x78,0x29,0x48,0x10
+DB      0xc5,0x78,0x29,0x50,0x20
+DB      0xc5,0x78,0x29,0x58,0x30
+DB      0xc5,0x78,0x29,0x60,0x40
+DB      0xc5,0x78,0x29,0x68,0x50
+DB      0xc5,0x78,0x29,0x70,0x60
+DB      0xc5,0x78,0x29,0x78,0x70
+        vmovdqa ymm0,YMMWORD[$L$Three]
+
+        vpxor   ymm2,ymm2,ymm2
+        vpxor   ymm3,ymm3,ymm3
+
+        vmovdqa ymm4,YMMWORD[$L$One]
+        vmovdqa ymm8,YMMWORD[$L$Two]
+        vmovdqa ymm12,YMMWORD[$L$Three]
+
+        vmovd   xmm1,r8d
+        vpermd  ymm1,ymm2,ymm1
+
+
+        mov     rax,21
+$L$select_loop_avx2_w7:
+
+        vmovdqa ymm5,YMMWORD[rdx]
+        vmovdqa ymm6,YMMWORD[32+rdx]
+
+        vmovdqa ymm9,YMMWORD[64+rdx]
+        vmovdqa ymm10,YMMWORD[96+rdx]
+
+        vmovdqa ymm13,YMMWORD[128+rdx]
+        vmovdqa ymm14,YMMWORD[160+rdx]
+
+        vpcmpeqd        ymm7,ymm4,ymm1
+        vpcmpeqd        ymm11,ymm8,ymm1
+        vpcmpeqd        ymm15,ymm12,ymm1
+
+        vpaddd  ymm4,ymm4,ymm0
+        vpaddd  ymm8,ymm8,ymm0
+        vpaddd  ymm12,ymm12,ymm0
+        lea     rdx,[192+rdx]
+
+        vpand   ymm5,ymm5,ymm7
+        vpand   ymm6,ymm6,ymm7
+        vpand   ymm9,ymm9,ymm11
+        vpand   ymm10,ymm10,ymm11
+        vpand   ymm13,ymm13,ymm15
+        vpand   ymm14,ymm14,ymm15
+
+        vpxor   ymm2,ymm2,ymm5
+        vpxor   ymm3,ymm3,ymm6
+        vpxor   ymm2,ymm2,ymm9
+        vpxor   ymm3,ymm3,ymm10
+        vpxor   ymm2,ymm2,ymm13
+        vpxor   ymm3,ymm3,ymm14
+
+        dec     rax
+        jnz     NEAR $L$select_loop_avx2_w7
+
+
+        vmovdqa ymm5,YMMWORD[rdx]
+        vmovdqa ymm6,YMMWORD[32+rdx]
+
+        vpcmpeqd        ymm7,ymm4,ymm1
+
+        vpand   ymm5,ymm5,ymm7
+        vpand   ymm6,ymm6,ymm7
+
+        vpxor   ymm2,ymm2,ymm5
+        vpxor   ymm3,ymm3,ymm6
+
+        vmovdqu YMMWORD[rcx],ymm2
+        vmovdqu YMMWORD[32+rcx],ymm3
+        vzeroupper
+        movaps  xmm6,XMMWORD[rsp]
+        movaps  xmm7,XMMWORD[16+rsp]
+        movaps  xmm8,XMMWORD[32+rsp]
+        movaps  xmm9,XMMWORD[48+rsp]
+        movaps  xmm10,XMMWORD[64+rsp]
+        movaps  xmm11,XMMWORD[80+rsp]
+        movaps  xmm12,XMMWORD[96+rsp]
+        movaps  xmm13,XMMWORD[112+rsp]
+        movaps  xmm14,XMMWORD[128+rsp]
+        movaps  xmm15,XMMWORD[144+rsp]
+        lea     rsp,[r11]
         DB      0F3h,0C3h               ;repret
 
 $L$SEH_end_ecp_nistz256_avx2_gather_w7:
+
 
 ALIGN   32
@@ -4482 +5467 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     ecx,0x80100
+        je      NEAR $L$point_doublex
         push    rbp
 
@@ -4715 +5704 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     ecx,0x80100
+        je      NEAR $L$point_addx
         push    rbp
 
@@ -5134 +6127 @@
 
 
+        mov     ecx,0x80100
+        and     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     ecx,0x80100
+        je      NEAR $L$point_add_affinex
         push    rbp
 
@@ -5453 +6450 @@
 
 $L$SEH_end_ecp_nistz256_point_add_affine:
+
+ALIGN   32
+__ecp_nistz256_add_tox:
+
+        xor     r11,r11
+        adc     r12,QWORD[rbx]
+        adc     r13,QWORD[8+rbx]
+        mov     rax,r12
+        adc     r8,QWORD[16+rbx]
+        adc     r9,QWORD[24+rbx]
+        mov     rbp,r13
+        adc     r11,0
+
+        xor     r10,r10
+        sbb     r12,-1
+        mov     rcx,r8
+        sbb     r13,r14
+        sbb     r8,0
+        mov     r10,r9
+        sbb     r9,r15
+        sbb     r11,0
+
+        cmovc   r12,rax
+        cmovc   r13,rbp
+        mov     QWORD[rdi],r12
+        cmovc   r8,rcx
+        mov     QWORD[8+rdi],r13
+        cmovc   r9,r10
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+__ecp_nistz256_sub_fromx:
+
+        xor     r11,r11
+        sbb     r12,QWORD[rbx]
+        sbb     r13,QWORD[8+rbx]
+        mov     rax,r12
+        sbb     r8,QWORD[16+rbx]
+        sbb     r9,QWORD[24+rbx]
+        mov     rbp,r13
+        sbb     r11,0
+
+        xor     r10,r10
+        adc     r12,-1
+        mov     rcx,r8
+        adc     r13,r14
+        adc     r8,0
+        mov     r10,r9
+        adc     r9,r15
+
+        bt      r11,0
+        cmovnc  r12,rax
+        cmovnc  r13,rbp
+        mov     QWORD[rdi],r12
+        cmovnc  r8,rcx
+        mov     QWORD[8+rdi],r13
+        cmovnc  r9,r10
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+__ecp_nistz256_subx:
+
+        xor     r11,r11
+        sbb     rax,r12
+        sbb     rbp,r13
+        mov     r12,rax
+        sbb     rcx,r8
+        sbb     r10,r9
+        mov     r13,rbp
+        sbb     r11,0
+
+        xor     r9,r9
+        adc     rax,-1
+        mov     r8,rcx
+        adc     rbp,r14
+        adc     rcx,0
+        mov     r9,r10
+        adc     r10,r15
+
+        bt      r11,0
+        cmovc   r12,rax
+        cmovc   r13,rbp
+        cmovc   r8,rcx
+        cmovc   r9,r10
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+__ecp_nistz256_mul_by_2x:
+
+        xor     r11,r11
+        adc     r12,r12
+        adc     r13,r13
+        mov     rax,r12
+        adc     r8,r8
+        adc     r9,r9
+        mov     rbp,r13
+        adc     r11,0
+
+        xor     r10,r10
+        sbb     r12,-1
+        mov     rcx,r8
+        sbb     r13,r14
+        sbb     r8,0
+        mov     r10,r9
+        sbb     r9,r15
+        sbb     r11,0
+
+        cmovc   r12,rax
+        cmovc   r13,rbp
+        mov     QWORD[rdi],r12
+        cmovc   r8,rcx
+        mov     QWORD[8+rdi],r13
+        cmovc   r9,r10
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+ALIGN   32
+ecp_nistz256_point_doublex:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ecp_nistz256_point_doublex:
+        mov     rdi,rcx
+        mov     rsi,rdx
+
+
+
+$L$point_doublex:
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,32*5+8
+
+$L$point_doublex_body:
+
+$L$point_double_shortcutx:
+        movdqu  xmm0,XMMWORD[rsi]
+        mov     rbx,rsi
+        movdqu  xmm1,XMMWORD[16+rsi]
+        mov     r12,QWORD[((32+0))+rsi]
+        mov     r13,QWORD[((32+8))+rsi]
+        mov     r8,QWORD[((32+16))+rsi]
+        mov     r9,QWORD[((32+24))+rsi]
+        mov     r14,QWORD[(($L$poly+8))]
+        mov     r15,QWORD[(($L$poly+24))]
+        movdqa  XMMWORD[96+rsp],xmm0
+        movdqa  XMMWORD[(96+16)+rsp],xmm1
+        lea     r10,[32+rdi]
+        lea     r11,[64+rdi]
+DB      102,72,15,110,199
+DB      102,73,15,110,202
+DB      102,73,15,110,211
+
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_mul_by_2x
+
+        mov     rdx,QWORD[((64+0))+rsi]
+        mov     r14,QWORD[((64+8))+rsi]
+        mov     r15,QWORD[((64+16))+rsi]
+        mov     r8,QWORD[((64+24))+rsi]
+        lea     rsi,[((64-128))+rsi]
+        lea     rdi,[64+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[((0+0))+rsp]
+        mov     r14,QWORD[((8+0))+rsp]
+        lea     rsi,[((-128+0))+rsp]
+        mov     r15,QWORD[((16+0))+rsp]
+        mov     r8,QWORD[((24+0))+rsp]
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[32+rbx]
+        mov     r9,QWORD[((64+0))+rbx]
+        mov     r10,QWORD[((64+8))+rbx]
+        mov     r11,QWORD[((64+16))+rbx]
+        mov     r12,QWORD[((64+24))+rbx]
+        lea     rsi,[((64-128))+rbx]
+        lea     rbx,[32+rbx]
+DB      102,72,15,126,215
+        call    __ecp_nistz256_mul_montx
+        call    __ecp_nistz256_mul_by_2x
+
+        mov     r12,QWORD[((96+0))+rsp]
+        mov     r13,QWORD[((96+8))+rsp]
+        lea     rbx,[64+rsp]
+        mov     r8,QWORD[((96+16))+rsp]
+        mov     r9,QWORD[((96+24))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_add_tox
+
+        mov     r12,QWORD[((96+0))+rsp]
+        mov     r13,QWORD[((96+8))+rsp]
+        lea     rbx,[64+rsp]
+        mov     r8,QWORD[((96+16))+rsp]
+        mov     r9,QWORD[((96+24))+rsp]
+        lea     rdi,[64+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rdx,QWORD[((0+0))+rsp]
+        mov     r14,QWORD[((8+0))+rsp]
+        lea     rsi,[((-128+0))+rsp]
+        mov     r15,QWORD[((16+0))+rsp]
+        mov     r8,QWORD[((24+0))+rsp]
+DB      102,72,15,126,207
+        call    __ecp_nistz256_sqr_montx
+        xor     r9,r9
+        mov     rax,r12
+        add     r12,-1
+        mov     r10,r13
+        adc     r13,rsi
+        mov     rcx,r14
+        adc     r14,0
+        mov     r8,r15
+        adc     r15,rbp
+        adc     r9,0
+        xor     rsi,rsi
+        test    rax,1
+
+        cmovz   r12,rax
+        cmovz   r13,r10
+        cmovz   r14,rcx
+        cmovz   r15,r8
+        cmovz   r9,rsi
+
+        mov     rax,r13
+        shr     r12,1
+        shl     rax,63
+        mov     r10,r14
+        shr     r13,1
+        or      r12,rax
+        shl     r10,63
+        mov     rcx,r15
+        shr     r14,1
+        or      r13,r10
+        shl     rcx,63
+        mov     QWORD[rdi],r12
+        shr     r15,1
+        mov     QWORD[8+rdi],r13
+        shl     r9,63
+        or      r14,rcx
+        or      r15,r9
+        mov     QWORD[16+rdi],r14
+        mov     QWORD[24+rdi],r15
+        mov     rdx,QWORD[64+rsp]
+        lea     rbx,[64+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rdi,[128+rsp]
+        call    __ecp_nistz256_mul_by_2x
+
+        lea     rbx,[32+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_add_tox
+
+        mov     rdx,QWORD[96+rsp]
+        lea     rbx,[96+rsp]
+        mov     r9,QWORD[((0+0))+rsp]
+        mov     r10,QWORD[((8+0))+rsp]
+        lea     rsi,[((-128+0))+rsp]
+        mov     r11,QWORD[((16+0))+rsp]
+        mov     r12,QWORD[((24+0))+rsp]
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rdi,[128+rsp]
+        call    __ecp_nistz256_mul_by_2x
+
+        mov     rdx,QWORD[((0+32))+rsp]
+        mov     r14,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r15,QWORD[((16+32))+rsp]
+        mov     r8,QWORD[((24+32))+rsp]
+DB      102,72,15,126,199
+        call    __ecp_nistz256_sqr_montx
+
+        lea     rbx,[128+rsp]
+        mov     r8,r14
+        mov     r9,r15
+        mov     r14,rsi
+        mov     r15,rbp
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rax,QWORD[((0+0))+rsp]
+        mov     rbp,QWORD[((0+8))+rsp]
+        mov     rcx,QWORD[((0+16))+rsp]
+        mov     r10,QWORD[((0+24))+rsp]
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_subx
+
+        mov     rdx,QWORD[32+rsp]
+        lea     rbx,[32+rsp]
+        mov     r14,r12
+        xor     ecx,ecx
+        mov     QWORD[((0+0))+rsp],r12
+        mov     r10,r13
+        mov     QWORD[((0+8))+rsp],r13
+        cmovz   r11,r8
+        mov     QWORD[((0+16))+rsp],r8
+        lea     rsi,[((0-128))+rsp]
+        cmovz   r12,r9
+        mov     QWORD[((0+24))+rsp],r9
+        mov     r9,r14
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_mul_montx
+
+DB      102,72,15,126,203
+DB      102,72,15,126,207
+        call    __ecp_nistz256_sub_fromx
+
+        lea     rsi,[((160+56))+rsp]
+
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbx,QWORD[((-16))+rsi]
+
+        mov     rbp,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$point_doublex_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_point_doublex:
+
+ALIGN   32
+ecp_nistz256_point_addx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ecp_nistz256_point_addx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$point_addx:
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,32*18+8
+
+$L$point_addx_body:
+
+        movdqu  xmm0,XMMWORD[rsi]
+        movdqu  xmm1,XMMWORD[16+rsi]
+        movdqu  xmm2,XMMWORD[32+rsi]
+        movdqu  xmm3,XMMWORD[48+rsi]
+        movdqu  xmm4,XMMWORD[64+rsi]
+        movdqu  xmm5,XMMWORD[80+rsi]
+        mov     rbx,rsi
+        mov     rsi,rdx
+        movdqa  XMMWORD[384+rsp],xmm0
+        movdqa  XMMWORD[(384+16)+rsp],xmm1
+        movdqa  XMMWORD[416+rsp],xmm2
+        movdqa  XMMWORD[(416+16)+rsp],xmm3
+        movdqa  XMMWORD[448+rsp],xmm4
+        movdqa  XMMWORD[(448+16)+rsp],xmm5
+        por     xmm5,xmm4
+
+        movdqu  xmm0,XMMWORD[rsi]
+        pshufd  xmm3,xmm5,0xb1
+        movdqu  xmm1,XMMWORD[16+rsi]
+        movdqu  xmm2,XMMWORD[32+rsi]
+        por     xmm5,xmm3
+        movdqu  xmm3,XMMWORD[48+rsi]
+        mov     rdx,QWORD[((64+0))+rsi]
+        mov     r14,QWORD[((64+8))+rsi]
+        mov     r15,QWORD[((64+16))+rsi]
+        mov     r8,QWORD[((64+24))+rsi]
+        movdqa  XMMWORD[480+rsp],xmm0
+        pshufd  xmm4,xmm5,0x1e
+        movdqa  XMMWORD[(480+16)+rsp],xmm1
+        movdqu  xmm0,XMMWORD[64+rsi]
+        movdqu  xmm1,XMMWORD[80+rsi]
+        movdqa  XMMWORD[512+rsp],xmm2
+        movdqa  XMMWORD[(512+16)+rsp],xmm3
+        por     xmm5,xmm4
+        pxor    xmm4,xmm4
+        por     xmm1,xmm0
+DB      102,72,15,110,199
+
+        lea     rsi,[((64-128))+rsi]
+        mov     QWORD[((544+0))+rsp],rdx
+        mov     QWORD[((544+8))+rsp],r14
+        mov     QWORD[((544+16))+rsp],r15
+        mov     QWORD[((544+24))+rsp],r8
+        lea     rdi,[96+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        pcmpeqd xmm5,xmm4
+        pshufd  xmm4,xmm1,0xb1
+        por     xmm4,xmm1
+        pshufd  xmm5,xmm5,0
+        pshufd  xmm3,xmm4,0x1e
+        por     xmm4,xmm3
+        pxor    xmm3,xmm3
+        pcmpeqd xmm4,xmm3
+        pshufd  xmm4,xmm4,0
+        mov     rdx,QWORD[((64+0))+rbx]
+        mov     r14,QWORD[((64+8))+rbx]
+        mov     r15,QWORD[((64+16))+rbx]
+        mov     r8,QWORD[((64+24))+rbx]
+DB      102,72,15,110,203
+
+        lea     rsi,[((64-128))+rbx]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[544+rsp]
+        lea     rbx,[544+rsp]
+        mov     r9,QWORD[((0+96))+rsp]
+        mov     r10,QWORD[((8+96))+rsp]
+        lea     rsi,[((-128+96))+rsp]
+        mov     r11,QWORD[((16+96))+rsp]
+        mov     r12,QWORD[((24+96))+rsp]
+        lea     rdi,[224+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[448+rsp]
+        lea     rbx,[448+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[256+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[416+rsp]
+        lea     rbx,[416+rsp]
+        mov     r9,QWORD[((0+224))+rsp]
+        mov     r10,QWORD[((8+224))+rsp]
+        lea     rsi,[((-128+224))+rsp]
+        mov     r11,QWORD[((16+224))+rsp]
+        mov     r12,QWORD[((24+224))+rsp]
+        lea     rdi,[224+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[512+rsp]
+        lea     rbx,[512+rsp]
+        mov     r9,QWORD[((0+256))+rsp]
+        mov     r10,QWORD[((8+256))+rsp]
+        lea     rsi,[((-128+256))+rsp]
+        mov     r11,QWORD[((16+256))+rsp]
+        mov     r12,QWORD[((24+256))+rsp]
+        lea     rdi,[256+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[224+rsp]
+        lea     rdi,[64+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        or      r12,r13
+        movdqa  xmm2,xmm4
+        or      r12,r8
+        or      r12,r9
+        por     xmm2,xmm5
+DB      102,73,15,110,220
+
+        mov     rdx,QWORD[384+rsp]
+        lea     rbx,[384+rsp]
+        mov     r9,QWORD[((0+96))+rsp]
+        mov     r10,QWORD[((8+96))+rsp]
+        lea     rsi,[((-128+96))+rsp]
+        mov     r11,QWORD[((16+96))+rsp]
+        mov     r12,QWORD[((24+96))+rsp]
+        lea     rdi,[160+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[480+rsp]
+        lea     rbx,[480+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[192+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[160+rsp]
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        or      r12,r13
+        or      r12,r8
+        or      r12,r9
+
+DB      102,73,15,126,208
+DB      102,73,15,126,217
+
+        or      r12,r8
+        or      r12,r9
+
+
+DB      0x3e
+        jnz     NEAR $L$add_proceedx
+
+$L$add_doublex:
+DB      102,72,15,126,206
+DB      102,72,15,126,199
+        add     rsp,416
+
+        jmp     NEAR $L$point_double_shortcutx
+
+
+ALIGN   32
+$L$add_proceedx:
+        mov     rdx,QWORD[((0+64))+rsp]
+        mov     r14,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r15,QWORD[((16+64))+rsp]
+        mov     r8,QWORD[((24+64))+rsp]
+        lea     rdi,[96+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[448+rsp]
+        lea     rbx,[448+rsp]
+        mov     r9,QWORD[((0+0))+rsp]
+        mov     r10,QWORD[((8+0))+rsp]
+        lea     rsi,[((-128+0))+rsp]
+        mov     r11,QWORD[((16+0))+rsp]
+        mov     r12,QWORD[((24+0))+rsp]
+        lea     rdi,[352+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[((0+0))+rsp]
+        mov     r14,QWORD[((8+0))+rsp]
+        lea     rsi,[((-128+0))+rsp]
+        mov     r15,QWORD[((16+0))+rsp]
+        mov     r8,QWORD[((24+0))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[544+rsp]
+        lea     rbx,[544+rsp]
+        mov     r9,QWORD[((0+352))+rsp]
+        mov     r10,QWORD[((8+352))+rsp]
+        lea     rsi,[((-128+352))+rsp]
+        mov     r11,QWORD[((16+352))+rsp]
+        mov     r12,QWORD[((24+352))+rsp]
+        lea     rdi,[352+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[rsp]
+        lea     rbx,[rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[128+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[160+rsp]
+        lea     rbx,[160+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[192+rsp]
+        call    __ecp_nistz256_mul_montx
+
+
+
+
+        xor     r11,r11
+        add     r12,r12
+        lea     rsi,[96+rsp]
+        adc     r13,r13
+        mov     rax,r12
+        adc     r8,r8
+        adc     r9,r9
+        mov     rbp,r13
+        adc     r11,0
+
+        sub     r12,-1
+        mov     rcx,r8
+        sbb     r13,r14
+        sbb     r8,0
+        mov     r10,r9
+        sbb     r9,r15
+        sbb     r11,0
+
+        cmovc   r12,rax
+        mov     rax,QWORD[rsi]
+        cmovc   r13,rbp
+        mov     rbp,QWORD[8+rsi]
+        cmovc   r8,rcx
+        mov     rcx,QWORD[16+rsi]
+        cmovc   r9,r10
+        mov     r10,QWORD[24+rsi]
+
+        call    __ecp_nistz256_subx
+
+        lea     rbx,[128+rsp]
+        lea     rdi,[288+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rax,QWORD[((192+0))+rsp]
+        mov     rbp,QWORD[((192+8))+rsp]
+        mov     rcx,QWORD[((192+16))+rsp]
+        mov     r10,QWORD[((192+24))+rsp]
+        lea     rdi,[320+rsp]
+
+        call    __ecp_nistz256_subx
+
+        mov     QWORD[rdi],r12
+        mov     QWORD[8+rdi],r13
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+        mov     rdx,QWORD[128+rsp]
+        lea     rbx,[128+rsp]
+        mov     r9,QWORD[((0+224))+rsp]
+        mov     r10,QWORD[((8+224))+rsp]
+        lea     rsi,[((-128+224))+rsp]
+        mov     r11,QWORD[((16+224))+rsp]
+        mov     r12,QWORD[((24+224))+rsp]
+        lea     rdi,[256+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[320+rsp]
+        lea     rbx,[320+rsp]
+        mov     r9,QWORD[((0+64))+rsp]
+        mov     r10,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r11,QWORD[((16+64))+rsp]
+        mov     r12,QWORD[((24+64))+rsp]
+        lea     rdi,[320+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[256+rsp]
+        lea     rdi,[320+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+DB      102,72,15,126,199
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[352+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((352+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[544+rsp]
+        pand    xmm3,XMMWORD[((544+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[448+rsp]
+        pand    xmm3,XMMWORD[((448+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[64+rdi],xmm2
+        movdqu  XMMWORD[80+rdi],xmm3
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[288+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((288+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[480+rsp]
+        pand    xmm3,XMMWORD[((480+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[384+rsp]
+        pand    xmm3,XMMWORD[((384+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[rdi],xmm2
+        movdqu  XMMWORD[16+rdi],xmm3
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[320+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((320+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[512+rsp]
+        pand    xmm3,XMMWORD[((512+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[416+rsp]
+        pand    xmm3,XMMWORD[((416+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[32+rdi],xmm2
+        movdqu  XMMWORD[48+rdi],xmm3
+
+$L$add_donex:
+        lea     rsi,[((576+56))+rsp]
+
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbx,QWORD[((-16))+rsi]
+
+        mov     rbp,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$point_addx_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_point_addx:
+
+ALIGN   32
+ecp_nistz256_point_add_affinex:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_ecp_nistz256_point_add_affinex:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$point_add_affinex:
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,32*15+8
+
+$L$add_affinex_body:
+
+        movdqu  xmm0,XMMWORD[rsi]
+        mov     rbx,rdx
+        movdqu  xmm1,XMMWORD[16+rsi]
+        movdqu  xmm2,XMMWORD[32+rsi]
+        movdqu  xmm3,XMMWORD[48+rsi]
+        movdqu  xmm4,XMMWORD[64+rsi]
+        movdqu  xmm5,XMMWORD[80+rsi]
+        mov     rdx,QWORD[((64+0))+rsi]
+        mov     r14,QWORD[((64+8))+rsi]
+        mov     r15,QWORD[((64+16))+rsi]
+        mov     r8,QWORD[((64+24))+rsi]
+        movdqa  XMMWORD[320+rsp],xmm0
+        movdqa  XMMWORD[(320+16)+rsp],xmm1
+        movdqa  XMMWORD[352+rsp],xmm2
+        movdqa  XMMWORD[(352+16)+rsp],xmm3
+        movdqa  XMMWORD[384+rsp],xmm4
+        movdqa  XMMWORD[(384+16)+rsp],xmm5
+        por     xmm5,xmm4
+
+        movdqu  xmm0,XMMWORD[rbx]
+        pshufd  xmm3,xmm5,0xb1
+        movdqu  xmm1,XMMWORD[16+rbx]
+        movdqu  xmm2,XMMWORD[32+rbx]
+        por     xmm5,xmm3
+        movdqu  xmm3,XMMWORD[48+rbx]
+        movdqa  XMMWORD[416+rsp],xmm0
+        pshufd  xmm4,xmm5,0x1e
+        movdqa  XMMWORD[(416+16)+rsp],xmm1
+        por     xmm1,xmm0
+DB      102,72,15,110,199
+        movdqa  XMMWORD[448+rsp],xmm2
+        movdqa  XMMWORD[(448+16)+rsp],xmm3
+        por     xmm3,xmm2
+        por     xmm5,xmm4
+        pxor    xmm4,xmm4
+        por     xmm3,xmm1
+
+        lea     rsi,[((64-128))+rsi]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        pcmpeqd xmm5,xmm4
+        pshufd  xmm4,xmm3,0xb1
+        mov     rdx,QWORD[rbx]
+
+        mov     r9,r12
+        por     xmm4,xmm3
+        pshufd  xmm5,xmm5,0
+        pshufd  xmm3,xmm4,0x1e
+        mov     r10,r13
+        por     xmm4,xmm3
+        pxor    xmm3,xmm3
+        mov     r11,r14
+        pcmpeqd xmm4,xmm3
+        pshufd  xmm4,xmm4,0
+
+        lea     rsi,[((32-128))+rsp]
+        mov     r12,r15
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[320+rsp]
+        lea     rdi,[64+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rdx,QWORD[384+rsp]
+        lea     rbx,[384+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[384+rsp]
+        lea     rbx,[384+rsp]
+        mov     r9,QWORD[((0+64))+rsp]
+        mov     r10,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r11,QWORD[((16+64))+rsp]
+        mov     r12,QWORD[((24+64))+rsp]
+        lea     rdi,[288+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[448+rsp]
+        lea     rbx,[448+rsp]
+        mov     r9,QWORD[((0+32))+rsp]
+        mov     r10,QWORD[((8+32))+rsp]
+        lea     rsi,[((-128+32))+rsp]
+        mov     r11,QWORD[((16+32))+rsp]
+        mov     r12,QWORD[((24+32))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[352+rsp]
+        lea     rdi,[96+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rdx,QWORD[((0+64))+rsp]
+        mov     r14,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r15,QWORD[((16+64))+rsp]
+        mov     r8,QWORD[((24+64))+rsp]
+        lea     rdi,[128+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[((0+96))+rsp]
+        mov     r14,QWORD[((8+96))+rsp]
+        lea     rsi,[((-128+96))+rsp]
+        mov     r15,QWORD[((16+96))+rsp]
+        mov     r8,QWORD[((24+96))+rsp]
+        lea     rdi,[192+rsp]
+        call    __ecp_nistz256_sqr_montx
+
+        mov     rdx,QWORD[128+rsp]
+        lea     rbx,[128+rsp]
+        mov     r9,QWORD[((0+64))+rsp]
+        mov     r10,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r11,QWORD[((16+64))+rsp]
+        mov     r12,QWORD[((24+64))+rsp]
+        lea     rdi,[160+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[320+rsp]
+        lea     rbx,[320+rsp]
+        mov     r9,QWORD[((0+128))+rsp]
+        mov     r10,QWORD[((8+128))+rsp]
+        lea     rsi,[((-128+128))+rsp]
+        mov     r11,QWORD[((16+128))+rsp]
+        mov     r12,QWORD[((24+128))+rsp]
+        lea     rdi,[rsp]
+        call    __ecp_nistz256_mul_montx
+
+
+
+
+        xor     r11,r11
+        add     r12,r12
+        lea     rsi,[192+rsp]
+        adc     r13,r13
+        mov     rax,r12
+        adc     r8,r8
+        adc     r9,r9
+        mov     rbp,r13
+        adc     r11,0
+
+        sub     r12,-1
+        mov     rcx,r8
+        sbb     r13,r14
+        sbb     r8,0
+        mov     r10,r9
+        sbb     r9,r15
+        sbb     r11,0
+
+        cmovc   r12,rax
+        mov     rax,QWORD[rsi]
+        cmovc   r13,rbp
+        mov     rbp,QWORD[8+rsi]
+        cmovc   r8,rcx
+        mov     rcx,QWORD[16+rsi]
+        cmovc   r9,r10
+        mov     r10,QWORD[24+rsi]
+
+        call    __ecp_nistz256_subx
+
+        lea     rbx,[160+rsp]
+        lea     rdi,[224+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+        mov     rax,QWORD[((0+0))+rsp]
+        mov     rbp,QWORD[((0+8))+rsp]
+        mov     rcx,QWORD[((0+16))+rsp]
+        mov     r10,QWORD[((0+24))+rsp]
+        lea     rdi,[64+rsp]
+
+        call    __ecp_nistz256_subx
+
+        mov     QWORD[rdi],r12
+        mov     QWORD[8+rdi],r13
+        mov     QWORD[16+rdi],r8
+        mov     QWORD[24+rdi],r9
+        mov     rdx,QWORD[352+rsp]
+        lea     rbx,[352+rsp]
+        mov     r9,QWORD[((0+160))+rsp]
+        mov     r10,QWORD[((8+160))+rsp]
+        lea     rsi,[((-128+160))+rsp]
+        mov     r11,QWORD[((16+160))+rsp]
+        mov     r12,QWORD[((24+160))+rsp]
+        lea     rdi,[32+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        mov     rdx,QWORD[96+rsp]
+        lea     rbx,[96+rsp]
+        mov     r9,QWORD[((0+64))+rsp]
+        mov     r10,QWORD[((8+64))+rsp]
+        lea     rsi,[((-128+64))+rsp]
+        mov     r11,QWORD[((16+64))+rsp]
+        mov     r12,QWORD[((24+64))+rsp]
+        lea     rdi,[64+rsp]
+        call    __ecp_nistz256_mul_montx
+
+        lea     rbx,[32+rsp]
+        lea     rdi,[256+rsp]
+        call    __ecp_nistz256_sub_fromx
+
+DB      102,72,15,126,199
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[288+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((288+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[$L$ONE_mont]
+        pand    xmm3,XMMWORD[(($L$ONE_mont+16))]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[384+rsp]
+        pand    xmm3,XMMWORD[((384+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[64+rdi],xmm2
+        movdqu  XMMWORD[80+rdi],xmm3
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[224+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((224+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[416+rsp]
+        pand    xmm3,XMMWORD[((416+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[320+rsp]
+        pand    xmm3,XMMWORD[((320+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[rdi],xmm2
+        movdqu  XMMWORD[16+rdi],xmm3
+
+        movdqa  xmm0,xmm5
+        movdqa  xmm1,xmm5
+        pandn   xmm0,XMMWORD[256+rsp]
+        movdqa  xmm2,xmm5
+        pandn   xmm1,XMMWORD[((256+16))+rsp]
+        movdqa  xmm3,xmm5
+        pand    xmm2,XMMWORD[448+rsp]
+        pand    xmm3,XMMWORD[((448+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+
+        movdqa  xmm0,xmm4
+        movdqa  xmm1,xmm4
+        pandn   xmm0,xmm2
+        movdqa  xmm2,xmm4
+        pandn   xmm1,xmm3
+        movdqa  xmm3,xmm4
+        pand    xmm2,XMMWORD[352+rsp]
+        pand    xmm3,XMMWORD[((352+16))+rsp]
+        por     xmm2,xmm0
+        por     xmm3,xmm1
+        movdqu  XMMWORD[32+rdi],xmm2
+        movdqu  XMMWORD[48+rdi],xmm3
+
+        lea     rsi,[((480+56))+rsp]
+
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbx,QWORD[((-16))+rsi]
+
+        mov     rbp,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$add_affinex_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_ecp_nistz256_point_add_affinex:
 EXTERN  __imp_RtlVirtualUnwind
 
@@ -5618 +7734 @@
         DD      $L$SEH_end_ecp_nistz256_ord_sqr_mont wrt ..imagebase
         DD      $L$SEH_info_ecp_nistz256_ord_sqr_mont wrt ..imagebase
+        DD      $L$SEH_begin_ecp_nistz256_ord_mul_montx wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_ord_mul_montx wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_ord_mul_montx wrt ..imagebase
+
+        DD      $L$SEH_begin_ecp_nistz256_ord_sqr_montx wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_ord_sqr_montx wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_ord_sqr_montx wrt ..imagebase
         DD      $L$SEH_begin_ecp_nistz256_to_mont wrt ..imagebase
         DD      $L$SEH_end_ecp_nistz256_to_mont wrt ..imagebase
@@ -5641 +7764 @@
         DD      $L$SEH_end_ecp_nistz256_gather_w7 wrt ..imagebase
         DD      $L$SEH_info_ecp_nistz256_gather_wX wrt ..imagebase
+        DD      $L$SEH_begin_ecp_nistz256_avx2_gather_w5 wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_avx2_gather_w5 wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_avx2_gather_wX wrt ..imagebase
+
+        DD      $L$SEH_begin_ecp_nistz256_avx2_gather_w7 wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_avx2_gather_w7 wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_avx2_gather_wX wrt ..imagebase
         DD      $L$SEH_begin_ecp_nistz256_point_double wrt ..imagebase
         DD      $L$SEH_end_ecp_nistz256_point_double wrt ..imagebase
@@ -5652 +7782 @@
         DD      $L$SEH_end_ecp_nistz256_point_add_affine wrt ..imagebase
         DD      $L$SEH_info_ecp_nistz256_point_add_affine wrt ..imagebase
+        DD      $L$SEH_begin_ecp_nistz256_point_doublex wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_point_doublex wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_point_doublex wrt ..imagebase
+
+        DD      $L$SEH_begin_ecp_nistz256_point_addx wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_point_addx wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_point_addx wrt ..imagebase
+
+        DD      $L$SEH_begin_ecp_nistz256_point_add_affinex wrt ..imagebase
+        DD      $L$SEH_end_ecp_nistz256_point_add_affinex wrt ..imagebase
+        DD      $L$SEH_info_ecp_nistz256_point_add_affinex wrt ..imagebase
 
 section .xdata rdata align=8
@@ -5689 +7830 @@
         DD      $L$ord_sqr_body wrt ..imagebase,$L$ord_sqr_epilogue wrt ..imagebase
         DD      48,0
+$L$SEH_info_ecp_nistz256_ord_mul_montx:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$ord_mulx_body wrt ..imagebase,$L$ord_mulx_epilogue wrt ..imagebase
+        DD      48,0
+$L$SEH_info_ecp_nistz256_ord_sqr_montx:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase
+        DD      48,0
 $L$SEH_info_ecp_nistz256_to_mont:
 DB      9,0,0,0
@@ -5722 +7873 @@
 DB      0x04,0x01,0x15,0x00
 ALIGN   8
+$L$SEH_info_ecp_nistz256_avx2_gather_wX:
+DB      0x01,0x36,0x17,0x0b
+DB      0x36,0xf8,0x09,0x00
+DB      0x31,0xe8,0x08,0x00
+DB      0x2c,0xd8,0x07,0x00
+DB      0x27,0xc8,0x06,0x00
+DB      0x22,0xb8,0x05,0x00
+DB      0x1d,0xa8,0x04,0x00
+DB      0x18,0x98,0x03,0x00
+DB      0x13,0x88,0x02,0x00
+DB      0x0e,0x78,0x01,0x00
+DB      0x09,0x68,0x00,0x00
+DB      0x04,0x01,0x15,0x00
+DB      0x00,0xb3,0x00,0x00
+ALIGN   8
 $L$SEH_info_ecp_nistz256_point_double:
 DB      9,0,0,0
@@ -5737 +7903 @@
         DD      $L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase
         DD      32*15+56,0
+ALIGN   8
+$L$SEH_info_ecp_nistz256_point_doublex:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase
+        DD      32*5+56,0
+$L$SEH_info_ecp_nistz256_point_addx:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase
+        DD      32*18+56,0
+$L$SEH_info_ecp_nistz256_point_add_affinex:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase
+        DD      32*15+56,0

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/ghash-x86_64.S

@@ -1355 +1355 @@
 gcm_init_avx:
 
-        jmp     NEAR $L$_init_clmul
+$L$SEH_begin_gcm_init_avx:
+
+DB      0x48,0x83,0xec,0x18
+DB      0x0f,0x29,0x34,0x24
+        vzeroupper
+
+        vmovdqu xmm2,XMMWORD[rdx]
+        vpshufd xmm2,xmm2,78
+
+
+        vpshufd xmm4,xmm2,255
+        vpsrlq  xmm3,xmm2,63
+        vpsllq  xmm2,xmm2,1
+        vpxor   xmm5,xmm5,xmm5
+        vpcmpgtd        xmm5,xmm5,xmm4
+        vpslldq xmm3,xmm3,8
+        vpor    xmm2,xmm2,xmm3
+
+
+        vpand   xmm5,xmm5,XMMWORD[$L$0x1c2_polynomial]
+        vpxor   xmm2,xmm2,xmm5
+
+        vpunpckhqdq     xmm6,xmm2,xmm2
+        vmovdqa xmm0,xmm2
+        vpxor   xmm6,xmm6,xmm2
+        mov     r10,4
+        jmp     NEAR $L$init_start_avx
+ALIGN   32
+$L$init_loop_avx:
+        vpalignr        xmm5,xmm4,xmm3,8
+        vmovdqu XMMWORD[(-16)+rcx],xmm5
+        vpunpckhqdq     xmm3,xmm0,xmm0
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm1,xmm0,xmm2,0x11
+        vpclmulqdq      xmm0,xmm0,xmm2,0x00
+        vpclmulqdq      xmm3,xmm3,xmm6,0x00
+        vpxor   xmm4,xmm1,xmm0
+        vpxor   xmm3,xmm3,xmm4
+
+        vpslldq xmm4,xmm3,8
+        vpsrldq xmm3,xmm3,8
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm1,xmm1,xmm3
+        vpsllq  xmm3,xmm0,57
+        vpsllq  xmm4,xmm0,62
+        vpxor   xmm4,xmm4,xmm3
+        vpsllq  xmm3,xmm0,63
+        vpxor   xmm4,xmm4,xmm3
+        vpslldq xmm3,xmm4,8
+        vpsrldq xmm4,xmm4,8
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm1,xmm1,xmm4
+
+        vpsrlq  xmm4,xmm0,1
+        vpxor   xmm1,xmm1,xmm0
+        vpxor   xmm0,xmm0,xmm4
+        vpsrlq  xmm4,xmm4,5
+        vpxor   xmm0,xmm0,xmm4
+        vpsrlq  xmm0,xmm0,1
+        vpxor   xmm0,xmm0,xmm1
+$L$init_start_avx:
+        vmovdqa xmm5,xmm0
+        vpunpckhqdq     xmm3,xmm0,xmm0
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm1,xmm0,xmm2,0x11
+        vpclmulqdq      xmm0,xmm0,xmm2,0x00
+        vpclmulqdq      xmm3,xmm3,xmm6,0x00
+        vpxor   xmm4,xmm1,xmm0
+        vpxor   xmm3,xmm3,xmm4
+
+        vpslldq xmm4,xmm3,8
+        vpsrldq xmm3,xmm3,8
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm1,xmm1,xmm3
+        vpsllq  xmm3,xmm0,57
+        vpsllq  xmm4,xmm0,62
+        vpxor   xmm4,xmm4,xmm3
+        vpsllq  xmm3,xmm0,63
+        vpxor   xmm4,xmm4,xmm3
+        vpslldq xmm3,xmm4,8
+        vpsrldq xmm4,xmm4,8
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm1,xmm1,xmm4
+
+        vpsrlq  xmm4,xmm0,1
+        vpxor   xmm1,xmm1,xmm0
+        vpxor   xmm0,xmm0,xmm4
+        vpsrlq  xmm4,xmm4,5
+        vpxor   xmm0,xmm0,xmm4
+        vpsrlq  xmm0,xmm0,1
+        vpxor   xmm0,xmm0,xmm1
+        vpshufd xmm3,xmm5,78
+        vpshufd xmm4,xmm0,78
+        vpxor   xmm3,xmm3,xmm5
+        vmovdqu XMMWORD[rcx],xmm5
+        vpxor   xmm4,xmm4,xmm0
+        vmovdqu XMMWORD[16+rcx],xmm0
+        lea     rcx,[48+rcx]
+        sub     r10,1
+        jnz     NEAR $L$init_loop_avx
+
+        vpalignr        xmm5,xmm3,xmm4,8
+        vmovdqu XMMWORD[(-16)+rcx],xmm5
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[rsp]
+        lea     rsp,[24+rsp]
+$L$SEH_end_gcm_init_avx:
+        DB      0F3h,0C3h               ;repret
 
 
@@ -1373 +1481 @@
 
 DB      243,15,30,250
-        jmp     NEAR $L$_ghash_clmul
+        lea     rax,[((-136))+rsp]
+$L$SEH_begin_gcm_ghash_avx:
+
+DB      0x48,0x8d,0x60,0xe0
+DB      0x0f,0x29,0x70,0xe0
+DB      0x0f,0x29,0x78,0xf0
+DB      0x44,0x0f,0x29,0x00
+DB      0x44,0x0f,0x29,0x48,0x10
+DB      0x44,0x0f,0x29,0x50,0x20
+DB      0x44,0x0f,0x29,0x58,0x30
+DB      0x44,0x0f,0x29,0x60,0x40
+DB      0x44,0x0f,0x29,0x68,0x50
+DB      0x44,0x0f,0x29,0x70,0x60
+DB      0x44,0x0f,0x29,0x78,0x70
+        vzeroupper
+
+        vmovdqu xmm10,XMMWORD[rcx]
+        lea     r10,[$L$0x1c2_polynomial]
+        lea     rdx,[64+rdx]
+        vmovdqu xmm13,XMMWORD[$L$bswap_mask]
+        vpshufb xmm10,xmm10,xmm13
+        cmp     r9,0x80
+        jb      NEAR $L$short_avx
+        sub     r9,0x80
+
+        vmovdqu xmm14,XMMWORD[112+r8]
+        vmovdqu xmm6,XMMWORD[((0-64))+rdx]
+        vpshufb xmm14,xmm14,xmm13
+        vmovdqu xmm7,XMMWORD[((32-64))+rdx]
+
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vmovdqu xmm15,XMMWORD[96+r8]
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpxor   xmm9,xmm9,xmm14
+        vpshufb xmm15,xmm15,xmm13
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((16-64))+rdx]
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vmovdqu xmm14,XMMWORD[80+r8]
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vpxor   xmm8,xmm8,xmm15
+
+        vpshufb xmm14,xmm14,xmm13
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((48-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+        vmovdqu xmm15,XMMWORD[64+r8]
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((80-64))+rdx]
+
+        vpshufb xmm15,xmm15,xmm13
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpxor   xmm4,xmm4,xmm1
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((64-64))+rdx]
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vpxor   xmm8,xmm8,xmm15
+
+        vmovdqu xmm14,XMMWORD[48+r8]
+        vpxor   xmm0,xmm0,xmm3
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpxor   xmm1,xmm1,xmm4
+        vpshufb xmm14,xmm14,xmm13
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((96-64))+rdx]
+        vpxor   xmm2,xmm2,xmm5
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((128-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+
+        vmovdqu xmm15,XMMWORD[32+r8]
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpxor   xmm4,xmm4,xmm1
+        vpshufb xmm15,xmm15,xmm13
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((112-64))+rdx]
+        vpxor   xmm5,xmm5,xmm2
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vpxor   xmm8,xmm8,xmm15
+
+        vmovdqu xmm14,XMMWORD[16+r8]
+        vpxor   xmm0,xmm0,xmm3
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpxor   xmm1,xmm1,xmm4
+        vpshufb xmm14,xmm14,xmm13
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((144-64))+rdx]
+        vpxor   xmm2,xmm2,xmm5
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((176-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+
+        vmovdqu xmm15,XMMWORD[r8]
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpxor   xmm4,xmm4,xmm1
+        vpshufb xmm15,xmm15,xmm13
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((160-64))+rdx]
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm9,xmm7,0x10
+
+        lea     r8,[128+r8]
+        cmp     r9,0x80
+        jb      NEAR $L$tail_avx
+
+        vpxor   xmm15,xmm15,xmm10
+        sub     r9,0x80
+        jmp     NEAR $L$oop8x_avx
+
+ALIGN   32
+$L$oop8x_avx:
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vmovdqu xmm14,XMMWORD[112+r8]
+        vpxor   xmm3,xmm3,xmm0
+        vpxor   xmm8,xmm8,xmm15
+        vpclmulqdq      xmm10,xmm15,xmm6,0x00
+        vpshufb xmm14,xmm14,xmm13
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm11,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((0-64))+rdx]
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm12,xmm8,xmm7,0x00
+        vmovdqu xmm7,XMMWORD[((32-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+
+        vmovdqu xmm15,XMMWORD[96+r8]
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpxor   xmm10,xmm10,xmm3
+        vpshufb xmm15,xmm15,xmm13
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vxorps  xmm11,xmm11,xmm4
+        vmovdqu xmm6,XMMWORD[((16-64))+rdx]
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vpxor   xmm12,xmm12,xmm5
+        vxorps  xmm8,xmm8,xmm15
+
+        vmovdqu xmm14,XMMWORD[80+r8]
+        vpxor   xmm12,xmm12,xmm10
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpxor   xmm12,xmm12,xmm11
+        vpslldq xmm9,xmm12,8
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vpsrldq xmm12,xmm12,8
+        vpxor   xmm10,xmm10,xmm9
+        vmovdqu xmm6,XMMWORD[((48-64))+rdx]
+        vpshufb xmm14,xmm14,xmm13
+        vxorps  xmm11,xmm11,xmm12
+        vpxor   xmm4,xmm4,xmm1
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((80-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+        vpxor   xmm5,xmm5,xmm2
+
+        vmovdqu xmm15,XMMWORD[64+r8]
+        vpalignr        xmm12,xmm10,xmm10,8
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpshufb xmm15,xmm15,xmm13
+        vpxor   xmm0,xmm0,xmm3
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((64-64))+rdx]
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm1,xmm1,xmm4
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vxorps  xmm8,xmm8,xmm15
+        vpxor   xmm2,xmm2,xmm5
+
+        vmovdqu xmm14,XMMWORD[48+r8]
+        vpclmulqdq      xmm10,xmm10,XMMWORD[r10],0x10
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpshufb xmm14,xmm14,xmm13
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((96-64))+rdx]
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((128-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+        vpxor   xmm5,xmm5,xmm2
+
+        vmovdqu xmm15,XMMWORD[32+r8]
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpshufb xmm15,xmm15,xmm13
+        vpxor   xmm0,xmm0,xmm3
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((112-64))+rdx]
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm1,xmm1,xmm4
+        vpclmulqdq      xmm2,xmm9,xmm7,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vpxor   xmm2,xmm2,xmm5
+        vxorps  xmm10,xmm10,xmm12
+
+        vmovdqu xmm14,XMMWORD[16+r8]
+        vpalignr        xmm12,xmm10,xmm10,8
+        vpclmulqdq      xmm3,xmm15,xmm6,0x00
+        vpshufb xmm14,xmm14,xmm13
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm4,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((144-64))+rdx]
+        vpclmulqdq      xmm10,xmm10,XMMWORD[r10],0x10
+        vxorps  xmm12,xmm12,xmm11
+        vpunpckhqdq     xmm9,xmm14,xmm14
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm5,xmm8,xmm7,0x10
+        vmovdqu xmm7,XMMWORD[((176-64))+rdx]
+        vpxor   xmm9,xmm9,xmm14
+        vpxor   xmm5,xmm5,xmm2
+
+        vmovdqu xmm15,XMMWORD[r8]
+        vpclmulqdq      xmm0,xmm14,xmm6,0x00
+        vpshufb xmm15,xmm15,xmm13
+        vpclmulqdq      xmm1,xmm14,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((160-64))+rdx]
+        vpxor   xmm15,xmm15,xmm12
+        vpclmulqdq      xmm2,xmm9,xmm7,0x10
+        vpxor   xmm15,xmm15,xmm10
+
+        lea     r8,[128+r8]
+        sub     r9,0x80
+        jnc     NEAR $L$oop8x_avx
+
+        add     r9,0x80
+        jmp     NEAR $L$tail_no_xor_avx
+
+ALIGN   32
+$L$short_avx:
+        vmovdqu xmm14,XMMWORD[((-16))+r9*1+r8]
+        lea     r8,[r9*1+r8]
+        vmovdqu xmm6,XMMWORD[((0-64))+rdx]
+        vmovdqu xmm7,XMMWORD[((32-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+
+        vmovdqa xmm3,xmm0
+        vmovdqa xmm4,xmm1
+        vmovdqa xmm5,xmm2
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-32))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((16-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vpsrldq xmm7,xmm7,8
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-48))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((48-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vmovdqu xmm7,XMMWORD[((80-64))+rdx]
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-64))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((64-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vpsrldq xmm7,xmm7,8
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-80))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((96-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vmovdqu xmm7,XMMWORD[((128-64))+rdx]
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-96))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((112-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vpsrldq xmm7,xmm7,8
+        sub     r9,0x10
+        jz      NEAR $L$tail_avx
+
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vmovdqu xmm14,XMMWORD[((-112))+r8]
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vmovdqu xmm6,XMMWORD[((144-64))+rdx]
+        vpshufb xmm15,xmm14,xmm13
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+        vmovq   xmm7,QWORD[((184-64))+rdx]
+        sub     r9,0x10
+        jmp     NEAR $L$tail_avx
+
+ALIGN   32
+$L$tail_avx:
+        vpxor   xmm15,xmm15,xmm10
+$L$tail_no_xor_avx:
+        vpunpckhqdq     xmm8,xmm15,xmm15
+        vpxor   xmm3,xmm3,xmm0
+        vpclmulqdq      xmm0,xmm15,xmm6,0x00
+        vpxor   xmm8,xmm8,xmm15
+        vpxor   xmm4,xmm4,xmm1
+        vpclmulqdq      xmm1,xmm15,xmm6,0x11
+        vpxor   xmm5,xmm5,xmm2
+        vpclmulqdq      xmm2,xmm8,xmm7,0x00
+
+        vmovdqu xmm12,XMMWORD[r10]
+
+        vpxor   xmm10,xmm3,xmm0
+        vpxor   xmm11,xmm4,xmm1
+        vpxor   xmm5,xmm5,xmm2
+
+        vpxor   xmm5,xmm5,xmm10
+        vpxor   xmm5,xmm5,xmm11
+        vpslldq xmm9,xmm5,8
+        vpsrldq xmm5,xmm5,8
+        vpxor   xmm10,xmm10,xmm9
+        vpxor   xmm11,xmm11,xmm5
+
+        vpclmulqdq      xmm9,xmm10,xmm12,0x10
+        vpalignr        xmm10,xmm10,xmm10,8
+        vpxor   xmm10,xmm10,xmm9
+
+        vpclmulqdq      xmm9,xmm10,xmm12,0x10
+        vpalignr        xmm10,xmm10,xmm10,8
+        vpxor   xmm10,xmm10,xmm11
+        vpxor   xmm10,xmm10,xmm9
+
+        cmp     r9,0
+        jne     NEAR $L$short_avx
+
+        vpshufb xmm10,xmm10,xmm13
+        vmovdqu XMMWORD[rcx],xmm10
+        vzeroupper
+        movaps  xmm6,XMMWORD[rsp]
+        movaps  xmm7,XMMWORD[16+rsp]
+        movaps  xmm8,XMMWORD[32+rsp]
+        movaps  xmm9,XMMWORD[48+rsp]
+        movaps  xmm10,XMMWORD[64+rsp]
+        movaps  xmm11,XMMWORD[80+rsp]
+        movaps  xmm12,XMMWORD[96+rsp]
+        movaps  xmm13,XMMWORD[112+rsp]
+        movaps  xmm14,XMMWORD[128+rsp]
+        movaps  xmm15,XMMWORD[144+rsp]
+        lea     rsp,[168+rsp]
+$L$SEH_end_gcm_ghash_avx:
+        DB      0F3h,0C3h               ;repret
 
 
@@ -1537 +2041 @@
         DD      $L$SEH_end_gcm_ghash_clmul wrt ..imagebase
         DD      $L$SEH_info_gcm_ghash_clmul wrt ..imagebase
+        DD      $L$SEH_begin_gcm_init_avx wrt ..imagebase
+        DD      $L$SEH_end_gcm_init_avx wrt ..imagebase
+        DD      $L$SEH_info_gcm_init_clmul wrt ..imagebase
+
+        DD      $L$SEH_begin_gcm_ghash_avx wrt ..imagebase
+        DD      $L$SEH_end_gcm_ghash_avx wrt ..imagebase
+        DD      $L$SEH_info_gcm_ghash_clmul wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/poly1305-x86_64.S

@@ -38 +38 @@
         lea     r10,[poly1305_blocks]
         lea     r11,[poly1305_emit]
+        mov     r9,QWORD[((OPENSSL_ia32cap_P+4))]
+        lea     rax,[poly1305_blocks_avx]
+        lea     rcx,[poly1305_emit_avx]
+        bt      r9,28
+        cmovc   r10,rax
+        cmovc   r11,rcx
+        lea     rax,[poly1305_blocks_avx2]
+        bt      r9,37
+        cmovc   r10,rax
+        mov     rax,2149646336
+        shr     r9,32
+        and     r9,rax
+        cmp     r9,rax
+        je      NEAR $L$init_base2_44
         mov     rax,0x0ffffffc0fffffff
         mov     rcx,0x0ffffffc0ffffffc
@@ -212 +226 @@
 
 $L$SEH_end_poly1305_emit:
+
+ALIGN   32
+__poly1305_block:
+
+        mul     r14
+        mov     r9,rax
+        mov     rax,r11
+        mov     r10,rdx
+
+        mul     r14
+        mov     r14,rax
+        mov     rax,r11
+        mov     r8,rdx
+
+        mul     rbx
+        add     r9,rax
+        mov     rax,r13
+        adc     r10,rdx
+
+        mul     rbx
+        mov     rbx,rbp
+        add     r14,rax
+        adc     r8,rdx
+
+        imul    rbx,r13
+        add     r9,rbx
+        mov     rbx,r8
+        adc     r10,0
+
+        imul    rbp,r11
+        add     rbx,r9
+        mov     rax,-4
+        adc     r10,rbp
+
+        and     rax,r10
+        mov     rbp,r10
+        shr     r10,2
+        and     rbp,3
+        add     rax,r10
+        add     r14,rax
+        adc     rbx,0
+        adc     rbp,0
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+__poly1305_init_avx:
+
+        mov     r14,r11
+        mov     rbx,r12
+        xor     rbp,rbp
+
+        lea     rdi,[((48+64))+rdi]
+
+        mov     rax,r12
+        call    __poly1305_block
+
+        mov     eax,0x3ffffff
+        mov     edx,0x3ffffff
+        mov     r8,r14
+        and     eax,r14d
+        mov     r9,r11
+        and     edx,r11d
+        mov     DWORD[((-64))+rdi],eax
+        shr     r8,26
+        mov     DWORD[((-60))+rdi],edx
+        shr     r9,26
+
+        mov     eax,0x3ffffff
+        mov     edx,0x3ffffff
+        and     eax,r8d
+        and     edx,r9d
+        mov     DWORD[((-48))+rdi],eax
+        lea     eax,[rax*4+rax]
+        mov     DWORD[((-44))+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        mov     DWORD[((-32))+rdi],eax
+        shr     r8,26
+        mov     DWORD[((-28))+rdi],edx
+        shr     r9,26
+
+        mov     rax,rbx
+        mov     rdx,r12
+        shl     rax,12
+        shl     rdx,12
+        or      rax,r8
+        or      rdx,r9
+        and     eax,0x3ffffff
+        and     edx,0x3ffffff
+        mov     DWORD[((-16))+rdi],eax
+        lea     eax,[rax*4+rax]
+        mov     DWORD[((-12))+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        mov     DWORD[rdi],eax
+        mov     r8,rbx
+        mov     DWORD[4+rdi],edx
+        mov     r9,r12
+
+        mov     eax,0x3ffffff
+        mov     edx,0x3ffffff
+        shr     r8,14
+        shr     r9,14
+        and     eax,r8d
+        and     edx,r9d
+        mov     DWORD[16+rdi],eax
+        lea     eax,[rax*4+rax]
+        mov     DWORD[20+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        mov     DWORD[32+rdi],eax
+        shr     r8,26
+        mov     DWORD[36+rdi],edx
+        shr     r9,26
+
+        mov     rax,rbp
+        shl     rax,24
+        or      r8,rax
+        mov     DWORD[48+rdi],r8d
+        lea     r8,[r8*4+r8]
+        mov     DWORD[52+rdi],r9d
+        lea     r9,[r9*4+r9]
+        mov     DWORD[64+rdi],r8d
+        mov     DWORD[68+rdi],r9d
+
+        mov     rax,r12
+        call    __poly1305_block
+
+        mov     eax,0x3ffffff
+        mov     r8,r14
+        and     eax,r14d
+        shr     r8,26
+        mov     DWORD[((-52))+rdi],eax
+
+        mov     edx,0x3ffffff
+        and     edx,r8d
+        mov     DWORD[((-36))+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        shr     r8,26
+        mov     DWORD[((-20))+rdi],edx
+
+        mov     rax,rbx
+        shl     rax,12
+        or      rax,r8
+        and     eax,0x3ffffff
+        mov     DWORD[((-4))+rdi],eax
+        lea     eax,[rax*4+rax]
+        mov     r8,rbx
+        mov     DWORD[12+rdi],eax
+
+        mov     edx,0x3ffffff
+        shr     r8,14
+        and     edx,r8d
+        mov     DWORD[28+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        shr     r8,26
+        mov     DWORD[44+rdi],edx
+
+        mov     rax,rbp
+        shl     rax,24
+        or      r8,rax
+        mov     DWORD[60+rdi],r8d
+        lea     r8,[r8*4+r8]
+        mov     DWORD[76+rdi],r8d
+
+        mov     rax,r12
+        call    __poly1305_block
+
+        mov     eax,0x3ffffff
+        mov     r8,r14
+        and     eax,r14d
+        shr     r8,26
+        mov     DWORD[((-56))+rdi],eax
+
+        mov     edx,0x3ffffff
+        and     edx,r8d
+        mov     DWORD[((-40))+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        shr     r8,26
+        mov     DWORD[((-24))+rdi],edx
+
+        mov     rax,rbx
+        shl     rax,12
+        or      rax,r8
+        and     eax,0x3ffffff
+        mov     DWORD[((-8))+rdi],eax
+        lea     eax,[rax*4+rax]
+        mov     r8,rbx
+        mov     DWORD[8+rdi],eax
+
+        mov     edx,0x3ffffff
+        shr     r8,14
+        and     edx,r8d
+        mov     DWORD[24+rdi],edx
+        lea     edx,[rdx*4+rdx]
+        shr     r8,26
+        mov     DWORD[40+rdi],edx
+
+        mov     rax,rbp
+        shl     rax,24
+        or      r8,rax
+        mov     DWORD[56+rdi],r8d
+        lea     r8,[r8*4+r8]
+        mov     DWORD[72+rdi],r8d
+
+        lea     rdi,[((-48-64))+rdi]
+        DB      0F3h,0C3h               ;repret
+
+
+
+
+ALIGN   32
+poly1305_blocks_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+        mov     r8d,DWORD[20+rdi]
+        cmp     rdx,128
+        jae     NEAR $L$blocks_avx
+        test    r8d,r8d
+        jz      NEAR $L$blocks
+
+$L$blocks_avx:
+        and     rdx,-16
+        jz      NEAR $L$no_data_avx
+
+        vzeroupper
+
+        test    r8d,r8d
+        jz      NEAR $L$base2_64_avx
+
+        test    rdx,31
+        jz      NEAR $L$even_avx
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$blocks_avx_body:
+
+        mov     r15,rdx
+
+        mov     r8,QWORD[rdi]
+        mov     r9,QWORD[8+rdi]
+        mov     ebp,DWORD[16+rdi]
+
+        mov     r11,QWORD[24+rdi]
+        mov     r13,QWORD[32+rdi]
+
+
+        mov     r14d,r8d
+        and     r8,-2147483648
+        mov     r12,r9
+        mov     ebx,r9d
+        and     r9,-2147483648
+
+        shr     r8,6
+        shl     r12,52
+        add     r14,r8
+        shr     rbx,12
+        shr     r9,18
+        add     r14,r12
+        adc     rbx,r9
+
+        mov     r8,rbp
+        shl     r8,40
+        shr     rbp,24
+        add     rbx,r8
+        adc     rbp,0
+
+        mov     r9,-4
+        mov     r8,rbp
+        and     r9,rbp
+        shr     r8,2
+        and     rbp,3
+        add     r8,r9
+        add     r14,r8
+        adc     rbx,0
+        adc     rbp,0
+
+        mov     r12,r13
+        mov     rax,r13
+        shr     r13,2
+        add     r13,r12
+
+        add     r14,QWORD[rsi]
+        adc     rbx,QWORD[8+rsi]
+        lea     rsi,[16+rsi]
+        adc     rbp,rcx
+
+        call    __poly1305_block
+
+        test    rcx,rcx
+        jz      NEAR $L$store_base2_64_avx
+
+
+        mov     rax,r14
+        mov     rdx,r14
+        shr     r14,52
+        mov     r11,rbx
+        mov     r12,rbx
+        shr     rdx,26
+        and     rax,0x3ffffff
+        shl     r11,12
+        and     rdx,0x3ffffff
+        shr     rbx,14
+        or      r14,r11
+        shl     rbp,24
+        and     r14,0x3ffffff
+        shr     r12,40
+        and     rbx,0x3ffffff
+        or      rbp,r12
+
+        sub     r15,16
+        jz      NEAR $L$store_base2_26_avx
+
+        vmovd   xmm0,eax
+        vmovd   xmm1,edx
+        vmovd   xmm2,r14d
+        vmovd   xmm3,ebx
+        vmovd   xmm4,ebp
+        jmp     NEAR $L$proceed_avx
+
+ALIGN   32
+$L$store_base2_64_avx:
+        mov     QWORD[rdi],r14
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rbp
+        jmp     NEAR $L$done_avx
+
+ALIGN   16
+$L$store_base2_26_avx:
+        mov     DWORD[rdi],eax
+        mov     DWORD[4+rdi],edx
+        mov     DWORD[8+rdi],r14d
+        mov     DWORD[12+rdi],ebx
+        mov     DWORD[16+rdi],ebp
+ALIGN   16
+$L$done_avx:
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbp,QWORD[32+rsp]
+
+        mov     rbx,QWORD[40+rsp]
+
+        lea     rsp,[48+rsp]
+
+$L$no_data_avx:
+$L$blocks_avx_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+
+ALIGN   32
+$L$base2_64_avx:
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$base2_64_avx_body:
+
+        mov     r15,rdx
+
+        mov     r11,QWORD[24+rdi]
+        mov     r13,QWORD[32+rdi]
+
+        mov     r14,QWORD[rdi]
+        mov     rbx,QWORD[8+rdi]
+        mov     ebp,DWORD[16+rdi]
+
+        mov     r12,r13
+        mov     rax,r13
+        shr     r13,2
+        add     r13,r12
+
+        test    rdx,31
+        jz      NEAR $L$init_avx
+
+        add     r14,QWORD[rsi]
+        adc     rbx,QWORD[8+rsi]
+        lea     rsi,[16+rsi]
+        adc     rbp,rcx
+        sub     r15,16
+
+        call    __poly1305_block
+
+$L$init_avx:
+
+        mov     rax,r14
+        mov     rdx,r14
+        shr     r14,52
+        mov     r8,rbx
+        mov     r9,rbx
+        shr     rdx,26
+        and     rax,0x3ffffff
+        shl     r8,12
+        and     rdx,0x3ffffff
+        shr     rbx,14
+        or      r14,r8
+        shl     rbp,24
+        and     r14,0x3ffffff
+        shr     r9,40
+        and     rbx,0x3ffffff
+        or      rbp,r9
+
+        vmovd   xmm0,eax
+        vmovd   xmm1,edx
+        vmovd   xmm2,r14d
+        vmovd   xmm3,ebx
+        vmovd   xmm4,ebp
+        mov     DWORD[20+rdi],1
+
+        call    __poly1305_init_avx
+
+$L$proceed_avx:
+        mov     rdx,r15
+
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbp,QWORD[32+rsp]
+
+        mov     rbx,QWORD[40+rsp]
+
+        lea     rax,[48+rsp]
+        lea     rsp,[48+rsp]
+
+$L$base2_64_avx_epilogue:
+        jmp     NEAR $L$do_avx
+
+
+ALIGN   32
+$L$even_avx:
+
+        vmovd   xmm0,DWORD[rdi]
+        vmovd   xmm1,DWORD[4+rdi]
+        vmovd   xmm2,DWORD[8+rdi]
+        vmovd   xmm3,DWORD[12+rdi]
+        vmovd   xmm4,DWORD[16+rdi]
+
+$L$do_avx:
+        lea     r11,[((-248))+rsp]
+        sub     rsp,0x218
+        vmovdqa XMMWORD[80+r11],xmm6
+        vmovdqa XMMWORD[96+r11],xmm7
+        vmovdqa XMMWORD[112+r11],xmm8
+        vmovdqa XMMWORD[128+r11],xmm9
+        vmovdqa XMMWORD[144+r11],xmm10
+        vmovdqa XMMWORD[160+r11],xmm11
+        vmovdqa XMMWORD[176+r11],xmm12
+        vmovdqa XMMWORD[192+r11],xmm13
+        vmovdqa XMMWORD[208+r11],xmm14
+        vmovdqa XMMWORD[224+r11],xmm15
+$L$do_avx_body:
+        sub     rdx,64
+        lea     rax,[((-32))+rsi]
+        cmovc   rsi,rax
+
+        vmovdqu xmm14,XMMWORD[48+rdi]
+        lea     rdi,[112+rdi]
+        lea     rcx,[$L$const]
+
+
+
+        vmovdqu xmm5,XMMWORD[32+rsi]
+        vmovdqu xmm6,XMMWORD[48+rsi]
+        vmovdqa xmm15,XMMWORD[64+rcx]
+
+        vpsrldq xmm7,xmm5,6
+        vpsrldq xmm8,xmm6,6
+        vpunpckhqdq     xmm9,xmm5,xmm6
+        vpunpcklqdq     xmm5,xmm5,xmm6
+        vpunpcklqdq     xmm8,xmm7,xmm8
+
+        vpsrlq  xmm9,xmm9,40
+        vpsrlq  xmm6,xmm5,26
+        vpand   xmm5,xmm5,xmm15
+        vpsrlq  xmm7,xmm8,4
+        vpand   xmm6,xmm6,xmm15
+        vpsrlq  xmm8,xmm8,30
+        vpand   xmm7,xmm7,xmm15
+        vpand   xmm8,xmm8,xmm15
+        vpor    xmm9,xmm9,XMMWORD[32+rcx]
+
+        jbe     NEAR $L$skip_loop_avx
+
+
+        vmovdqu xmm11,XMMWORD[((-48))+rdi]
+        vmovdqu xmm12,XMMWORD[((-32))+rdi]
+        vpshufd xmm13,xmm14,0xEE
+        vpshufd xmm10,xmm14,0x44
+        vmovdqa XMMWORD[(-144)+r11],xmm13
+        vmovdqa XMMWORD[rsp],xmm10
+        vpshufd xmm14,xmm11,0xEE
+        vmovdqu xmm10,XMMWORD[((-16))+rdi]
+        vpshufd xmm11,xmm11,0x44
+        vmovdqa XMMWORD[(-128)+r11],xmm14
+        vmovdqa XMMWORD[16+rsp],xmm11
+        vpshufd xmm13,xmm12,0xEE
+        vmovdqu xmm11,XMMWORD[rdi]
+        vpshufd xmm12,xmm12,0x44
+        vmovdqa XMMWORD[(-112)+r11],xmm13
+        vmovdqa XMMWORD[32+rsp],xmm12
+        vpshufd xmm14,xmm10,0xEE
+        vmovdqu xmm12,XMMWORD[16+rdi]
+        vpshufd xmm10,xmm10,0x44
+        vmovdqa XMMWORD[(-96)+r11],xmm14
+        vmovdqa XMMWORD[48+rsp],xmm10
+        vpshufd xmm13,xmm11,0xEE
+        vmovdqu xmm10,XMMWORD[32+rdi]
+        vpshufd xmm11,xmm11,0x44
+        vmovdqa XMMWORD[(-80)+r11],xmm13
+        vmovdqa XMMWORD[64+rsp],xmm11
+        vpshufd xmm14,xmm12,0xEE
+        vmovdqu xmm11,XMMWORD[48+rdi]
+        vpshufd xmm12,xmm12,0x44
+        vmovdqa XMMWORD[(-64)+r11],xmm14
+        vmovdqa XMMWORD[80+rsp],xmm12
+        vpshufd xmm13,xmm10,0xEE
+        vmovdqu xmm12,XMMWORD[64+rdi]
+        vpshufd xmm10,xmm10,0x44
+        vmovdqa XMMWORD[(-48)+r11],xmm13
+        vmovdqa XMMWORD[96+rsp],xmm10
+        vpshufd xmm14,xmm11,0xEE
+        vpshufd xmm11,xmm11,0x44
+        vmovdqa XMMWORD[(-32)+r11],xmm14
+        vmovdqa XMMWORD[112+rsp],xmm11
+        vpshufd xmm13,xmm12,0xEE
+        vmovdqa xmm14,XMMWORD[rsp]
+        vpshufd xmm12,xmm12,0x44
+        vmovdqa XMMWORD[(-16)+r11],xmm13
+        vmovdqa XMMWORD[128+rsp],xmm12
+
+        jmp     NEAR $L$oop_avx
+
+ALIGN   32
+$L$oop_avx:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        vpmuludq        xmm10,xmm14,xmm5
+        vpmuludq        xmm11,xmm14,xmm6
+        vmovdqa XMMWORD[32+r11],xmm2
+        vpmuludq        xmm12,xmm14,xmm7
+        vmovdqa xmm2,XMMWORD[16+rsp]
+        vpmuludq        xmm13,xmm14,xmm8
+        vpmuludq        xmm14,xmm14,xmm9
+
+        vmovdqa XMMWORD[r11],xmm0
+        vpmuludq        xmm0,xmm9,XMMWORD[32+rsp]
+        vmovdqa XMMWORD[16+r11],xmm1
+        vpmuludq        xmm1,xmm2,xmm8
+        vpaddq  xmm10,xmm10,xmm0
+        vpaddq  xmm14,xmm14,xmm1
+        vmovdqa XMMWORD[48+r11],xmm3
+        vpmuludq        xmm0,xmm2,xmm7
+        vpmuludq        xmm1,xmm2,xmm6
+        vpaddq  xmm13,xmm13,xmm0
+        vmovdqa xmm3,XMMWORD[48+rsp]
+        vpaddq  xmm12,xmm12,xmm1
+        vmovdqa XMMWORD[64+r11],xmm4
+        vpmuludq        xmm2,xmm2,xmm5
+        vpmuludq        xmm0,xmm3,xmm7
+        vpaddq  xmm11,xmm11,xmm2
+
+        vmovdqa xmm4,XMMWORD[64+rsp]
+        vpaddq  xmm14,xmm14,xmm0
+        vpmuludq        xmm1,xmm3,xmm6
+        vpmuludq        xmm3,xmm3,xmm5
+        vpaddq  xmm13,xmm13,xmm1
+        vmovdqa xmm2,XMMWORD[80+rsp]
+        vpaddq  xmm12,xmm12,xmm3
+        vpmuludq        xmm0,xmm4,xmm9
+        vpmuludq        xmm4,xmm4,xmm8
+        vpaddq  xmm11,xmm11,xmm0
+        vmovdqa xmm3,XMMWORD[96+rsp]
+        vpaddq  xmm10,xmm10,xmm4
+
+        vmovdqa xmm4,XMMWORD[128+rsp]
+        vpmuludq        xmm1,xmm2,xmm6
+        vpmuludq        xmm2,xmm2,xmm5
+        vpaddq  xmm14,xmm14,xmm1
+        vpaddq  xmm13,xmm13,xmm2
+        vpmuludq        xmm0,xmm3,xmm9
+        vpmuludq        xmm1,xmm3,xmm8
+        vpaddq  xmm12,xmm12,xmm0
+        vmovdqu xmm0,XMMWORD[rsi]
+        vpaddq  xmm11,xmm11,xmm1
+        vpmuludq        xmm3,xmm3,xmm7
+        vpmuludq        xmm7,xmm4,xmm7
+        vpaddq  xmm10,xmm10,xmm3
+
+        vmovdqu xmm1,XMMWORD[16+rsi]
+        vpaddq  xmm11,xmm11,xmm7
+        vpmuludq        xmm8,xmm4,xmm8
+        vpmuludq        xmm9,xmm4,xmm9
+        vpsrldq xmm2,xmm0,6
+        vpaddq  xmm12,xmm12,xmm8
+        vpaddq  xmm13,xmm13,xmm9
+        vpsrldq xmm3,xmm1,6
+        vpmuludq        xmm9,xmm5,XMMWORD[112+rsp]
+        vpmuludq        xmm5,xmm4,xmm6
+        vpunpckhqdq     xmm4,xmm0,xmm1
+        vpaddq  xmm14,xmm14,xmm9
+        vmovdqa xmm9,XMMWORD[((-144))+r11]
+        vpaddq  xmm10,xmm10,xmm5
+
+        vpunpcklqdq     xmm0,xmm0,xmm1
+        vpunpcklqdq     xmm3,xmm2,xmm3
+
+
+        vpsrldq xmm4,xmm4,5
+        vpsrlq  xmm1,xmm0,26
+        vpand   xmm0,xmm0,xmm15
+        vpsrlq  xmm2,xmm3,4
+        vpand   xmm1,xmm1,xmm15
+        vpand   xmm4,xmm4,XMMWORD[rcx]
+        vpsrlq  xmm3,xmm3,30
+        vpand   xmm2,xmm2,xmm15
+        vpand   xmm3,xmm3,xmm15
+        vpor    xmm4,xmm4,XMMWORD[32+rcx]
+
+        vpaddq  xmm0,xmm0,XMMWORD[r11]
+        vpaddq  xmm1,xmm1,XMMWORD[16+r11]
+        vpaddq  xmm2,xmm2,XMMWORD[32+r11]
+        vpaddq  xmm3,xmm3,XMMWORD[48+r11]
+        vpaddq  xmm4,xmm4,XMMWORD[64+r11]
+
+        lea     rax,[32+rsi]
+        lea     rsi,[64+rsi]
+        sub     rdx,64
+        cmovc   rsi,rax
+
+
+
+
+
+
+
+
+
+
+        vpmuludq        xmm5,xmm9,xmm0
+        vpmuludq        xmm6,xmm9,xmm1
+        vpaddq  xmm10,xmm10,xmm5
+        vpaddq  xmm11,xmm11,xmm6
+        vmovdqa xmm7,XMMWORD[((-128))+r11]
+        vpmuludq        xmm5,xmm9,xmm2
+        vpmuludq        xmm6,xmm9,xmm3
+        vpaddq  xmm12,xmm12,xmm5
+        vpaddq  xmm13,xmm13,xmm6
+        vpmuludq        xmm9,xmm9,xmm4
+        vpmuludq        xmm5,xmm4,XMMWORD[((-112))+r11]
+        vpaddq  xmm14,xmm14,xmm9
+
+        vpaddq  xmm10,xmm10,xmm5
+        vpmuludq        xmm6,xmm7,xmm2
+        vpmuludq        xmm5,xmm7,xmm3
+        vpaddq  xmm13,xmm13,xmm6
+        vmovdqa xmm8,XMMWORD[((-96))+r11]
+        vpaddq  xmm14,xmm14,xmm5
+        vpmuludq        xmm6,xmm7,xmm1
+        vpmuludq        xmm7,xmm7,xmm0
+        vpaddq  xmm12,xmm12,xmm6
+        vpaddq  xmm11,xmm11,xmm7
+
+        vmovdqa xmm9,XMMWORD[((-80))+r11]
+        vpmuludq        xmm5,xmm8,xmm2
+        vpmuludq        xmm6,xmm8,xmm1
+        vpaddq  xmm14,xmm14,xmm5
+        vpaddq  xmm13,xmm13,xmm6
+        vmovdqa xmm7,XMMWORD[((-64))+r11]
+        vpmuludq        xmm8,xmm8,xmm0
+        vpmuludq        xmm5,xmm9,xmm4
+        vpaddq  xmm12,xmm12,xmm8
+        vpaddq  xmm11,xmm11,xmm5
+        vmovdqa xmm8,XMMWORD[((-48))+r11]
+        vpmuludq        xmm9,xmm9,xmm3
+        vpmuludq        xmm6,xmm7,xmm1
+        vpaddq  xmm10,xmm10,xmm9
+
+        vmovdqa xmm9,XMMWORD[((-16))+r11]
+        vpaddq  xmm14,xmm14,xmm6
+        vpmuludq        xmm7,xmm7,xmm0
+        vpmuludq        xmm5,xmm8,xmm4
+        vpaddq  xmm13,xmm13,xmm7
+        vpaddq  xmm12,xmm12,xmm5
+        vmovdqu xmm5,XMMWORD[32+rsi]
+        vpmuludq        xmm7,xmm8,xmm3
+        vpmuludq        xmm8,xmm8,xmm2
+        vpaddq  xmm11,xmm11,xmm7
+        vmovdqu xmm6,XMMWORD[48+rsi]
+        vpaddq  xmm10,xmm10,xmm8
+
+        vpmuludq        xmm2,xmm9,xmm2
+        vpmuludq        xmm3,xmm9,xmm3
+        vpsrldq xmm7,xmm5,6
+        vpaddq  xmm11,xmm11,xmm2
+        vpmuludq        xmm4,xmm9,xmm4
+        vpsrldq xmm8,xmm6,6
+        vpaddq  xmm2,xmm12,xmm3
+        vpaddq  xmm3,xmm13,xmm4
+        vpmuludq        xmm4,xmm0,XMMWORD[((-32))+r11]
+        vpmuludq        xmm0,xmm9,xmm1
+        vpunpckhqdq     xmm9,xmm5,xmm6
+        vpaddq  xmm4,xmm14,xmm4
+        vpaddq  xmm0,xmm10,xmm0
+
+        vpunpcklqdq     xmm5,xmm5,xmm6
+        vpunpcklqdq     xmm8,xmm7,xmm8
+
+
+        vpsrldq xmm9,xmm9,5
+        vpsrlq  xmm6,xmm5,26
+        vmovdqa xmm14,XMMWORD[rsp]
+        vpand   xmm5,xmm5,xmm15
+        vpsrlq  xmm7,xmm8,4
+        vpand   xmm6,xmm6,xmm15
+        vpand   xmm9,xmm9,XMMWORD[rcx]
+        vpsrlq  xmm8,xmm8,30
+        vpand   xmm7,xmm7,xmm15
+        vpand   xmm8,xmm8,xmm15
+        vpor    xmm9,xmm9,XMMWORD[32+rcx]
+
+
+
+
+
+        vpsrlq  xmm13,xmm3,26
+        vpand   xmm3,xmm3,xmm15
+        vpaddq  xmm4,xmm4,xmm13
+
+        vpsrlq  xmm10,xmm0,26
+        vpand   xmm0,xmm0,xmm15
+        vpaddq  xmm1,xmm11,xmm10
+
+        vpsrlq  xmm10,xmm4,26
+        vpand   xmm4,xmm4,xmm15
+
+        vpsrlq  xmm11,xmm1,26
+        vpand   xmm1,xmm1,xmm15
+        vpaddq  xmm2,xmm2,xmm11
+
+        vpaddq  xmm0,xmm0,xmm10
+        vpsllq  xmm10,xmm10,2
+        vpaddq  xmm0,xmm0,xmm10
+
+        vpsrlq  xmm12,xmm2,26
+        vpand   xmm2,xmm2,xmm15
+        vpaddq  xmm3,xmm3,xmm12
+
+        vpsrlq  xmm10,xmm0,26
+        vpand   xmm0,xmm0,xmm15
+        vpaddq  xmm1,xmm1,xmm10
+
+        vpsrlq  xmm13,xmm3,26
+        vpand   xmm3,xmm3,xmm15
+        vpaddq  xmm4,xmm4,xmm13
+
+        ja      NEAR $L$oop_avx
+
+$L$skip_loop_avx:
+
+
+
+        vpshufd xmm14,xmm14,0x10
+        add     rdx,32
+        jnz     NEAR $L$ong_tail_avx
+
+        vpaddq  xmm7,xmm7,xmm2
+        vpaddq  xmm5,xmm5,xmm0
+        vpaddq  xmm6,xmm6,xmm1
+        vpaddq  xmm8,xmm8,xmm3
+        vpaddq  xmm9,xmm9,xmm4
+
+$L$ong_tail_avx:
+        vmovdqa XMMWORD[32+r11],xmm2
+        vmovdqa XMMWORD[r11],xmm0
+        vmovdqa XMMWORD[16+r11],xmm1
+        vmovdqa XMMWORD[48+r11],xmm3
+        vmovdqa XMMWORD[64+r11],xmm4
+
+
+
+
+
+
+
+        vpmuludq        xmm12,xmm14,xmm7
+        vpmuludq        xmm10,xmm14,xmm5
+        vpshufd xmm2,XMMWORD[((-48))+rdi],0x10
+        vpmuludq        xmm11,xmm14,xmm6
+        vpmuludq        xmm13,xmm14,xmm8
+        vpmuludq        xmm14,xmm14,xmm9
+
+        vpmuludq        xmm0,xmm2,xmm8
+        vpaddq  xmm14,xmm14,xmm0
+        vpshufd xmm3,XMMWORD[((-32))+rdi],0x10
+        vpmuludq        xmm1,xmm2,xmm7
+        vpaddq  xmm13,xmm13,xmm1
+        vpshufd xmm4,XMMWORD[((-16))+rdi],0x10
+        vpmuludq        xmm0,xmm2,xmm6
+        vpaddq  xmm12,xmm12,xmm0
+        vpmuludq        xmm2,xmm2,xmm5
+        vpaddq  xmm11,xmm11,xmm2
+        vpmuludq        xmm3,xmm3,xmm9
+        vpaddq  xmm10,xmm10,xmm3
+
+        vpshufd xmm2,XMMWORD[rdi],0x10
+        vpmuludq        xmm1,xmm4,xmm7
+        vpaddq  xmm14,xmm14,xmm1
+        vpmuludq        xmm0,xmm4,xmm6
+        vpaddq  xmm13,xmm13,xmm0
+        vpshufd xmm3,XMMWORD[16+rdi],0x10
+        vpmuludq        xmm4,xmm4,xmm5
+        vpaddq  xmm12,xmm12,xmm4
+        vpmuludq        xmm1,xmm2,xmm9
+        vpaddq  xmm11,xmm11,xmm1
+        vpshufd xmm4,XMMWORD[32+rdi],0x10
+        vpmuludq        xmm2,xmm2,xmm8
+        vpaddq  xmm10,xmm10,xmm2
+
+        vpmuludq        xmm0,xmm3,xmm6
+        vpaddq  xmm14,xmm14,xmm0
+        vpmuludq        xmm3,xmm3,xmm5
+        vpaddq  xmm13,xmm13,xmm3
+        vpshufd xmm2,XMMWORD[48+rdi],0x10
+        vpmuludq        xmm1,xmm4,xmm9
+        vpaddq  xmm12,xmm12,xmm1
+        vpshufd xmm3,XMMWORD[64+rdi],0x10
+        vpmuludq        xmm0,xmm4,xmm8
+        vpaddq  xmm11,xmm11,xmm0
+        vpmuludq        xmm4,xmm4,xmm7
+        vpaddq  xmm10,xmm10,xmm4
+
+        vpmuludq        xmm2,xmm2,xmm5
+        vpaddq  xmm14,xmm14,xmm2
+        vpmuludq        xmm1,xmm3,xmm9
+        vpaddq  xmm13,xmm13,xmm1
+        vpmuludq        xmm0,xmm3,xmm8
+        vpaddq  xmm12,xmm12,xmm0
+        vpmuludq        xmm1,xmm3,xmm7
+        vpaddq  xmm11,xmm11,xmm1
+        vpmuludq        xmm3,xmm3,xmm6
+        vpaddq  xmm10,xmm10,xmm3
+
+        jz      NEAR $L$short_tail_avx
+
+        vmovdqu xmm0,XMMWORD[rsi]
+        vmovdqu xmm1,XMMWORD[16+rsi]
+
+        vpsrldq xmm2,xmm0,6
+        vpsrldq xmm3,xmm1,6
+        vpunpckhqdq     xmm4,xmm0,xmm1
+        vpunpcklqdq     xmm0,xmm0,xmm1
+        vpunpcklqdq     xmm3,xmm2,xmm3
+
+        vpsrlq  xmm4,xmm4,40
+        vpsrlq  xmm1,xmm0,26
+        vpand   xmm0,xmm0,xmm15
+        vpsrlq  xmm2,xmm3,4
+        vpand   xmm1,xmm1,xmm15
+        vpsrlq  xmm3,xmm3,30
+        vpand   xmm2,xmm2,xmm15
+        vpand   xmm3,xmm3,xmm15
+        vpor    xmm4,xmm4,XMMWORD[32+rcx]
+
+        vpshufd xmm9,XMMWORD[((-64))+rdi],0x32
+        vpaddq  xmm0,xmm0,XMMWORD[r11]
+        vpaddq  xmm1,xmm1,XMMWORD[16+r11]
+        vpaddq  xmm2,xmm2,XMMWORD[32+r11]
+        vpaddq  xmm3,xmm3,XMMWORD[48+r11]
+        vpaddq  xmm4,xmm4,XMMWORD[64+r11]
+
+
+
+
+        vpmuludq        xmm5,xmm9,xmm0
+        vpaddq  xmm10,xmm10,xmm5
+        vpmuludq        xmm6,xmm9,xmm1
+        vpaddq  xmm11,xmm11,xmm6
+        vpmuludq        xmm5,xmm9,xmm2
+        vpaddq  xmm12,xmm12,xmm5
+        vpshufd xmm7,XMMWORD[((-48))+rdi],0x32
+        vpmuludq        xmm6,xmm9,xmm3
+        vpaddq  xmm13,xmm13,xmm6
+        vpmuludq        xmm9,xmm9,xmm4
+        vpaddq  xmm14,xmm14,xmm9
+
+        vpmuludq        xmm5,xmm7,xmm3
+        vpaddq  xmm14,xmm14,xmm5
+        vpshufd xmm8,XMMWORD[((-32))+rdi],0x32
+        vpmuludq        xmm6,xmm7,xmm2
+        vpaddq  xmm13,xmm13,xmm6
+        vpshufd xmm9,XMMWORD[((-16))+rdi],0x32
+        vpmuludq        xmm5,xmm7,xmm1
+        vpaddq  xmm12,xmm12,xmm5
+        vpmuludq        xmm7,xmm7,xmm0
+        vpaddq  xmm11,xmm11,xmm7
+        vpmuludq        xmm8,xmm8,xmm4
+        vpaddq  xmm10,xmm10,xmm8
+
+        vpshufd xmm7,XMMWORD[rdi],0x32
+        vpmuludq        xmm6,xmm9,xmm2
+        vpaddq  xmm14,xmm14,xmm6
+        vpmuludq        xmm5,xmm9,xmm1
+        vpaddq  xmm13,xmm13,xmm5
+        vpshufd xmm8,XMMWORD[16+rdi],0x32
+        vpmuludq        xmm9,xmm9,xmm0
+        vpaddq  xmm12,xmm12,xmm9
+        vpmuludq        xmm6,xmm7,xmm4
+        vpaddq  xmm11,xmm11,xmm6
+        vpshufd xmm9,XMMWORD[32+rdi],0x32
+        vpmuludq        xmm7,xmm7,xmm3
+        vpaddq  xmm10,xmm10,xmm7
+
+        vpmuludq        xmm5,xmm8,xmm1
+        vpaddq  xmm14,xmm14,xmm5
+        vpmuludq        xmm8,xmm8,xmm0
+        vpaddq  xmm13,xmm13,xmm8
+        vpshufd xmm7,XMMWORD[48+rdi],0x32
+        vpmuludq        xmm6,xmm9,xmm4
+        vpaddq  xmm12,xmm12,xmm6
+        vpshufd xmm8,XMMWORD[64+rdi],0x32
+        vpmuludq        xmm5,xmm9,xmm3
+        vpaddq  xmm11,xmm11,xmm5
+        vpmuludq        xmm9,xmm9,xmm2
+        vpaddq  xmm10,xmm10,xmm9
+
+        vpmuludq        xmm7,xmm7,xmm0
+        vpaddq  xmm14,xmm14,xmm7
+        vpmuludq        xmm6,xmm8,xmm4
+        vpaddq  xmm13,xmm13,xmm6
+        vpmuludq        xmm5,xmm8,xmm3
+        vpaddq  xmm12,xmm12,xmm5
+        vpmuludq        xmm6,xmm8,xmm2
+        vpaddq  xmm11,xmm11,xmm6
+        vpmuludq        xmm8,xmm8,xmm1
+        vpaddq  xmm10,xmm10,xmm8
+
+$L$short_tail_avx:
+
+
+
+        vpsrldq xmm9,xmm14,8
+        vpsrldq xmm8,xmm13,8
+        vpsrldq xmm6,xmm11,8
+        vpsrldq xmm5,xmm10,8
+        vpsrldq xmm7,xmm12,8
+        vpaddq  xmm13,xmm13,xmm8
+        vpaddq  xmm14,xmm14,xmm9
+        vpaddq  xmm10,xmm10,xmm5
+        vpaddq  xmm11,xmm11,xmm6
+        vpaddq  xmm12,xmm12,xmm7
+
+
+
+
+        vpsrlq  xmm3,xmm13,26
+        vpand   xmm13,xmm13,xmm15
+        vpaddq  xmm14,xmm14,xmm3
+
+        vpsrlq  xmm0,xmm10,26
+        vpand   xmm10,xmm10,xmm15
+        vpaddq  xmm11,xmm11,xmm0
+
+        vpsrlq  xmm4,xmm14,26
+        vpand   xmm14,xmm14,xmm15
+
+        vpsrlq  xmm1,xmm11,26
+        vpand   xmm11,xmm11,xmm15
+        vpaddq  xmm12,xmm12,xmm1
+
+        vpaddq  xmm10,xmm10,xmm4
+        vpsllq  xmm4,xmm4,2
+        vpaddq  xmm10,xmm10,xmm4
+
+        vpsrlq  xmm2,xmm12,26
+        vpand   xmm12,xmm12,xmm15
+        vpaddq  xmm13,xmm13,xmm2
+
+        vpsrlq  xmm0,xmm10,26
+        vpand   xmm10,xmm10,xmm15
+        vpaddq  xmm11,xmm11,xmm0
+
+        vpsrlq  xmm3,xmm13,26
+        vpand   xmm13,xmm13,xmm15
+        vpaddq  xmm14,xmm14,xmm3
+
+        vmovd   DWORD[(-112)+rdi],xmm10
+        vmovd   DWORD[(-108)+rdi],xmm11
+        vmovd   DWORD[(-104)+rdi],xmm12
+        vmovd   DWORD[(-100)+rdi],xmm13
+        vmovd   DWORD[(-96)+rdi],xmm14
+        vmovdqa xmm6,XMMWORD[80+r11]
+        vmovdqa xmm7,XMMWORD[96+r11]
+        vmovdqa xmm8,XMMWORD[112+r11]
+        vmovdqa xmm9,XMMWORD[128+r11]
+        vmovdqa xmm10,XMMWORD[144+r11]
+        vmovdqa xmm11,XMMWORD[160+r11]
+        vmovdqa xmm12,XMMWORD[176+r11]
+        vmovdqa xmm13,XMMWORD[192+r11]
+        vmovdqa xmm14,XMMWORD[208+r11]
+        vmovdqa xmm15,XMMWORD[224+r11]
+        lea     rsp,[248+r11]
+$L$do_avx_epilogue:
+        vzeroupper
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_avx:
+
+
+ALIGN   32
+poly1305_emit_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_emit_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+        cmp     DWORD[20+rdi],0
+        je      NEAR $L$emit
+
+        mov     eax,DWORD[rdi]
+        mov     ecx,DWORD[4+rdi]
+        mov     r8d,DWORD[8+rdi]
+        mov     r11d,DWORD[12+rdi]
+        mov     r10d,DWORD[16+rdi]
+
+        shl     rcx,26
+        mov     r9,r8
+        shl     r8,52
+        add     rax,rcx
+        shr     r9,12
+        add     r8,rax
+        adc     r9,0
+
+        shl     r11,14
+        mov     rax,r10
+        shr     r10,24
+        add     r9,r11
+        shl     rax,40
+        add     r9,rax
+        adc     r10,0
+
+        mov     rax,r10
+        mov     rcx,r10
+        and     r10,3
+        shr     rax,2
+        and     rcx,-4
+        add     rax,rcx
+        add     r8,rax
+        adc     r9,0
+        adc     r10,0
+
+        mov     rax,r8
+        add     r8,5
+        mov     rcx,r9
+        adc     r9,0
+        adc     r10,0
+        shr     r10,2
+        cmovnz  rax,r8
+        cmovnz  rcx,r9
+
+        add     rax,QWORD[rdx]
+        adc     rcx,QWORD[8+rdx]
+        mov     QWORD[rsi],rax
+        mov     QWORD[8+rsi],rcx
+
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_emit_avx:
+
+ALIGN   32
+poly1305_blocks_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+        mov     r8d,DWORD[20+rdi]
+        cmp     rdx,128
+        jae     NEAR $L$blocks_avx2
+        test    r8d,r8d
+        jz      NEAR $L$blocks
+
+$L$blocks_avx2:
+        and     rdx,-16
+        jz      NEAR $L$no_data_avx2
+
+        vzeroupper
+
+        test    r8d,r8d
+        jz      NEAR $L$base2_64_avx2
+
+        test    rdx,63
+        jz      NEAR $L$even_avx2
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$blocks_avx2_body:
+
+        mov     r15,rdx
+
+        mov     r8,QWORD[rdi]
+        mov     r9,QWORD[8+rdi]
+        mov     ebp,DWORD[16+rdi]
+
+        mov     r11,QWORD[24+rdi]
+        mov     r13,QWORD[32+rdi]
+
+
+        mov     r14d,r8d
+        and     r8,-2147483648
+        mov     r12,r9
+        mov     ebx,r9d
+        and     r9,-2147483648
+
+        shr     r8,6
+        shl     r12,52
+        add     r14,r8
+        shr     rbx,12
+        shr     r9,18
+        add     r14,r12
+        adc     rbx,r9
+
+        mov     r8,rbp
+        shl     r8,40
+        shr     rbp,24
+        add     rbx,r8
+        adc     rbp,0
+
+        mov     r9,-4
+        mov     r8,rbp
+        and     r9,rbp
+        shr     r8,2
+        and     rbp,3
+        add     r8,r9
+        add     r14,r8
+        adc     rbx,0
+        adc     rbp,0
+
+        mov     r12,r13
+        mov     rax,r13
+        shr     r13,2
+        add     r13,r12
+
+$L$base2_26_pre_avx2:
+        add     r14,QWORD[rsi]
+        adc     rbx,QWORD[8+rsi]
+        lea     rsi,[16+rsi]
+        adc     rbp,rcx
+        sub     r15,16
+
+        call    __poly1305_block
+        mov     rax,r12
+
+        test    r15,63
+        jnz     NEAR $L$base2_26_pre_avx2
+
+        test    rcx,rcx
+        jz      NEAR $L$store_base2_64_avx2
+
+
+        mov     rax,r14
+        mov     rdx,r14
+        shr     r14,52
+        mov     r11,rbx
+        mov     r12,rbx
+        shr     rdx,26
+        and     rax,0x3ffffff
+        shl     r11,12
+        and     rdx,0x3ffffff
+        shr     rbx,14
+        or      r14,r11
+        shl     rbp,24
+        and     r14,0x3ffffff
+        shr     r12,40
+        and     rbx,0x3ffffff
+        or      rbp,r12
+
+        test    r15,r15
+        jz      NEAR $L$store_base2_26_avx2
+
+        vmovd   xmm0,eax
+        vmovd   xmm1,edx
+        vmovd   xmm2,r14d
+        vmovd   xmm3,ebx
+        vmovd   xmm4,ebp
+        jmp     NEAR $L$proceed_avx2
+
+ALIGN   32
+$L$store_base2_64_avx2:
+        mov     QWORD[rdi],r14
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rbp
+        jmp     NEAR $L$done_avx2
+
+ALIGN   16
+$L$store_base2_26_avx2:
+        mov     DWORD[rdi],eax
+        mov     DWORD[4+rdi],edx
+        mov     DWORD[8+rdi],r14d
+        mov     DWORD[12+rdi],ebx
+        mov     DWORD[16+rdi],ebp
+ALIGN   16
+$L$done_avx2:
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbp,QWORD[32+rsp]
+
+        mov     rbx,QWORD[40+rsp]
+
+        lea     rsp,[48+rsp]
+
+$L$no_data_avx2:
+$L$blocks_avx2_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+
+ALIGN   32
+$L$base2_64_avx2:
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$base2_64_avx2_body:
+
+        mov     r15,rdx
+
+        mov     r11,QWORD[24+rdi]
+        mov     r13,QWORD[32+rdi]
+
+        mov     r14,QWORD[rdi]
+        mov     rbx,QWORD[8+rdi]
+        mov     ebp,DWORD[16+rdi]
+
+        mov     r12,r13
+        mov     rax,r13
+        shr     r13,2
+        add     r13,r12
+
+        test    rdx,63
+        jz      NEAR $L$init_avx2
+
+$L$base2_64_pre_avx2:
+        add     r14,QWORD[rsi]
+        adc     rbx,QWORD[8+rsi]
+        lea     rsi,[16+rsi]
+        adc     rbp,rcx
+        sub     r15,16
+
+        call    __poly1305_block
+        mov     rax,r12
+
+        test    r15,63
+        jnz     NEAR $L$base2_64_pre_avx2
+
+$L$init_avx2:
+
+        mov     rax,r14
+        mov     rdx,r14
+        shr     r14,52
+        mov     r8,rbx
+        mov     r9,rbx
+        shr     rdx,26
+        and     rax,0x3ffffff
+        shl     r8,12
+        and     rdx,0x3ffffff
+        shr     rbx,14
+        or      r14,r8
+        shl     rbp,24
+        and     r14,0x3ffffff
+        shr     r9,40
+        and     rbx,0x3ffffff
+        or      rbp,r9
+
+        vmovd   xmm0,eax
+        vmovd   xmm1,edx
+        vmovd   xmm2,r14d
+        vmovd   xmm3,ebx
+        vmovd   xmm4,ebp
+        mov     DWORD[20+rdi],1
+
+        call    __poly1305_init_avx
+
+$L$proceed_avx2:
+        mov     rdx,r15
+        mov     r10d,DWORD[((OPENSSL_ia32cap_P+8))]
+        mov     r11d,3221291008
+
+        mov     r15,QWORD[rsp]
+
+        mov     r14,QWORD[8+rsp]
+
+        mov     r13,QWORD[16+rsp]
+
+        mov     r12,QWORD[24+rsp]
+
+        mov     rbp,QWORD[32+rsp]
+
+        mov     rbx,QWORD[40+rsp]
+
+        lea     rax,[48+rsp]
+        lea     rsp,[48+rsp]
+
+$L$base2_64_avx2_epilogue:
+        jmp     NEAR $L$do_avx2
+
+
+ALIGN   32
+$L$even_avx2:
+
+        mov     r10d,DWORD[((OPENSSL_ia32cap_P+8))]
+        vmovd   xmm0,DWORD[rdi]
+        vmovd   xmm1,DWORD[4+rdi]
+        vmovd   xmm2,DWORD[8+rdi]
+        vmovd   xmm3,DWORD[12+rdi]
+        vmovd   xmm4,DWORD[16+rdi]
+
+$L$do_avx2:
+        cmp     rdx,512
+        jb      NEAR $L$skip_avx512
+        and     r10d,r11d
+        test    r10d,65536
+        jnz     NEAR $L$blocks_avx512
+$L$skip_avx512:
+        lea     r11,[((-248))+rsp]
+        sub     rsp,0x1c8
+        vmovdqa XMMWORD[80+r11],xmm6
+        vmovdqa XMMWORD[96+r11],xmm7
+        vmovdqa XMMWORD[112+r11],xmm8
+        vmovdqa XMMWORD[128+r11],xmm9
+        vmovdqa XMMWORD[144+r11],xmm10
+        vmovdqa XMMWORD[160+r11],xmm11
+        vmovdqa XMMWORD[176+r11],xmm12
+        vmovdqa XMMWORD[192+r11],xmm13
+        vmovdqa XMMWORD[208+r11],xmm14
+        vmovdqa XMMWORD[224+r11],xmm15
+$L$do_avx2_body:
+        lea     rcx,[$L$const]
+        lea     rdi,[((48+64))+rdi]
+        vmovdqa ymm7,YMMWORD[96+rcx]
+
+
+        vmovdqu xmm9,XMMWORD[((-64))+rdi]
+        and     rsp,-512
+        vmovdqu xmm10,XMMWORD[((-48))+rdi]
+        vmovdqu xmm6,XMMWORD[((-32))+rdi]
+        vmovdqu xmm11,XMMWORD[((-16))+rdi]
+        vmovdqu xmm12,XMMWORD[rdi]
+        vmovdqu xmm13,XMMWORD[16+rdi]
+        lea     rax,[144+rsp]
+        vmovdqu xmm14,XMMWORD[32+rdi]
+        vpermd  ymm9,ymm7,ymm9
+        vmovdqu xmm15,XMMWORD[48+rdi]
+        vpermd  ymm10,ymm7,ymm10
+        vmovdqu xmm5,XMMWORD[64+rdi]
+        vpermd  ymm6,ymm7,ymm6
+        vmovdqa YMMWORD[rsp],ymm9
+        vpermd  ymm11,ymm7,ymm11
+        vmovdqa YMMWORD[(32-144)+rax],ymm10
+        vpermd  ymm12,ymm7,ymm12
+        vmovdqa YMMWORD[(64-144)+rax],ymm6
+        vpermd  ymm13,ymm7,ymm13
+        vmovdqa YMMWORD[(96-144)+rax],ymm11
+        vpermd  ymm14,ymm7,ymm14
+        vmovdqa YMMWORD[(128-144)+rax],ymm12
+        vpermd  ymm15,ymm7,ymm15
+        vmovdqa YMMWORD[(160-144)+rax],ymm13
+        vpermd  ymm5,ymm7,ymm5
+        vmovdqa YMMWORD[(192-144)+rax],ymm14
+        vmovdqa YMMWORD[(224-144)+rax],ymm15
+        vmovdqa YMMWORD[(256-144)+rax],ymm5
+        vmovdqa ymm5,YMMWORD[64+rcx]
+
+
+
+        vmovdqu xmm7,XMMWORD[rsi]
+        vmovdqu xmm8,XMMWORD[16+rsi]
+        vinserti128     ymm7,ymm7,XMMWORD[32+rsi],1
+        vinserti128     ymm8,ymm8,XMMWORD[48+rsi],1
+        lea     rsi,[64+rsi]
+
+        vpsrldq ymm9,ymm7,6
+        vpsrldq ymm10,ymm8,6
+        vpunpckhqdq     ymm6,ymm7,ymm8
+        vpunpcklqdq     ymm9,ymm9,ymm10
+        vpunpcklqdq     ymm7,ymm7,ymm8
+
+        vpsrlq  ymm10,ymm9,30
+        vpsrlq  ymm9,ymm9,4
+        vpsrlq  ymm8,ymm7,26
+        vpsrlq  ymm6,ymm6,40
+        vpand   ymm9,ymm9,ymm5
+        vpand   ymm7,ymm7,ymm5
+        vpand   ymm8,ymm8,ymm5
+        vpand   ymm10,ymm10,ymm5
+        vpor    ymm6,ymm6,YMMWORD[32+rcx]
+
+        vpaddq  ymm2,ymm9,ymm2
+        sub     rdx,64
+        jz      NEAR $L$tail_avx2
+        jmp     NEAR $L$oop_avx2
+
+ALIGN   32
+$L$oop_avx2:
+
+
+
+
+
+
+
+
+        vpaddq  ymm0,ymm7,ymm0
+        vmovdqa ymm7,YMMWORD[rsp]
+        vpaddq  ymm1,ymm8,ymm1
+        vmovdqa ymm8,YMMWORD[32+rsp]
+        vpaddq  ymm3,ymm10,ymm3
+        vmovdqa ymm9,YMMWORD[96+rsp]
+        vpaddq  ymm4,ymm6,ymm4
+        vmovdqa ymm10,YMMWORD[48+rax]
+        vmovdqa ymm5,YMMWORD[112+rax]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        vpmuludq        ymm13,ymm7,ymm2
+        vpmuludq        ymm14,ymm8,ymm2
+        vpmuludq        ymm15,ymm9,ymm2
+        vpmuludq        ymm11,ymm10,ymm2
+        vpmuludq        ymm12,ymm5,ymm2
+
+        vpmuludq        ymm6,ymm8,ymm0
+        vpmuludq        ymm2,ymm8,ymm1
+        vpaddq  ymm12,ymm12,ymm6
+        vpaddq  ymm13,ymm13,ymm2
+        vpmuludq        ymm6,ymm8,ymm3
+        vpmuludq        ymm2,ymm4,YMMWORD[64+rsp]
+        vpaddq  ymm15,ymm15,ymm6
+        vpaddq  ymm11,ymm11,ymm2
+        vmovdqa ymm8,YMMWORD[((-16))+rax]
+
+        vpmuludq        ymm6,ymm7,ymm0
+        vpmuludq        ymm2,ymm7,ymm1
+        vpaddq  ymm11,ymm11,ymm6
+        vpaddq  ymm12,ymm12,ymm2
+        vpmuludq        ymm6,ymm7,ymm3
+        vpmuludq        ymm2,ymm7,ymm4
+        vmovdqu xmm7,XMMWORD[rsi]
+        vpaddq  ymm14,ymm14,ymm6
+        vpaddq  ymm15,ymm15,ymm2
+        vinserti128     ymm7,ymm7,XMMWORD[32+rsi],1
+
+        vpmuludq        ymm6,ymm8,ymm3
+        vpmuludq        ymm2,ymm8,ymm4
+        vmovdqu xmm8,XMMWORD[16+rsi]
+        vpaddq  ymm11,ymm11,ymm6
+        vpaddq  ymm12,ymm12,ymm2
+        vmovdqa ymm2,YMMWORD[16+rax]
+        vpmuludq        ymm6,ymm9,ymm1
+        vpmuludq        ymm9,ymm9,ymm0
+        vpaddq  ymm14,ymm14,ymm6
+        vpaddq  ymm13,ymm13,ymm9
+        vinserti128     ymm8,ymm8,XMMWORD[48+rsi],1
+        lea     rsi,[64+rsi]
+
+        vpmuludq        ymm6,ymm2,ymm1
+        vpmuludq        ymm2,ymm2,ymm0
+        vpsrldq ymm9,ymm7,6
+        vpaddq  ymm15,ymm15,ymm6
+        vpaddq  ymm14,ymm14,ymm2
+        vpmuludq        ymm6,ymm10,ymm3
+        vpmuludq        ymm2,ymm10,ymm4
+        vpsrldq ymm10,ymm8,6
+        vpaddq  ymm12,ymm12,ymm6
+        vpaddq  ymm13,ymm13,ymm2
+        vpunpckhqdq     ymm6,ymm7,ymm8
+
+        vpmuludq        ymm3,ymm5,ymm3
+        vpmuludq        ymm4,ymm5,ymm4
+        vpunpcklqdq     ymm7,ymm7,ymm8
+        vpaddq  ymm2,ymm13,ymm3
+        vpaddq  ymm3,ymm14,ymm4
+        vpunpcklqdq     ymm10,ymm9,ymm10
+        vpmuludq        ymm4,ymm0,YMMWORD[80+rax]
+        vpmuludq        ymm0,ymm5,ymm1
+        vmovdqa ymm5,YMMWORD[64+rcx]
+        vpaddq  ymm4,ymm15,ymm4
+        vpaddq  ymm0,ymm11,ymm0
+
+
+
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpaddq  ymm4,ymm4,ymm14
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpaddq  ymm1,ymm12,ymm11
+
+        vpsrlq  ymm15,ymm4,26
+        vpand   ymm4,ymm4,ymm5
+
+        vpsrlq  ymm9,ymm10,4
+
+        vpsrlq  ymm12,ymm1,26
+        vpand   ymm1,ymm1,ymm5
+        vpaddq  ymm2,ymm2,ymm12
+
+        vpaddq  ymm0,ymm0,ymm15
+        vpsllq  ymm15,ymm15,2
+        vpaddq  ymm0,ymm0,ymm15
+
+        vpand   ymm9,ymm9,ymm5
+        vpsrlq  ymm8,ymm7,26
+
+        vpsrlq  ymm13,ymm2,26
+        vpand   ymm2,ymm2,ymm5
+        vpaddq  ymm3,ymm3,ymm13
+
+        vpaddq  ymm2,ymm2,ymm9
+        vpsrlq  ymm10,ymm10,30
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpaddq  ymm1,ymm1,ymm11
+
+        vpsrlq  ymm6,ymm6,40
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpaddq  ymm4,ymm4,ymm14
+
+        vpand   ymm7,ymm7,ymm5
+        vpand   ymm8,ymm8,ymm5
+        vpand   ymm10,ymm10,ymm5
+        vpor    ymm6,ymm6,YMMWORD[32+rcx]
+
+        sub     rdx,64
+        jnz     NEAR $L$oop_avx2
+
+DB      0x66,0x90
+$L$tail_avx2:
+
+
+
+
+
+
+
+        vpaddq  ymm0,ymm7,ymm0
+        vmovdqu ymm7,YMMWORD[4+rsp]
+        vpaddq  ymm1,ymm8,ymm1
+        vmovdqu ymm8,YMMWORD[36+rsp]
+        vpaddq  ymm3,ymm10,ymm3
+        vmovdqu ymm9,YMMWORD[100+rsp]
+        vpaddq  ymm4,ymm6,ymm4
+        vmovdqu ymm10,YMMWORD[52+rax]
+        vmovdqu ymm5,YMMWORD[116+rax]
+
+        vpmuludq        ymm13,ymm7,ymm2
+        vpmuludq        ymm14,ymm8,ymm2
+        vpmuludq        ymm15,ymm9,ymm2
+        vpmuludq        ymm11,ymm10,ymm2
+        vpmuludq        ymm12,ymm5,ymm2
+
+        vpmuludq        ymm6,ymm8,ymm0
+        vpmuludq        ymm2,ymm8,ymm1
+        vpaddq  ymm12,ymm12,ymm6
+        vpaddq  ymm13,ymm13,ymm2
+        vpmuludq        ymm6,ymm8,ymm3
+        vpmuludq        ymm2,ymm4,YMMWORD[68+rsp]
+        vpaddq  ymm15,ymm15,ymm6
+        vpaddq  ymm11,ymm11,ymm2
+
+        vpmuludq        ymm6,ymm7,ymm0
+        vpmuludq        ymm2,ymm7,ymm1
+        vpaddq  ymm11,ymm11,ymm6
+        vmovdqu ymm8,YMMWORD[((-12))+rax]
+        vpaddq  ymm12,ymm12,ymm2
+        vpmuludq        ymm6,ymm7,ymm3
+        vpmuludq        ymm2,ymm7,ymm4
+        vpaddq  ymm14,ymm14,ymm6
+        vpaddq  ymm15,ymm15,ymm2
+
+        vpmuludq        ymm6,ymm8,ymm3
+        vpmuludq        ymm2,ymm8,ymm4
+        vpaddq  ymm11,ymm11,ymm6
+        vpaddq  ymm12,ymm12,ymm2
+        vmovdqu ymm2,YMMWORD[20+rax]
+        vpmuludq        ymm6,ymm9,ymm1
+        vpmuludq        ymm9,ymm9,ymm0
+        vpaddq  ymm14,ymm14,ymm6
+        vpaddq  ymm13,ymm13,ymm9
+
+        vpmuludq        ymm6,ymm2,ymm1
+        vpmuludq        ymm2,ymm2,ymm0
+        vpaddq  ymm15,ymm15,ymm6
+        vpaddq  ymm14,ymm14,ymm2
+        vpmuludq        ymm6,ymm10,ymm3
+        vpmuludq        ymm2,ymm10,ymm4
+        vpaddq  ymm12,ymm12,ymm6
+        vpaddq  ymm13,ymm13,ymm2
+
+        vpmuludq        ymm3,ymm5,ymm3
+        vpmuludq        ymm4,ymm5,ymm4
+        vpaddq  ymm2,ymm13,ymm3
+        vpaddq  ymm3,ymm14,ymm4
+        vpmuludq        ymm4,ymm0,YMMWORD[84+rax]
+        vpmuludq        ymm0,ymm5,ymm1
+        vmovdqa ymm5,YMMWORD[64+rcx]
+        vpaddq  ymm4,ymm15,ymm4
+        vpaddq  ymm0,ymm11,ymm0
+
+
+
+
+        vpsrldq ymm8,ymm12,8
+        vpsrldq ymm9,ymm2,8
+        vpsrldq ymm10,ymm3,8
+        vpsrldq ymm6,ymm4,8
+        vpsrldq ymm7,ymm0,8
+        vpaddq  ymm12,ymm12,ymm8
+        vpaddq  ymm2,ymm2,ymm9
+        vpaddq  ymm3,ymm3,ymm10
+        vpaddq  ymm4,ymm4,ymm6
+        vpaddq  ymm0,ymm0,ymm7
+
+        vpermq  ymm10,ymm3,0x2
+        vpermq  ymm6,ymm4,0x2
+        vpermq  ymm7,ymm0,0x2
+        vpermq  ymm8,ymm12,0x2
+        vpermq  ymm9,ymm2,0x2
+        vpaddq  ymm3,ymm3,ymm10
+        vpaddq  ymm4,ymm4,ymm6
+        vpaddq  ymm0,ymm0,ymm7
+        vpaddq  ymm12,ymm12,ymm8
+        vpaddq  ymm2,ymm2,ymm9
+
+
+
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpaddq  ymm4,ymm4,ymm14
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpaddq  ymm1,ymm12,ymm11
+
+        vpsrlq  ymm15,ymm4,26
+        vpand   ymm4,ymm4,ymm5
+
+        vpsrlq  ymm12,ymm1,26
+        vpand   ymm1,ymm1,ymm5
+        vpaddq  ymm2,ymm2,ymm12
+
+        vpaddq  ymm0,ymm0,ymm15
+        vpsllq  ymm15,ymm15,2
+        vpaddq  ymm0,ymm0,ymm15
+
+        vpsrlq  ymm13,ymm2,26
+        vpand   ymm2,ymm2,ymm5
+        vpaddq  ymm3,ymm3,ymm13
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpaddq  ymm1,ymm1,ymm11
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpaddq  ymm4,ymm4,ymm14
+
+        vmovd   DWORD[(-112)+rdi],xmm0
+        vmovd   DWORD[(-108)+rdi],xmm1
+        vmovd   DWORD[(-104)+rdi],xmm2
+        vmovd   DWORD[(-100)+rdi],xmm3
+        vmovd   DWORD[(-96)+rdi],xmm4
+        vmovdqa xmm6,XMMWORD[80+r11]
+        vmovdqa xmm7,XMMWORD[96+r11]
+        vmovdqa xmm8,XMMWORD[112+r11]
+        vmovdqa xmm9,XMMWORD[128+r11]
+        vmovdqa xmm10,XMMWORD[144+r11]
+        vmovdqa xmm11,XMMWORD[160+r11]
+        vmovdqa xmm12,XMMWORD[176+r11]
+        vmovdqa xmm13,XMMWORD[192+r11]
+        vmovdqa xmm14,XMMWORD[208+r11]
+        vmovdqa xmm15,XMMWORD[224+r11]
+        lea     rsp,[248+r11]
+$L$do_avx2_epilogue:
+        vzeroupper
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_avx2:
+
+ALIGN   32
+poly1305_blocks_avx512:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_avx512:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+$L$blocks_avx512:
+        mov     eax,15
+        kmovw   k2,eax
+        lea     r11,[((-248))+rsp]
+        sub     rsp,0x1c8
+        vmovdqa XMMWORD[80+r11],xmm6
+        vmovdqa XMMWORD[96+r11],xmm7
+        vmovdqa XMMWORD[112+r11],xmm8
+        vmovdqa XMMWORD[128+r11],xmm9
+        vmovdqa XMMWORD[144+r11],xmm10
+        vmovdqa XMMWORD[160+r11],xmm11
+        vmovdqa XMMWORD[176+r11],xmm12
+        vmovdqa XMMWORD[192+r11],xmm13
+        vmovdqa XMMWORD[208+r11],xmm14
+        vmovdqa XMMWORD[224+r11],xmm15
+$L$do_avx512_body:
+        lea     rcx,[$L$const]
+        lea     rdi,[((48+64))+rdi]
+        vmovdqa ymm9,YMMWORD[96+rcx]
+
+
+        vmovdqu xmm11,XMMWORD[((-64))+rdi]
+        and     rsp,-512
+        vmovdqu xmm12,XMMWORD[((-48))+rdi]
+        mov     rax,0x20
+        vmovdqu xmm7,XMMWORD[((-32))+rdi]
+        vmovdqu xmm13,XMMWORD[((-16))+rdi]
+        vmovdqu xmm8,XMMWORD[rdi]
+        vmovdqu xmm14,XMMWORD[16+rdi]
+        vmovdqu xmm10,XMMWORD[32+rdi]
+        vmovdqu xmm15,XMMWORD[48+rdi]
+        vmovdqu xmm6,XMMWORD[64+rdi]
+        vpermd  zmm16,zmm9,zmm11
+        vpbroadcastq    zmm5,QWORD[64+rcx]
+        vpermd  zmm17,zmm9,zmm12
+        vpermd  zmm21,zmm9,zmm7
+        vpermd  zmm18,zmm9,zmm13
+        vmovdqa64       ZMMWORD[rsp]{k2},zmm16
+        vpsrlq  zmm7,zmm16,32
+        vpermd  zmm22,zmm9,zmm8
+        vmovdqu64       ZMMWORD[rax*1+rsp]{k2},zmm17
+        vpsrlq  zmm8,zmm17,32
+        vpermd  zmm19,zmm9,zmm14
+        vmovdqa64       ZMMWORD[64+rsp]{k2},zmm21
+        vpermd  zmm23,zmm9,zmm10
+        vpermd  zmm20,zmm9,zmm15
+        vmovdqu64       ZMMWORD[64+rax*1+rsp]{k2},zmm18
+        vpermd  zmm24,zmm9,zmm6
+        vmovdqa64       ZMMWORD[128+rsp]{k2},zmm22
+        vmovdqu64       ZMMWORD[128+rax*1+rsp]{k2},zmm19
+        vmovdqa64       ZMMWORD[192+rsp]{k2},zmm23
+        vmovdqu64       ZMMWORD[192+rax*1+rsp]{k2},zmm20
+        vmovdqa64       ZMMWORD[256+rsp]{k2},zmm24
+
+
+
+
+
+
+
+
+
+
+        vpmuludq        zmm11,zmm16,zmm7
+        vpmuludq        zmm12,zmm17,zmm7
+        vpmuludq        zmm13,zmm18,zmm7
+        vpmuludq        zmm14,zmm19,zmm7
+        vpmuludq        zmm15,zmm20,zmm7
+        vpsrlq  zmm9,zmm18,32
+
+        vpmuludq        zmm25,zmm24,zmm8
+        vpmuludq        zmm26,zmm16,zmm8
+        vpmuludq        zmm27,zmm17,zmm8
+        vpmuludq        zmm28,zmm18,zmm8
+        vpmuludq        zmm29,zmm19,zmm8
+        vpsrlq  zmm10,zmm19,32
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+
+        vpmuludq        zmm25,zmm23,zmm9
+        vpmuludq        zmm26,zmm24,zmm9
+        vpmuludq        zmm28,zmm17,zmm9
+        vpmuludq        zmm29,zmm18,zmm9
+        vpmuludq        zmm27,zmm16,zmm9
+        vpsrlq  zmm6,zmm20,32
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpmuludq        zmm25,zmm22,zmm10
+        vpmuludq        zmm28,zmm16,zmm10
+        vpmuludq        zmm29,zmm17,zmm10
+        vpmuludq        zmm26,zmm23,zmm10
+        vpmuludq        zmm27,zmm24,zmm10
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpmuludq        zmm28,zmm24,zmm6
+        vpmuludq        zmm29,zmm16,zmm6
+        vpmuludq        zmm25,zmm21,zmm6
+        vpmuludq        zmm26,zmm22,zmm6
+        vpmuludq        zmm27,zmm23,zmm6
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+
+
+        vmovdqu64       zmm10,ZMMWORD[rsi]
+        vmovdqu64       zmm6,ZMMWORD[64+rsi]
+        lea     rsi,[128+rsi]
+
+
+
+
+        vpsrlq  zmm28,zmm14,26
+        vpandq  zmm14,zmm14,zmm5
+        vpaddq  zmm15,zmm15,zmm28
+
+        vpsrlq  zmm25,zmm11,26
+        vpandq  zmm11,zmm11,zmm5
+        vpaddq  zmm12,zmm12,zmm25
+
+        vpsrlq  zmm29,zmm15,26
+        vpandq  zmm15,zmm15,zmm5
+
+        vpsrlq  zmm26,zmm12,26
+        vpandq  zmm12,zmm12,zmm5
+        vpaddq  zmm13,zmm13,zmm26
+
+        vpaddq  zmm11,zmm11,zmm29
+        vpsllq  zmm29,zmm29,2
+        vpaddq  zmm11,zmm11,zmm29
+
+        vpsrlq  zmm27,zmm13,26
+        vpandq  zmm13,zmm13,zmm5
+        vpaddq  zmm14,zmm14,zmm27
+
+        vpsrlq  zmm25,zmm11,26
+        vpandq  zmm11,zmm11,zmm5
+        vpaddq  zmm12,zmm12,zmm25
+
+        vpsrlq  zmm28,zmm14,26
+        vpandq  zmm14,zmm14,zmm5
+        vpaddq  zmm15,zmm15,zmm28
+
+
+
+
+
+        vpunpcklqdq     zmm7,zmm10,zmm6
+        vpunpckhqdq     zmm6,zmm10,zmm6
+
+
+
+
+
+
+        vmovdqa32       zmm25,ZMMWORD[128+rcx]
+        mov     eax,0x7777
+        kmovw   k1,eax
+
+        vpermd  zmm16,zmm25,zmm16
+        vpermd  zmm17,zmm25,zmm17
+        vpermd  zmm18,zmm25,zmm18
+        vpermd  zmm19,zmm25,zmm19
+        vpermd  zmm20,zmm25,zmm20
+
+        vpermd  zmm16{k1},zmm25,zmm11
+        vpermd  zmm17{k1},zmm25,zmm12
+        vpermd  zmm18{k1},zmm25,zmm13
+        vpermd  zmm19{k1},zmm25,zmm14
+        vpermd  zmm20{k1},zmm25,zmm15
+
+        vpslld  zmm21,zmm17,2
+        vpslld  zmm22,zmm18,2
+        vpslld  zmm23,zmm19,2
+        vpslld  zmm24,zmm20,2
+        vpaddd  zmm21,zmm21,zmm17
+        vpaddd  zmm22,zmm22,zmm18
+        vpaddd  zmm23,zmm23,zmm19
+        vpaddd  zmm24,zmm24,zmm20
+
+        vpbroadcastq    zmm30,QWORD[32+rcx]
+
+        vpsrlq  zmm9,zmm7,52
+        vpsllq  zmm10,zmm6,12
+        vporq   zmm9,zmm9,zmm10
+        vpsrlq  zmm8,zmm7,26
+        vpsrlq  zmm10,zmm6,14
+        vpsrlq  zmm6,zmm6,40
+        vpandq  zmm9,zmm9,zmm5
+        vpandq  zmm7,zmm7,zmm5
+
+
+
+
+        vpaddq  zmm2,zmm9,zmm2
+        sub     rdx,192
+        jbe     NEAR $L$tail_avx512
+        jmp     NEAR $L$oop_avx512
+
+ALIGN   32
+$L$oop_avx512:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        vpmuludq        zmm14,zmm17,zmm2
+        vpaddq  zmm0,zmm7,zmm0
+        vpmuludq        zmm15,zmm18,zmm2
+        vpandq  zmm8,zmm8,zmm5
+        vpmuludq        zmm11,zmm23,zmm2
+        vpandq  zmm10,zmm10,zmm5
+        vpmuludq        zmm12,zmm24,zmm2
+        vporq   zmm6,zmm6,zmm30
+        vpmuludq        zmm13,zmm16,zmm2
+        vpaddq  zmm1,zmm8,zmm1
+        vpaddq  zmm3,zmm10,zmm3
+        vpaddq  zmm4,zmm6,zmm4
+
+        vmovdqu64       zmm10,ZMMWORD[rsi]
+        vmovdqu64       zmm6,ZMMWORD[64+rsi]
+        lea     rsi,[128+rsi]
+        vpmuludq        zmm28,zmm19,zmm0
+        vpmuludq        zmm29,zmm20,zmm0
+        vpmuludq        zmm25,zmm16,zmm0
+        vpmuludq        zmm26,zmm17,zmm0
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+
+        vpmuludq        zmm28,zmm18,zmm1
+        vpmuludq        zmm29,zmm19,zmm1
+        vpmuludq        zmm25,zmm24,zmm1
+        vpmuludq        zmm27,zmm18,zmm0
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpunpcklqdq     zmm7,zmm10,zmm6
+        vpunpckhqdq     zmm6,zmm10,zmm6
+
+        vpmuludq        zmm28,zmm16,zmm3
+        vpmuludq        zmm29,zmm17,zmm3
+        vpmuludq        zmm26,zmm16,zmm1
+        vpmuludq        zmm27,zmm17,zmm1
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpmuludq        zmm28,zmm24,zmm4
+        vpmuludq        zmm29,zmm16,zmm4
+        vpmuludq        zmm25,zmm22,zmm3
+        vpmuludq        zmm26,zmm23,zmm3
+        vpaddq  zmm14,zmm14,zmm28
+        vpmuludq        zmm27,zmm24,zmm3
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpmuludq        zmm25,zmm21,zmm4
+        vpmuludq        zmm26,zmm22,zmm4
+        vpmuludq        zmm27,zmm23,zmm4
+        vpaddq  zmm0,zmm11,zmm25
+        vpaddq  zmm1,zmm12,zmm26
+        vpaddq  zmm2,zmm13,zmm27
+
+
+
+
+        vpsrlq  zmm9,zmm7,52
+        vpsllq  zmm10,zmm6,12
+
+        vpsrlq  zmm3,zmm14,26
+        vpandq  zmm14,zmm14,zmm5
+        vpaddq  zmm4,zmm15,zmm3
+
+        vporq   zmm9,zmm9,zmm10
+
+        vpsrlq  zmm11,zmm0,26
+        vpandq  zmm0,zmm0,zmm5
+        vpaddq  zmm1,zmm1,zmm11
+
+        vpandq  zmm9,zmm9,zmm5
+
+        vpsrlq  zmm15,zmm4,26
+        vpandq  zmm4,zmm4,zmm5
+
+        vpsrlq  zmm12,zmm1,26
+        vpandq  zmm1,zmm1,zmm5
+        vpaddq  zmm2,zmm2,zmm12
+
+        vpaddq  zmm0,zmm0,zmm15
+        vpsllq  zmm15,zmm15,2
+        vpaddq  zmm0,zmm0,zmm15
+
+        vpaddq  zmm2,zmm2,zmm9
+        vpsrlq  zmm8,zmm7,26
+
+        vpsrlq  zmm13,zmm2,26
+        vpandq  zmm2,zmm2,zmm5
+        vpaddq  zmm3,zmm14,zmm13
+
+        vpsrlq  zmm10,zmm6,14
+
+        vpsrlq  zmm11,zmm0,26
+        vpandq  zmm0,zmm0,zmm5
+        vpaddq  zmm1,zmm1,zmm11
+
+        vpsrlq  zmm6,zmm6,40
+
+        vpsrlq  zmm14,zmm3,26
+        vpandq  zmm3,zmm3,zmm5
+        vpaddq  zmm4,zmm4,zmm14
+
+        vpandq  zmm7,zmm7,zmm5
+
+
+
+
+        sub     rdx,128
+        ja      NEAR $L$oop_avx512
+
+$L$tail_avx512:
+
+
+
+
+
+        vpsrlq  zmm16,zmm16,32
+        vpsrlq  zmm17,zmm17,32
+        vpsrlq  zmm18,zmm18,32
+        vpsrlq  zmm23,zmm23,32
+        vpsrlq  zmm24,zmm24,32
+        vpsrlq  zmm19,zmm19,32
+        vpsrlq  zmm20,zmm20,32
+        vpsrlq  zmm21,zmm21,32
+        vpsrlq  zmm22,zmm22,32
+
+
+
+        lea     rsi,[rdx*1+rsi]
+
+
+        vpaddq  zmm0,zmm7,zmm0
+
+        vpmuludq        zmm14,zmm17,zmm2
+        vpmuludq        zmm15,zmm18,zmm2
+        vpmuludq        zmm11,zmm23,zmm2
+        vpandq  zmm8,zmm8,zmm5
+        vpmuludq        zmm12,zmm24,zmm2
+        vpandq  zmm10,zmm10,zmm5
+        vpmuludq        zmm13,zmm16,zmm2
+        vporq   zmm6,zmm6,zmm30
+        vpaddq  zmm1,zmm8,zmm1
+        vpaddq  zmm3,zmm10,zmm3
+        vpaddq  zmm4,zmm6,zmm4
+
+        vmovdqu xmm7,XMMWORD[rsi]
+        vpmuludq        zmm28,zmm19,zmm0
+        vpmuludq        zmm29,zmm20,zmm0
+        vpmuludq        zmm25,zmm16,zmm0
+        vpmuludq        zmm26,zmm17,zmm0
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+
+        vmovdqu xmm8,XMMWORD[16+rsi]
+        vpmuludq        zmm28,zmm18,zmm1
+        vpmuludq        zmm29,zmm19,zmm1
+        vpmuludq        zmm25,zmm24,zmm1
+        vpmuludq        zmm27,zmm18,zmm0
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm13,zmm13,zmm27
+
+        vinserti128     ymm7,ymm7,XMMWORD[32+rsi],1
+        vpmuludq        zmm28,zmm16,zmm3
+        vpmuludq        zmm29,zmm17,zmm3
+        vpmuludq        zmm26,zmm16,zmm1
+        vpmuludq        zmm27,zmm17,zmm1
+        vpaddq  zmm14,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+        vinserti128     ymm8,ymm8,XMMWORD[48+rsi],1
+        vpmuludq        zmm28,zmm24,zmm4
+        vpmuludq        zmm29,zmm16,zmm4
+        vpmuludq        zmm25,zmm22,zmm3
+        vpmuludq        zmm26,zmm23,zmm3
+        vpmuludq        zmm27,zmm24,zmm3
+        vpaddq  zmm3,zmm14,zmm28
+        vpaddq  zmm15,zmm15,zmm29
+        vpaddq  zmm11,zmm11,zmm25
+        vpaddq  zmm12,zmm12,zmm26
+        vpaddq  zmm13,zmm13,zmm27
+
+        vpmuludq        zmm25,zmm21,zmm4
+        vpmuludq        zmm26,zmm22,zmm4
+        vpmuludq        zmm27,zmm23,zmm4
+        vpaddq  zmm0,zmm11,zmm25
+        vpaddq  zmm1,zmm12,zmm26
+        vpaddq  zmm2,zmm13,zmm27
+
+
+
+
+        mov     eax,1
+        vpermq  zmm14,zmm3,0xb1
+        vpermq  zmm4,zmm15,0xb1
+        vpermq  zmm11,zmm0,0xb1
+        vpermq  zmm12,zmm1,0xb1
+        vpermq  zmm13,zmm2,0xb1
+        vpaddq  zmm3,zmm3,zmm14
+        vpaddq  zmm4,zmm4,zmm15
+        vpaddq  zmm0,zmm0,zmm11
+        vpaddq  zmm1,zmm1,zmm12
+        vpaddq  zmm2,zmm2,zmm13
+
+        kmovw   k3,eax
+        vpermq  zmm14,zmm3,0x2
+        vpermq  zmm15,zmm4,0x2
+        vpermq  zmm11,zmm0,0x2
+        vpermq  zmm12,zmm1,0x2
+        vpermq  zmm13,zmm2,0x2
+        vpaddq  zmm3,zmm3,zmm14
+        vpaddq  zmm4,zmm4,zmm15
+        vpaddq  zmm0,zmm0,zmm11
+        vpaddq  zmm1,zmm1,zmm12
+        vpaddq  zmm2,zmm2,zmm13
+
+        vextracti64x4   ymm14,zmm3,0x1
+        vextracti64x4   ymm15,zmm4,0x1
+        vextracti64x4   ymm11,zmm0,0x1
+        vextracti64x4   ymm12,zmm1,0x1
+        vextracti64x4   ymm13,zmm2,0x1
+        vpaddq  zmm3{k3}{z},zmm3,zmm14
+        vpaddq  zmm4{k3}{z},zmm4,zmm15
+        vpaddq  zmm0{k3}{z},zmm0,zmm11
+        vpaddq  zmm1{k3}{z},zmm1,zmm12
+        vpaddq  zmm2{k3}{z},zmm2,zmm13
+
+
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpsrldq ymm9,ymm7,6
+        vpsrldq ymm10,ymm8,6
+        vpunpckhqdq     ymm6,ymm7,ymm8
+        vpaddq  ymm4,ymm4,ymm14
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpunpcklqdq     ymm9,ymm9,ymm10
+        vpunpcklqdq     ymm7,ymm7,ymm8
+        vpaddq  ymm1,ymm1,ymm11
+
+        vpsrlq  ymm15,ymm4,26
+        vpand   ymm4,ymm4,ymm5
+
+        vpsrlq  ymm12,ymm1,26
+        vpand   ymm1,ymm1,ymm5
+        vpsrlq  ymm10,ymm9,30
+        vpsrlq  ymm9,ymm9,4
+        vpaddq  ymm2,ymm2,ymm12
+
+        vpaddq  ymm0,ymm0,ymm15
+        vpsllq  ymm15,ymm15,2
+        vpsrlq  ymm8,ymm7,26
+        vpsrlq  ymm6,ymm6,40
+        vpaddq  ymm0,ymm0,ymm15
+
+        vpsrlq  ymm13,ymm2,26
+        vpand   ymm2,ymm2,ymm5
+        vpand   ymm9,ymm9,ymm5
+        vpand   ymm7,ymm7,ymm5
+        vpaddq  ymm3,ymm3,ymm13
+
+        vpsrlq  ymm11,ymm0,26
+        vpand   ymm0,ymm0,ymm5
+        vpaddq  ymm2,ymm9,ymm2
+        vpand   ymm8,ymm8,ymm5
+        vpaddq  ymm1,ymm1,ymm11
+
+        vpsrlq  ymm14,ymm3,26
+        vpand   ymm3,ymm3,ymm5
+        vpand   ymm10,ymm10,ymm5
+        vpor    ymm6,ymm6,YMMWORD[32+rcx]
+        vpaddq  ymm4,ymm4,ymm14
+
+        lea     rax,[144+rsp]
+        add     rdx,64
+        jnz     NEAR $L$tail_avx2
+
+        vpsubq  ymm2,ymm2,ymm9
+        vmovd   DWORD[(-112)+rdi],xmm0
+        vmovd   DWORD[(-108)+rdi],xmm1
+        vmovd   DWORD[(-104)+rdi],xmm2
+        vmovd   DWORD[(-100)+rdi],xmm3
+        vmovd   DWORD[(-96)+rdi],xmm4
+        vzeroall
+        movdqa  xmm6,XMMWORD[80+r11]
+        movdqa  xmm7,XMMWORD[96+r11]
+        movdqa  xmm8,XMMWORD[112+r11]
+        movdqa  xmm9,XMMWORD[128+r11]
+        movdqa  xmm10,XMMWORD[144+r11]
+        movdqa  xmm11,XMMWORD[160+r11]
+        movdqa  xmm12,XMMWORD[176+r11]
+        movdqa  xmm13,XMMWORD[192+r11]
+        movdqa  xmm14,XMMWORD[208+r11]
+        movdqa  xmm15,XMMWORD[224+r11]
+        lea     rsp,[248+r11]
+$L$do_avx512_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_avx512:
+
+ALIGN   32
+poly1305_init_base2_44:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_init_base2_44:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+        xor     rax,rax
+        mov     QWORD[rdi],rax
+        mov     QWORD[8+rdi],rax
+        mov     QWORD[16+rdi],rax
+
+$L$init_base2_44:
+        lea     r10,[poly1305_blocks_vpmadd52]
+        lea     r11,[poly1305_emit_base2_44]
+
+        mov     rax,0x0ffffffc0fffffff
+        mov     rcx,0x0ffffffc0ffffffc
+        and     rax,QWORD[rsi]
+        mov     r8,0x00000fffffffffff
+        and     rcx,QWORD[8+rsi]
+        mov     r9,0x00000fffffffffff
+        and     r8,rax
+        shrd    rax,rcx,44
+        mov     QWORD[40+rdi],r8
+        and     rax,r9
+        shr     rcx,24
+        mov     QWORD[48+rdi],rax
+        lea     rax,[rax*4+rax]
+        mov     QWORD[56+rdi],rcx
+        shl     rax,2
+        lea     rcx,[rcx*4+rcx]
+        shl     rcx,2
+        mov     QWORD[24+rdi],rax
+        mov     QWORD[32+rdi],rcx
+        mov     QWORD[64+rdi],-1
+        mov     QWORD[rdx],r10
+        mov     QWORD[8+rdx],r11
+        mov     eax,1
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_init_base2_44:
+
+ALIGN   32
+poly1305_blocks_vpmadd52:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_vpmadd52:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+DB      243,15,30,250
+        shr     rdx,4
+        jz      NEAR $L$no_data_vpmadd52
+
+        shl     rcx,40
+        mov     r8,QWORD[64+rdi]
+
+
+
+
+
+
+        mov     rax,3
+        mov     r10,1
+        cmp     rdx,4
+        cmovae  rax,r10
+        test    r8,r8
+        cmovns  rax,r10
+
+        and     rax,rdx
+        jz      NEAR $L$blocks_vpmadd52_4x
+
+        sub     rdx,rax
+        mov     r10d,7
+        mov     r11d,1
+        kmovw   k7,r10d
+        lea     r10,[$L$2_44_inp_permd]
+        kmovw   k1,r11d
+
+        vmovq   xmm21,rcx
+        vmovdqa64       ymm19,YMMWORD[r10]
+        vmovdqa64       ymm20,YMMWORD[32+r10]
+        vpermq  ymm21,ymm21,0xcf
+        vmovdqa64       ymm22,YMMWORD[64+r10]
+
+        vmovdqu64       ymm16{k7}{z},[rdi]
+        vmovdqu64       ymm3{k7}{z},[40+rdi]
+        vmovdqu64       ymm4{k7}{z},[32+rdi]
+        vmovdqu64       ymm5{k7}{z},[24+rdi]
+
+        vmovdqa64       ymm23,YMMWORD[96+r10]
+        vmovdqa64       ymm24,YMMWORD[128+r10]
+
+        jmp     NEAR $L$oop_vpmadd52
+
+ALIGN   32
+$L$oop_vpmadd52:
+        vmovdqu32       xmm18,XMMWORD[rsi]
+        lea     rsi,[16+rsi]
+
+        vpermd  ymm18,ymm19,ymm18
+        vpsrlvq ymm18,ymm18,ymm20
+        vpandq  ymm18,ymm18,ymm22
+        vporq   ymm18,ymm18,ymm21
+
+        vpaddq  ymm16,ymm16,ymm18
+
+        vpermq  ymm0{k7}{z},ymm16,0
+        vpermq  ymm1{k7}{z},ymm16,85
+        vpermq  ymm2{k7}{z},ymm16,170
+
+        vpxord  ymm16,ymm16,ymm16
+        vpxord  ymm17,ymm17,ymm17
+
+        vpmadd52luq     ymm16,ymm0,ymm3
+        vpmadd52huq     ymm17,ymm0,ymm3
+
+        vpmadd52luq     ymm16,ymm1,ymm4
+        vpmadd52huq     ymm17,ymm1,ymm4
+
+        vpmadd52luq     ymm16,ymm2,ymm5
+        vpmadd52huq     ymm17,ymm2,ymm5
+
+        vpsrlvq ymm18,ymm16,ymm23
+        vpsllvq ymm17,ymm17,ymm24
+        vpandq  ymm16,ymm16,ymm22
+
+        vpaddq  ymm17,ymm17,ymm18
+
+        vpermq  ymm17,ymm17,147
+
+        vpaddq  ymm16,ymm16,ymm17
+
+        vpsrlvq ymm18,ymm16,ymm23
+        vpandq  ymm16,ymm16,ymm22
+
+        vpermq  ymm18,ymm18,147
+
+        vpaddq  ymm16,ymm16,ymm18
+
+        vpermq  ymm18{k1}{z},ymm16,147
+
+        vpaddq  ymm16,ymm16,ymm18
+        vpsllq  ymm18,ymm18,2
+
+        vpaddq  ymm16,ymm16,ymm18
+
+        dec     rax
+        jnz     NEAR $L$oop_vpmadd52
+
+        vmovdqu64       YMMWORD[rdi]{k7},ymm16
+
+        test    rdx,rdx
+        jnz     NEAR $L$blocks_vpmadd52_4x
+
+$L$no_data_vpmadd52:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_vpmadd52:
+
+ALIGN   32
+poly1305_blocks_vpmadd52_4x:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_vpmadd52_4x:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+        shr     rdx,4
+        jz      NEAR $L$no_data_vpmadd52_4x
+
+        shl     rcx,40
+        mov     r8,QWORD[64+rdi]
+
+$L$blocks_vpmadd52_4x:
+        vpbroadcastq    ymm31,rcx
+
+        vmovdqa64       ymm28,YMMWORD[$L$x_mask44]
+        mov     eax,5
+        vmovdqa64       ymm29,YMMWORD[$L$x_mask42]
+        kmovw   k1,eax
+
+        test    r8,r8
+        js      NEAR $L$init_vpmadd52
+
+        vmovq   xmm0,QWORD[rdi]
+        vmovq   xmm1,QWORD[8+rdi]
+        vmovq   xmm2,QWORD[16+rdi]
+
+        test    rdx,3
+        jnz     NEAR $L$blocks_vpmadd52_2x_do
+
+$L$blocks_vpmadd52_4x_do:
+        vpbroadcastq    ymm3,QWORD[64+rdi]
+        vpbroadcastq    ymm4,QWORD[96+rdi]
+        vpbroadcastq    ymm5,QWORD[128+rdi]
+        vpbroadcastq    ymm16,QWORD[160+rdi]
+
+$L$blocks_vpmadd52_4x_key_loaded:
+        vpsllq  ymm17,ymm5,2
+        vpaddq  ymm17,ymm17,ymm5
+        vpsllq  ymm17,ymm17,2
+
+        test    rdx,7
+        jz      NEAR $L$blocks_vpmadd52_8x
+
+        vmovdqu64       ymm26,YMMWORD[rsi]
+        vmovdqu64       ymm27,YMMWORD[32+rsi]
+        lea     rsi,[64+rsi]
+
+        vpunpcklqdq     ymm25,ymm26,ymm27
+        vpunpckhqdq     ymm27,ymm26,ymm27
+
+
+
+        vpsrlq  ymm26,ymm27,24
+        vporq   ymm26,ymm26,ymm31
+        vpaddq  ymm2,ymm2,ymm26
+        vpandq  ymm24,ymm25,ymm28
+        vpsrlq  ymm25,ymm25,44
+        vpsllq  ymm27,ymm27,20
+        vporq   ymm25,ymm25,ymm27
+        vpandq  ymm25,ymm25,ymm28
+
+        sub     rdx,4
+        jz      NEAR $L$tail_vpmadd52_4x
+        jmp     NEAR $L$oop_vpmadd52_4x
+        ud2
+
+ALIGN   32
+$L$init_vpmadd52:
+        vmovq   xmm16,QWORD[24+rdi]
+        vmovq   xmm2,QWORD[56+rdi]
+        vmovq   xmm17,QWORD[32+rdi]
+        vmovq   xmm3,QWORD[40+rdi]
+        vmovq   xmm4,QWORD[48+rdi]
+
+        vmovdqa ymm0,ymm3
+        vmovdqa ymm1,ymm4
+        vmovdqa ymm5,ymm2
+
+        mov     eax,2
+
+$L$mul_init_vpmadd52:
+        vpxorq  ymm18,ymm18,ymm18
+        vpmadd52luq     ymm18,ymm16,ymm2
+        vpxorq  ymm19,ymm19,ymm19
+        vpmadd52huq     ymm19,ymm16,ymm2
+        vpxorq  ymm20,ymm20,ymm20
+        vpmadd52luq     ymm20,ymm17,ymm2
+        vpxorq  ymm21,ymm21,ymm21
+        vpmadd52huq     ymm21,ymm17,ymm2
+        vpxorq  ymm22,ymm22,ymm22
+        vpmadd52luq     ymm22,ymm3,ymm2
+        vpxorq  ymm23,ymm23,ymm23
+        vpmadd52huq     ymm23,ymm3,ymm2
+
+        vpmadd52luq     ymm18,ymm3,ymm0
+        vpmadd52huq     ymm19,ymm3,ymm0
+        vpmadd52luq     ymm20,ymm4,ymm0
+        vpmadd52huq     ymm21,ymm4,ymm0
+        vpmadd52luq     ymm22,ymm5,ymm0
+        vpmadd52huq     ymm23,ymm5,ymm0
+
+        vpmadd52luq     ymm18,ymm17,ymm1
+        vpmadd52huq     ymm19,ymm17,ymm1
+        vpmadd52luq     ymm20,ymm3,ymm1
+        vpmadd52huq     ymm21,ymm3,ymm1
+        vpmadd52luq     ymm22,ymm4,ymm1
+        vpmadd52huq     ymm23,ymm4,ymm1
+
+
+
+        vpsrlq  ymm30,ymm18,44
+        vpsllq  ymm19,ymm19,8
+        vpandq  ymm0,ymm18,ymm28
+        vpaddq  ymm19,ymm19,ymm30
+
+        vpaddq  ymm20,ymm20,ymm19
+
+        vpsrlq  ymm30,ymm20,44
+        vpsllq  ymm21,ymm21,8
+        vpandq  ymm1,ymm20,ymm28
+        vpaddq  ymm21,ymm21,ymm30
+
+        vpaddq  ymm22,ymm22,ymm21
+
+        vpsrlq  ymm30,ymm22,42
+        vpsllq  ymm23,ymm23,10
+        vpandq  ymm2,ymm22,ymm29
+        vpaddq  ymm23,ymm23,ymm30
+
+        vpaddq  ymm0,ymm0,ymm23
+        vpsllq  ymm23,ymm23,2
+
+        vpaddq  ymm0,ymm0,ymm23
+
+        vpsrlq  ymm30,ymm0,44
+        vpandq  ymm0,ymm0,ymm28
+
+        vpaddq  ymm1,ymm1,ymm30
+
+        dec     eax
+        jz      NEAR $L$done_init_vpmadd52
+
+        vpunpcklqdq     ymm4,ymm1,ymm4
+        vpbroadcastq    xmm1,xmm1
+        vpunpcklqdq     ymm5,ymm2,ymm5
+        vpbroadcastq    xmm2,xmm2
+        vpunpcklqdq     ymm3,ymm0,ymm3
+        vpbroadcastq    xmm0,xmm0
+
+        vpsllq  ymm16,ymm4,2
+        vpsllq  ymm17,ymm5,2
+        vpaddq  ymm16,ymm16,ymm4
+        vpaddq  ymm17,ymm17,ymm5
+        vpsllq  ymm16,ymm16,2
+        vpsllq  ymm17,ymm17,2
+
+        jmp     NEAR $L$mul_init_vpmadd52
+        ud2
+
+ALIGN   32
+$L$done_init_vpmadd52:
+        vinserti128     ymm4,ymm1,xmm4,1
+        vinserti128     ymm5,ymm2,xmm5,1
+        vinserti128     ymm3,ymm0,xmm3,1
+
+        vpermq  ymm4,ymm4,216
+        vpermq  ymm5,ymm5,216
+        vpermq  ymm3,ymm3,216
+
+        vpsllq  ymm16,ymm4,2
+        vpaddq  ymm16,ymm16,ymm4
+        vpsllq  ymm16,ymm16,2
+
+        vmovq   xmm0,QWORD[rdi]
+        vmovq   xmm1,QWORD[8+rdi]
+        vmovq   xmm2,QWORD[16+rdi]
+
+        test    rdx,3
+        jnz     NEAR $L$done_init_vpmadd52_2x
+
+        vmovdqu64       YMMWORD[64+rdi],ymm3
+        vpbroadcastq    ymm3,xmm3
+        vmovdqu64       YMMWORD[96+rdi],ymm4
+        vpbroadcastq    ymm4,xmm4
+        vmovdqu64       YMMWORD[128+rdi],ymm5
+        vpbroadcastq    ymm5,xmm5
+        vmovdqu64       YMMWORD[160+rdi],ymm16
+        vpbroadcastq    ymm16,xmm16
+
+        jmp     NEAR $L$blocks_vpmadd52_4x_key_loaded
+        ud2
+
+ALIGN   32
+$L$done_init_vpmadd52_2x:
+        vmovdqu64       YMMWORD[64+rdi],ymm3
+        vpsrldq ymm3,ymm3,8
+        vmovdqu64       YMMWORD[96+rdi],ymm4
+        vpsrldq ymm4,ymm4,8
+        vmovdqu64       YMMWORD[128+rdi],ymm5
+        vpsrldq ymm5,ymm5,8
+        vmovdqu64       YMMWORD[160+rdi],ymm16
+        vpsrldq ymm16,ymm16,8
+        jmp     NEAR $L$blocks_vpmadd52_2x_key_loaded
+        ud2
+
+ALIGN   32
+$L$blocks_vpmadd52_2x_do:
+        vmovdqu64       ymm5{k1}{z},[((128+8))+rdi]
+        vmovdqu64       ymm16{k1}{z},[((160+8))+rdi]
+        vmovdqu64       ymm3{k1}{z},[((64+8))+rdi]
+        vmovdqu64       ymm4{k1}{z},[((96+8))+rdi]
+
+$L$blocks_vpmadd52_2x_key_loaded:
+        vmovdqu64       ymm26,YMMWORD[rsi]
+        vpxorq  ymm27,ymm27,ymm27
+        lea     rsi,[32+rsi]
+
+        vpunpcklqdq     ymm25,ymm26,ymm27
+        vpunpckhqdq     ymm27,ymm26,ymm27
+
+
+
+        vpsrlq  ymm26,ymm27,24
+        vporq   ymm26,ymm26,ymm31
+        vpaddq  ymm2,ymm2,ymm26
+        vpandq  ymm24,ymm25,ymm28
+        vpsrlq  ymm25,ymm25,44
+        vpsllq  ymm27,ymm27,20
+        vporq   ymm25,ymm25,ymm27
+        vpandq  ymm25,ymm25,ymm28
+
+        jmp     NEAR $L$tail_vpmadd52_2x
+        ud2
+
+ALIGN   32
+$L$oop_vpmadd52_4x:
+
+        vpaddq  ymm0,ymm0,ymm24
+        vpaddq  ymm1,ymm1,ymm25
+
+        vpxorq  ymm18,ymm18,ymm18
+        vpmadd52luq     ymm18,ymm16,ymm2
+        vpxorq  ymm19,ymm19,ymm19
+        vpmadd52huq     ymm19,ymm16,ymm2
+        vpxorq  ymm20,ymm20,ymm20
+        vpmadd52luq     ymm20,ymm17,ymm2
+        vpxorq  ymm21,ymm21,ymm21
+        vpmadd52huq     ymm21,ymm17,ymm2
+        vpxorq  ymm22,ymm22,ymm22
+        vpmadd52luq     ymm22,ymm3,ymm2
+        vpxorq  ymm23,ymm23,ymm23
+        vpmadd52huq     ymm23,ymm3,ymm2
+
+        vmovdqu64       ymm26,YMMWORD[rsi]
+        vmovdqu64       ymm27,YMMWORD[32+rsi]
+        lea     rsi,[64+rsi]
+        vpmadd52luq     ymm18,ymm3,ymm0
+        vpmadd52huq     ymm19,ymm3,ymm0
+        vpmadd52luq     ymm20,ymm4,ymm0
+        vpmadd52huq     ymm21,ymm4,ymm0
+        vpmadd52luq     ymm22,ymm5,ymm0
+        vpmadd52huq     ymm23,ymm5,ymm0
+
+        vpunpcklqdq     ymm25,ymm26,ymm27
+        vpunpckhqdq     ymm27,ymm26,ymm27
+        vpmadd52luq     ymm18,ymm17,ymm1
+        vpmadd52huq     ymm19,ymm17,ymm1
+        vpmadd52luq     ymm20,ymm3,ymm1
+        vpmadd52huq     ymm21,ymm3,ymm1
+        vpmadd52luq     ymm22,ymm4,ymm1
+        vpmadd52huq     ymm23,ymm4,ymm1
+
+
+
+        vpsrlq  ymm30,ymm18,44
+        vpsllq  ymm19,ymm19,8
+        vpandq  ymm0,ymm18,ymm28
+        vpaddq  ymm19,ymm19,ymm30
+
+        vpsrlq  ymm26,ymm27,24
+        vporq   ymm26,ymm26,ymm31
+        vpaddq  ymm20,ymm20,ymm19
+
+        vpsrlq  ymm30,ymm20,44
+        vpsllq  ymm21,ymm21,8
+        vpandq  ymm1,ymm20,ymm28
+        vpaddq  ymm21,ymm21,ymm30
+
+        vpandq  ymm24,ymm25,ymm28
+        vpsrlq  ymm25,ymm25,44
+        vpsllq  ymm27,ymm27,20
+        vpaddq  ymm22,ymm22,ymm21
+
+        vpsrlq  ymm30,ymm22,42
+        vpsllq  ymm23,ymm23,10
+        vpandq  ymm2,ymm22,ymm29
+        vpaddq  ymm23,ymm23,ymm30
+
+        vpaddq  ymm2,ymm2,ymm26
+        vpaddq  ymm0,ymm0,ymm23
+        vpsllq  ymm23,ymm23,2
+
+        vpaddq  ymm0,ymm0,ymm23
+        vporq   ymm25,ymm25,ymm27
+        vpandq  ymm25,ymm25,ymm28
+
+        vpsrlq  ymm30,ymm0,44
+        vpandq  ymm0,ymm0,ymm28
+
+        vpaddq  ymm1,ymm1,ymm30
+
+        sub     rdx,4
+        jnz     NEAR $L$oop_vpmadd52_4x
+
+$L$tail_vpmadd52_4x:
+        vmovdqu64       ymm5,YMMWORD[128+rdi]
+        vmovdqu64       ymm16,YMMWORD[160+rdi]
+        vmovdqu64       ymm3,YMMWORD[64+rdi]
+        vmovdqu64       ymm4,YMMWORD[96+rdi]
+
+$L$tail_vpmadd52_2x:
+        vpsllq  ymm17,ymm5,2
+        vpaddq  ymm17,ymm17,ymm5
+        vpsllq  ymm17,ymm17,2
+
+
+        vpaddq  ymm0,ymm0,ymm24
+        vpaddq  ymm1,ymm1,ymm25
+
+        vpxorq  ymm18,ymm18,ymm18
+        vpmadd52luq     ymm18,ymm16,ymm2
+        vpxorq  ymm19,ymm19,ymm19
+        vpmadd52huq     ymm19,ymm16,ymm2
+        vpxorq  ymm20,ymm20,ymm20
+        vpmadd52luq     ymm20,ymm17,ymm2
+        vpxorq  ymm21,ymm21,ymm21
+        vpmadd52huq     ymm21,ymm17,ymm2
+        vpxorq  ymm22,ymm22,ymm22
+        vpmadd52luq     ymm22,ymm3,ymm2
+        vpxorq  ymm23,ymm23,ymm23
+        vpmadd52huq     ymm23,ymm3,ymm2
+
+        vpmadd52luq     ymm18,ymm3,ymm0
+        vpmadd52huq     ymm19,ymm3,ymm0
+        vpmadd52luq     ymm20,ymm4,ymm0
+        vpmadd52huq     ymm21,ymm4,ymm0
+        vpmadd52luq     ymm22,ymm5,ymm0
+        vpmadd52huq     ymm23,ymm5,ymm0
+
+        vpmadd52luq     ymm18,ymm17,ymm1
+        vpmadd52huq     ymm19,ymm17,ymm1
+        vpmadd52luq     ymm20,ymm3,ymm1
+        vpmadd52huq     ymm21,ymm3,ymm1
+        vpmadd52luq     ymm22,ymm4,ymm1
+        vpmadd52huq     ymm23,ymm4,ymm1
+
+
+
+
+        mov     eax,1
+        kmovw   k1,eax
+        vpsrldq ymm24,ymm18,8
+        vpsrldq ymm0,ymm19,8
+        vpsrldq ymm25,ymm20,8
+        vpsrldq ymm1,ymm21,8
+        vpaddq  ymm18,ymm18,ymm24
+        vpaddq  ymm19,ymm19,ymm0
+        vpsrldq ymm26,ymm22,8
+        vpsrldq ymm2,ymm23,8
+        vpaddq  ymm20,ymm20,ymm25
+        vpaddq  ymm21,ymm21,ymm1
+        vpermq  ymm24,ymm18,0x2
+        vpermq  ymm0,ymm19,0x2
+        vpaddq  ymm22,ymm22,ymm26
+        vpaddq  ymm23,ymm23,ymm2
+
+        vpermq  ymm25,ymm20,0x2
+        vpermq  ymm1,ymm21,0x2
+        vpaddq  ymm18{k1}{z},ymm18,ymm24
+        vpaddq  ymm19{k1}{z},ymm19,ymm0
+        vpermq  ymm26,ymm22,0x2
+        vpermq  ymm2,ymm23,0x2
+        vpaddq  ymm20{k1}{z},ymm20,ymm25
+        vpaddq  ymm21{k1}{z},ymm21,ymm1
+        vpaddq  ymm22{k1}{z},ymm22,ymm26
+        vpaddq  ymm23{k1}{z},ymm23,ymm2
+
+
+
+        vpsrlq  ymm30,ymm18,44
+        vpsllq  ymm19,ymm19,8
+        vpandq  ymm0,ymm18,ymm28
+        vpaddq  ymm19,ymm19,ymm30
+
+        vpaddq  ymm20,ymm20,ymm19
+
+        vpsrlq  ymm30,ymm20,44
+        vpsllq  ymm21,ymm21,8
+        vpandq  ymm1,ymm20,ymm28
+        vpaddq  ymm21,ymm21,ymm30
+
+        vpaddq  ymm22,ymm22,ymm21
+
+        vpsrlq  ymm30,ymm22,42
+        vpsllq  ymm23,ymm23,10
+        vpandq  ymm2,ymm22,ymm29
+        vpaddq  ymm23,ymm23,ymm30
+
+        vpaddq  ymm0,ymm0,ymm23
+        vpsllq  ymm23,ymm23,2
+
+        vpaddq  ymm0,ymm0,ymm23
+
+        vpsrlq  ymm30,ymm0,44
+        vpandq  ymm0,ymm0,ymm28
+
+        vpaddq  ymm1,ymm1,ymm30
+
+
+        sub     rdx,2
+        ja      NEAR $L$blocks_vpmadd52_4x_do
+
+        vmovq   QWORD[rdi],xmm0
+        vmovq   QWORD[8+rdi],xmm1
+        vmovq   QWORD[16+rdi],xmm2
+        vzeroall
+
+$L$no_data_vpmadd52_4x:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_vpmadd52_4x:
+
+ALIGN   32
+poly1305_blocks_vpmadd52_8x:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_blocks_vpmadd52_8x:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+
+
+
+        shr     rdx,4
+        jz      NEAR $L$no_data_vpmadd52_8x
+
+        shl     rcx,40
+        mov     r8,QWORD[64+rdi]
+
+        vmovdqa64       ymm28,YMMWORD[$L$x_mask44]
+        vmovdqa64       ymm29,YMMWORD[$L$x_mask42]
+
+        test    r8,r8
+        js      NEAR $L$init_vpmadd52
+
+        vmovq   xmm0,QWORD[rdi]
+        vmovq   xmm1,QWORD[8+rdi]
+        vmovq   xmm2,QWORD[16+rdi]
+
+$L$blocks_vpmadd52_8x:
+
+
+
+        vmovdqu64       ymm5,YMMWORD[128+rdi]
+        vmovdqu64       ymm16,YMMWORD[160+rdi]
+        vmovdqu64       ymm3,YMMWORD[64+rdi]
+        vmovdqu64       ymm4,YMMWORD[96+rdi]
+
+        vpsllq  ymm17,ymm5,2
+        vpaddq  ymm17,ymm17,ymm5
+        vpsllq  ymm17,ymm17,2
+
+        vpbroadcastq    ymm8,xmm5
+        vpbroadcastq    ymm6,xmm3
+        vpbroadcastq    ymm7,xmm4
+
+        vpxorq  ymm18,ymm18,ymm18
+        vpmadd52luq     ymm18,ymm16,ymm8
+        vpxorq  ymm19,ymm19,ymm19
+        vpmadd52huq     ymm19,ymm16,ymm8
+        vpxorq  ymm20,ymm20,ymm20
+        vpmadd52luq     ymm20,ymm17,ymm8
+        vpxorq  ymm21,ymm21,ymm21
+        vpmadd52huq     ymm21,ymm17,ymm8
+        vpxorq  ymm22,ymm22,ymm22
+        vpmadd52luq     ymm22,ymm3,ymm8
+        vpxorq  ymm23,ymm23,ymm23
+        vpmadd52huq     ymm23,ymm3,ymm8
+
+        vpmadd52luq     ymm18,ymm3,ymm6
+        vpmadd52huq     ymm19,ymm3,ymm6
+        vpmadd52luq     ymm20,ymm4,ymm6
+        vpmadd52huq     ymm21,ymm4,ymm6
+        vpmadd52luq     ymm22,ymm5,ymm6
+        vpmadd52huq     ymm23,ymm5,ymm6
+
+        vpmadd52luq     ymm18,ymm17,ymm7
+        vpmadd52huq     ymm19,ymm17,ymm7
+        vpmadd52luq     ymm20,ymm3,ymm7
+        vpmadd52huq     ymm21,ymm3,ymm7
+        vpmadd52luq     ymm22,ymm4,ymm7
+        vpmadd52huq     ymm23,ymm4,ymm7
+
+
+
+        vpsrlq  ymm30,ymm18,44
+        vpsllq  ymm19,ymm19,8
+        vpandq  ymm6,ymm18,ymm28
+        vpaddq  ymm19,ymm19,ymm30
+
+        vpaddq  ymm20,ymm20,ymm19
+
+        vpsrlq  ymm30,ymm20,44
+        vpsllq  ymm21,ymm21,8
+        vpandq  ymm7,ymm20,ymm28
+        vpaddq  ymm21,ymm21,ymm30
+
+        vpaddq  ymm22,ymm22,ymm21
+
+        vpsrlq  ymm30,ymm22,42
+        vpsllq  ymm23,ymm23,10
+        vpandq  ymm8,ymm22,ymm29
+        vpaddq  ymm23,ymm23,ymm30
+
+        vpaddq  ymm6,ymm6,ymm23
+        vpsllq  ymm23,ymm23,2
+
+        vpaddq  ymm6,ymm6,ymm23
+
+        vpsrlq  ymm30,ymm6,44
+        vpandq  ymm6,ymm6,ymm28
+
+        vpaddq  ymm7,ymm7,ymm30
+
+
+
+
+
+        vpunpcklqdq     ymm26,ymm8,ymm5
+        vpunpckhqdq     ymm5,ymm8,ymm5
+        vpunpcklqdq     ymm24,ymm6,ymm3
+        vpunpckhqdq     ymm3,ymm6,ymm3
+        vpunpcklqdq     ymm25,ymm7,ymm4
+        vpunpckhqdq     ymm4,ymm7,ymm4
+        vshufi64x2      zmm8,zmm26,zmm5,0x44
+        vshufi64x2      zmm6,zmm24,zmm3,0x44
+        vshufi64x2      zmm7,zmm25,zmm4,0x44
+
+        vmovdqu64       zmm26,ZMMWORD[rsi]
+        vmovdqu64       zmm27,ZMMWORD[64+rsi]
+        lea     rsi,[128+rsi]
+
+        vpsllq  zmm10,zmm8,2
+        vpsllq  zmm9,zmm7,2
+        vpaddq  zmm10,zmm10,zmm8
+        vpaddq  zmm9,zmm9,zmm7
+        vpsllq  zmm10,zmm10,2
+        vpsllq  zmm9,zmm9,2
+
+        vpbroadcastq    zmm31,rcx
+        vpbroadcastq    zmm28,xmm28
+        vpbroadcastq    zmm29,xmm29
+
+        vpbroadcastq    zmm16,xmm9
+        vpbroadcastq    zmm17,xmm10
+        vpbroadcastq    zmm3,xmm6
+        vpbroadcastq    zmm4,xmm7
+        vpbroadcastq    zmm5,xmm8
+
+        vpunpcklqdq     zmm25,zmm26,zmm27
+        vpunpckhqdq     zmm27,zmm26,zmm27
+
+
+
+        vpsrlq  zmm26,zmm27,24
+        vporq   zmm26,zmm26,zmm31
+        vpaddq  zmm2,zmm2,zmm26
+        vpandq  zmm24,zmm25,zmm28
+        vpsrlq  zmm25,zmm25,44
+        vpsllq  zmm27,zmm27,20
+        vporq   zmm25,zmm25,zmm27
+        vpandq  zmm25,zmm25,zmm28
+
+        sub     rdx,8
+        jz      NEAR $L$tail_vpmadd52_8x
+        jmp     NEAR $L$oop_vpmadd52_8x
+
+ALIGN   32
+$L$oop_vpmadd52_8x:
+
+        vpaddq  zmm0,zmm0,zmm24
+        vpaddq  zmm1,zmm1,zmm25
+
+        vpxorq  zmm18,zmm18,zmm18
+        vpmadd52luq     zmm18,zmm16,zmm2
+        vpxorq  zmm19,zmm19,zmm19
+        vpmadd52huq     zmm19,zmm16,zmm2
+        vpxorq  zmm20,zmm20,zmm20
+        vpmadd52luq     zmm20,zmm17,zmm2
+        vpxorq  zmm21,zmm21,zmm21
+        vpmadd52huq     zmm21,zmm17,zmm2
+        vpxorq  zmm22,zmm22,zmm22
+        vpmadd52luq     zmm22,zmm3,zmm2
+        vpxorq  zmm23,zmm23,zmm23
+        vpmadd52huq     zmm23,zmm3,zmm2
+
+        vmovdqu64       zmm26,ZMMWORD[rsi]
+        vmovdqu64       zmm27,ZMMWORD[64+rsi]
+        lea     rsi,[128+rsi]
+        vpmadd52luq     zmm18,zmm3,zmm0
+        vpmadd52huq     zmm19,zmm3,zmm0
+        vpmadd52luq     zmm20,zmm4,zmm0
+        vpmadd52huq     zmm21,zmm4,zmm0
+        vpmadd52luq     zmm22,zmm5,zmm0
+        vpmadd52huq     zmm23,zmm5,zmm0
+
+        vpunpcklqdq     zmm25,zmm26,zmm27
+        vpunpckhqdq     zmm27,zmm26,zmm27
+        vpmadd52luq     zmm18,zmm17,zmm1
+        vpmadd52huq     zmm19,zmm17,zmm1
+        vpmadd52luq     zmm20,zmm3,zmm1
+        vpmadd52huq     zmm21,zmm3,zmm1
+        vpmadd52luq     zmm22,zmm4,zmm1
+        vpmadd52huq     zmm23,zmm4,zmm1
+
+
+
+        vpsrlq  zmm30,zmm18,44
+        vpsllq  zmm19,zmm19,8
+        vpandq  zmm0,zmm18,zmm28
+        vpaddq  zmm19,zmm19,zmm30
+
+        vpsrlq  zmm26,zmm27,24
+        vporq   zmm26,zmm26,zmm31
+        vpaddq  zmm20,zmm20,zmm19
+
+        vpsrlq  zmm30,zmm20,44
+        vpsllq  zmm21,zmm21,8
+        vpandq  zmm1,zmm20,zmm28
+        vpaddq  zmm21,zmm21,zmm30
+
+        vpandq  zmm24,zmm25,zmm28
+        vpsrlq  zmm25,zmm25,44
+        vpsllq  zmm27,zmm27,20
+        vpaddq  zmm22,zmm22,zmm21
+
+        vpsrlq  zmm30,zmm22,42
+        vpsllq  zmm23,zmm23,10
+        vpandq  zmm2,zmm22,zmm29
+        vpaddq  zmm23,zmm23,zmm30
+
+        vpaddq  zmm2,zmm2,zmm26
+        vpaddq  zmm0,zmm0,zmm23
+        vpsllq  zmm23,zmm23,2
+
+        vpaddq  zmm0,zmm0,zmm23
+        vporq   zmm25,zmm25,zmm27
+        vpandq  zmm25,zmm25,zmm28
+
+        vpsrlq  zmm30,zmm0,44
+        vpandq  zmm0,zmm0,zmm28
+
+        vpaddq  zmm1,zmm1,zmm30
+
+        sub     rdx,8
+        jnz     NEAR $L$oop_vpmadd52_8x
+
+$L$tail_vpmadd52_8x:
+
+        vpaddq  zmm0,zmm0,zmm24
+        vpaddq  zmm1,zmm1,zmm25
+
+        vpxorq  zmm18,zmm18,zmm18
+        vpmadd52luq     zmm18,zmm9,zmm2
+        vpxorq  zmm19,zmm19,zmm19
+        vpmadd52huq     zmm19,zmm9,zmm2
+        vpxorq  zmm20,zmm20,zmm20
+        vpmadd52luq     zmm20,zmm10,zmm2
+        vpxorq  zmm21,zmm21,zmm21
+        vpmadd52huq     zmm21,zmm10,zmm2
+        vpxorq  zmm22,zmm22,zmm22
+        vpmadd52luq     zmm22,zmm6,zmm2
+        vpxorq  zmm23,zmm23,zmm23
+        vpmadd52huq     zmm23,zmm6,zmm2
+
+        vpmadd52luq     zmm18,zmm6,zmm0
+        vpmadd52huq     zmm19,zmm6,zmm0
+        vpmadd52luq     zmm20,zmm7,zmm0
+        vpmadd52huq     zmm21,zmm7,zmm0
+        vpmadd52luq     zmm22,zmm8,zmm0
+        vpmadd52huq     zmm23,zmm8,zmm0
+
+        vpmadd52luq     zmm18,zmm10,zmm1
+        vpmadd52huq     zmm19,zmm10,zmm1
+        vpmadd52luq     zmm20,zmm6,zmm1
+        vpmadd52huq     zmm21,zmm6,zmm1
+        vpmadd52luq     zmm22,zmm7,zmm1
+        vpmadd52huq     zmm23,zmm7,zmm1
+
+
+
+
+        mov     eax,1
+        kmovw   k1,eax
+        vpsrldq zmm24,zmm18,8
+        vpsrldq zmm0,zmm19,8
+        vpsrldq zmm25,zmm20,8
+        vpsrldq zmm1,zmm21,8
+        vpaddq  zmm18,zmm18,zmm24
+        vpaddq  zmm19,zmm19,zmm0
+        vpsrldq zmm26,zmm22,8
+        vpsrldq zmm2,zmm23,8
+        vpaddq  zmm20,zmm20,zmm25
+        vpaddq  zmm21,zmm21,zmm1
+        vpermq  zmm24,zmm18,0x2
+        vpermq  zmm0,zmm19,0x2
+        vpaddq  zmm22,zmm22,zmm26
+        vpaddq  zmm23,zmm23,zmm2
+
+        vpermq  zmm25,zmm20,0x2
+        vpermq  zmm1,zmm21,0x2
+        vpaddq  zmm18,zmm18,zmm24
+        vpaddq  zmm19,zmm19,zmm0
+        vpermq  zmm26,zmm22,0x2
+        vpermq  zmm2,zmm23,0x2
+        vpaddq  zmm20,zmm20,zmm25
+        vpaddq  zmm21,zmm21,zmm1
+        vextracti64x4   ymm24,zmm18,1
+        vextracti64x4   ymm0,zmm19,1
+        vpaddq  zmm22,zmm22,zmm26
+        vpaddq  zmm23,zmm23,zmm2
+
+        vextracti64x4   ymm25,zmm20,1
+        vextracti64x4   ymm1,zmm21,1
+        vextracti64x4   ymm26,zmm22,1
+        vextracti64x4   ymm2,zmm23,1
+        vpaddq  ymm18{k1}{z},ymm18,ymm24
+        vpaddq  ymm19{k1}{z},ymm19,ymm0
+        vpaddq  ymm20{k1}{z},ymm20,ymm25
+        vpaddq  ymm21{k1}{z},ymm21,ymm1
+        vpaddq  ymm22{k1}{z},ymm22,ymm26
+        vpaddq  ymm23{k1}{z},ymm23,ymm2
+
+
+
+        vpsrlq  ymm30,ymm18,44
+        vpsllq  ymm19,ymm19,8
+        vpandq  ymm0,ymm18,ymm28
+        vpaddq  ymm19,ymm19,ymm30
+
+        vpaddq  ymm20,ymm20,ymm19
+
+        vpsrlq  ymm30,ymm20,44
+        vpsllq  ymm21,ymm21,8
+        vpandq  ymm1,ymm20,ymm28
+        vpaddq  ymm21,ymm21,ymm30
+
+        vpaddq  ymm22,ymm22,ymm21
+
+        vpsrlq  ymm30,ymm22,42
+        vpsllq  ymm23,ymm23,10
+        vpandq  ymm2,ymm22,ymm29
+        vpaddq  ymm23,ymm23,ymm30
+
+        vpaddq  ymm0,ymm0,ymm23
+        vpsllq  ymm23,ymm23,2
+
+        vpaddq  ymm0,ymm0,ymm23
+
+        vpsrlq  ymm30,ymm0,44
+        vpandq  ymm0,ymm0,ymm28
+
+        vpaddq  ymm1,ymm1,ymm30
+
+
+
+        vmovq   QWORD[rdi],xmm0
+        vmovq   QWORD[8+rdi],xmm1
+        vmovq   QWORD[16+rdi],xmm2
+        vzeroall
+
+$L$no_data_vpmadd52_8x:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_blocks_vpmadd52_8x:
+
+ALIGN   32
+poly1305_emit_base2_44:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_poly1305_emit_base2_44:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+DB      243,15,30,250
+        mov     r8,QWORD[rdi]
+        mov     r9,QWORD[8+rdi]
+        mov     r10,QWORD[16+rdi]
+
+        mov     rax,r9
+        shr     r9,20
+        shl     rax,44
+        mov     rcx,r10
+        shr     r10,40
+        shl     rcx,24
+
+        add     r8,rax
+        adc     r9,rcx
+        adc     r10,0
+
+        mov     rax,r8
+        add     r8,5
+        mov     rcx,r9
+        adc     r9,0
+        adc     r10,0
+        shr     r10,2
+        cmovnz  rax,r8
+        cmovnz  rcx,r9
+
+        add     rax,QWORD[rdx]
+        adc     rcx,QWORD[8+rdx]
+        mov     QWORD[rsi],rax
+        mov     QWORD[8+rsi],rcx
+
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_poly1305_emit_base2_44:
+ALIGN   64
+$L$const:
+$L$mask24:
+        DD      0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0
+$L$129:
+        DD      16777216,0,16777216,0,16777216,0,16777216,0
+$L$mask26:
+        DD      0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0
+$L$permd_avx2:
+        DD      2,2,2,3,2,0,2,1
+$L$permd_avx512:
+        DD      0,0,0,1,0,2,0,3,0,4,0,5,0,6,0,7
+
+$L$2_44_inp_permd:
+        DD      0,1,1,2,2,3,7,7
+$L$2_44_inp_shift:
+        DQ      0,12,24,64
+$L$2_44_mask:
+        DQ      0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff
+$L$2_44_shift_rgt:
+        DQ      44,44,42,64
+$L$2_44_shift_lft:
+        DQ      8,8,10,64
+
+ALIGN   64
+$L$x_mask44:
+        DQ      0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
+        DQ      0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
+$L$x_mask42:
+        DQ      0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
+        DQ      0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
 DB      80,111,108,121,49,51,48,53,32,102,111,114,32,120,56,54
 DB      95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32
@@ -462 +3910 @@
         DD      $L$SEH_end_poly1305_emit wrt ..imagebase
         DD      $L$SEH_info_poly1305_emit wrt ..imagebase
+        DD      $L$SEH_begin_poly1305_blocks_avx wrt ..imagebase
+        DD      $L$base2_64_avx wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx_1 wrt ..imagebase
+
+        DD      $L$base2_64_avx wrt ..imagebase
+        DD      $L$even_avx wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx_2 wrt ..imagebase
+
+        DD      $L$even_avx wrt ..imagebase
+        DD      $L$SEH_end_poly1305_blocks_avx wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx_3 wrt ..imagebase
+
+        DD      $L$SEH_begin_poly1305_emit_avx wrt ..imagebase
+        DD      $L$SEH_end_poly1305_emit_avx wrt ..imagebase
+        DD      $L$SEH_info_poly1305_emit_avx wrt ..imagebase
+        DD      $L$SEH_begin_poly1305_blocks_avx2 wrt ..imagebase
+        DD      $L$base2_64_avx2 wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx2_1 wrt ..imagebase
+
+        DD      $L$base2_64_avx2 wrt ..imagebase
+        DD      $L$even_avx2 wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx2_2 wrt ..imagebase
+
+        DD      $L$even_avx2 wrt ..imagebase
+        DD      $L$SEH_end_poly1305_blocks_avx2 wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx2_3 wrt ..imagebase
+        DD      $L$SEH_begin_poly1305_blocks_avx512 wrt ..imagebase
+        DD      $L$SEH_end_poly1305_blocks_avx512 wrt ..imagebase
+        DD      $L$SEH_info_poly1305_blocks_avx512 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -478 +3955 @@
         DD      se_handler wrt ..imagebase
         DD      $L$SEH_begin_poly1305_emit wrt ..imagebase,$L$SEH_begin_poly1305_emit wrt ..imagebase
+$L$SEH_info_poly1305_blocks_avx_1:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$blocks_avx_body wrt ..imagebase,$L$blocks_avx_epilogue wrt ..imagebase
+
+$L$SEH_info_poly1305_blocks_avx_2:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$base2_64_avx_body wrt ..imagebase,$L$base2_64_avx_epilogue wrt ..imagebase
+
+$L$SEH_info_poly1305_blocks_avx_3:
+DB      9,0,0,0
+        DD      avx_handler wrt ..imagebase
+        DD      $L$do_avx_body wrt ..imagebase,$L$do_avx_epilogue wrt ..imagebase
+
+$L$SEH_info_poly1305_emit_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$SEH_begin_poly1305_emit_avx wrt ..imagebase,$L$SEH_begin_poly1305_emit_avx wrt ..imagebase
+$L$SEH_info_poly1305_blocks_avx2_1:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$blocks_avx2_body wrt ..imagebase,$L$blocks_avx2_epilogue wrt ..imagebase
+
+$L$SEH_info_poly1305_blocks_avx2_2:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$base2_64_avx2_body wrt ..imagebase,$L$base2_64_avx2_epilogue wrt ..imagebase
+
+$L$SEH_info_poly1305_blocks_avx2_3:
+DB      9,0,0,0
+        DD      avx_handler wrt ..imagebase
+        DD      $L$do_avx2_body wrt ..imagebase,$L$do_avx2_epilogue wrt ..imagebase
+$L$SEH_info_poly1305_blocks_avx512:
+DB      9,0,0,0
+        DD      avx_handler wrt ..imagebase
+        DD      $L$do_avx512_body wrt ..imagebase,$L$do_avx512_epilogue wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/rsaz-avx2.S

@@ -6 +6 @@
 
 
+global  rsaz_1024_sqr_avx2
+
+ALIGN   64
+rsaz_1024_sqr_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_rsaz_1024_sqr_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+        lea     rax,[rsp]
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        vzeroupper
+        lea     rsp,[((-168))+rsp]
+        vmovaps XMMWORD[(-216)+rax],xmm6
+        vmovaps XMMWORD[(-200)+rax],xmm7
+        vmovaps XMMWORD[(-184)+rax],xmm8
+        vmovaps XMMWORD[(-168)+rax],xmm9
+        vmovaps XMMWORD[(-152)+rax],xmm10
+        vmovaps XMMWORD[(-136)+rax],xmm11
+        vmovaps XMMWORD[(-120)+rax],xmm12
+        vmovaps XMMWORD[(-104)+rax],xmm13
+        vmovaps XMMWORD[(-88)+rax],xmm14
+        vmovaps XMMWORD[(-72)+rax],xmm15
+$L$sqr_1024_body:
+        mov     rbp,rax
+
+        mov     r13,rdx
+        sub     rsp,832
+        mov     r15,r13
+        sub     rdi,-128
+        sub     rsi,-128
+        sub     r13,-128
+
+        and     r15,4095
+        add     r15,32*10
+        shr     r15,12
+        vpxor   ymm9,ymm9,ymm9
+        jz      NEAR $L$sqr_1024_no_n_copy
+
+
+
+
+
+        sub     rsp,32*10
+        vmovdqu ymm0,YMMWORD[((0-128))+r13]
+        and     rsp,-2048
+        vmovdqu ymm1,YMMWORD[((32-128))+r13]
+        vmovdqu ymm2,YMMWORD[((64-128))+r13]
+        vmovdqu ymm3,YMMWORD[((96-128))+r13]
+        vmovdqu ymm4,YMMWORD[((128-128))+r13]
+        vmovdqu ymm5,YMMWORD[((160-128))+r13]
+        vmovdqu ymm6,YMMWORD[((192-128))+r13]
+        vmovdqu ymm7,YMMWORD[((224-128))+r13]
+        vmovdqu ymm8,YMMWORD[((256-128))+r13]
+        lea     r13,[((832+128))+rsp]
+        vmovdqu YMMWORD[(0-128)+r13],ymm0
+        vmovdqu YMMWORD[(32-128)+r13],ymm1
+        vmovdqu YMMWORD[(64-128)+r13],ymm2
+        vmovdqu YMMWORD[(96-128)+r13],ymm3
+        vmovdqu YMMWORD[(128-128)+r13],ymm4
+        vmovdqu YMMWORD[(160-128)+r13],ymm5
+        vmovdqu YMMWORD[(192-128)+r13],ymm6
+        vmovdqu YMMWORD[(224-128)+r13],ymm7
+        vmovdqu YMMWORD[(256-128)+r13],ymm8
+        vmovdqu YMMWORD[(288-128)+r13],ymm9
+
+$L$sqr_1024_no_n_copy:
+        and     rsp,-1024
+
+        vmovdqu ymm1,YMMWORD[((32-128))+rsi]
+        vmovdqu ymm2,YMMWORD[((64-128))+rsi]
+        vmovdqu ymm3,YMMWORD[((96-128))+rsi]
+        vmovdqu ymm4,YMMWORD[((128-128))+rsi]
+        vmovdqu ymm5,YMMWORD[((160-128))+rsi]
+        vmovdqu ymm6,YMMWORD[((192-128))+rsi]
+        vmovdqu ymm7,YMMWORD[((224-128))+rsi]
+        vmovdqu ymm8,YMMWORD[((256-128))+rsi]
+
+        lea     rbx,[192+rsp]
+        vmovdqu ymm15,YMMWORD[$L$and_mask]
+        jmp     NEAR $L$OOP_GRANDE_SQR_1024
+
+ALIGN   32
+$L$OOP_GRANDE_SQR_1024:
+        lea     r9,[((576+128))+rsp]
+        lea     r12,[448+rsp]
+
+
+
+
+        vpaddq  ymm1,ymm1,ymm1
+        vpbroadcastq    ymm10,QWORD[((0-128))+rsi]
+        vpaddq  ymm2,ymm2,ymm2
+        vmovdqa YMMWORD[(0-128)+r9],ymm1
+        vpaddq  ymm3,ymm3,ymm3
+        vmovdqa YMMWORD[(32-128)+r9],ymm2
+        vpaddq  ymm4,ymm4,ymm4
+        vmovdqa YMMWORD[(64-128)+r9],ymm3
+        vpaddq  ymm5,ymm5,ymm5
+        vmovdqa YMMWORD[(96-128)+r9],ymm4
+        vpaddq  ymm6,ymm6,ymm6
+        vmovdqa YMMWORD[(128-128)+r9],ymm5
+        vpaddq  ymm7,ymm7,ymm7
+        vmovdqa YMMWORD[(160-128)+r9],ymm6
+        vpaddq  ymm8,ymm8,ymm8
+        vmovdqa YMMWORD[(192-128)+r9],ymm7
+        vpxor   ymm9,ymm9,ymm9
+        vmovdqa YMMWORD[(224-128)+r9],ymm8
+
+        vpmuludq        ymm0,ymm10,YMMWORD[((0-128))+rsi]
+        vpbroadcastq    ymm11,QWORD[((32-128))+rsi]
+        vmovdqu YMMWORD[(288-192)+rbx],ymm9
+        vpmuludq        ymm1,ymm1,ymm10
+        vmovdqu YMMWORD[(320-448)+r12],ymm9
+        vpmuludq        ymm2,ymm2,ymm10
+        vmovdqu YMMWORD[(352-448)+r12],ymm9
+        vpmuludq        ymm3,ymm3,ymm10
+        vmovdqu YMMWORD[(384-448)+r12],ymm9
+        vpmuludq        ymm4,ymm4,ymm10
+        vmovdqu YMMWORD[(416-448)+r12],ymm9
+        vpmuludq        ymm5,ymm5,ymm10
+        vmovdqu YMMWORD[(448-448)+r12],ymm9
+        vpmuludq        ymm6,ymm6,ymm10
+        vmovdqu YMMWORD[(480-448)+r12],ymm9
+        vpmuludq        ymm7,ymm7,ymm10
+        vmovdqu YMMWORD[(512-448)+r12],ymm9
+        vpmuludq        ymm8,ymm8,ymm10
+        vpbroadcastq    ymm10,QWORD[((64-128))+rsi]
+        vmovdqu YMMWORD[(544-448)+r12],ymm9
+
+        mov     r15,rsi
+        mov     r14d,4
+        jmp     NEAR $L$sqr_entry_1024
+ALIGN   32
+$L$OOP_SQR_1024:
+        vpbroadcastq    ymm11,QWORD[((32-128))+r15]
+        vpmuludq        ymm0,ymm10,YMMWORD[((0-128))+rsi]
+        vpaddq  ymm0,ymm0,YMMWORD[((0-192))+rbx]
+        vpmuludq        ymm1,ymm10,YMMWORD[((0-128))+r9]
+        vpaddq  ymm1,ymm1,YMMWORD[((32-192))+rbx]
+        vpmuludq        ymm2,ymm10,YMMWORD[((32-128))+r9]
+        vpaddq  ymm2,ymm2,YMMWORD[((64-192))+rbx]
+        vpmuludq        ymm3,ymm10,YMMWORD[((64-128))+r9]
+        vpaddq  ymm3,ymm3,YMMWORD[((96-192))+rbx]
+        vpmuludq        ymm4,ymm10,YMMWORD[((96-128))+r9]
+        vpaddq  ymm4,ymm4,YMMWORD[((128-192))+rbx]
+        vpmuludq        ymm5,ymm10,YMMWORD[((128-128))+r9]
+        vpaddq  ymm5,ymm5,YMMWORD[((160-192))+rbx]
+        vpmuludq        ymm6,ymm10,YMMWORD[((160-128))+r9]
+        vpaddq  ymm6,ymm6,YMMWORD[((192-192))+rbx]
+        vpmuludq        ymm7,ymm10,YMMWORD[((192-128))+r9]
+        vpaddq  ymm7,ymm7,YMMWORD[((224-192))+rbx]
+        vpmuludq        ymm8,ymm10,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm10,QWORD[((64-128))+r15]
+        vpaddq  ymm8,ymm8,YMMWORD[((256-192))+rbx]
+$L$sqr_entry_1024:
+        vmovdqu YMMWORD[(0-192)+rbx],ymm0
+        vmovdqu YMMWORD[(32-192)+rbx],ymm1
+
+        vpmuludq        ymm12,ymm11,YMMWORD[((32-128))+rsi]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm14,ymm11,YMMWORD[((32-128))+r9]
+        vpaddq  ymm3,ymm3,ymm14
+        vpmuludq        ymm13,ymm11,YMMWORD[((64-128))+r9]
+        vpaddq  ymm4,ymm4,ymm13
+        vpmuludq        ymm12,ymm11,YMMWORD[((96-128))+r9]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm14,ymm11,YMMWORD[((128-128))+r9]
+        vpaddq  ymm6,ymm6,ymm14
+        vpmuludq        ymm13,ymm11,YMMWORD[((160-128))+r9]
+        vpaddq  ymm7,ymm7,ymm13
+        vpmuludq        ymm12,ymm11,YMMWORD[((192-128))+r9]
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm0,ymm11,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm11,QWORD[((96-128))+r15]
+        vpaddq  ymm0,ymm0,YMMWORD[((288-192))+rbx]
+
+        vmovdqu YMMWORD[(64-192)+rbx],ymm2
+        vmovdqu YMMWORD[(96-192)+rbx],ymm3
+
+        vpmuludq        ymm13,ymm10,YMMWORD[((64-128))+rsi]
+        vpaddq  ymm4,ymm4,ymm13
+        vpmuludq        ymm12,ymm10,YMMWORD[((64-128))+r9]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm14,ymm10,YMMWORD[((96-128))+r9]
+        vpaddq  ymm6,ymm6,ymm14
+        vpmuludq        ymm13,ymm10,YMMWORD[((128-128))+r9]
+        vpaddq  ymm7,ymm7,ymm13
+        vpmuludq        ymm12,ymm10,YMMWORD[((160-128))+r9]
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm14,ymm10,YMMWORD[((192-128))+r9]
+        vpaddq  ymm0,ymm0,ymm14
+        vpmuludq        ymm1,ymm10,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm10,QWORD[((128-128))+r15]
+        vpaddq  ymm1,ymm1,YMMWORD[((320-448))+r12]
+
+        vmovdqu YMMWORD[(128-192)+rbx],ymm4
+        vmovdqu YMMWORD[(160-192)+rbx],ymm5
+
+        vpmuludq        ymm12,ymm11,YMMWORD[((96-128))+rsi]
+        vpaddq  ymm6,ymm6,ymm12
+        vpmuludq        ymm14,ymm11,YMMWORD[((96-128))+r9]
+        vpaddq  ymm7,ymm7,ymm14
+        vpmuludq        ymm13,ymm11,YMMWORD[((128-128))+r9]
+        vpaddq  ymm8,ymm8,ymm13
+        vpmuludq        ymm12,ymm11,YMMWORD[((160-128))+r9]
+        vpaddq  ymm0,ymm0,ymm12
+        vpmuludq        ymm14,ymm11,YMMWORD[((192-128))+r9]
+        vpaddq  ymm1,ymm1,ymm14
+        vpmuludq        ymm2,ymm11,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm11,QWORD[((160-128))+r15]
+        vpaddq  ymm2,ymm2,YMMWORD[((352-448))+r12]
+
+        vmovdqu YMMWORD[(192-192)+rbx],ymm6
+        vmovdqu YMMWORD[(224-192)+rbx],ymm7
+
+        vpmuludq        ymm12,ymm10,YMMWORD[((128-128))+rsi]
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm14,ymm10,YMMWORD[((128-128))+r9]
+        vpaddq  ymm0,ymm0,ymm14
+        vpmuludq        ymm13,ymm10,YMMWORD[((160-128))+r9]
+        vpaddq  ymm1,ymm1,ymm13
+        vpmuludq        ymm12,ymm10,YMMWORD[((192-128))+r9]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm3,ymm10,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm10,QWORD[((192-128))+r15]
+        vpaddq  ymm3,ymm3,YMMWORD[((384-448))+r12]
+
+        vmovdqu YMMWORD[(256-192)+rbx],ymm8
+        vmovdqu YMMWORD[(288-192)+rbx],ymm0
+        lea     rbx,[8+rbx]
+
+        vpmuludq        ymm13,ymm11,YMMWORD[((160-128))+rsi]
+        vpaddq  ymm1,ymm1,ymm13
+        vpmuludq        ymm12,ymm11,YMMWORD[((160-128))+r9]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm14,ymm11,YMMWORD[((192-128))+r9]
+        vpaddq  ymm3,ymm3,ymm14
+        vpmuludq        ymm4,ymm11,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm11,QWORD[((224-128))+r15]
+        vpaddq  ymm4,ymm4,YMMWORD[((416-448))+r12]
+
+        vmovdqu YMMWORD[(320-448)+r12],ymm1
+        vmovdqu YMMWORD[(352-448)+r12],ymm2
+
+        vpmuludq        ymm12,ymm10,YMMWORD[((192-128))+rsi]
+        vpaddq  ymm3,ymm3,ymm12
+        vpmuludq        ymm14,ymm10,YMMWORD[((192-128))+r9]
+        vpbroadcastq    ymm0,QWORD[((256-128))+r15]
+        vpaddq  ymm4,ymm4,ymm14
+        vpmuludq        ymm5,ymm10,YMMWORD[((224-128))+r9]
+        vpbroadcastq    ymm10,QWORD[((0+8-128))+r15]
+        vpaddq  ymm5,ymm5,YMMWORD[((448-448))+r12]
+
+        vmovdqu YMMWORD[(384-448)+r12],ymm3
+        vmovdqu YMMWORD[(416-448)+r12],ymm4
+        lea     r15,[8+r15]
+
+        vpmuludq        ymm12,ymm11,YMMWORD[((224-128))+rsi]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm6,ymm11,YMMWORD[((224-128))+r9]
+        vpaddq  ymm6,ymm6,YMMWORD[((480-448))+r12]
+
+        vpmuludq        ymm7,ymm0,YMMWORD[((256-128))+rsi]
+        vmovdqu YMMWORD[(448-448)+r12],ymm5
+        vpaddq  ymm7,ymm7,YMMWORD[((512-448))+r12]
+        vmovdqu YMMWORD[(480-448)+r12],ymm6
+        vmovdqu YMMWORD[(512-448)+r12],ymm7
+        lea     r12,[8+r12]
+
+        dec     r14d
+        jnz     NEAR $L$OOP_SQR_1024
+
+        vmovdqu ymm8,YMMWORD[256+rsp]
+        vmovdqu ymm1,YMMWORD[288+rsp]
+        vmovdqu ymm2,YMMWORD[320+rsp]
+        lea     rbx,[192+rsp]
+
+        vpsrlq  ymm14,ymm8,29
+        vpand   ymm8,ymm8,ymm15
+        vpsrlq  ymm11,ymm1,29
+        vpand   ymm1,ymm1,ymm15
+
+        vpermq  ymm14,ymm14,0x93
+        vpxor   ymm9,ymm9,ymm9
+        vpermq  ymm11,ymm11,0x93
+
+        vpblendd        ymm10,ymm14,ymm9,3
+        vpblendd        ymm14,ymm11,ymm14,3
+        vpaddq  ymm8,ymm8,ymm10
+        vpblendd        ymm11,ymm9,ymm11,3
+        vpaddq  ymm1,ymm1,ymm14
+        vpaddq  ymm2,ymm2,ymm11
+        vmovdqu YMMWORD[(288-192)+rbx],ymm1
+        vmovdqu YMMWORD[(320-192)+rbx],ymm2
+
+        mov     rax,QWORD[rsp]
+        mov     r10,QWORD[8+rsp]
+        mov     r11,QWORD[16+rsp]
+        mov     r12,QWORD[24+rsp]
+        vmovdqu ymm1,YMMWORD[32+rsp]
+        vmovdqu ymm2,YMMWORD[((64-192))+rbx]
+        vmovdqu ymm3,YMMWORD[((96-192))+rbx]
+        vmovdqu ymm4,YMMWORD[((128-192))+rbx]
+        vmovdqu ymm5,YMMWORD[((160-192))+rbx]
+        vmovdqu ymm6,YMMWORD[((192-192))+rbx]
+        vmovdqu ymm7,YMMWORD[((224-192))+rbx]
+
+        mov     r9,rax
+        imul    eax,ecx
+        and     eax,0x1fffffff
+        vmovd   xmm12,eax
+
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+r13]
+        vpbroadcastq    ymm12,xmm12
+        add     r9,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+r13]
+        shr     r9,29
+        add     r10,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((16-128))+r13]
+        add     r10,r9
+        add     r11,rax
+        imul    rdx,QWORD[((24-128))+r13]
+        add     r12,rdx
+
+        mov     rax,r10
+        imul    eax,ecx
+        and     eax,0x1fffffff
+
+        mov     r14d,9
+        jmp     NEAR $L$OOP_REDUCE_1024
+
+ALIGN   32
+$L$OOP_REDUCE_1024:
+        vmovd   xmm13,eax
+        vpbroadcastq    ymm13,xmm13
+
+        vpmuludq        ymm10,ymm12,YMMWORD[((32-128))+r13]
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+r13]
+        vpaddq  ymm1,ymm1,ymm10
+        add     r10,rax
+        vpmuludq        ymm14,ymm12,YMMWORD[((64-128))+r13]
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+r13]
+        vpaddq  ymm2,ymm2,ymm14
+        vpmuludq        ymm11,ymm12,YMMWORD[((96-128))+r13]
+DB      0x67
+        add     r11,rax
+DB      0x67
+        mov     rax,rdx
+        imul    rax,QWORD[((16-128))+r13]
+        shr     r10,29
+        vpaddq  ymm3,ymm3,ymm11
+        vpmuludq        ymm10,ymm12,YMMWORD[((128-128))+r13]
+        add     r12,rax
+        add     r11,r10
+        vpaddq  ymm4,ymm4,ymm10
+        vpmuludq        ymm14,ymm12,YMMWORD[((160-128))+r13]
+        mov     rax,r11
+        imul    eax,ecx
+        vpaddq  ymm5,ymm5,ymm14
+        vpmuludq        ymm11,ymm12,YMMWORD[((192-128))+r13]
+        and     eax,0x1fffffff
+        vpaddq  ymm6,ymm6,ymm11
+        vpmuludq        ymm10,ymm12,YMMWORD[((224-128))+r13]
+        vpaddq  ymm7,ymm7,ymm10
+        vpmuludq        ymm14,ymm12,YMMWORD[((256-128))+r13]
+        vmovd   xmm12,eax
+
+        vpaddq  ymm8,ymm8,ymm14
+
+        vpbroadcastq    ymm12,xmm12
+
+        vpmuludq        ymm11,ymm13,YMMWORD[((32-8-128))+r13]
+        vmovdqu ymm14,YMMWORD[((96-8-128))+r13]
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+r13]
+        vpaddq  ymm1,ymm1,ymm11
+        vpmuludq        ymm10,ymm13,YMMWORD[((64-8-128))+r13]
+        vmovdqu ymm11,YMMWORD[((128-8-128))+r13]
+        add     r11,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+r13]
+        vpaddq  ymm2,ymm2,ymm10
+        add     rax,r12
+        shr     r11,29
+        vpmuludq        ymm14,ymm14,ymm13
+        vmovdqu ymm10,YMMWORD[((160-8-128))+r13]
+        add     rax,r11
+        vpaddq  ymm3,ymm3,ymm14
+        vpmuludq        ymm11,ymm11,ymm13
+        vmovdqu ymm14,YMMWORD[((192-8-128))+r13]
+DB      0x67
+        mov     r12,rax
+        imul    eax,ecx
+        vpaddq  ymm4,ymm4,ymm11
+        vpmuludq        ymm10,ymm10,ymm13
+DB      0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00
+        and     eax,0x1fffffff
+        vpaddq  ymm5,ymm5,ymm10
+        vpmuludq        ymm14,ymm14,ymm13
+        vmovdqu ymm10,YMMWORD[((256-8-128))+r13]
+        vpaddq  ymm6,ymm6,ymm14
+        vpmuludq        ymm11,ymm11,ymm13
+        vmovdqu ymm9,YMMWORD[((288-8-128))+r13]
+        vmovd   xmm0,eax
+        imul    rax,QWORD[((-128))+r13]
+        vpaddq  ymm7,ymm7,ymm11
+        vpmuludq        ymm10,ymm10,ymm13
+        vmovdqu ymm14,YMMWORD[((32-16-128))+r13]
+        vpbroadcastq    ymm0,xmm0
+        vpaddq  ymm8,ymm8,ymm10
+        vpmuludq        ymm9,ymm9,ymm13
+        vmovdqu ymm11,YMMWORD[((64-16-128))+r13]
+        add     r12,rax
+
+        vmovdqu ymm13,YMMWORD[((32-24-128))+r13]
+        vpmuludq        ymm14,ymm14,ymm12
+        vmovdqu ymm10,YMMWORD[((96-16-128))+r13]
+        vpaddq  ymm1,ymm1,ymm14
+        vpmuludq        ymm13,ymm13,ymm0
+        vpmuludq        ymm11,ymm11,ymm12
+DB      0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff
+        vpaddq  ymm13,ymm13,ymm1
+        vpaddq  ymm2,ymm2,ymm11
+        vpmuludq        ymm10,ymm10,ymm12
+        vmovdqu ymm11,YMMWORD[((160-16-128))+r13]
+DB      0x67
+        vmovq   rax,xmm13
+        vmovdqu YMMWORD[rsp],ymm13
+        vpaddq  ymm3,ymm3,ymm10
+        vpmuludq        ymm14,ymm14,ymm12
+        vmovdqu ymm10,YMMWORD[((192-16-128))+r13]
+        vpaddq  ymm4,ymm4,ymm14
+        vpmuludq        ymm11,ymm11,ymm12
+        vmovdqu ymm14,YMMWORD[((224-16-128))+r13]
+        vpaddq  ymm5,ymm5,ymm11
+        vpmuludq        ymm10,ymm10,ymm12
+        vmovdqu ymm11,YMMWORD[((256-16-128))+r13]
+        vpaddq  ymm6,ymm6,ymm10
+        vpmuludq        ymm14,ymm14,ymm12
+        shr     r12,29
+        vmovdqu ymm10,YMMWORD[((288-16-128))+r13]
+        add     rax,r12
+        vpaddq  ymm7,ymm7,ymm14
+        vpmuludq        ymm11,ymm11,ymm12
+
+        mov     r9,rax
+        imul    eax,ecx
+        vpaddq  ymm8,ymm8,ymm11
+        vpmuludq        ymm10,ymm10,ymm12
+        and     eax,0x1fffffff
+        vmovd   xmm12,eax
+        vmovdqu ymm11,YMMWORD[((96-24-128))+r13]
+DB      0x67
+        vpaddq  ymm9,ymm9,ymm10
+        vpbroadcastq    ymm12,xmm12
+
+        vpmuludq        ymm14,ymm0,YMMWORD[((64-24-128))+r13]
+        vmovdqu ymm10,YMMWORD[((128-24-128))+r13]
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+r13]
+        mov     r10,QWORD[8+rsp]
+        vpaddq  ymm1,ymm2,ymm14
+        vpmuludq        ymm11,ymm11,ymm0
+        vmovdqu ymm14,YMMWORD[((160-24-128))+r13]
+        add     r9,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+r13]
+DB      0x67
+        shr     r9,29
+        mov     r11,QWORD[16+rsp]
+        vpaddq  ymm2,ymm3,ymm11
+        vpmuludq        ymm10,ymm10,ymm0
+        vmovdqu ymm11,YMMWORD[((192-24-128))+r13]
+        add     r10,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((16-128))+r13]
+        vpaddq  ymm3,ymm4,ymm10
+        vpmuludq        ymm14,ymm14,ymm0
+        vmovdqu ymm10,YMMWORD[((224-24-128))+r13]
+        imul    rdx,QWORD[((24-128))+r13]
+        add     r11,rax
+        lea     rax,[r10*1+r9]
+        vpaddq  ymm4,ymm5,ymm14
+        vpmuludq        ymm11,ymm11,ymm0
+        vmovdqu ymm14,YMMWORD[((256-24-128))+r13]
+        mov     r10,rax
+        imul    eax,ecx
+        vpmuludq        ymm10,ymm10,ymm0
+        vpaddq  ymm5,ymm6,ymm11
+        vmovdqu ymm11,YMMWORD[((288-24-128))+r13]
+        and     eax,0x1fffffff
+        vpaddq  ymm6,ymm7,ymm10
+        vpmuludq        ymm14,ymm14,ymm0
+        add     rdx,QWORD[24+rsp]
+        vpaddq  ymm7,ymm8,ymm14
+        vpmuludq        ymm11,ymm11,ymm0
+        vpaddq  ymm8,ymm9,ymm11
+        vmovq   xmm9,r12
+        mov     r12,rdx
+
+        dec     r14d
+        jnz     NEAR $L$OOP_REDUCE_1024
+        lea     r12,[448+rsp]
+        vpaddq  ymm0,ymm13,ymm9
+        vpxor   ymm9,ymm9,ymm9
+
+        vpaddq  ymm0,ymm0,YMMWORD[((288-192))+rbx]
+        vpaddq  ymm1,ymm1,YMMWORD[((320-448))+r12]
+        vpaddq  ymm2,ymm2,YMMWORD[((352-448))+r12]
+        vpaddq  ymm3,ymm3,YMMWORD[((384-448))+r12]
+        vpaddq  ymm4,ymm4,YMMWORD[((416-448))+r12]
+        vpaddq  ymm5,ymm5,YMMWORD[((448-448))+r12]
+        vpaddq  ymm6,ymm6,YMMWORD[((480-448))+r12]
+        vpaddq  ymm7,ymm7,YMMWORD[((512-448))+r12]
+        vpaddq  ymm8,ymm8,YMMWORD[((544-448))+r12]
+
+        vpsrlq  ymm14,ymm0,29
+        vpand   ymm0,ymm0,ymm15
+        vpsrlq  ymm11,ymm1,29
+        vpand   ymm1,ymm1,ymm15
+        vpsrlq  ymm12,ymm2,29
+        vpermq  ymm14,ymm14,0x93
+        vpand   ymm2,ymm2,ymm15
+        vpsrlq  ymm13,ymm3,29
+        vpermq  ymm11,ymm11,0x93
+        vpand   ymm3,ymm3,ymm15
+        vpermq  ymm12,ymm12,0x93
+
+        vpblendd        ymm10,ymm14,ymm9,3
+        vpermq  ymm13,ymm13,0x93
+        vpblendd        ymm14,ymm11,ymm14,3
+        vpaddq  ymm0,ymm0,ymm10
+        vpblendd        ymm11,ymm12,ymm11,3
+        vpaddq  ymm1,ymm1,ymm14
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm2,ymm2,ymm11
+        vpblendd        ymm13,ymm9,ymm13,3
+        vpaddq  ymm3,ymm3,ymm12
+        vpaddq  ymm4,ymm4,ymm13
+
+        vpsrlq  ymm14,ymm0,29
+        vpand   ymm0,ymm0,ymm15
+        vpsrlq  ymm11,ymm1,29
+        vpand   ymm1,ymm1,ymm15
+        vpsrlq  ymm12,ymm2,29
+        vpermq  ymm14,ymm14,0x93
+        vpand   ymm2,ymm2,ymm15
+        vpsrlq  ymm13,ymm3,29
+        vpermq  ymm11,ymm11,0x93
+        vpand   ymm3,ymm3,ymm15
+        vpermq  ymm12,ymm12,0x93
+
+        vpblendd        ymm10,ymm14,ymm9,3
+        vpermq  ymm13,ymm13,0x93
+        vpblendd        ymm14,ymm11,ymm14,3
+        vpaddq  ymm0,ymm0,ymm10
+        vpblendd        ymm11,ymm12,ymm11,3
+        vpaddq  ymm1,ymm1,ymm14
+        vmovdqu YMMWORD[(0-128)+rdi],ymm0
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm2,ymm2,ymm11
+        vmovdqu YMMWORD[(32-128)+rdi],ymm1
+        vpblendd        ymm13,ymm9,ymm13,3
+        vpaddq  ymm3,ymm3,ymm12
+        vmovdqu YMMWORD[(64-128)+rdi],ymm2
+        vpaddq  ymm4,ymm4,ymm13
+        vmovdqu YMMWORD[(96-128)+rdi],ymm3
+        vpsrlq  ymm14,ymm4,29
+        vpand   ymm4,ymm4,ymm15
+        vpsrlq  ymm11,ymm5,29
+        vpand   ymm5,ymm5,ymm15
+        vpsrlq  ymm12,ymm6,29
+        vpermq  ymm14,ymm14,0x93
+        vpand   ymm6,ymm6,ymm15
+        vpsrlq  ymm13,ymm7,29
+        vpermq  ymm11,ymm11,0x93
+        vpand   ymm7,ymm7,ymm15
+        vpsrlq  ymm0,ymm8,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm8,ymm8,ymm15
+        vpermq  ymm13,ymm13,0x93
+
+        vpblendd        ymm10,ymm14,ymm9,3
+        vpermq  ymm0,ymm0,0x93
+        vpblendd        ymm14,ymm11,ymm14,3
+        vpaddq  ymm4,ymm4,ymm10
+        vpblendd        ymm11,ymm12,ymm11,3
+        vpaddq  ymm5,ymm5,ymm14
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm6,ymm6,ymm11
+        vpblendd        ymm13,ymm0,ymm13,3
+        vpaddq  ymm7,ymm7,ymm12
+        vpaddq  ymm8,ymm8,ymm13
+
+        vpsrlq  ymm14,ymm4,29
+        vpand   ymm4,ymm4,ymm15
+        vpsrlq  ymm11,ymm5,29
+        vpand   ymm5,ymm5,ymm15
+        vpsrlq  ymm12,ymm6,29
+        vpermq  ymm14,ymm14,0x93
+        vpand   ymm6,ymm6,ymm15
+        vpsrlq  ymm13,ymm7,29
+        vpermq  ymm11,ymm11,0x93
+        vpand   ymm7,ymm7,ymm15
+        vpsrlq  ymm0,ymm8,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm8,ymm8,ymm15
+        vpermq  ymm13,ymm13,0x93
+
+        vpblendd        ymm10,ymm14,ymm9,3
+        vpermq  ymm0,ymm0,0x93
+        vpblendd        ymm14,ymm11,ymm14,3
+        vpaddq  ymm4,ymm4,ymm10
+        vpblendd        ymm11,ymm12,ymm11,3
+        vpaddq  ymm5,ymm5,ymm14
+        vmovdqu YMMWORD[(128-128)+rdi],ymm4
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm6,ymm6,ymm11
+        vmovdqu YMMWORD[(160-128)+rdi],ymm5
+        vpblendd        ymm13,ymm0,ymm13,3
+        vpaddq  ymm7,ymm7,ymm12
+        vmovdqu YMMWORD[(192-128)+rdi],ymm6
+        vpaddq  ymm8,ymm8,ymm13
+        vmovdqu YMMWORD[(224-128)+rdi],ymm7
+        vmovdqu YMMWORD[(256-128)+rdi],ymm8
+
+        mov     rsi,rdi
+        dec     r8d
+        jne     NEAR $L$OOP_GRANDE_SQR_1024
+
+        vzeroall
+        mov     rax,rbp
+
+$L$sqr_1024_in_tail:
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$sqr_1024_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_rsaz_1024_sqr_avx2:
+global  rsaz_1024_mul_avx2
+
+ALIGN   64
+rsaz_1024_mul_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_rsaz_1024_mul_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+
+
+
+        lea     rax,[rsp]
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        vzeroupper
+        lea     rsp,[((-168))+rsp]
+        vmovaps XMMWORD[(-216)+rax],xmm6
+        vmovaps XMMWORD[(-200)+rax],xmm7
+        vmovaps XMMWORD[(-184)+rax],xmm8
+        vmovaps XMMWORD[(-168)+rax],xmm9
+        vmovaps XMMWORD[(-152)+rax],xmm10
+        vmovaps XMMWORD[(-136)+rax],xmm11
+        vmovaps XMMWORD[(-120)+rax],xmm12
+        vmovaps XMMWORD[(-104)+rax],xmm13
+        vmovaps XMMWORD[(-88)+rax],xmm14
+        vmovaps XMMWORD[(-72)+rax],xmm15
+$L$mul_1024_body:
+        mov     rbp,rax
+
+        vzeroall
+        mov     r13,rdx
+        sub     rsp,64
+
+
+
+
+
+
+DB      0x67,0x67
+        mov     r15,rsi
+        and     r15,4095
+        add     r15,32*10
+        shr     r15,12
+        mov     r15,rsi
+        cmovnz  rsi,r13
+        cmovnz  r13,r15
+
+        mov     r15,rcx
+        sub     rsi,-128
+        sub     rcx,-128
+        sub     rdi,-128
+
+        and     r15,4095
+        add     r15,32*10
+DB      0x67,0x67
+        shr     r15,12
+        jz      NEAR $L$mul_1024_no_n_copy
+
+
+
+
+
+        sub     rsp,32*10
+        vmovdqu ymm0,YMMWORD[((0-128))+rcx]
+        and     rsp,-512
+        vmovdqu ymm1,YMMWORD[((32-128))+rcx]
+        vmovdqu ymm2,YMMWORD[((64-128))+rcx]
+        vmovdqu ymm3,YMMWORD[((96-128))+rcx]
+        vmovdqu ymm4,YMMWORD[((128-128))+rcx]
+        vmovdqu ymm5,YMMWORD[((160-128))+rcx]
+        vmovdqu ymm6,YMMWORD[((192-128))+rcx]
+        vmovdqu ymm7,YMMWORD[((224-128))+rcx]
+        vmovdqu ymm8,YMMWORD[((256-128))+rcx]
+        lea     rcx,[((64+128))+rsp]
+        vmovdqu YMMWORD[(0-128)+rcx],ymm0
+        vpxor   ymm0,ymm0,ymm0
+        vmovdqu YMMWORD[(32-128)+rcx],ymm1
+        vpxor   ymm1,ymm1,ymm1
+        vmovdqu YMMWORD[(64-128)+rcx],ymm2
+        vpxor   ymm2,ymm2,ymm2
+        vmovdqu YMMWORD[(96-128)+rcx],ymm3
+        vpxor   ymm3,ymm3,ymm3
+        vmovdqu YMMWORD[(128-128)+rcx],ymm4
+        vpxor   ymm4,ymm4,ymm4
+        vmovdqu YMMWORD[(160-128)+rcx],ymm5
+        vpxor   ymm5,ymm5,ymm5
+        vmovdqu YMMWORD[(192-128)+rcx],ymm6
+        vpxor   ymm6,ymm6,ymm6
+        vmovdqu YMMWORD[(224-128)+rcx],ymm7
+        vpxor   ymm7,ymm7,ymm7
+        vmovdqu YMMWORD[(256-128)+rcx],ymm8
+        vmovdqa ymm8,ymm0
+        vmovdqu YMMWORD[(288-128)+rcx],ymm9
+$L$mul_1024_no_n_copy:
+        and     rsp,-64
+
+        mov     rbx,QWORD[r13]
+        vpbroadcastq    ymm10,QWORD[r13]
+        vmovdqu YMMWORD[rsp],ymm0
+        xor     r9,r9
+DB      0x67
+        xor     r10,r10
+        xor     r11,r11
+        xor     r12,r12
+
+        vmovdqu ymm15,YMMWORD[$L$and_mask]
+        mov     r14d,9
+        vmovdqu YMMWORD[(288-128)+rdi],ymm9
+        jmp     NEAR $L$oop_mul_1024
+
+ALIGN   32
+$L$oop_mul_1024:
+        vpsrlq  ymm9,ymm3,29
+        mov     rax,rbx
+        imul    rax,QWORD[((-128))+rsi]
+        add     rax,r9
+        mov     r10,rbx
+        imul    r10,QWORD[((8-128))+rsi]
+        add     r10,QWORD[8+rsp]
+
+        mov     r9,rax
+        imul    eax,r8d
+        and     eax,0x1fffffff
+
+        mov     r11,rbx
+        imul    r11,QWORD[((16-128))+rsi]
+        add     r11,QWORD[16+rsp]
+
+        mov     r12,rbx
+        imul    r12,QWORD[((24-128))+rsi]
+        add     r12,QWORD[24+rsp]
+        vpmuludq        ymm0,ymm10,YMMWORD[((32-128))+rsi]
+        vmovd   xmm11,eax
+        vpaddq  ymm1,ymm1,ymm0
+        vpmuludq        ymm12,ymm10,YMMWORD[((64-128))+rsi]
+        vpbroadcastq    ymm11,xmm11
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm13,ymm10,YMMWORD[((96-128))+rsi]
+        vpand   ymm3,ymm3,ymm15
+        vpaddq  ymm3,ymm3,ymm13
+        vpmuludq        ymm0,ymm10,YMMWORD[((128-128))+rsi]
+        vpaddq  ymm4,ymm4,ymm0
+        vpmuludq        ymm12,ymm10,YMMWORD[((160-128))+rsi]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm13,ymm10,YMMWORD[((192-128))+rsi]
+        vpaddq  ymm6,ymm6,ymm13
+        vpmuludq        ymm0,ymm10,YMMWORD[((224-128))+rsi]
+        vpermq  ymm9,ymm9,0x93
+        vpaddq  ymm7,ymm7,ymm0
+        vpmuludq        ymm12,ymm10,YMMWORD[((256-128))+rsi]
+        vpbroadcastq    ymm10,QWORD[8+r13]
+        vpaddq  ymm8,ymm8,ymm12
+
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+rcx]
+        add     r9,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+rcx]
+        add     r10,rax
+        mov     rax,rdx
+        imul    rax,QWORD[((16-128))+rcx]
+        add     r11,rax
+        shr     r9,29
+        imul    rdx,QWORD[((24-128))+rcx]
+        add     r12,rdx
+        add     r10,r9
+
+        vpmuludq        ymm13,ymm11,YMMWORD[((32-128))+rcx]
+        vmovq   rbx,xmm10
+        vpaddq  ymm1,ymm1,ymm13
+        vpmuludq        ymm0,ymm11,YMMWORD[((64-128))+rcx]
+        vpaddq  ymm2,ymm2,ymm0
+        vpmuludq        ymm12,ymm11,YMMWORD[((96-128))+rcx]
+        vpaddq  ymm3,ymm3,ymm12
+        vpmuludq        ymm13,ymm11,YMMWORD[((128-128))+rcx]
+        vpaddq  ymm4,ymm4,ymm13
+        vpmuludq        ymm0,ymm11,YMMWORD[((160-128))+rcx]
+        vpaddq  ymm5,ymm5,ymm0
+        vpmuludq        ymm12,ymm11,YMMWORD[((192-128))+rcx]
+        vpaddq  ymm6,ymm6,ymm12
+        vpmuludq        ymm13,ymm11,YMMWORD[((224-128))+rcx]
+        vpblendd        ymm12,ymm9,ymm14,3
+        vpaddq  ymm7,ymm7,ymm13
+        vpmuludq        ymm0,ymm11,YMMWORD[((256-128))+rcx]
+        vpaddq  ymm3,ymm3,ymm12
+        vpaddq  ymm8,ymm8,ymm0
+
+        mov     rax,rbx
+        imul    rax,QWORD[((-128))+rsi]
+        add     r10,rax
+        vmovdqu ymm12,YMMWORD[((-8+32-128))+rsi]
+        mov     rax,rbx
+        imul    rax,QWORD[((8-128))+rsi]
+        add     r11,rax
+        vmovdqu ymm13,YMMWORD[((-8+64-128))+rsi]
+
+        mov     rax,r10
+        vpblendd        ymm9,ymm9,ymm14,0xfc
+        imul    eax,r8d
+        vpaddq  ymm4,ymm4,ymm9
+        and     eax,0x1fffffff
+
+        imul    rbx,QWORD[((16-128))+rsi]
+        add     r12,rbx
+        vpmuludq        ymm12,ymm12,ymm10
+        vmovd   xmm11,eax
+        vmovdqu ymm0,YMMWORD[((-8+96-128))+rsi]
+        vpaddq  ymm1,ymm1,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vpbroadcastq    ymm11,xmm11
+        vmovdqu ymm12,YMMWORD[((-8+128-128))+rsi]
+        vpaddq  ymm2,ymm2,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-8+160-128))+rsi]
+        vpaddq  ymm3,ymm3,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vmovdqu ymm0,YMMWORD[((-8+192-128))+rsi]
+        vpaddq  ymm4,ymm4,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vmovdqu ymm12,YMMWORD[((-8+224-128))+rsi]
+        vpaddq  ymm5,ymm5,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-8+256-128))+rsi]
+        vpaddq  ymm6,ymm6,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vmovdqu ymm9,YMMWORD[((-8+288-128))+rsi]
+        vpaddq  ymm7,ymm7,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vpaddq  ymm8,ymm8,ymm13
+        vpmuludq        ymm9,ymm9,ymm10
+        vpbroadcastq    ymm10,QWORD[16+r13]
+
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+rcx]
+        add     r10,rax
+        vmovdqu ymm0,YMMWORD[((-8+32-128))+rcx]
+        mov     rax,rdx
+        imul    rax,QWORD[((8-128))+rcx]
+        add     r11,rax
+        vmovdqu ymm12,YMMWORD[((-8+64-128))+rcx]
+        shr     r10,29
+        imul    rdx,QWORD[((16-128))+rcx]
+        add     r12,rdx
+        add     r11,r10
+
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovq   rbx,xmm10
+        vmovdqu ymm13,YMMWORD[((-8+96-128))+rcx]
+        vpaddq  ymm1,ymm1,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-8+128-128))+rcx]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-8+160-128))+rcx]
+        vpaddq  ymm3,ymm3,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-8+192-128))+rcx]
+        vpaddq  ymm4,ymm4,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-8+224-128))+rcx]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-8+256-128))+rcx]
+        vpaddq  ymm6,ymm6,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-8+288-128))+rcx]
+        vpaddq  ymm7,ymm7,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vpaddq  ymm9,ymm9,ymm13
+
+        vmovdqu ymm0,YMMWORD[((-16+32-128))+rsi]
+        mov     rax,rbx
+        imul    rax,QWORD[((-128))+rsi]
+        add     rax,r11
+
+        vmovdqu ymm12,YMMWORD[((-16+64-128))+rsi]
+        mov     r11,rax
+        imul    eax,r8d
+        and     eax,0x1fffffff
+
+        imul    rbx,QWORD[((8-128))+rsi]
+        add     r12,rbx
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovd   xmm11,eax
+        vmovdqu ymm13,YMMWORD[((-16+96-128))+rsi]
+        vpaddq  ymm1,ymm1,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vpbroadcastq    ymm11,xmm11
+        vmovdqu ymm0,YMMWORD[((-16+128-128))+rsi]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vmovdqu ymm12,YMMWORD[((-16+160-128))+rsi]
+        vpaddq  ymm3,ymm3,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-16+192-128))+rsi]
+        vpaddq  ymm4,ymm4,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vmovdqu ymm0,YMMWORD[((-16+224-128))+rsi]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vmovdqu ymm12,YMMWORD[((-16+256-128))+rsi]
+        vpaddq  ymm6,ymm6,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-16+288-128))+rsi]
+        vpaddq  ymm7,ymm7,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vpbroadcastq    ymm10,QWORD[24+r13]
+        vpaddq  ymm9,ymm9,ymm13
+
+        vmovdqu ymm0,YMMWORD[((-16+32-128))+rcx]
+        mov     rdx,rax
+        imul    rax,QWORD[((-128))+rcx]
+        add     r11,rax
+        vmovdqu ymm12,YMMWORD[((-16+64-128))+rcx]
+        imul    rdx,QWORD[((8-128))+rcx]
+        add     r12,rdx
+        shr     r11,29
+
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovq   rbx,xmm10
+        vmovdqu ymm13,YMMWORD[((-16+96-128))+rcx]
+        vpaddq  ymm1,ymm1,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-16+128-128))+rcx]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-16+160-128))+rcx]
+        vpaddq  ymm3,ymm3,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-16+192-128))+rcx]
+        vpaddq  ymm4,ymm4,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-16+224-128))+rcx]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-16+256-128))+rcx]
+        vpaddq  ymm6,ymm6,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-16+288-128))+rcx]
+        vpaddq  ymm7,ymm7,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-24+32-128))+rsi]
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-24+64-128))+rsi]
+        vpaddq  ymm9,ymm9,ymm13
+
+        add     r12,r11
+        imul    rbx,QWORD[((-128))+rsi]
+        add     r12,rbx
+
+        mov     rax,r12
+        imul    eax,r8d
+        and     eax,0x1fffffff
+
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovd   xmm11,eax
+        vmovdqu ymm13,YMMWORD[((-24+96-128))+rsi]
+        vpaddq  ymm1,ymm1,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vpbroadcastq    ymm11,xmm11
+        vmovdqu ymm0,YMMWORD[((-24+128-128))+rsi]
+        vpaddq  ymm2,ymm2,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vmovdqu ymm12,YMMWORD[((-24+160-128))+rsi]
+        vpaddq  ymm3,ymm3,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-24+192-128))+rsi]
+        vpaddq  ymm4,ymm4,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vmovdqu ymm0,YMMWORD[((-24+224-128))+rsi]
+        vpaddq  ymm5,ymm5,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vmovdqu ymm12,YMMWORD[((-24+256-128))+rsi]
+        vpaddq  ymm6,ymm6,ymm13
+        vpmuludq        ymm0,ymm0,ymm10
+        vmovdqu ymm13,YMMWORD[((-24+288-128))+rsi]
+        vpaddq  ymm7,ymm7,ymm0
+        vpmuludq        ymm12,ymm12,ymm10
+        vpaddq  ymm8,ymm8,ymm12
+        vpmuludq        ymm13,ymm13,ymm10
+        vpbroadcastq    ymm10,QWORD[32+r13]
+        vpaddq  ymm9,ymm9,ymm13
+        add     r13,32
+
+        vmovdqu ymm0,YMMWORD[((-24+32-128))+rcx]
+        imul    rax,QWORD[((-128))+rcx]
+        add     r12,rax
+        shr     r12,29
+
+        vmovdqu ymm12,YMMWORD[((-24+64-128))+rcx]
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovq   rbx,xmm10
+        vmovdqu ymm13,YMMWORD[((-24+96-128))+rcx]
+        vpaddq  ymm0,ymm1,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu YMMWORD[rsp],ymm0
+        vpaddq  ymm1,ymm2,ymm12
+        vmovdqu ymm0,YMMWORD[((-24+128-128))+rcx]
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-24+160-128))+rcx]
+        vpaddq  ymm2,ymm3,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-24+192-128))+rcx]
+        vpaddq  ymm3,ymm4,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        vmovdqu ymm0,YMMWORD[((-24+224-128))+rcx]
+        vpaddq  ymm4,ymm5,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovdqu ymm12,YMMWORD[((-24+256-128))+rcx]
+        vpaddq  ymm5,ymm6,ymm13
+        vpmuludq        ymm0,ymm0,ymm11
+        vmovdqu ymm13,YMMWORD[((-24+288-128))+rcx]
+        mov     r9,r12
+        vpaddq  ymm6,ymm7,ymm0
+        vpmuludq        ymm12,ymm12,ymm11
+        add     r9,QWORD[rsp]
+        vpaddq  ymm7,ymm8,ymm12
+        vpmuludq        ymm13,ymm13,ymm11
+        vmovq   xmm12,r12
+        vpaddq  ymm8,ymm9,ymm13
+
+        dec     r14d
+        jnz     NEAR $L$oop_mul_1024
+        vpaddq  ymm0,ymm12,YMMWORD[rsp]
+
+        vpsrlq  ymm12,ymm0,29
+        vpand   ymm0,ymm0,ymm15
+        vpsrlq  ymm13,ymm1,29
+        vpand   ymm1,ymm1,ymm15
+        vpsrlq  ymm10,ymm2,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm2,ymm2,ymm15
+        vpsrlq  ymm11,ymm3,29
+        vpermq  ymm13,ymm13,0x93
+        vpand   ymm3,ymm3,ymm15
+
+        vpblendd        ymm9,ymm12,ymm14,3
+        vpermq  ymm10,ymm10,0x93
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpermq  ymm11,ymm11,0x93
+        vpaddq  ymm0,ymm0,ymm9
+        vpblendd        ymm13,ymm10,ymm13,3
+        vpaddq  ymm1,ymm1,ymm12
+        vpblendd        ymm10,ymm11,ymm10,3
+        vpaddq  ymm2,ymm2,ymm13
+        vpblendd        ymm11,ymm14,ymm11,3
+        vpaddq  ymm3,ymm3,ymm10
+        vpaddq  ymm4,ymm4,ymm11
+
+        vpsrlq  ymm12,ymm0,29
+        vpand   ymm0,ymm0,ymm15
+        vpsrlq  ymm13,ymm1,29
+        vpand   ymm1,ymm1,ymm15
+        vpsrlq  ymm10,ymm2,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm2,ymm2,ymm15
+        vpsrlq  ymm11,ymm3,29
+        vpermq  ymm13,ymm13,0x93
+        vpand   ymm3,ymm3,ymm15
+        vpermq  ymm10,ymm10,0x93
+
+        vpblendd        ymm9,ymm12,ymm14,3
+        vpermq  ymm11,ymm11,0x93
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm0,ymm0,ymm9
+        vpblendd        ymm13,ymm10,ymm13,3
+        vpaddq  ymm1,ymm1,ymm12
+        vpblendd        ymm10,ymm11,ymm10,3
+        vpaddq  ymm2,ymm2,ymm13
+        vpblendd        ymm11,ymm14,ymm11,3
+        vpaddq  ymm3,ymm3,ymm10
+        vpaddq  ymm4,ymm4,ymm11
+
+        vmovdqu YMMWORD[(0-128)+rdi],ymm0
+        vmovdqu YMMWORD[(32-128)+rdi],ymm1
+        vmovdqu YMMWORD[(64-128)+rdi],ymm2
+        vmovdqu YMMWORD[(96-128)+rdi],ymm3
+        vpsrlq  ymm12,ymm4,29
+        vpand   ymm4,ymm4,ymm15
+        vpsrlq  ymm13,ymm5,29
+        vpand   ymm5,ymm5,ymm15
+        vpsrlq  ymm10,ymm6,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm6,ymm6,ymm15
+        vpsrlq  ymm11,ymm7,29
+        vpermq  ymm13,ymm13,0x93
+        vpand   ymm7,ymm7,ymm15
+        vpsrlq  ymm0,ymm8,29
+        vpermq  ymm10,ymm10,0x93
+        vpand   ymm8,ymm8,ymm15
+        vpermq  ymm11,ymm11,0x93
+
+        vpblendd        ymm9,ymm12,ymm14,3
+        vpermq  ymm0,ymm0,0x93
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm4,ymm4,ymm9
+        vpblendd        ymm13,ymm10,ymm13,3
+        vpaddq  ymm5,ymm5,ymm12
+        vpblendd        ymm10,ymm11,ymm10,3
+        vpaddq  ymm6,ymm6,ymm13
+        vpblendd        ymm11,ymm0,ymm11,3
+        vpaddq  ymm7,ymm7,ymm10
+        vpaddq  ymm8,ymm8,ymm11
+
+        vpsrlq  ymm12,ymm4,29
+        vpand   ymm4,ymm4,ymm15
+        vpsrlq  ymm13,ymm5,29
+        vpand   ymm5,ymm5,ymm15
+        vpsrlq  ymm10,ymm6,29
+        vpermq  ymm12,ymm12,0x93
+        vpand   ymm6,ymm6,ymm15
+        vpsrlq  ymm11,ymm7,29
+        vpermq  ymm13,ymm13,0x93
+        vpand   ymm7,ymm7,ymm15
+        vpsrlq  ymm0,ymm8,29
+        vpermq  ymm10,ymm10,0x93
+        vpand   ymm8,ymm8,ymm15
+        vpermq  ymm11,ymm11,0x93
+
+        vpblendd        ymm9,ymm12,ymm14,3
+        vpermq  ymm0,ymm0,0x93
+        vpblendd        ymm12,ymm13,ymm12,3
+        vpaddq  ymm4,ymm4,ymm9
+        vpblendd        ymm13,ymm10,ymm13,3
+        vpaddq  ymm5,ymm5,ymm12
+        vpblendd        ymm10,ymm11,ymm10,3
+        vpaddq  ymm6,ymm6,ymm13
+        vpblendd        ymm11,ymm0,ymm11,3
+        vpaddq  ymm7,ymm7,ymm10
+        vpaddq  ymm8,ymm8,ymm11
+
+        vmovdqu YMMWORD[(128-128)+rdi],ymm4
+        vmovdqu YMMWORD[(160-128)+rdi],ymm5
+        vmovdqu YMMWORD[(192-128)+rdi],ymm6
+        vmovdqu YMMWORD[(224-128)+rdi],ymm7
+        vmovdqu YMMWORD[(256-128)+rdi],ymm8
+        vzeroupper
+
+        mov     rax,rbp
+
+$L$mul_1024_in_tail:
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$mul_1024_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_rsaz_1024_mul_avx2:
+global  rsaz_1024_red2norm_avx2
+
+ALIGN   32
+rsaz_1024_red2norm_avx2:
+
+        sub     rdx,-128
+        xor     rax,rax
+        mov     r8,QWORD[((-128))+rdx]
+        mov     r9,QWORD[((-120))+rdx]
+        mov     r10,QWORD[((-112))+rdx]
+        shl     r8,0
+        shl     r9,29
+        mov     r11,r10
+        shl     r10,58
+        shr     r11,6
+        add     rax,r8
+        add     rax,r9
+        add     rax,r10
+        adc     r11,0
+        mov     QWORD[rcx],rax
+        mov     rax,r11
+        mov     r8,QWORD[((-104))+rdx]
+        mov     r9,QWORD[((-96))+rdx]
+        shl     r8,23
+        mov     r10,r9
+        shl     r9,52
+        shr     r10,12
+        add     rax,r8
+        add     rax,r9
+        adc     r10,0
+        mov     QWORD[8+rcx],rax
+        mov     rax,r10
+        mov     r11,QWORD[((-88))+rdx]
+        mov     r8,QWORD[((-80))+rdx]
+        shl     r11,17
+        mov     r9,r8
+        shl     r8,46
+        shr     r9,18
+        add     rax,r11
+        add     rax,r8
+        adc     r9,0
+        mov     QWORD[16+rcx],rax
+        mov     rax,r9
+        mov     r10,QWORD[((-72))+rdx]
+        mov     r11,QWORD[((-64))+rdx]
+        shl     r10,11
+        mov     r8,r11
+        shl     r11,40
+        shr     r8,24
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[24+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[((-56))+rdx]
+        mov     r10,QWORD[((-48))+rdx]
+        mov     r11,QWORD[((-40))+rdx]
+        shl     r9,5
+        shl     r10,34
+        mov     r8,r11
+        shl     r11,63
+        shr     r8,1
+        add     rax,r9
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[32+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[((-32))+rdx]
+        mov     r10,QWORD[((-24))+rdx]
+        shl     r9,28
+        mov     r11,r10
+        shl     r10,57
+        shr     r11,7
+        add     rax,r9
+        add     rax,r10
+        adc     r11,0
+        mov     QWORD[40+rcx],rax
+        mov     rax,r11
+        mov     r8,QWORD[((-16))+rdx]
+        mov     r9,QWORD[((-8))+rdx]
+        shl     r8,22
+        mov     r10,r9
+        shl     r9,51
+        shr     r10,13
+        add     rax,r8
+        add     rax,r9
+        adc     r10,0
+        mov     QWORD[48+rcx],rax
+        mov     rax,r10
+        mov     r11,QWORD[rdx]
+        mov     r8,QWORD[8+rdx]
+        shl     r11,16
+        mov     r9,r8
+        shl     r8,45
+        shr     r9,19
+        add     rax,r11
+        add     rax,r8
+        adc     r9,0
+        mov     QWORD[56+rcx],rax
+        mov     rax,r9
+        mov     r10,QWORD[16+rdx]
+        mov     r11,QWORD[24+rdx]
+        shl     r10,10
+        mov     r8,r11
+        shl     r11,39
+        shr     r8,25
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[64+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[32+rdx]
+        mov     r10,QWORD[40+rdx]
+        mov     r11,QWORD[48+rdx]
+        shl     r9,4
+        shl     r10,33
+        mov     r8,r11
+        shl     r11,62
+        shr     r8,2
+        add     rax,r9
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[72+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[56+rdx]
+        mov     r10,QWORD[64+rdx]
+        shl     r9,27
+        mov     r11,r10
+        shl     r10,56
+        shr     r11,8
+        add     rax,r9
+        add     rax,r10
+        adc     r11,0
+        mov     QWORD[80+rcx],rax
+        mov     rax,r11
+        mov     r8,QWORD[72+rdx]
+        mov     r9,QWORD[80+rdx]
+        shl     r8,21
+        mov     r10,r9
+        shl     r9,50
+        shr     r10,14
+        add     rax,r8
+        add     rax,r9
+        adc     r10,0
+        mov     QWORD[88+rcx],rax
+        mov     rax,r10
+        mov     r11,QWORD[88+rdx]
+        mov     r8,QWORD[96+rdx]
+        shl     r11,15
+        mov     r9,r8
+        shl     r8,44
+        shr     r9,20
+        add     rax,r11
+        add     rax,r8
+        adc     r9,0
+        mov     QWORD[96+rcx],rax
+        mov     rax,r9
+        mov     r10,QWORD[104+rdx]
+        mov     r11,QWORD[112+rdx]
+        shl     r10,9
+        mov     r8,r11
+        shl     r11,38
+        shr     r8,26
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[104+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[120+rdx]
+        mov     r10,QWORD[128+rdx]
+        mov     r11,QWORD[136+rdx]
+        shl     r9,3
+        shl     r10,32
+        mov     r8,r11
+        shl     r11,61
+        shr     r8,3
+        add     rax,r9
+        add     rax,r10
+        add     rax,r11
+        adc     r8,0
+        mov     QWORD[112+rcx],rax
+        mov     rax,r8
+        mov     r9,QWORD[144+rdx]
+        mov     r10,QWORD[152+rdx]
+        shl     r9,26
+        mov     r11,r10
+        shl     r10,55
+        shr     r11,9
+        add     rax,r9
+        add     rax,r10
+        adc     r11,0
+        mov     QWORD[120+rcx],rax
+        mov     rax,r11
+        DB      0F3h,0C3h               ;repret
+
+
+
+global  rsaz_1024_norm2red_avx2
+
+ALIGN   32
+rsaz_1024_norm2red_avx2:
+
+        sub     rcx,-128
+        mov     r8,QWORD[rdx]
+        mov     eax,0x1fffffff
+        mov     r9,QWORD[8+rdx]
+        mov     r11,r8
+        shr     r11,0
+        and     r11,rax
+        mov     QWORD[((-128))+rcx],r11
+        mov     r10,r8
+        shr     r10,29
+        and     r10,rax
+        mov     QWORD[((-120))+rcx],r10
+        shrd    r8,r9,58
+        and     r8,rax
+        mov     QWORD[((-112))+rcx],r8
+        mov     r10,QWORD[16+rdx]
+        mov     r8,r9
+        shr     r8,23
+        and     r8,rax
+        mov     QWORD[((-104))+rcx],r8
+        shrd    r9,r10,52
+        and     r9,rax
+        mov     QWORD[((-96))+rcx],r9
+        mov     r11,QWORD[24+rdx]
+        mov     r9,r10
+        shr     r9,17
+        and     r9,rax
+        mov     QWORD[((-88))+rcx],r9
+        shrd    r10,r11,46
+        and     r10,rax
+        mov     QWORD[((-80))+rcx],r10
+        mov     r8,QWORD[32+rdx]
+        mov     r10,r11
+        shr     r10,11
+        and     r10,rax
+        mov     QWORD[((-72))+rcx],r10
+        shrd    r11,r8,40
+        and     r11,rax
+        mov     QWORD[((-64))+rcx],r11
+        mov     r9,QWORD[40+rdx]
+        mov     r11,r8
+        shr     r11,5
+        and     r11,rax
+        mov     QWORD[((-56))+rcx],r11
+        mov     r10,r8
+        shr     r10,34
+        and     r10,rax
+        mov     QWORD[((-48))+rcx],r10
+        shrd    r8,r9,63
+        and     r8,rax
+        mov     QWORD[((-40))+rcx],r8
+        mov     r10,QWORD[48+rdx]
+        mov     r8,r9
+        shr     r8,28
+        and     r8,rax
+        mov     QWORD[((-32))+rcx],r8
+        shrd    r9,r10,57
+        and     r9,rax
+        mov     QWORD[((-24))+rcx],r9
+        mov     r11,QWORD[56+rdx]
+        mov     r9,r10
+        shr     r9,22
+        and     r9,rax
+        mov     QWORD[((-16))+rcx],r9
+        shrd    r10,r11,51
+        and     r10,rax
+        mov     QWORD[((-8))+rcx],r10
+        mov     r8,QWORD[64+rdx]
+        mov     r10,r11
+        shr     r10,16
+        and     r10,rax
+        mov     QWORD[rcx],r10
+        shrd    r11,r8,45
+        and     r11,rax
+        mov     QWORD[8+rcx],r11
+        mov     r9,QWORD[72+rdx]
+        mov     r11,r8
+        shr     r11,10
+        and     r11,rax
+        mov     QWORD[16+rcx],r11
+        shrd    r8,r9,39
+        and     r8,rax
+        mov     QWORD[24+rcx],r8
+        mov     r10,QWORD[80+rdx]
+        mov     r8,r9
+        shr     r8,4
+        and     r8,rax
+        mov     QWORD[32+rcx],r8
+        mov     r11,r9
+        shr     r11,33
+        and     r11,rax
+        mov     QWORD[40+rcx],r11
+        shrd    r9,r10,62
+        and     r9,rax
+        mov     QWORD[48+rcx],r9
+        mov     r11,QWORD[88+rdx]
+        mov     r9,r10
+        shr     r9,27
+        and     r9,rax
+        mov     QWORD[56+rcx],r9
+        shrd    r10,r11,56
+        and     r10,rax
+        mov     QWORD[64+rcx],r10
+        mov     r8,QWORD[96+rdx]
+        mov     r10,r11
+        shr     r10,21
+        and     r10,rax
+        mov     QWORD[72+rcx],r10
+        shrd    r11,r8,50
+        and     r11,rax
+        mov     QWORD[80+rcx],r11
+        mov     r9,QWORD[104+rdx]
+        mov     r11,r8
+        shr     r11,15
+        and     r11,rax
+        mov     QWORD[88+rcx],r11
+        shrd    r8,r9,44
+        and     r8,rax
+        mov     QWORD[96+rcx],r8
+        mov     r10,QWORD[112+rdx]
+        mov     r8,r9
+        shr     r8,9
+        and     r8,rax
+        mov     QWORD[104+rcx],r8
+        shrd    r9,r10,38
+        and     r9,rax
+        mov     QWORD[112+rcx],r9
+        mov     r11,QWORD[120+rdx]
+        mov     r9,r10
+        shr     r9,3
+        and     r9,rax
+        mov     QWORD[120+rcx],r9
+        mov     r8,r10
+        shr     r8,32
+        and     r8,rax
+        mov     QWORD[128+rcx],r8
+        shrd    r10,r11,61
+        and     r10,rax
+        mov     QWORD[136+rcx],r10
+        xor     r8,r8
+        mov     r10,r11
+        shr     r10,26
+        and     r10,rax
+        mov     QWORD[144+rcx],r10
+        shrd    r11,r8,55
+        and     r11,rax
+        mov     QWORD[152+rcx],r11
+        mov     QWORD[160+rcx],r8
+        mov     QWORD[168+rcx],r8
+        mov     QWORD[176+rcx],r8
+        mov     QWORD[184+rcx],r8
+        DB      0F3h,0C3h               ;repret
+
+
+global  rsaz_1024_scatter5_avx2
+
+ALIGN   32
+rsaz_1024_scatter5_avx2:
+
+        vzeroupper
+        vmovdqu ymm5,YMMWORD[$L$scatter_permd]
+        shl     r8d,4
+        lea     rcx,[r8*1+rcx]
+        mov     eax,9
+        jmp     NEAR $L$oop_scatter_1024
+
+ALIGN   32
+$L$oop_scatter_1024:
+        vmovdqu ymm0,YMMWORD[rdx]
+        lea     rdx,[32+rdx]
+        vpermd  ymm0,ymm5,ymm0
+        vmovdqu XMMWORD[rcx],xmm0
+        lea     rcx,[512+rcx]
+        dec     eax
+        jnz     NEAR $L$oop_scatter_1024
+
+        vzeroupper
+        DB      0F3h,0C3h               ;repret
+
+
+
+global  rsaz_1024_gather5_avx2
+
+ALIGN   32
+rsaz_1024_gather5_avx2:
+
+        vzeroupper
+        mov     r11,rsp
+
+        lea     rax,[((-136))+rsp]
+$L$SEH_begin_rsaz_1024_gather5:
+
+DB      0x48,0x8d,0x60,0xe0
+DB      0xc5,0xf8,0x29,0x70,0xe0
+DB      0xc5,0xf8,0x29,0x78,0xf0
+DB      0xc5,0x78,0x29,0x40,0x00
+DB      0xc5,0x78,0x29,0x48,0x10
+DB      0xc5,0x78,0x29,0x50,0x20
+DB      0xc5,0x78,0x29,0x58,0x30
+DB      0xc5,0x78,0x29,0x60,0x40
+DB      0xc5,0x78,0x29,0x68,0x50
+DB      0xc5,0x78,0x29,0x70,0x60
+DB      0xc5,0x78,0x29,0x78,0x70
+        lea     rsp,[((-256))+rsp]
+        and     rsp,-32
+        lea     r10,[$L$inc]
+        lea     rax,[((-128))+rsp]
+
+        vmovd   xmm4,r8d
+        vmovdqa ymm0,YMMWORD[r10]
+        vmovdqa ymm1,YMMWORD[32+r10]
+        vmovdqa ymm5,YMMWORD[64+r10]
+        vpbroadcastd    ymm4,xmm4
+
+        vpaddd  ymm2,ymm0,ymm5
+        vpcmpeqd        ymm0,ymm0,ymm4
+        vpaddd  ymm3,ymm1,ymm5
+        vpcmpeqd        ymm1,ymm1,ymm4
+        vmovdqa YMMWORD[(0+128)+rax],ymm0
+        vpaddd  ymm0,ymm2,ymm5
+        vpcmpeqd        ymm2,ymm2,ymm4
+        vmovdqa YMMWORD[(32+128)+rax],ymm1
+        vpaddd  ymm1,ymm3,ymm5
+        vpcmpeqd        ymm3,ymm3,ymm4
+        vmovdqa YMMWORD[(64+128)+rax],ymm2
+        vpaddd  ymm2,ymm0,ymm5
+        vpcmpeqd        ymm0,ymm0,ymm4
+        vmovdqa YMMWORD[(96+128)+rax],ymm3
+        vpaddd  ymm3,ymm1,ymm5
+        vpcmpeqd        ymm1,ymm1,ymm4
+        vmovdqa YMMWORD[(128+128)+rax],ymm0
+        vpaddd  ymm8,ymm2,ymm5
+        vpcmpeqd        ymm2,ymm2,ymm4
+        vmovdqa YMMWORD[(160+128)+rax],ymm1
+        vpaddd  ymm9,ymm3,ymm5
+        vpcmpeqd        ymm3,ymm3,ymm4
+        vmovdqa YMMWORD[(192+128)+rax],ymm2
+        vpaddd  ymm10,ymm8,ymm5
+        vpcmpeqd        ymm8,ymm8,ymm4
+        vmovdqa YMMWORD[(224+128)+rax],ymm3
+        vpaddd  ymm11,ymm9,ymm5
+        vpcmpeqd        ymm9,ymm9,ymm4
+        vpaddd  ymm12,ymm10,ymm5
+        vpcmpeqd        ymm10,ymm10,ymm4
+        vpaddd  ymm13,ymm11,ymm5
+        vpcmpeqd        ymm11,ymm11,ymm4
+        vpaddd  ymm14,ymm12,ymm5
+        vpcmpeqd        ymm12,ymm12,ymm4
+        vpaddd  ymm15,ymm13,ymm5
+        vpcmpeqd        ymm13,ymm13,ymm4
+        vpcmpeqd        ymm14,ymm14,ymm4
+        vpcmpeqd        ymm15,ymm15,ymm4
+
+        vmovdqa ymm7,YMMWORD[((-32))+r10]
+        lea     rdx,[128+rdx]
+        mov     r8d,9
+
+$L$oop_gather_1024:
+        vmovdqa ymm0,YMMWORD[((0-128))+rdx]
+        vmovdqa ymm1,YMMWORD[((32-128))+rdx]
+        vmovdqa ymm2,YMMWORD[((64-128))+rdx]
+        vmovdqa ymm3,YMMWORD[((96-128))+rdx]
+        vpand   ymm0,ymm0,YMMWORD[((0+128))+rax]
+        vpand   ymm1,ymm1,YMMWORD[((32+128))+rax]
+        vpand   ymm2,ymm2,YMMWORD[((64+128))+rax]
+        vpor    ymm4,ymm1,ymm0
+        vpand   ymm3,ymm3,YMMWORD[((96+128))+rax]
+        vmovdqa ymm0,YMMWORD[((128-128))+rdx]
+        vmovdqa ymm1,YMMWORD[((160-128))+rdx]
+        vpor    ymm5,ymm3,ymm2
+        vmovdqa ymm2,YMMWORD[((192-128))+rdx]
+        vmovdqa ymm3,YMMWORD[((224-128))+rdx]
+        vpand   ymm0,ymm0,YMMWORD[((128+128))+rax]
+        vpand   ymm1,ymm1,YMMWORD[((160+128))+rax]
+        vpand   ymm2,ymm2,YMMWORD[((192+128))+rax]
+        vpor    ymm4,ymm4,ymm0
+        vpand   ymm3,ymm3,YMMWORD[((224+128))+rax]
+        vpand   ymm0,ymm8,YMMWORD[((256-128))+rdx]
+        vpor    ymm5,ymm5,ymm1
+        vpand   ymm1,ymm9,YMMWORD[((288-128))+rdx]
+        vpor    ymm4,ymm4,ymm2
+        vpand   ymm2,ymm10,YMMWORD[((320-128))+rdx]
+        vpor    ymm5,ymm5,ymm3
+        vpand   ymm3,ymm11,YMMWORD[((352-128))+rdx]
+        vpor    ymm4,ymm4,ymm0
+        vpand   ymm0,ymm12,YMMWORD[((384-128))+rdx]
+        vpor    ymm5,ymm5,ymm1
+        vpand   ymm1,ymm13,YMMWORD[((416-128))+rdx]
+        vpor    ymm4,ymm4,ymm2
+        vpand   ymm2,ymm14,YMMWORD[((448-128))+rdx]
+        vpor    ymm5,ymm5,ymm3
+        vpand   ymm3,ymm15,YMMWORD[((480-128))+rdx]
+        lea     rdx,[512+rdx]
+        vpor    ymm4,ymm4,ymm0
+        vpor    ymm5,ymm5,ymm1
+        vpor    ymm4,ymm4,ymm2
+        vpor    ymm5,ymm5,ymm3
+
+        vpor    ymm4,ymm4,ymm5
+        vextracti128    xmm5,ymm4,1
+        vpor    xmm5,xmm5,xmm4
+        vpermd  ymm5,ymm7,ymm5
+        vmovdqu YMMWORD[rcx],ymm5
+        lea     rcx,[32+rcx]
+        dec     r8d
+        jnz     NEAR $L$oop_gather_1024
+
+        vpxor   ymm0,ymm0,ymm0
+        vmovdqu YMMWORD[rcx],ymm0
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-168))+r11]
+        movaps  xmm7,XMMWORD[((-152))+r11]
+        movaps  xmm8,XMMWORD[((-136))+r11]
+        movaps  xmm9,XMMWORD[((-120))+r11]
+        movaps  xmm10,XMMWORD[((-104))+r11]
+        movaps  xmm11,XMMWORD[((-88))+r11]
+        movaps  xmm12,XMMWORD[((-72))+r11]
+        movaps  xmm13,XMMWORD[((-56))+r11]
+        movaps  xmm14,XMMWORD[((-40))+r11]
+        movaps  xmm15,XMMWORD[((-24))+r11]
+        lea     rsp,[r11]
+
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_rsaz_1024_gather5:
+
+EXTERN  OPENSSL_ia32cap_P
 global  rsaz_avx2_eligible
 
+ALIGN   32
 rsaz_avx2_eligible:
-        xor     eax,eax
+        mov     eax,DWORD[((OPENSSL_ia32cap_P+8))]
+        mov     ecx,524544
+        mov     edx,0
+        and     ecx,eax
+        cmp     ecx,524544
+        cmove   eax,edx
+        and     eax,32
+        shr     eax,5
         DB      0F3h,0C3h               ;repret
 
 
-global  rsaz_1024_sqr_avx2
-global  rsaz_1024_mul_avx2
-global  rsaz_1024_norm2red_avx2
-global  rsaz_1024_red2norm_avx2
-global  rsaz_1024_scatter5_avx2
-global  rsaz_1024_gather5_avx2
-
-rsaz_1024_sqr_avx2:
-rsaz_1024_mul_avx2:
-rsaz_1024_norm2red_avx2:
-rsaz_1024_red2norm_avx2:
-rsaz_1024_scatter5_avx2:
-rsaz_1024_gather5_avx2:
-DB      0x0f,0x0b
+ALIGN   64
+$L$and_mask:
+        DQ      0x1fffffff,0x1fffffff,0x1fffffff,0x1fffffff
+$L$scatter_permd:
+        DD      0,2,4,6,7,7,7,7
+$L$gather_permd:
+        DD      0,7,1,7,2,7,3,7
+$L$inc:
+        DD      0,0,0,0,1,1,1,1
+        DD      2,2,2,2,3,3,3,3
+        DD      4,4,4,4,4,4,4,4
+ALIGN   64
+EXTERN  __imp_RtlVirtualUnwind
+
+ALIGN   16
+rsaz_se_handler:
+        push    rsi
+        push    rdi
+        push    rbx
+        push    rbp
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        pushfq
+        sub     rsp,64
+
+        mov     rax,QWORD[120+r8]
+        mov     rbx,QWORD[248+r8]
+
+        mov     rsi,QWORD[8+r9]
+        mov     r11,QWORD[56+r9]
+
+        mov     r10d,DWORD[r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jb      NEAR $L$common_seh_tail
+
+        mov     r10d,DWORD[4+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jae     NEAR $L$common_seh_tail
+
+        mov     rbp,QWORD[160+r8]
+
+        mov     r10d,DWORD[8+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        cmovc   rax,rbp
+
+        mov     r15,QWORD[((-48))+rax]
+        mov     r14,QWORD[((-40))+rax]
+        mov     r13,QWORD[((-32))+rax]
+        mov     r12,QWORD[((-24))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+        mov     rbx,QWORD[((-8))+rax]
+        mov     QWORD[240+r8],r15
+        mov     QWORD[232+r8],r14
+        mov     QWORD[224+r8],r13
+        mov     QWORD[216+r8],r12
+        mov     QWORD[160+r8],rbp
+        mov     QWORD[144+r8],rbx
+
+        lea     rsi,[((-216))+rax]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+
+$L$common_seh_tail:
+        mov     rdi,QWORD[8+rax]
+        mov     rsi,QWORD[16+rax]
+        mov     QWORD[152+r8],rax
+        mov     QWORD[168+r8],rsi
+        mov     QWORD[176+r8],rdi
+
+        mov     rdi,QWORD[40+r9]
+        mov     rsi,r8
+        mov     ecx,154
+        DD      0xa548f3fc
+
+        mov     rsi,r9
+        xor     rcx,rcx
+        mov     rdx,QWORD[8+rsi]
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[16+rsi]
+        mov     r10,QWORD[40+rsi]
+        lea     r11,[56+rsi]
+        lea     r12,[24+rsi]
+        mov     QWORD[32+rsp],r10
+        mov     QWORD[40+rsp],r11
+        mov     QWORD[48+rsp],r12
+        mov     QWORD[56+rsp],rcx
+        call    QWORD[__imp_RtlVirtualUnwind]
+
+        mov     eax,1
+        add     rsp,64
+        popfq
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     rbp
+        pop     rbx
+        pop     rdi
+        pop     rsi
         DB      0F3h,0C3h               ;repret
 
+
+section .pdata rdata align=4
+ALIGN   4
+        DD      $L$SEH_begin_rsaz_1024_sqr_avx2 wrt ..imagebase
+        DD      $L$SEH_end_rsaz_1024_sqr_avx2 wrt ..imagebase
+        DD      $L$SEH_info_rsaz_1024_sqr_avx2 wrt ..imagebase
+
+        DD      $L$SEH_begin_rsaz_1024_mul_avx2 wrt ..imagebase
+        DD      $L$SEH_end_rsaz_1024_mul_avx2 wrt ..imagebase
+        DD      $L$SEH_info_rsaz_1024_mul_avx2 wrt ..imagebase
+
+        DD      $L$SEH_begin_rsaz_1024_gather5 wrt ..imagebase
+        DD      $L$SEH_end_rsaz_1024_gather5 wrt ..imagebase
+        DD      $L$SEH_info_rsaz_1024_gather5 wrt ..imagebase
+section .xdata rdata align=8
+ALIGN   8
+$L$SEH_info_rsaz_1024_sqr_avx2:
+DB      9,0,0,0
+        DD      rsaz_se_handler wrt ..imagebase
+        DD      $L$sqr_1024_body wrt ..imagebase,$L$sqr_1024_epilogue wrt ..imagebase,$L$sqr_1024_in_tail wrt ..imagebase
+        DD      0
+$L$SEH_info_rsaz_1024_mul_avx2:
+DB      9,0,0,0
+        DD      rsaz_se_handler wrt ..imagebase
+        DD      $L$mul_1024_body wrt ..imagebase,$L$mul_1024_epilogue wrt ..imagebase,$L$mul_1024_in_tail wrt ..imagebase
+        DD      0
+$L$SEH_info_rsaz_1024_gather5:
+DB      0x01,0x36,0x17,0x0b
+DB      0x36,0xf8,0x09,0x00
+DB      0x31,0xe8,0x08,0x00
+DB      0x2c,0xd8,0x07,0x00
+DB      0x27,0xc8,0x06,0x00
+DB      0x22,0xb8,0x05,0x00
+DB      0x1d,0xa8,0x04,0x00
+DB      0x18,0x98,0x03,0x00
+DB      0x13,0x88,0x02,0x00
+DB      0x0e,0x78,0x01,0x00
+DB      0x09,0x68,0x00,0x00
+DB      0x04,0x01,0x15,0x00
+DB      0x00,0xb3,0x00,0x00

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/rsaz-x86_64.S

@@ -44 +44 @@
         mov     rax,QWORD[8+rsi]
         mov     QWORD[128+rsp],rcx
+        mov     r11d,0x80100
+        and     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     r11d,0x80100
+        je      NEAR $L$oop_sqrx
         jmp     NEAR $L$oop_sqr
 
@@ -414 +418 @@
         dec     r8d
         jnz     NEAR $L$oop_sqr
+        jmp     NEAR $L$sqr_tail
+
+ALIGN   32
+$L$oop_sqrx:
+        mov     DWORD[((128+8))+rsp],r8d
+DB      102,72,15,110,199
+
+        mulx    r9,r8,rax
+        mov     rbx,rax
+
+        mulx    r10,rcx,QWORD[16+rsi]
+        xor     rbp,rbp
+
+        mulx    r11,rax,QWORD[24+rsi]
+        adcx    r9,rcx
+
+DB      0xc4,0x62,0xf3,0xf6,0xa6,0x20,0x00,0x00,0x00
+        adcx    r10,rax
+
+DB      0xc4,0x62,0xfb,0xf6,0xae,0x28,0x00,0x00,0x00
+        adcx    r11,rcx
+
+        mulx    r14,rcx,QWORD[48+rsi]
+        adcx    r12,rax
+        adcx    r13,rcx
+
+        mulx    r15,rax,QWORD[56+rsi]
+        adcx    r14,rax
+        adcx    r15,rbp
+
+        mulx    rdi,rax,rdx
+        mov     rdx,rbx
+        xor     rcx,rcx
+        adox    r8,r8
+        adcx    r8,rdi
+        adox    rcx,rbp
+        adcx    rcx,rbp
+
+        mov     QWORD[rsp],rax
+        mov     QWORD[8+rsp],r8
+
+
+DB      0xc4,0xe2,0xfb,0xf6,0x9e,0x10,0x00,0x00,0x00
+        adox    r10,rax
+        adcx    r11,rbx
+
+        mulx    r8,rdi,QWORD[24+rsi]
+        adox    r11,rdi
+DB      0x66
+        adcx    r12,r8
+
+        mulx    rbx,rax,QWORD[32+rsi]
+        adox    r12,rax
+        adcx    r13,rbx
+
+        mulx    r8,rdi,QWORD[40+rsi]
+        adox    r13,rdi
+        adcx    r14,r8
+
+DB      0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00
+        adox    r14,rax
+        adcx    r15,rbx
+
+DB      0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00
+        adox    r15,rdi
+        adcx    r8,rbp
+        mulx    rdi,rax,rdx
+        adox    r8,rbp
+DB      0x48,0x8b,0x96,0x10,0x00,0x00,0x00
+
+        xor     rbx,rbx
+        adox    r9,r9
+
+        adcx    rax,rcx
+        adox    r10,r10
+        adcx    r9,rax
+        adox    rbx,rbp
+        adcx    r10,rdi
+        adcx    rbx,rbp
+
+        mov     QWORD[16+rsp],r9
+DB      0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00
+
+
+        mulx    r9,rdi,QWORD[24+rsi]
+        adox    r12,rdi
+        adcx    r13,r9
+
+        mulx    rcx,rax,QWORD[32+rsi]
+        adox    r13,rax
+        adcx    r14,rcx
+
+DB      0xc4,0x62,0xc3,0xf6,0x8e,0x28,0x00,0x00,0x00
+        adox    r14,rdi
+        adcx    r15,r9
+
+DB      0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00
+        adox    r15,rax
+        adcx    r8,rcx
+
+        mulx    r9,rdi,QWORD[56+rsi]
+        adox    r8,rdi
+        adcx    r9,rbp
+        mulx    rdi,rax,rdx
+        adox    r9,rbp
+        mov     rdx,QWORD[24+rsi]
+
+        xor     rcx,rcx
+        adox    r11,r11
+
+        adcx    rax,rbx
+        adox    r12,r12
+        adcx    r11,rax
+        adox    rcx,rbp
+        adcx    r12,rdi
+        adcx    rcx,rbp
+
+        mov     QWORD[32+rsp],r11
+        mov     QWORD[40+rsp],r12
+
+
+        mulx    rbx,rax,QWORD[32+rsi]
+        adox    r14,rax
+        adcx    r15,rbx
+
+        mulx    r10,rdi,QWORD[40+rsi]
+        adox    r15,rdi
+        adcx    r8,r10
+
+        mulx    rbx,rax,QWORD[48+rsi]
+        adox    r8,rax
+        adcx    r9,rbx
+
+        mulx    r10,rdi,QWORD[56+rsi]
+        adox    r9,rdi
+        adcx    r10,rbp
+        mulx    rdi,rax,rdx
+        adox    r10,rbp
+        mov     rdx,QWORD[32+rsi]
+
+        xor     rbx,rbx
+        adox    r13,r13
+
+        adcx    rax,rcx
+        adox    r14,r14
+        adcx    r13,rax
+        adox    rbx,rbp
+        adcx    r14,rdi
+        adcx    rbx,rbp
+
+        mov     QWORD[48+rsp],r13
+        mov     QWORD[56+rsp],r14
+
+
+        mulx    r11,rdi,QWORD[40+rsi]
+        adox    r8,rdi
+        adcx    r9,r11
+
+        mulx    rcx,rax,QWORD[48+rsi]
+        adox    r9,rax
+        adcx    r10,rcx
+
+        mulx    r11,rdi,QWORD[56+rsi]
+        adox    r10,rdi
+        adcx    r11,rbp
+        mulx    rdi,rax,rdx
+        mov     rdx,QWORD[40+rsi]
+        adox    r11,rbp
+
+        xor     rcx,rcx
+        adox    r15,r15
+
+        adcx    rax,rbx
+        adox    r8,r8
+        adcx    r15,rax
+        adox    rcx,rbp
+        adcx    r8,rdi
+        adcx    rcx,rbp
+
+        mov     QWORD[64+rsp],r15
+        mov     QWORD[72+rsp],r8
+
+
+DB      0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00
+        adox    r10,rax
+        adcx    r11,rbx
+
+DB      0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00
+        adox    r11,rdi
+        adcx    r12,rbp
+        mulx    rdi,rax,rdx
+        adox    r12,rbp
+        mov     rdx,QWORD[48+rsi]
+
+        xor     rbx,rbx
+        adox    r9,r9
+
+        adcx    rax,rcx
+        adox    r10,r10
+        adcx    r9,rax
+        adcx    r10,rdi
+        adox    rbx,rbp
+        adcx    rbx,rbp
+
+        mov     QWORD[80+rsp],r9
+        mov     QWORD[88+rsp],r10
+
+
+DB      0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00
+        adox    r12,rax
+        adox    r13,rbp
+
+        mulx    rdi,rax,rdx
+        xor     rcx,rcx
+        mov     rdx,QWORD[56+rsi]
+        adox    r11,r11
+
+        adcx    rax,rbx
+        adox    r12,r12
+        adcx    r11,rax
+        adox    rcx,rbp
+        adcx    r12,rdi
+        adcx    rcx,rbp
+
+DB      0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00
+DB      0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00
+
+
+        mulx    rdx,rax,rdx
+        xor     rbx,rbx
+        adox    r13,r13
+
+        adcx    rax,rcx
+        adox    rbx,rbp
+        adcx    rax,r13
+        adcx    rbx,rdx
+
+DB      102,72,15,126,199
+DB      102,72,15,126,205
+
+        mov     rdx,QWORD[128+rsp]
+        mov     r8,QWORD[rsp]
+        mov     r9,QWORD[8+rsp]
+        mov     r10,QWORD[16+rsp]
+        mov     r11,QWORD[24+rsp]
+        mov     r12,QWORD[32+rsp]
+        mov     r13,QWORD[40+rsp]
+        mov     r14,QWORD[48+rsp]
+        mov     r15,QWORD[56+rsp]
+
+        mov     QWORD[112+rsp],rax
+        mov     QWORD[120+rsp],rbx
+
+        call    __rsaz_512_reducex
+
+        add     r8,QWORD[64+rsp]
+        adc     r9,QWORD[72+rsp]
+        adc     r10,QWORD[80+rsp]
+        adc     r11,QWORD[88+rsp]
+        adc     r12,QWORD[96+rsp]
+        adc     r13,QWORD[104+rsp]
+        adc     r14,QWORD[112+rsp]
+        adc     r15,QWORD[120+rsp]
+        sbb     rcx,rcx
+
+        call    __rsaz_512_subtract
+
+        mov     rdx,r8
+        mov     rax,r9
+        mov     r8d,DWORD[((128+8))+rsp]
+        mov     rsi,rdi
+
+        dec     r8d
+        jnz     NEAR $L$oop_sqrx
+
+$L$sqr_tail:
 
         lea     rax,[((128+24+48))+rsp]
@@ -472 +752 @@
 DB      102,72,15,110,201
         mov     QWORD[128+rsp],r8
+        mov     r11d,0x80100
+        and     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     r11d,0x80100
+        je      NEAR $L$mulx
         mov     rbx,QWORD[rdx]
         mov     rbp,rdx
@@ -489 +773 @@
 
         call    __rsaz_512_reduce
+        jmp     NEAR $L$mul_tail
+
+ALIGN   32
+$L$mulx:
+        mov     rbp,rdx
+        mov     rdx,QWORD[rdx]
+        call    __rsaz_512_mulx
+
+DB      102,72,15,126,199
+DB      102,72,15,126,205
+
+        mov     rdx,QWORD[128+rsp]
+        mov     r8,QWORD[rsp]
+        mov     r9,QWORD[8+rsp]
+        mov     r10,QWORD[16+rsp]
+        mov     r11,QWORD[24+rsp]
+        mov     r12,QWORD[32+rsp]
+        mov     r13,QWORD[40+rsp]
+        mov     r14,QWORD[48+rsp]
+        mov     r15,QWORD[56+rsp]
+
+        call    __rsaz_512_reducex
+$L$mul_tail:
         add     r8,QWORD[64+rsp]
         adc     r9,QWORD[72+rsp]
@@ -620 +927 @@
         pshufd  xmm9,xmm8,0x4e
         por     xmm8,xmm9
+        mov     r11d,0x80100
+        and     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     r11d,0x80100
+        je      NEAR $L$mulx_gather
 DB      102,76,15,126,195
 
@@ -800 +1111 @@
 
         call    __rsaz_512_reduce
+        jmp     NEAR $L$mul_gather_tail
+
+ALIGN   32
+$L$mulx_gather:
+DB      102,76,15,126,194
+
+        mov     QWORD[128+rsp],r8
+        mov     QWORD[((128+8))+rsp],rdi
+        mov     QWORD[((128+16))+rsp],rcx
+
+        mulx    r8,rbx,QWORD[rsi]
+        mov     QWORD[rsp],rbx
+        xor     edi,edi
+
+        mulx    r9,rax,QWORD[8+rsi]
+
+        mulx    r10,rbx,QWORD[16+rsi]
+        adcx    r8,rax
+
+        mulx    r11,rax,QWORD[24+rsi]
+        adcx    r9,rbx
+
+        mulx    r12,rbx,QWORD[32+rsi]
+        adcx    r10,rax
+
+        mulx    r13,rax,QWORD[40+rsi]
+        adcx    r11,rbx
+
+        mulx    r14,rbx,QWORD[48+rsi]
+        adcx    r12,rax
+
+        mulx    r15,rax,QWORD[56+rsi]
+        adcx    r13,rbx
+        adcx    r14,rax
+DB      0x67
+        mov     rbx,r8
+        adcx    r15,rdi
+
+        mov     rcx,-7
+        jmp     NEAR $L$oop_mulx_gather
+
+ALIGN   32
+$L$oop_mulx_gather:
+        movdqa  xmm8,XMMWORD[rbp]
+        movdqa  xmm9,XMMWORD[16+rbp]
+        movdqa  xmm10,XMMWORD[32+rbp]
+        movdqa  xmm11,XMMWORD[48+rbp]
+        pand    xmm8,xmm0
+        movdqa  xmm12,XMMWORD[64+rbp]
+        pand    xmm9,xmm1
+        movdqa  xmm13,XMMWORD[80+rbp]
+        pand    xmm10,xmm2
+        movdqa  xmm14,XMMWORD[96+rbp]
+        pand    xmm11,xmm3
+        movdqa  xmm15,XMMWORD[112+rbp]
+        lea     rbp,[128+rbp]
+        pand    xmm12,xmm4
+        pand    xmm13,xmm5
+        pand    xmm14,xmm6
+        pand    xmm15,xmm7
+        por     xmm8,xmm10
+        por     xmm9,xmm11
+        por     xmm8,xmm12
+        por     xmm9,xmm13
+        por     xmm8,xmm14
+        por     xmm9,xmm15
+
+        por     xmm8,xmm9
+        pshufd  xmm9,xmm8,0x4e
+        por     xmm8,xmm9
+DB      102,76,15,126,194
+
+DB      0xc4,0x62,0xfb,0xf6,0x86,0x00,0x00,0x00,0x00
+        adcx    rbx,rax
+        adox    r8,r9
+
+        mulx    r9,rax,QWORD[8+rsi]
+        adcx    r8,rax
+        adox    r9,r10
+
+        mulx    r10,rax,QWORD[16+rsi]
+        adcx    r9,rax
+        adox    r10,r11
+
+DB      0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00
+        adcx    r10,rax
+        adox    r11,r12
+
+        mulx    r12,rax,QWORD[32+rsi]
+        adcx    r11,rax
+        adox    r12,r13
+
+        mulx    r13,rax,QWORD[40+rsi]
+        adcx    r12,rax
+        adox    r13,r14
+
+DB      0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00
+        adcx    r13,rax
+DB      0x67
+        adox    r14,r15
+
+        mulx    r15,rax,QWORD[56+rsi]
+        mov     QWORD[64+rcx*8+rsp],rbx
+        adcx    r14,rax
+        adox    r15,rdi
+        mov     rbx,r8
+        adcx    r15,rdi
+
+        inc     rcx
+        jnz     NEAR $L$oop_mulx_gather
+
+        mov     QWORD[64+rsp],r8
+        mov     QWORD[((64+8))+rsp],r9
+        mov     QWORD[((64+16))+rsp],r10
+        mov     QWORD[((64+24))+rsp],r11
+        mov     QWORD[((64+32))+rsp],r12
+        mov     QWORD[((64+40))+rsp],r13
+        mov     QWORD[((64+48))+rsp],r14
+        mov     QWORD[((64+56))+rsp],r15
+
+        mov     rdx,QWORD[128+rsp]
+        mov     rdi,QWORD[((128+8))+rsp]
+        mov     rbp,QWORD[((128+16))+rsp]
+
+        mov     r8,QWORD[rsp]
+        mov     r9,QWORD[8+rsp]
+        mov     r10,QWORD[16+rsp]
+        mov     r11,QWORD[24+rsp]
+        mov     r12,QWORD[32+rsp]
+        mov     r13,QWORD[40+rsp]
+        mov     r14,QWORD[48+rsp]
+        mov     r15,QWORD[56+rsp]
+
+        call    __rsaz_512_reducex
+
+$L$mul_gather_tail:
         add     r8,QWORD[64+rsp]
         adc     r9,QWORD[72+rsp]
@@ -886 +1333 @@
 
         mov     rbp,rdi
+        mov     r11d,0x80100
+        and     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
+        cmp     r11d,0x80100
+        je      NEAR $L$mulx_scatter
         mov     rbx,QWORD[rdi]
         call    __rsaz_512_mul
@@ -902 +1353 @@
 
         call    __rsaz_512_reduce
+        jmp     NEAR $L$mul_scatter_tail
+
+ALIGN   32
+$L$mulx_scatter:
+        mov     rdx,QWORD[rdi]
+        call    __rsaz_512_mulx
+
+DB      102,72,15,126,199
+DB      102,72,15,126,205
+
+        mov     rdx,QWORD[128+rsp]
+        mov     r8,QWORD[rsp]
+        mov     r9,QWORD[8+rsp]
+        mov     r10,QWORD[16+rsp]
+        mov     r11,QWORD[24+rsp]
+        mov     r12,QWORD[32+rsp]
+        mov     r13,QWORD[40+rsp]
+        mov     r14,QWORD[48+rsp]
+        mov     r15,QWORD[56+rsp]
+
+        call    __rsaz_512_reducex
+
+$L$mul_scatter_tail:
         add     r8,QWORD[64+rsp]
         adc     r9,QWORD[72+rsp]
@@ -977 +1451 @@
 
 $L$mul_by_one_body:
+        mov     eax,DWORD[((OPENSSL_ia32cap_P+8))]
         mov     rbp,rdx
         mov     QWORD[128+rsp],rcx
@@ -997 +1472 @@
         movdqa  XMMWORD[80+rsp],xmm0
         movdqa  XMMWORD[96+rsp],xmm0
+        and     eax,0x80100
+        cmp     eax,0x80100
+        je      NEAR $L$by_one_callx
         call    __rsaz_512_reduce
+        jmp     NEAR $L$by_one_tail
+ALIGN   32
+$L$by_one_callx:
+        mov     rdx,QWORD[128+rsp]
+        call    __rsaz_512_reducex
+$L$by_one_tail:
         mov     QWORD[rdi],r8
         mov     QWORD[8+rdi],r9
@@ -1110 +1594 @@
         dec     ecx
         jne     NEAR $L$reduction_loop
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+ALIGN   32
+__rsaz_512_reducex:
+
+
+        imul    rdx,r8
+        xor     rsi,rsi
+        mov     ecx,8
+        jmp     NEAR $L$reduction_loopx
+
+ALIGN   32
+$L$reduction_loopx:
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rbp]
+        adcx    rax,rbx
+        adox    r8,r9
+
+        mulx    r9,rax,QWORD[8+rbp]
+        adcx    r8,rax
+        adox    r9,r10
+
+        mulx    r10,rbx,QWORD[16+rbp]
+        adcx    r9,rbx
+        adox    r10,r11
+
+        mulx    r11,rbx,QWORD[24+rbp]
+        adcx    r10,rbx
+        adox    r11,r12
+
+DB      0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00
+        mov     rax,rdx
+        mov     rdx,r8
+        adcx    r11,rbx
+        adox    r12,r13
+
+        mulx    rdx,rbx,QWORD[((128+8))+rsp]
+        mov     rdx,rax
+
+        mulx    r13,rax,QWORD[40+rbp]
+        adcx    r12,rax
+        adox    r13,r14
+
+DB      0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00
+        adcx    r13,rax
+        adox    r14,r15
+
+        mulx    r15,rax,QWORD[56+rbp]
+        mov     rdx,rbx
+        adcx    r14,rax
+        adox    r15,rsi
+        adcx    r15,rsi
+
+        dec     ecx
+        jne     NEAR $L$reduction_loopx
 
         DB      0F3h,0C3h               ;repret
@@ -1313 +1855 @@
         mov     QWORD[48+rdi],r14
         mov     QWORD[56+rdi],r15
+
+        DB      0F3h,0C3h               ;repret
+
+
+
+ALIGN   32
+__rsaz_512_mulx:
+
+        mulx    r8,rbx,QWORD[rsi]
+        mov     rcx,-6
+
+        mulx    r9,rax,QWORD[8+rsi]
+        mov     QWORD[8+rsp],rbx
+
+        mulx    r10,rbx,QWORD[16+rsi]
+        adc     r8,rax
+
+        mulx    r11,rax,QWORD[24+rsi]
+        adc     r9,rbx
+
+        mulx    r12,rbx,QWORD[32+rsi]
+        adc     r10,rax
+
+        mulx    r13,rax,QWORD[40+rsi]
+        adc     r11,rbx
+
+        mulx    r14,rbx,QWORD[48+rsi]
+        adc     r12,rax
+
+        mulx    r15,rax,QWORD[56+rsi]
+        mov     rdx,QWORD[8+rbp]
+        adc     r13,rbx
+        adc     r14,rax
+        adc     r15,0
+
+        xor     rdi,rdi
+        jmp     NEAR $L$oop_mulx
+
+ALIGN   32
+$L$oop_mulx:
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rsi]
+        adcx    rbx,rax
+        adox    r8,r9
+
+        mulx    r9,rax,QWORD[8+rsi]
+        adcx    r8,rax
+        adox    r9,r10
+
+        mulx    r10,rax,QWORD[16+rsi]
+        adcx    r9,rax
+        adox    r10,r11
+
+        mulx    r11,rax,QWORD[24+rsi]
+        adcx    r10,rax
+        adox    r11,r12
+
+DB      0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00
+        adcx    r11,rax
+        adox    r12,r13
+
+        mulx    r13,rax,QWORD[40+rsi]
+        adcx    r12,rax
+        adox    r13,r14
+
+        mulx    r14,rax,QWORD[48+rsi]
+        adcx    r13,rax
+        adox    r14,r15
+
+        mulx    r15,rax,QWORD[56+rsi]
+        mov     rdx,QWORD[64+rcx*8+rbp]
+        mov     QWORD[((8+64-8))+rcx*8+rsp],rbx
+        adcx    r14,rax
+        adox    r15,rdi
+        adcx    r15,rdi
+
+        inc     rcx
+        jnz     NEAR $L$oop_mulx
+
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rsi]
+        adcx    rbx,rax
+        adox    r8,r9
+
+DB      0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00
+        adcx    r8,rax
+        adox    r9,r10
+
+DB      0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00
+        adcx    r9,rax
+        adox    r10,r11
+
+        mulx    r11,rax,QWORD[24+rsi]
+        adcx    r10,rax
+        adox    r11,r12
+
+        mulx    r12,rax,QWORD[32+rsi]
+        adcx    r11,rax
+        adox    r12,r13
+
+        mulx    r13,rax,QWORD[40+rsi]
+        adcx    r12,rax
+        adox    r13,r14
+
+DB      0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00
+        adcx    r13,rax
+        adox    r14,r15
+
+DB      0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00
+        adcx    r14,rax
+        adox    r15,rdi
+        adcx    r15,rdi
+
+        mov     QWORD[((8+64-8))+rsp],rbx
+        mov     QWORD[((8+64))+rsp],r8
+        mov     QWORD[((8+64+8))+rsp],r9
+        mov     QWORD[((8+64+16))+rsp],r10
+        mov     QWORD[((8+64+24))+rsp],r11
+        mov     QWORD[((8+64+32))+rsp],r12
+        mov     QWORD[((8+64+40))+rsp],r13
+        mov     QWORD[((8+64+48))+rsp],r14
+        mov     QWORD[((8+64+56))+rsp],r15
 
         DB      0F3h,0C3h               ;repret

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/sha1-mb-x86_64.S

@@ -25 +25 @@
         bt      rcx,61
         jc      NEAR _shaext_shortcut
+        test    ecx,268435456
+        jnz     NEAR _avx_shortcut
         mov     rax,rsp
 
@@ -3018 +3020 @@
 $L$SEH_end_sha1_multi_block_shaext:
 
+ALIGN   32
+sha1_multi_block_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha1_multi_block_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx_shortcut:
+        shr     rcx,32
+        cmp     edx,2
+        jb      NEAR $L$avx
+        test    ecx,32
+        jnz     NEAR _avx2_shortcut
+        jmp     NEAR $L$avx
+ALIGN   32
+$L$avx:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[(-120)+rax],xmm10
+        movaps  XMMWORD[(-104)+rax],xmm11
+        movaps  XMMWORD[(-88)+rax],xmm12
+        movaps  XMMWORD[(-72)+rax],xmm13
+        movaps  XMMWORD[(-56)+rax],xmm14
+        movaps  XMMWORD[(-40)+rax],xmm15
+        sub     rsp,288
+        and     rsp,-256
+        mov     QWORD[272+rsp],rax
+
+$L$body_avx:
+        lea     rbp,[K_XX_XX]
+        lea     rbx,[256+rsp]
+
+        vzeroupper
+$L$oop_grande_avx:
+        mov     DWORD[280+rsp],edx
+        xor     edx,edx
+
+        mov     r8,QWORD[rsi]
+
+        mov     ecx,DWORD[8+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[rbx],ecx
+        cmovle  r8,rbp
+
+        mov     r9,QWORD[16+rsi]
+
+        mov     ecx,DWORD[24+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[4+rbx],ecx
+        cmovle  r9,rbp
+
+        mov     r10,QWORD[32+rsi]
+
+        mov     ecx,DWORD[40+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[8+rbx],ecx
+        cmovle  r10,rbp
+
+        mov     r11,QWORD[48+rsi]
+
+        mov     ecx,DWORD[56+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[12+rbx],ecx
+        cmovle  r11,rbp
+        test    edx,edx
+        jz      NEAR $L$done_avx
+
+        vmovdqu xmm10,XMMWORD[rdi]
+        lea     rax,[128+rsp]
+        vmovdqu xmm11,XMMWORD[32+rdi]
+        vmovdqu xmm12,XMMWORD[64+rdi]
+        vmovdqu xmm13,XMMWORD[96+rdi]
+        vmovdqu xmm14,XMMWORD[128+rdi]
+        vmovdqu xmm5,XMMWORD[96+rbp]
+        jmp     NEAR $L$oop_avx
+
+ALIGN   32
+$L$oop_avx:
+        vmovdqa xmm15,XMMWORD[((-32))+rbp]
+        vmovd   xmm0,DWORD[r8]
+        lea     r8,[64+r8]
+        vmovd   xmm2,DWORD[r9]
+        lea     r9,[64+r9]
+        vpinsrd xmm0,xmm0,DWORD[r10],1
+        lea     r10,[64+r10]
+        vpinsrd xmm2,xmm2,DWORD[r11],1
+        lea     r11,[64+r11]
+        vmovd   xmm1,DWORD[((-60))+r8]
+        vpunpckldq      xmm0,xmm0,xmm2
+        vmovd   xmm9,DWORD[((-60))+r9]
+        vpshufb xmm0,xmm0,xmm5
+        vpinsrd xmm1,xmm1,DWORD[((-60))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-60))+r11],1
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpandn  xmm7,xmm11,xmm13
+        vpand   xmm6,xmm11,xmm12
+
+        vmovdqa XMMWORD[(0-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpunpckldq      xmm1,xmm1,xmm9
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm2,DWORD[((-56))+r8]
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-56))+r9]
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpshufb xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpinsrd xmm2,xmm2,DWORD[((-56))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-56))+r11],1
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpandn  xmm7,xmm10,xmm12
+        vpand   xmm6,xmm10,xmm11
+
+        vmovdqa XMMWORD[(16-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpunpckldq      xmm2,xmm2,xmm9
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm3,DWORD[((-52))+r8]
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-52))+r9]
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpshufb xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpinsrd xmm3,xmm3,DWORD[((-52))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-52))+r11],1
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpandn  xmm7,xmm14,xmm11
+        vpand   xmm6,xmm14,xmm10
+
+        vmovdqa XMMWORD[(32-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpunpckldq      xmm3,xmm3,xmm9
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm4,DWORD[((-48))+r8]
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-48))+r9]
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpshufb xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpinsrd xmm4,xmm4,DWORD[((-48))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-48))+r11],1
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpandn  xmm7,xmm13,xmm10
+        vpand   xmm6,xmm13,xmm14
+
+        vmovdqa XMMWORD[(48-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpunpckldq      xmm4,xmm4,xmm9
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm0,DWORD[((-44))+r8]
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-44))+r9]
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpshufb xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpinsrd xmm0,xmm0,DWORD[((-44))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-44))+r11],1
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpandn  xmm7,xmm12,xmm14
+        vpand   xmm6,xmm12,xmm13
+
+        vmovdqa XMMWORD[(64-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpunpckldq      xmm0,xmm0,xmm9
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm1,DWORD[((-40))+r8]
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-40))+r9]
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpshufb xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpinsrd xmm1,xmm1,DWORD[((-40))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-40))+r11],1
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpandn  xmm7,xmm11,xmm13
+        vpand   xmm6,xmm11,xmm12
+
+        vmovdqa XMMWORD[(80-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpunpckldq      xmm1,xmm1,xmm9
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm2,DWORD[((-36))+r8]
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-36))+r9]
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpshufb xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpinsrd xmm2,xmm2,DWORD[((-36))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-36))+r11],1
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpandn  xmm7,xmm10,xmm12
+        vpand   xmm6,xmm10,xmm11
+
+        vmovdqa XMMWORD[(96-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpunpckldq      xmm2,xmm2,xmm9
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm3,DWORD[((-32))+r8]
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-32))+r9]
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpshufb xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpinsrd xmm3,xmm3,DWORD[((-32))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-32))+r11],1
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpandn  xmm7,xmm14,xmm11
+        vpand   xmm6,xmm14,xmm10
+
+        vmovdqa XMMWORD[(112-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpunpckldq      xmm3,xmm3,xmm9
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm4,DWORD[((-28))+r8]
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-28))+r9]
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpshufb xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpinsrd xmm4,xmm4,DWORD[((-28))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-28))+r11],1
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpandn  xmm7,xmm13,xmm10
+        vpand   xmm6,xmm13,xmm14
+
+        vmovdqa XMMWORD[(128-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpunpckldq      xmm4,xmm4,xmm9
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm0,DWORD[((-24))+r8]
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-24))+r9]
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpshufb xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpinsrd xmm0,xmm0,DWORD[((-24))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-24))+r11],1
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpandn  xmm7,xmm12,xmm14
+        vpand   xmm6,xmm12,xmm13
+
+        vmovdqa XMMWORD[(144-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpunpckldq      xmm0,xmm0,xmm9
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm1,DWORD[((-20))+r8]
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-20))+r9]
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpshufb xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpinsrd xmm1,xmm1,DWORD[((-20))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-20))+r11],1
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpandn  xmm7,xmm11,xmm13
+        vpand   xmm6,xmm11,xmm12
+
+        vmovdqa XMMWORD[(160-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpunpckldq      xmm1,xmm1,xmm9
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm2,DWORD[((-16))+r8]
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-16))+r9]
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpshufb xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpinsrd xmm2,xmm2,DWORD[((-16))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-16))+r11],1
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpandn  xmm7,xmm10,xmm12
+        vpand   xmm6,xmm10,xmm11
+
+        vmovdqa XMMWORD[(176-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpunpckldq      xmm2,xmm2,xmm9
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm3,DWORD[((-12))+r8]
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-12))+r9]
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpshufb xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpinsrd xmm3,xmm3,DWORD[((-12))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-12))+r11],1
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpandn  xmm7,xmm14,xmm11
+        vpand   xmm6,xmm14,xmm10
+
+        vmovdqa XMMWORD[(192-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpunpckldq      xmm3,xmm3,xmm9
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm4,DWORD[((-8))+r8]
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-8))+r9]
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpshufb xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpinsrd xmm4,xmm4,DWORD[((-8))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-8))+r11],1
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpandn  xmm7,xmm13,xmm10
+        vpand   xmm6,xmm13,xmm14
+
+        vmovdqa XMMWORD[(208-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpunpckldq      xmm4,xmm4,xmm9
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm7
+        vmovd   xmm0,DWORD[((-4))+r8]
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vmovd   xmm9,DWORD[((-4))+r9]
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpshufb xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vmovdqa xmm1,XMMWORD[((0-128))+rax]
+        vpinsrd xmm0,xmm0,DWORD[((-4))+r10],1
+        vpinsrd xmm9,xmm9,DWORD[((-4))+r11],1
+        vpaddd  xmm10,xmm10,xmm15
+        prefetcht0      [63+r8]
+        vpslld  xmm8,xmm11,5
+        vpandn  xmm7,xmm12,xmm14
+        vpand   xmm6,xmm12,xmm13
+
+        vmovdqa XMMWORD[(224-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpunpckldq      xmm0,xmm0,xmm9
+        vpsrld  xmm9,xmm11,27
+        prefetcht0      [63+r9]
+        vpxor   xmm6,xmm6,xmm7
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        prefetcht0      [63+r10]
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        prefetcht0      [63+r11]
+        vpshufb xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vmovdqa xmm2,XMMWORD[((16-128))+rax]
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((32-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpandn  xmm7,xmm11,xmm13
+
+        vpand   xmm6,xmm11,xmm12
+
+        vmovdqa XMMWORD[(240-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((128-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm7
+        vpxor   xmm1,xmm1,xmm3
+
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((48-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpandn  xmm7,xmm10,xmm12
+
+        vpand   xmm6,xmm10,xmm11
+
+        vmovdqa XMMWORD[(0-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((144-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm7
+        vpxor   xmm2,xmm2,xmm4
+
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((64-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpandn  xmm7,xmm14,xmm11
+
+        vpand   xmm6,xmm14,xmm10
+
+        vmovdqa XMMWORD[(16-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((160-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm7
+        vpxor   xmm3,xmm3,xmm0
+
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((80-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpandn  xmm7,xmm13,xmm10
+
+        vpand   xmm6,xmm13,xmm14
+
+        vmovdqa XMMWORD[(32-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((176-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm7
+        vpxor   xmm4,xmm4,xmm1
+
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((96-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpandn  xmm7,xmm12,xmm14
+
+        vpand   xmm6,xmm12,xmm13
+
+        vmovdqa XMMWORD[(48-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((192-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm7
+        vpxor   xmm0,xmm0,xmm2
+
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vmovdqa xmm15,XMMWORD[rbp]
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((112-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(64-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((208-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((128-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(80-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((224-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((144-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(96-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((240-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((160-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(112-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((0-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((176-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(128-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((16-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((192-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(144-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((32-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((208-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(160-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((48-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((224-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(176-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((64-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((240-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(192-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((80-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((0-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(208-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((96-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((16-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(224-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((112-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((32-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(240-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((128-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((48-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(0-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((144-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((64-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(16-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((160-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((80-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(32-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((176-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((96-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(48-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((192-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((112-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(64-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((208-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((128-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(80-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((224-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((144-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(96-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((240-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((160-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(112-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((0-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vmovdqa xmm15,XMMWORD[32+rbp]
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((176-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpand   xmm7,xmm13,xmm12
+        vpxor   xmm1,xmm1,XMMWORD[((16-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm7
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm13,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vmovdqu XMMWORD[(128-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm1,31
+        vpand   xmm6,xmm6,xmm11
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpslld  xmm7,xmm11,30
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((192-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpand   xmm7,xmm12,xmm11
+        vpxor   xmm2,xmm2,XMMWORD[((32-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm7
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm12,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vmovdqu XMMWORD[(144-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm2,31
+        vpand   xmm6,xmm6,xmm10
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpslld  xmm7,xmm10,30
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((208-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpand   xmm7,xmm11,xmm10
+        vpxor   xmm3,xmm3,XMMWORD[((48-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm7
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm11,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vmovdqu XMMWORD[(160-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm3,31
+        vpand   xmm6,xmm6,xmm14
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpslld  xmm7,xmm14,30
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((224-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpand   xmm7,xmm10,xmm14
+        vpxor   xmm4,xmm4,XMMWORD[((64-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm7
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm10,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vmovdqu XMMWORD[(176-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm4,31
+        vpand   xmm6,xmm6,xmm13
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpslld  xmm7,xmm13,30
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((240-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpand   xmm7,xmm14,xmm13
+        vpxor   xmm0,xmm0,XMMWORD[((80-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm7
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm14,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vmovdqu XMMWORD[(192-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm0,31
+        vpand   xmm6,xmm6,xmm12
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpslld  xmm7,xmm12,30
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((0-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpand   xmm7,xmm13,xmm12
+        vpxor   xmm1,xmm1,XMMWORD[((96-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm7
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm13,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vmovdqu XMMWORD[(208-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm1,31
+        vpand   xmm6,xmm6,xmm11
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpslld  xmm7,xmm11,30
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((16-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpand   xmm7,xmm12,xmm11
+        vpxor   xmm2,xmm2,XMMWORD[((112-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm7
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm12,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vmovdqu XMMWORD[(224-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm2,31
+        vpand   xmm6,xmm6,xmm10
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpslld  xmm7,xmm10,30
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((32-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpand   xmm7,xmm11,xmm10
+        vpxor   xmm3,xmm3,XMMWORD[((128-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm7
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm11,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vmovdqu XMMWORD[(240-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm3,31
+        vpand   xmm6,xmm6,xmm14
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpslld  xmm7,xmm14,30
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((48-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpand   xmm7,xmm10,xmm14
+        vpxor   xmm4,xmm4,XMMWORD[((144-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm7
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm10,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vmovdqu XMMWORD[(0-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm4,31
+        vpand   xmm6,xmm6,xmm13
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpslld  xmm7,xmm13,30
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((64-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpand   xmm7,xmm14,xmm13
+        vpxor   xmm0,xmm0,XMMWORD[((160-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm7
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm14,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vmovdqu XMMWORD[(16-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm0,31
+        vpand   xmm6,xmm6,xmm12
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpslld  xmm7,xmm12,30
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((80-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpand   xmm7,xmm13,xmm12
+        vpxor   xmm1,xmm1,XMMWORD[((176-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm7
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm13,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vmovdqu XMMWORD[(32-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm1,31
+        vpand   xmm6,xmm6,xmm11
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpslld  xmm7,xmm11,30
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((96-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpand   xmm7,xmm12,xmm11
+        vpxor   xmm2,xmm2,XMMWORD[((192-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm7
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm12,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vmovdqu XMMWORD[(48-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm2,31
+        vpand   xmm6,xmm6,xmm10
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpslld  xmm7,xmm10,30
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((112-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpand   xmm7,xmm11,xmm10
+        vpxor   xmm3,xmm3,XMMWORD[((208-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm7
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm11,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vmovdqu XMMWORD[(64-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm3,31
+        vpand   xmm6,xmm6,xmm14
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpslld  xmm7,xmm14,30
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((128-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpand   xmm7,xmm10,xmm14
+        vpxor   xmm4,xmm4,XMMWORD[((224-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm7
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm10,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vmovdqu XMMWORD[(80-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm4,31
+        vpand   xmm6,xmm6,xmm13
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpslld  xmm7,xmm13,30
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((144-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpand   xmm7,xmm14,xmm13
+        vpxor   xmm0,xmm0,XMMWORD[((240-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm7
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm14,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vmovdqu XMMWORD[(96-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm0,31
+        vpand   xmm6,xmm6,xmm12
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpslld  xmm7,xmm12,30
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((160-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm15
+        vpslld  xmm8,xmm10,5
+        vpand   xmm7,xmm13,xmm12
+        vpxor   xmm1,xmm1,XMMWORD[((0-128))+rax]
+
+        vpaddd  xmm14,xmm14,xmm7
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm13,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vmovdqu XMMWORD[(112-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm1,31
+        vpand   xmm6,xmm6,xmm11
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpslld  xmm7,xmm11,30
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((176-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm15
+        vpslld  xmm8,xmm14,5
+        vpand   xmm7,xmm12,xmm11
+        vpxor   xmm2,xmm2,XMMWORD[((16-128))+rax]
+
+        vpaddd  xmm13,xmm13,xmm7
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm12,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vmovdqu XMMWORD[(128-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm2,31
+        vpand   xmm6,xmm6,xmm10
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpslld  xmm7,xmm10,30
+        vpaddd  xmm13,xmm13,xmm6
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((192-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm15
+        vpslld  xmm8,xmm13,5
+        vpand   xmm7,xmm11,xmm10
+        vpxor   xmm3,xmm3,XMMWORD[((32-128))+rax]
+
+        vpaddd  xmm12,xmm12,xmm7
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm11,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vmovdqu XMMWORD[(144-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm3,31
+        vpand   xmm6,xmm6,xmm14
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpslld  xmm7,xmm14,30
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((208-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm15
+        vpslld  xmm8,xmm12,5
+        vpand   xmm7,xmm10,xmm14
+        vpxor   xmm4,xmm4,XMMWORD[((48-128))+rax]
+
+        vpaddd  xmm11,xmm11,xmm7
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm10,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vmovdqu XMMWORD[(160-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm4,31
+        vpand   xmm6,xmm6,xmm13
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpslld  xmm7,xmm13,30
+        vpaddd  xmm11,xmm11,xmm6
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((224-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm15
+        vpslld  xmm8,xmm11,5
+        vpand   xmm7,xmm14,xmm13
+        vpxor   xmm0,xmm0,XMMWORD[((64-128))+rax]
+
+        vpaddd  xmm10,xmm10,xmm7
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm14,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vmovdqu XMMWORD[(176-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpor    xmm8,xmm8,xmm9
+        vpsrld  xmm5,xmm0,31
+        vpand   xmm6,xmm6,xmm12
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpslld  xmm7,xmm12,30
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vmovdqa xmm15,XMMWORD[64+rbp]
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((240-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(192-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((80-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((0-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(208-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((96-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((16-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(224-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((112-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((32-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(240-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((128-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((48-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(0-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((144-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((64-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(16-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((160-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((80-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(32-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((176-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((96-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vmovdqa XMMWORD[(48-128)+rax],xmm2
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((192-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((112-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vmovdqa XMMWORD[(64-128)+rax],xmm3
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((208-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((128-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vmovdqa XMMWORD[(80-128)+rax],xmm4
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((224-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((144-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vmovdqa XMMWORD[(96-128)+rax],xmm0
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((240-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((160-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vmovdqa XMMWORD[(112-128)+rax],xmm1
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((0-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((176-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((16-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((192-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((32-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpxor   xmm0,xmm0,xmm2
+        vmovdqa xmm2,XMMWORD[((208-128))+rax]
+
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm0,xmm0,XMMWORD[((48-128))+rax]
+        vpsrld  xmm9,xmm11,27
+        vpxor   xmm6,xmm6,xmm13
+        vpxor   xmm0,xmm0,xmm2
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+        vpsrld  xmm5,xmm0,31
+        vpaddd  xmm0,xmm0,xmm0
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm0,xmm0,xmm5
+        vpor    xmm12,xmm12,xmm7
+        vpxor   xmm1,xmm1,xmm3
+        vmovdqa xmm3,XMMWORD[((224-128))+rax]
+
+        vpslld  xmm8,xmm10,5
+        vpaddd  xmm14,xmm14,xmm15
+        vpxor   xmm6,xmm13,xmm11
+        vpaddd  xmm14,xmm14,xmm0
+        vpxor   xmm1,xmm1,XMMWORD[((64-128))+rax]
+        vpsrld  xmm9,xmm10,27
+        vpxor   xmm6,xmm6,xmm12
+        vpxor   xmm1,xmm1,xmm3
+
+        vpslld  xmm7,xmm11,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm14,xmm14,xmm6
+        vpsrld  xmm5,xmm1,31
+        vpaddd  xmm1,xmm1,xmm1
+
+        vpsrld  xmm11,xmm11,2
+        vpaddd  xmm14,xmm14,xmm8
+        vpor    xmm1,xmm1,xmm5
+        vpor    xmm11,xmm11,xmm7
+        vpxor   xmm2,xmm2,xmm4
+        vmovdqa xmm4,XMMWORD[((240-128))+rax]
+
+        vpslld  xmm8,xmm14,5
+        vpaddd  xmm13,xmm13,xmm15
+        vpxor   xmm6,xmm12,xmm10
+        vpaddd  xmm13,xmm13,xmm1
+        vpxor   xmm2,xmm2,XMMWORD[((80-128))+rax]
+        vpsrld  xmm9,xmm14,27
+        vpxor   xmm6,xmm6,xmm11
+        vpxor   xmm2,xmm2,xmm4
+
+        vpslld  xmm7,xmm10,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm13,xmm13,xmm6
+        vpsrld  xmm5,xmm2,31
+        vpaddd  xmm2,xmm2,xmm2
+
+        vpsrld  xmm10,xmm10,2
+        vpaddd  xmm13,xmm13,xmm8
+        vpor    xmm2,xmm2,xmm5
+        vpor    xmm10,xmm10,xmm7
+        vpxor   xmm3,xmm3,xmm0
+        vmovdqa xmm0,XMMWORD[((0-128))+rax]
+
+        vpslld  xmm8,xmm13,5
+        vpaddd  xmm12,xmm12,xmm15
+        vpxor   xmm6,xmm11,xmm14
+        vpaddd  xmm12,xmm12,xmm2
+        vpxor   xmm3,xmm3,XMMWORD[((96-128))+rax]
+        vpsrld  xmm9,xmm13,27
+        vpxor   xmm6,xmm6,xmm10
+        vpxor   xmm3,xmm3,xmm0
+
+        vpslld  xmm7,xmm14,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm12,xmm12,xmm6
+        vpsrld  xmm5,xmm3,31
+        vpaddd  xmm3,xmm3,xmm3
+
+        vpsrld  xmm14,xmm14,2
+        vpaddd  xmm12,xmm12,xmm8
+        vpor    xmm3,xmm3,xmm5
+        vpor    xmm14,xmm14,xmm7
+        vpxor   xmm4,xmm4,xmm1
+        vmovdqa xmm1,XMMWORD[((16-128))+rax]
+
+        vpslld  xmm8,xmm12,5
+        vpaddd  xmm11,xmm11,xmm15
+        vpxor   xmm6,xmm10,xmm13
+        vpaddd  xmm11,xmm11,xmm3
+        vpxor   xmm4,xmm4,XMMWORD[((112-128))+rax]
+        vpsrld  xmm9,xmm12,27
+        vpxor   xmm6,xmm6,xmm14
+        vpxor   xmm4,xmm4,xmm1
+
+        vpslld  xmm7,xmm13,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm11,xmm11,xmm6
+        vpsrld  xmm5,xmm4,31
+        vpaddd  xmm4,xmm4,xmm4
+
+        vpsrld  xmm13,xmm13,2
+        vpaddd  xmm11,xmm11,xmm8
+        vpor    xmm4,xmm4,xmm5
+        vpor    xmm13,xmm13,xmm7
+        vpslld  xmm8,xmm11,5
+        vpaddd  xmm10,xmm10,xmm15
+        vpxor   xmm6,xmm14,xmm12
+
+        vpsrld  xmm9,xmm11,27
+        vpaddd  xmm10,xmm10,xmm4
+        vpxor   xmm6,xmm6,xmm13
+
+        vpslld  xmm7,xmm12,30
+        vpor    xmm8,xmm8,xmm9
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpsrld  xmm12,xmm12,2
+        vpaddd  xmm10,xmm10,xmm8
+        vpor    xmm12,xmm12,xmm7
+        mov     ecx,1
+        cmp     ecx,DWORD[rbx]
+        cmovge  r8,rbp
+        cmp     ecx,DWORD[4+rbx]
+        cmovge  r9,rbp
+        cmp     ecx,DWORD[8+rbx]
+        cmovge  r10,rbp
+        cmp     ecx,DWORD[12+rbx]
+        cmovge  r11,rbp
+        vmovdqu xmm6,XMMWORD[rbx]
+        vpxor   xmm8,xmm8,xmm8
+        vmovdqa xmm7,xmm6
+        vpcmpgtd        xmm7,xmm7,xmm8
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpand   xmm10,xmm10,xmm7
+        vpand   xmm11,xmm11,xmm7
+        vpaddd  xmm10,xmm10,XMMWORD[rdi]
+        vpand   xmm12,xmm12,xmm7
+        vpaddd  xmm11,xmm11,XMMWORD[32+rdi]
+        vpand   xmm13,xmm13,xmm7
+        vpaddd  xmm12,xmm12,XMMWORD[64+rdi]
+        vpand   xmm14,xmm14,xmm7
+        vpaddd  xmm13,xmm13,XMMWORD[96+rdi]
+        vpaddd  xmm14,xmm14,XMMWORD[128+rdi]
+        vmovdqu XMMWORD[rdi],xmm10
+        vmovdqu XMMWORD[32+rdi],xmm11
+        vmovdqu XMMWORD[64+rdi],xmm12
+        vmovdqu XMMWORD[96+rdi],xmm13
+        vmovdqu XMMWORD[128+rdi],xmm14
+
+        vmovdqu XMMWORD[rbx],xmm6
+        vmovdqu xmm5,XMMWORD[96+rbp]
+        dec     edx
+        jnz     NEAR $L$oop_avx
+
+        mov     edx,DWORD[280+rsp]
+        lea     rdi,[16+rdi]
+        lea     rsi,[64+rsi]
+        dec     edx
+        jnz     NEAR $L$oop_grande_avx
+
+$L$done_avx:
+        mov     rax,QWORD[272+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-184))+rax]
+        movaps  xmm7,XMMWORD[((-168))+rax]
+        movaps  xmm8,XMMWORD[((-152))+rax]
+        movaps  xmm9,XMMWORD[((-136))+rax]
+        movaps  xmm10,XMMWORD[((-120))+rax]
+        movaps  xmm11,XMMWORD[((-104))+rax]
+        movaps  xmm12,XMMWORD[((-88))+rax]
+        movaps  xmm13,XMMWORD[((-72))+rax]
+        movaps  xmm14,XMMWORD[((-56))+rax]
+        movaps  xmm15,XMMWORD[((-40))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha1_multi_block_avx:
+
+ALIGN   32
+sha1_multi_block_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha1_multi_block_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx2_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[64+rsp],xmm10
+        movaps  XMMWORD[80+rsp],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+        sub     rsp,576
+        and     rsp,-256
+        mov     QWORD[544+rsp],rax
+
+$L$body_avx2:
+        lea     rbp,[K_XX_XX]
+        shr     edx,1
+
+        vzeroupper
+$L$oop_grande_avx2:
+        mov     DWORD[552+rsp],edx
+        xor     edx,edx
+        lea     rbx,[512+rsp]
+
+        mov     r12,QWORD[rsi]
+
+        mov     ecx,DWORD[8+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[rbx],ecx
+        cmovle  r12,rbp
+
+        mov     r13,QWORD[16+rsi]
+
+        mov     ecx,DWORD[24+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[4+rbx],ecx
+        cmovle  r13,rbp
+
+        mov     r14,QWORD[32+rsi]
+
+        mov     ecx,DWORD[40+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[8+rbx],ecx
+        cmovle  r14,rbp
+
+        mov     r15,QWORD[48+rsi]
+
+        mov     ecx,DWORD[56+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[12+rbx],ecx
+        cmovle  r15,rbp
+
+        mov     r8,QWORD[64+rsi]
+
+        mov     ecx,DWORD[72+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[16+rbx],ecx
+        cmovle  r8,rbp
+
+        mov     r9,QWORD[80+rsi]
+
+        mov     ecx,DWORD[88+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[20+rbx],ecx
+        cmovle  r9,rbp
+
+        mov     r10,QWORD[96+rsi]
+
+        mov     ecx,DWORD[104+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[24+rbx],ecx
+        cmovle  r10,rbp
+
+        mov     r11,QWORD[112+rsi]
+
+        mov     ecx,DWORD[120+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[28+rbx],ecx
+        cmovle  r11,rbp
+        vmovdqu ymm0,YMMWORD[rdi]
+        lea     rax,[128+rsp]
+        vmovdqu ymm1,YMMWORD[32+rdi]
+        lea     rbx,[((256+128))+rsp]
+        vmovdqu ymm2,YMMWORD[64+rdi]
+        vmovdqu ymm3,YMMWORD[96+rdi]
+        vmovdqu ymm4,YMMWORD[128+rdi]
+        vmovdqu ymm9,YMMWORD[96+rbp]
+        jmp     NEAR $L$oop_avx2
+
+ALIGN   32
+$L$oop_avx2:
+        vmovdqa ymm15,YMMWORD[((-32))+rbp]
+        vmovd   xmm10,DWORD[r12]
+        lea     r12,[64+r12]
+        vmovd   xmm12,DWORD[r8]
+        lea     r8,[64+r8]
+        vmovd   xmm7,DWORD[r13]
+        lea     r13,[64+r13]
+        vmovd   xmm6,DWORD[r9]
+        lea     r9,[64+r9]
+        vpinsrd xmm10,xmm10,DWORD[r14],1
+        lea     r14,[64+r14]
+        vpinsrd xmm12,xmm12,DWORD[r10],1
+        lea     r10,[64+r10]
+        vpinsrd xmm7,xmm7,DWORD[r15],1
+        lea     r15,[64+r15]
+        vpunpckldq      ymm10,ymm10,ymm7
+        vpinsrd xmm6,xmm6,DWORD[r11],1
+        lea     r11,[64+r11]
+        vpunpckldq      ymm12,ymm12,ymm6
+        vmovd   xmm11,DWORD[((-60))+r12]
+        vinserti128     ymm10,ymm10,xmm12,1
+        vmovd   xmm8,DWORD[((-60))+r8]
+        vpshufb ymm10,ymm10,ymm9
+        vmovd   xmm7,DWORD[((-60))+r13]
+        vmovd   xmm6,DWORD[((-60))+r9]
+        vpinsrd xmm11,xmm11,DWORD[((-60))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-60))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-60))+r15],1
+        vpunpckldq      ymm11,ymm11,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-60))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpandn  ymm6,ymm1,ymm3
+        vpand   ymm5,ymm1,ymm2
+
+        vmovdqa YMMWORD[(0-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vinserti128     ymm11,ymm11,xmm8,1
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm12,DWORD[((-56))+r12]
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-56))+r8]
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpshufb ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vmovd   xmm7,DWORD[((-56))+r13]
+        vmovd   xmm6,DWORD[((-56))+r9]
+        vpinsrd xmm12,xmm12,DWORD[((-56))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-56))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-56))+r15],1
+        vpunpckldq      ymm12,ymm12,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-56))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpandn  ymm6,ymm0,ymm2
+        vpand   ymm5,ymm0,ymm1
+
+        vmovdqa YMMWORD[(32-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vinserti128     ymm12,ymm12,xmm8,1
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm13,DWORD[((-52))+r12]
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-52))+r8]
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpshufb ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vmovd   xmm7,DWORD[((-52))+r13]
+        vmovd   xmm6,DWORD[((-52))+r9]
+        vpinsrd xmm13,xmm13,DWORD[((-52))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-52))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-52))+r15],1
+        vpunpckldq      ymm13,ymm13,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-52))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpandn  ymm6,ymm4,ymm1
+        vpand   ymm5,ymm4,ymm0
+
+        vmovdqa YMMWORD[(64-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vinserti128     ymm13,ymm13,xmm8,1
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm14,DWORD[((-48))+r12]
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-48))+r8]
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpshufb ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vmovd   xmm7,DWORD[((-48))+r13]
+        vmovd   xmm6,DWORD[((-48))+r9]
+        vpinsrd xmm14,xmm14,DWORD[((-48))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-48))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-48))+r15],1
+        vpunpckldq      ymm14,ymm14,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-48))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpandn  ymm6,ymm3,ymm0
+        vpand   ymm5,ymm3,ymm4
+
+        vmovdqa YMMWORD[(96-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vinserti128     ymm14,ymm14,xmm8,1
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm10,DWORD[((-44))+r12]
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-44))+r8]
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpshufb ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vmovd   xmm7,DWORD[((-44))+r13]
+        vmovd   xmm6,DWORD[((-44))+r9]
+        vpinsrd xmm10,xmm10,DWORD[((-44))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-44))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-44))+r15],1
+        vpunpckldq      ymm10,ymm10,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-44))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpandn  ymm6,ymm2,ymm4
+        vpand   ymm5,ymm2,ymm3
+
+        vmovdqa YMMWORD[(128-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vinserti128     ymm10,ymm10,xmm8,1
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm11,DWORD[((-40))+r12]
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-40))+r8]
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpshufb ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovd   xmm7,DWORD[((-40))+r13]
+        vmovd   xmm6,DWORD[((-40))+r9]
+        vpinsrd xmm11,xmm11,DWORD[((-40))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-40))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-40))+r15],1
+        vpunpckldq      ymm11,ymm11,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-40))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpandn  ymm6,ymm1,ymm3
+        vpand   ymm5,ymm1,ymm2
+
+        vmovdqa YMMWORD[(160-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vinserti128     ymm11,ymm11,xmm8,1
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm12,DWORD[((-36))+r12]
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-36))+r8]
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpshufb ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vmovd   xmm7,DWORD[((-36))+r13]
+        vmovd   xmm6,DWORD[((-36))+r9]
+        vpinsrd xmm12,xmm12,DWORD[((-36))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-36))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-36))+r15],1
+        vpunpckldq      ymm12,ymm12,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-36))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpandn  ymm6,ymm0,ymm2
+        vpand   ymm5,ymm0,ymm1
+
+        vmovdqa YMMWORD[(192-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vinserti128     ymm12,ymm12,xmm8,1
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm13,DWORD[((-32))+r12]
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-32))+r8]
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpshufb ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vmovd   xmm7,DWORD[((-32))+r13]
+        vmovd   xmm6,DWORD[((-32))+r9]
+        vpinsrd xmm13,xmm13,DWORD[((-32))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-32))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-32))+r15],1
+        vpunpckldq      ymm13,ymm13,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-32))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpandn  ymm6,ymm4,ymm1
+        vpand   ymm5,ymm4,ymm0
+
+        vmovdqa YMMWORD[(224-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vinserti128     ymm13,ymm13,xmm8,1
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm14,DWORD[((-28))+r12]
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-28))+r8]
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpshufb ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vmovd   xmm7,DWORD[((-28))+r13]
+        vmovd   xmm6,DWORD[((-28))+r9]
+        vpinsrd xmm14,xmm14,DWORD[((-28))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-28))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-28))+r15],1
+        vpunpckldq      ymm14,ymm14,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-28))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpandn  ymm6,ymm3,ymm0
+        vpand   ymm5,ymm3,ymm4
+
+        vmovdqa YMMWORD[(256-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vinserti128     ymm14,ymm14,xmm8,1
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm10,DWORD[((-24))+r12]
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-24))+r8]
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpshufb ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vmovd   xmm7,DWORD[((-24))+r13]
+        vmovd   xmm6,DWORD[((-24))+r9]
+        vpinsrd xmm10,xmm10,DWORD[((-24))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-24))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-24))+r15],1
+        vpunpckldq      ymm10,ymm10,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-24))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpandn  ymm6,ymm2,ymm4
+        vpand   ymm5,ymm2,ymm3
+
+        vmovdqa YMMWORD[(288-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vinserti128     ymm10,ymm10,xmm8,1
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm11,DWORD[((-20))+r12]
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-20))+r8]
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpshufb ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovd   xmm7,DWORD[((-20))+r13]
+        vmovd   xmm6,DWORD[((-20))+r9]
+        vpinsrd xmm11,xmm11,DWORD[((-20))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-20))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-20))+r15],1
+        vpunpckldq      ymm11,ymm11,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-20))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpandn  ymm6,ymm1,ymm3
+        vpand   ymm5,ymm1,ymm2
+
+        vmovdqa YMMWORD[(320-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vinserti128     ymm11,ymm11,xmm8,1
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm12,DWORD[((-16))+r12]
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-16))+r8]
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpshufb ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vmovd   xmm7,DWORD[((-16))+r13]
+        vmovd   xmm6,DWORD[((-16))+r9]
+        vpinsrd xmm12,xmm12,DWORD[((-16))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-16))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-16))+r15],1
+        vpunpckldq      ymm12,ymm12,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-16))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpandn  ymm6,ymm0,ymm2
+        vpand   ymm5,ymm0,ymm1
+
+        vmovdqa YMMWORD[(352-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vinserti128     ymm12,ymm12,xmm8,1
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm13,DWORD[((-12))+r12]
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-12))+r8]
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpshufb ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vmovd   xmm7,DWORD[((-12))+r13]
+        vmovd   xmm6,DWORD[((-12))+r9]
+        vpinsrd xmm13,xmm13,DWORD[((-12))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-12))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-12))+r15],1
+        vpunpckldq      ymm13,ymm13,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-12))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpandn  ymm6,ymm4,ymm1
+        vpand   ymm5,ymm4,ymm0
+
+        vmovdqa YMMWORD[(384-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vinserti128     ymm13,ymm13,xmm8,1
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm14,DWORD[((-8))+r12]
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-8))+r8]
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpshufb ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vmovd   xmm7,DWORD[((-8))+r13]
+        vmovd   xmm6,DWORD[((-8))+r9]
+        vpinsrd xmm14,xmm14,DWORD[((-8))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-8))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-8))+r15],1
+        vpunpckldq      ymm14,ymm14,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-8))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpandn  ymm6,ymm3,ymm0
+        vpand   ymm5,ymm3,ymm4
+
+        vmovdqa YMMWORD[(416-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vinserti128     ymm14,ymm14,xmm8,1
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm6
+        vmovd   xmm10,DWORD[((-4))+r12]
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vmovd   xmm8,DWORD[((-4))+r8]
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpshufb ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vmovdqa ymm11,YMMWORD[((0-128))+rax]
+        vmovd   xmm7,DWORD[((-4))+r13]
+        vmovd   xmm6,DWORD[((-4))+r9]
+        vpinsrd xmm10,xmm10,DWORD[((-4))+r14],1
+        vpinsrd xmm8,xmm8,DWORD[((-4))+r10],1
+        vpinsrd xmm7,xmm7,DWORD[((-4))+r15],1
+        vpunpckldq      ymm10,ymm10,ymm7
+        vpinsrd xmm6,xmm6,DWORD[((-4))+r11],1
+        vpunpckldq      ymm8,ymm8,ymm6
+        vpaddd  ymm0,ymm0,ymm15
+        prefetcht0      [63+r12]
+        vpslld  ymm7,ymm1,5
+        vpandn  ymm6,ymm2,ymm4
+        vpand   ymm5,ymm2,ymm3
+
+        vmovdqa YMMWORD[(448-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vinserti128     ymm10,ymm10,xmm8,1
+        vpsrld  ymm8,ymm1,27
+        prefetcht0      [63+r13]
+        vpxor   ymm5,ymm5,ymm6
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        prefetcht0      [63+r14]
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        prefetcht0      [63+r15]
+        vpshufb ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovdqa ymm12,YMMWORD[((32-128))+rax]
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((64-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpandn  ymm6,ymm1,ymm3
+        prefetcht0      [63+r8]
+        vpand   ymm5,ymm1,ymm2
+
+        vmovdqa YMMWORD[(480-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((256-256-128))+rbx]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        prefetcht0      [63+r9]
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        prefetcht0      [63+r10]
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        prefetcht0      [63+r11]
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((96-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpandn  ymm6,ymm0,ymm2
+
+        vpand   ymm5,ymm0,ymm1
+
+        vmovdqa YMMWORD[(0-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((288-256-128))+rbx]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm6
+        vpxor   ymm12,ymm12,ymm14
+
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((128-128))+rax]
+
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpandn  ymm6,ymm4,ymm1
+
+        vpand   ymm5,ymm4,ymm0
+
+        vmovdqa YMMWORD[(32-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((320-256-128))+rbx]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm6
+        vpxor   ymm13,ymm13,ymm10
+
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((160-128))+rax]
+
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpandn  ymm6,ymm3,ymm0
+
+        vpand   ymm5,ymm3,ymm4
+
+        vmovdqa YMMWORD[(64-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((352-256-128))+rbx]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm6
+        vpxor   ymm14,ymm14,ymm11
+
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((192-128))+rax]
+
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpandn  ymm6,ymm2,ymm4
+
+        vpand   ymm5,ymm2,ymm3
+
+        vmovdqa YMMWORD[(96-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((384-256-128))+rbx]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm6
+        vpxor   ymm10,ymm10,ymm12
+
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovdqa ymm15,YMMWORD[rbp]
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((224-128))+rax]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(128-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((416-256-128))+rbx]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((256-256-128))+rbx]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(160-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((448-256-128))+rbx]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((288-256-128))+rbx]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(192-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((480-256-128))+rbx]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((320-256-128))+rbx]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(224-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((0-128))+rax]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((352-256-128))+rbx]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(256-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((32-128))+rax]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((384-256-128))+rbx]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(288-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((64-128))+rax]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((416-256-128))+rbx]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(320-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((96-128))+rax]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((448-256-128))+rbx]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(352-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((128-128))+rax]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((480-256-128))+rbx]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(384-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((160-128))+rax]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((0-128))+rax]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(416-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((192-128))+rax]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((32-128))+rax]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(448-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((224-128))+rax]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((64-128))+rax]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(480-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((256-256-128))+rbx]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((96-128))+rax]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(0-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((288-256-128))+rbx]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((128-128))+rax]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(32-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((320-256-128))+rbx]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((160-128))+rax]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(64-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((352-256-128))+rbx]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((192-128))+rax]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(96-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((384-256-128))+rbx]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((224-128))+rax]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(128-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((416-256-128))+rbx]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((256-256-128))+rbx]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(160-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((448-256-128))+rbx]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((288-256-128))+rbx]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(192-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((480-256-128))+rbx]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((320-256-128))+rbx]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(224-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((0-128))+rax]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovdqa ymm15,YMMWORD[32+rbp]
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((352-256-128))+rbx]
+
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpand   ymm6,ymm3,ymm2
+        vpxor   ymm11,ymm11,YMMWORD[((32-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm6
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm3,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vmovdqu YMMWORD[(256-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm11,31
+        vpand   ymm5,ymm5,ymm1
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpslld  ymm6,ymm1,30
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((384-256-128))+rbx]
+
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpand   ymm6,ymm2,ymm1
+        vpxor   ymm12,ymm12,YMMWORD[((64-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm6
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm2,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vmovdqu YMMWORD[(288-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm12,31
+        vpand   ymm5,ymm5,ymm0
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpslld  ymm6,ymm0,30
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((416-256-128))+rbx]
+
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpand   ymm6,ymm1,ymm0
+        vpxor   ymm13,ymm13,YMMWORD[((96-128))+rax]
+
+        vpaddd  ymm2,ymm2,ymm6
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm1,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vmovdqu YMMWORD[(320-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm13,31
+        vpand   ymm5,ymm5,ymm4
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpslld  ymm6,ymm4,30
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((448-256-128))+rbx]
+
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpand   ymm6,ymm0,ymm4
+        vpxor   ymm14,ymm14,YMMWORD[((128-128))+rax]
+
+        vpaddd  ymm1,ymm1,ymm6
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm0,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vmovdqu YMMWORD[(352-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm14,31
+        vpand   ymm5,ymm5,ymm3
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpslld  ymm6,ymm3,30
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((480-256-128))+rbx]
+
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpand   ymm6,ymm4,ymm3
+        vpxor   ymm10,ymm10,YMMWORD[((160-128))+rax]
+
+        vpaddd  ymm0,ymm0,ymm6
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm4,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vmovdqu YMMWORD[(384-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm10,31
+        vpand   ymm5,ymm5,ymm2
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpslld  ymm6,ymm2,30
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((0-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpand   ymm6,ymm3,ymm2
+        vpxor   ymm11,ymm11,YMMWORD[((192-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm6
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm3,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vmovdqu YMMWORD[(416-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm11,31
+        vpand   ymm5,ymm5,ymm1
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpslld  ymm6,ymm1,30
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((32-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpand   ymm6,ymm2,ymm1
+        vpxor   ymm12,ymm12,YMMWORD[((224-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm6
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm2,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vmovdqu YMMWORD[(448-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm12,31
+        vpand   ymm5,ymm5,ymm0
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpslld  ymm6,ymm0,30
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((64-128))+rax]
+
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpand   ymm6,ymm1,ymm0
+        vpxor   ymm13,ymm13,YMMWORD[((256-256-128))+rbx]
+
+        vpaddd  ymm2,ymm2,ymm6
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm1,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vmovdqu YMMWORD[(480-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm13,31
+        vpand   ymm5,ymm5,ymm4
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpslld  ymm6,ymm4,30
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((96-128))+rax]
+
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpand   ymm6,ymm0,ymm4
+        vpxor   ymm14,ymm14,YMMWORD[((288-256-128))+rbx]
+
+        vpaddd  ymm1,ymm1,ymm6
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm0,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vmovdqu YMMWORD[(0-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm14,31
+        vpand   ymm5,ymm5,ymm3
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpslld  ymm6,ymm3,30
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((128-128))+rax]
+
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpand   ymm6,ymm4,ymm3
+        vpxor   ymm10,ymm10,YMMWORD[((320-256-128))+rbx]
+
+        vpaddd  ymm0,ymm0,ymm6
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm4,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vmovdqu YMMWORD[(32-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm10,31
+        vpand   ymm5,ymm5,ymm2
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpslld  ymm6,ymm2,30
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((160-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpand   ymm6,ymm3,ymm2
+        vpxor   ymm11,ymm11,YMMWORD[((352-256-128))+rbx]
+
+        vpaddd  ymm4,ymm4,ymm6
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm3,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vmovdqu YMMWORD[(64-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm11,31
+        vpand   ymm5,ymm5,ymm1
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpslld  ymm6,ymm1,30
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((192-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpand   ymm6,ymm2,ymm1
+        vpxor   ymm12,ymm12,YMMWORD[((384-256-128))+rbx]
+
+        vpaddd  ymm3,ymm3,ymm6
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm2,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vmovdqu YMMWORD[(96-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm12,31
+        vpand   ymm5,ymm5,ymm0
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpslld  ymm6,ymm0,30
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((224-128))+rax]
+
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpand   ymm6,ymm1,ymm0
+        vpxor   ymm13,ymm13,YMMWORD[((416-256-128))+rbx]
+
+        vpaddd  ymm2,ymm2,ymm6
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm1,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vmovdqu YMMWORD[(128-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm13,31
+        vpand   ymm5,ymm5,ymm4
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpslld  ymm6,ymm4,30
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((256-256-128))+rbx]
+
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpand   ymm6,ymm0,ymm4
+        vpxor   ymm14,ymm14,YMMWORD[((448-256-128))+rbx]
+
+        vpaddd  ymm1,ymm1,ymm6
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm0,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vmovdqu YMMWORD[(160-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm14,31
+        vpand   ymm5,ymm5,ymm3
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpslld  ymm6,ymm3,30
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((288-256-128))+rbx]
+
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpand   ymm6,ymm4,ymm3
+        vpxor   ymm10,ymm10,YMMWORD[((480-256-128))+rbx]
+
+        vpaddd  ymm0,ymm0,ymm6
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm4,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vmovdqu YMMWORD[(192-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm10,31
+        vpand   ymm5,ymm5,ymm2
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpslld  ymm6,ymm2,30
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((320-256-128))+rbx]
+
+        vpaddd  ymm4,ymm4,ymm15
+        vpslld  ymm7,ymm0,5
+        vpand   ymm6,ymm3,ymm2
+        vpxor   ymm11,ymm11,YMMWORD[((0-128))+rax]
+
+        vpaddd  ymm4,ymm4,ymm6
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm3,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vmovdqu YMMWORD[(224-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm11,31
+        vpand   ymm5,ymm5,ymm1
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpslld  ymm6,ymm1,30
+        vpaddd  ymm4,ymm4,ymm5
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((352-256-128))+rbx]
+
+        vpaddd  ymm3,ymm3,ymm15
+        vpslld  ymm7,ymm4,5
+        vpand   ymm6,ymm2,ymm1
+        vpxor   ymm12,ymm12,YMMWORD[((32-128))+rax]
+
+        vpaddd  ymm3,ymm3,ymm6
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm2,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vmovdqu YMMWORD[(256-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm12,31
+        vpand   ymm5,ymm5,ymm0
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpslld  ymm6,ymm0,30
+        vpaddd  ymm3,ymm3,ymm5
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((384-256-128))+rbx]
+
+        vpaddd  ymm2,ymm2,ymm15
+        vpslld  ymm7,ymm3,5
+        vpand   ymm6,ymm1,ymm0
+        vpxor   ymm13,ymm13,YMMWORD[((64-128))+rax]
+
+        vpaddd  ymm2,ymm2,ymm6
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm1,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vmovdqu YMMWORD[(288-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm13,31
+        vpand   ymm5,ymm5,ymm4
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpslld  ymm6,ymm4,30
+        vpaddd  ymm2,ymm2,ymm5
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((416-256-128))+rbx]
+
+        vpaddd  ymm1,ymm1,ymm15
+        vpslld  ymm7,ymm2,5
+        vpand   ymm6,ymm0,ymm4
+        vpxor   ymm14,ymm14,YMMWORD[((96-128))+rax]
+
+        vpaddd  ymm1,ymm1,ymm6
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm0,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vmovdqu YMMWORD[(320-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm14,31
+        vpand   ymm5,ymm5,ymm3
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpslld  ymm6,ymm3,30
+        vpaddd  ymm1,ymm1,ymm5
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((448-256-128))+rbx]
+
+        vpaddd  ymm0,ymm0,ymm15
+        vpslld  ymm7,ymm1,5
+        vpand   ymm6,ymm4,ymm3
+        vpxor   ymm10,ymm10,YMMWORD[((128-128))+rax]
+
+        vpaddd  ymm0,ymm0,ymm6
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm4,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vmovdqu YMMWORD[(352-256-128)+rbx],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpor    ymm7,ymm7,ymm8
+        vpsrld  ymm9,ymm10,31
+        vpand   ymm5,ymm5,ymm2
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpslld  ymm6,ymm2,30
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vmovdqa ymm15,YMMWORD[64+rbp]
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((480-256-128))+rbx]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(384-256-128)+rbx],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((160-128))+rax]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((0-128))+rax]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(416-256-128)+rbx],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((192-128))+rax]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((32-128))+rax]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(448-256-128)+rbx],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((224-128))+rax]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((64-128))+rax]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(480-256-128)+rbx],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((256-256-128))+rbx]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((96-128))+rax]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(0-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((288-256-128))+rbx]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((128-128))+rax]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(32-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((320-256-128))+rbx]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((160-128))+rax]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(64-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((352-256-128))+rbx]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((192-128))+rax]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vmovdqa YMMWORD[(96-128)+rax],ymm12
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((384-256-128))+rbx]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((224-128))+rax]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vmovdqa YMMWORD[(128-128)+rax],ymm13
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((416-256-128))+rbx]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((256-256-128))+rbx]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vmovdqa YMMWORD[(160-128)+rax],ymm14
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((448-256-128))+rbx]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((288-256-128))+rbx]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vmovdqa YMMWORD[(192-128)+rax],ymm10
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((480-256-128))+rbx]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((320-256-128))+rbx]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vmovdqa YMMWORD[(224-128)+rax],ymm11
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((0-128))+rax]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((352-256-128))+rbx]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((32-128))+rax]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((384-256-128))+rbx]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((64-128))+rax]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpxor   ymm10,ymm10,ymm12
+        vmovdqa ymm12,YMMWORD[((416-256-128))+rbx]
+
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm10,ymm10,YMMWORD[((96-128))+rax]
+        vpsrld  ymm8,ymm1,27
+        vpxor   ymm5,ymm5,ymm3
+        vpxor   ymm10,ymm10,ymm12
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+        vpsrld  ymm9,ymm10,31
+        vpaddd  ymm10,ymm10,ymm10
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm10,ymm10,ymm9
+        vpor    ymm2,ymm2,ymm6
+        vpxor   ymm11,ymm11,ymm13
+        vmovdqa ymm13,YMMWORD[((448-256-128))+rbx]
+
+        vpslld  ymm7,ymm0,5
+        vpaddd  ymm4,ymm4,ymm15
+        vpxor   ymm5,ymm3,ymm1
+        vpaddd  ymm4,ymm4,ymm10
+        vpxor   ymm11,ymm11,YMMWORD[((128-128))+rax]
+        vpsrld  ymm8,ymm0,27
+        vpxor   ymm5,ymm5,ymm2
+        vpxor   ymm11,ymm11,ymm13
+
+        vpslld  ymm6,ymm1,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm4,ymm4,ymm5
+        vpsrld  ymm9,ymm11,31
+        vpaddd  ymm11,ymm11,ymm11
+
+        vpsrld  ymm1,ymm1,2
+        vpaddd  ymm4,ymm4,ymm7
+        vpor    ymm11,ymm11,ymm9
+        vpor    ymm1,ymm1,ymm6
+        vpxor   ymm12,ymm12,ymm14
+        vmovdqa ymm14,YMMWORD[((480-256-128))+rbx]
+
+        vpslld  ymm7,ymm4,5
+        vpaddd  ymm3,ymm3,ymm15
+        vpxor   ymm5,ymm2,ymm0
+        vpaddd  ymm3,ymm3,ymm11
+        vpxor   ymm12,ymm12,YMMWORD[((160-128))+rax]
+        vpsrld  ymm8,ymm4,27
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm12,ymm12,ymm14
+
+        vpslld  ymm6,ymm0,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm3,ymm3,ymm5
+        vpsrld  ymm9,ymm12,31
+        vpaddd  ymm12,ymm12,ymm12
+
+        vpsrld  ymm0,ymm0,2
+        vpaddd  ymm3,ymm3,ymm7
+        vpor    ymm12,ymm12,ymm9
+        vpor    ymm0,ymm0,ymm6
+        vpxor   ymm13,ymm13,ymm10
+        vmovdqa ymm10,YMMWORD[((0-128))+rax]
+
+        vpslld  ymm7,ymm3,5
+        vpaddd  ymm2,ymm2,ymm15
+        vpxor   ymm5,ymm1,ymm4
+        vpaddd  ymm2,ymm2,ymm12
+        vpxor   ymm13,ymm13,YMMWORD[((192-128))+rax]
+        vpsrld  ymm8,ymm3,27
+        vpxor   ymm5,ymm5,ymm0
+        vpxor   ymm13,ymm13,ymm10
+
+        vpslld  ymm6,ymm4,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm2,ymm2,ymm5
+        vpsrld  ymm9,ymm13,31
+        vpaddd  ymm13,ymm13,ymm13
+
+        vpsrld  ymm4,ymm4,2
+        vpaddd  ymm2,ymm2,ymm7
+        vpor    ymm13,ymm13,ymm9
+        vpor    ymm4,ymm4,ymm6
+        vpxor   ymm14,ymm14,ymm11
+        vmovdqa ymm11,YMMWORD[((32-128))+rax]
+
+        vpslld  ymm7,ymm2,5
+        vpaddd  ymm1,ymm1,ymm15
+        vpxor   ymm5,ymm0,ymm3
+        vpaddd  ymm1,ymm1,ymm13
+        vpxor   ymm14,ymm14,YMMWORD[((224-128))+rax]
+        vpsrld  ymm8,ymm2,27
+        vpxor   ymm5,ymm5,ymm4
+        vpxor   ymm14,ymm14,ymm11
+
+        vpslld  ymm6,ymm3,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm1,ymm1,ymm5
+        vpsrld  ymm9,ymm14,31
+        vpaddd  ymm14,ymm14,ymm14
+
+        vpsrld  ymm3,ymm3,2
+        vpaddd  ymm1,ymm1,ymm7
+        vpor    ymm14,ymm14,ymm9
+        vpor    ymm3,ymm3,ymm6
+        vpslld  ymm7,ymm1,5
+        vpaddd  ymm0,ymm0,ymm15
+        vpxor   ymm5,ymm4,ymm2
+
+        vpsrld  ymm8,ymm1,27
+        vpaddd  ymm0,ymm0,ymm14
+        vpxor   ymm5,ymm5,ymm3
+
+        vpslld  ymm6,ymm2,30
+        vpor    ymm7,ymm7,ymm8
+        vpaddd  ymm0,ymm0,ymm5
+
+        vpsrld  ymm2,ymm2,2
+        vpaddd  ymm0,ymm0,ymm7
+        vpor    ymm2,ymm2,ymm6
+        mov     ecx,1
+        lea     rbx,[512+rsp]
+        cmp     ecx,DWORD[rbx]
+        cmovge  r12,rbp
+        cmp     ecx,DWORD[4+rbx]
+        cmovge  r13,rbp
+        cmp     ecx,DWORD[8+rbx]
+        cmovge  r14,rbp
+        cmp     ecx,DWORD[12+rbx]
+        cmovge  r15,rbp
+        cmp     ecx,DWORD[16+rbx]
+        cmovge  r8,rbp
+        cmp     ecx,DWORD[20+rbx]
+        cmovge  r9,rbp
+        cmp     ecx,DWORD[24+rbx]
+        cmovge  r10,rbp
+        cmp     ecx,DWORD[28+rbx]
+        cmovge  r11,rbp
+        vmovdqu ymm5,YMMWORD[rbx]
+        vpxor   ymm7,ymm7,ymm7
+        vmovdqa ymm6,ymm5
+        vpcmpgtd        ymm6,ymm6,ymm7
+        vpaddd  ymm5,ymm5,ymm6
+
+        vpand   ymm0,ymm0,ymm6
+        vpand   ymm1,ymm1,ymm6
+        vpaddd  ymm0,ymm0,YMMWORD[rdi]
+        vpand   ymm2,ymm2,ymm6
+        vpaddd  ymm1,ymm1,YMMWORD[32+rdi]
+        vpand   ymm3,ymm3,ymm6
+        vpaddd  ymm2,ymm2,YMMWORD[64+rdi]
+        vpand   ymm4,ymm4,ymm6
+        vpaddd  ymm3,ymm3,YMMWORD[96+rdi]
+        vpaddd  ymm4,ymm4,YMMWORD[128+rdi]
+        vmovdqu YMMWORD[rdi],ymm0
+        vmovdqu YMMWORD[32+rdi],ymm1
+        vmovdqu YMMWORD[64+rdi],ymm2
+        vmovdqu YMMWORD[96+rdi],ymm3
+        vmovdqu YMMWORD[128+rdi],ymm4
+
+        vmovdqu YMMWORD[rbx],ymm5
+        lea     rbx,[((256+128))+rsp]
+        vmovdqu ymm9,YMMWORD[96+rbp]
+        dec     edx
+        jnz     NEAR $L$oop_avx2
+
+
+
+
+
+
+
+$L$done_avx2:
+        mov     rax,QWORD[544+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha1_multi_block_avx2:
+
 ALIGN   256
         DD      0x5a827999,0x5a827999,0x5a827999,0x5a827999
@@ -3120 +7523 @@
         DB      0F3h,0C3h               ;repret
 
+
+ALIGN   16
+avx2_handler:
+        push    rsi
+        push    rdi
+        push    rbx
+        push    rbp
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        pushfq
+        sub     rsp,64
+
+        mov     rax,QWORD[120+r8]
+        mov     rbx,QWORD[248+r8]
+
+        mov     rsi,QWORD[8+r9]
+        mov     r11,QWORD[56+r9]
+
+        mov     r10d,DWORD[r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jb      NEAR $L$in_prologue
+
+        mov     rax,QWORD[152+r8]
+
+        mov     r10d,DWORD[4+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jae     NEAR $L$in_prologue
+
+        mov     rax,QWORD[544+r8]
+
+        mov     rbx,QWORD[((-8))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+        mov     r12,QWORD[((-24))+rax]
+        mov     r13,QWORD[((-32))+rax]
+        mov     r14,QWORD[((-40))+rax]
+        mov     r15,QWORD[((-48))+rax]
+        mov     QWORD[144+r8],rbx
+        mov     QWORD[160+r8],rbp
+        mov     QWORD[216+r8],r12
+        mov     QWORD[224+r8],r13
+        mov     QWORD[232+r8],r14
+        mov     QWORD[240+r8],r15
+
+        lea     rsi,[((-56-160))+rax]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+
+        jmp     NEAR $L$in_prologue
+
 section .pdata rdata align=4
 ALIGN   4
@@ -3128 +7585 @@
         DD      $L$SEH_end_sha1_multi_block_shaext wrt ..imagebase
         DD      $L$SEH_info_sha1_multi_block_shaext wrt ..imagebase
+        DD      $L$SEH_begin_sha1_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_end_sha1_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_info_sha1_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_begin_sha1_multi_block_avx2 wrt ..imagebase
+        DD      $L$SEH_end_sha1_multi_block_avx2 wrt ..imagebase
+        DD      $L$SEH_info_sha1_multi_block_avx2 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -3138 +7601 @@
         DD      se_handler wrt ..imagebase
         DD      $L$body_shaext wrt ..imagebase,$L$epilogue_shaext wrt ..imagebase
+$L$SEH_info_sha1_multi_block_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$body_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_sha1_multi_block_avx2:
+DB      9,0,0,0
+        DD      avx2_handler wrt ..imagebase
+        DD      $L$body_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/sha1-x86_64.S

@@ -28 +28 @@
         test    r10d,536870912
         jnz     NEAR _shaext_shortcut
+        and     r10d,296
+        cmp     r10d,296
+        je      NEAR _avx2_shortcut
+        and     r8d,268435456
+        and     r9d,1073741824
+        or      r8d,r9d
+        cmp     r8d,1342177280
+        je      NEAR _avx_shortcut
         jmp     NEAR _ssse3_shortcut
 
@@ -2668 +2676 @@
 
 $L$SEH_end_sha1_block_data_order_ssse3:
+
+ALIGN   16
+sha1_block_data_order_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha1_block_data_order_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+_avx_shortcut:
+
+        mov     r11,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        lea     rsp,[((-160))+rsp]
+        vzeroupper
+        vmovaps XMMWORD[(-40-96)+r11],xmm6
+        vmovaps XMMWORD[(-40-80)+r11],xmm7
+        vmovaps XMMWORD[(-40-64)+r11],xmm8
+        vmovaps XMMWORD[(-40-48)+r11],xmm9
+        vmovaps XMMWORD[(-40-32)+r11],xmm10
+        vmovaps XMMWORD[(-40-16)+r11],xmm11
+$L$prologue_avx:
+        and     rsp,-64
+        mov     r8,rdi
+        mov     r9,rsi
+        mov     r10,rdx
+
+        shl     r10,6
+        add     r10,r9
+        lea     r14,[((K_XX_XX+64))]
+
+        mov     eax,DWORD[r8]
+        mov     ebx,DWORD[4+r8]
+        mov     ecx,DWORD[8+r8]
+        mov     edx,DWORD[12+r8]
+        mov     esi,ebx
+        mov     ebp,DWORD[16+r8]
+        mov     edi,ecx
+        xor     edi,edx
+        and     esi,edi
+
+        vmovdqa xmm6,XMMWORD[64+r14]
+        vmovdqa xmm11,XMMWORD[((-64))+r14]
+        vmovdqu xmm0,XMMWORD[r9]
+        vmovdqu xmm1,XMMWORD[16+r9]
+        vmovdqu xmm2,XMMWORD[32+r9]
+        vmovdqu xmm3,XMMWORD[48+r9]
+        vpshufb xmm0,xmm0,xmm6
+        add     r9,64
+        vpshufb xmm1,xmm1,xmm6
+        vpshufb xmm2,xmm2,xmm6
+        vpshufb xmm3,xmm3,xmm6
+        vpaddd  xmm4,xmm0,xmm11
+        vpaddd  xmm5,xmm1,xmm11
+        vpaddd  xmm6,xmm2,xmm11
+        vmovdqa XMMWORD[rsp],xmm4
+        vmovdqa XMMWORD[16+rsp],xmm5
+        vmovdqa XMMWORD[32+rsp],xmm6
+        jmp     NEAR $L$oop_avx
+ALIGN   16
+$L$oop_avx:
+        shrd    ebx,ebx,2
+        xor     esi,edx
+        vpalignr        xmm4,xmm1,xmm0,8
+        mov     edi,eax
+        add     ebp,DWORD[rsp]
+        vpaddd  xmm9,xmm11,xmm3
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vpsrldq xmm8,xmm3,4
+        add     ebp,esi
+        and     edi,ebx
+        vpxor   xmm4,xmm4,xmm0
+        xor     ebx,ecx
+        add     ebp,eax
+        vpxor   xmm8,xmm8,xmm2
+        shrd    eax,eax,7
+        xor     edi,ecx
+        mov     esi,ebp
+        add     edx,DWORD[4+rsp]
+        vpxor   xmm4,xmm4,xmm8
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vmovdqa XMMWORD[48+rsp],xmm9
+        add     edx,edi
+        and     esi,eax
+        vpsrld  xmm8,xmm4,31
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     esi,ebx
+        vpslldq xmm10,xmm4,12
+        vpaddd  xmm4,xmm4,xmm4
+        mov     edi,edx
+        add     ecx,DWORD[8+rsp]
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpsrld  xmm9,xmm10,30
+        vpor    xmm4,xmm4,xmm8
+        add     ecx,esi
+        and     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        vpslld  xmm10,xmm10,2
+        vpxor   xmm4,xmm4,xmm9
+        shrd    edx,edx,7
+        xor     edi,eax
+        mov     esi,ecx
+        add     ebx,DWORD[12+rsp]
+        vpxor   xmm4,xmm4,xmm10
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        and     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        shrd    ecx,ecx,7
+        xor     esi,ebp
+        vpalignr        xmm5,xmm2,xmm1,8
+        mov     edi,ebx
+        add     eax,DWORD[16+rsp]
+        vpaddd  xmm9,xmm11,xmm4
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vpsrldq xmm8,xmm4,4
+        add     eax,esi
+        and     edi,ecx
+        vpxor   xmm5,xmm5,xmm1
+        xor     ecx,edx
+        add     eax,ebx
+        vpxor   xmm8,xmm8,xmm3
+        shrd    ebx,ebx,7
+        xor     edi,edx
+        mov     esi,eax
+        add     ebp,DWORD[20+rsp]
+        vpxor   xmm5,xmm5,xmm8
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vmovdqa XMMWORD[rsp],xmm9
+        add     ebp,edi
+        and     esi,ebx
+        vpsrld  xmm8,xmm5,31
+        xor     ebx,ecx
+        add     ebp,eax
+        shrd    eax,eax,7
+        xor     esi,ecx
+        vpslldq xmm10,xmm5,12
+        vpaddd  xmm5,xmm5,xmm5
+        mov     edi,ebp
+        add     edx,DWORD[24+rsp]
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vpsrld  xmm9,xmm10,30
+        vpor    xmm5,xmm5,xmm8
+        add     edx,esi
+        and     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        vpslld  xmm10,xmm10,2
+        vpxor   xmm5,xmm5,xmm9
+        shrd    ebp,ebp,7
+        xor     edi,ebx
+        mov     esi,edx
+        add     ecx,DWORD[28+rsp]
+        vpxor   xmm5,xmm5,xmm10
+        xor     ebp,eax
+        shld    edx,edx,5
+        vmovdqa xmm11,XMMWORD[((-32))+r14]
+        add     ecx,edi
+        and     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        shrd    edx,edx,7
+        xor     esi,eax
+        vpalignr        xmm6,xmm3,xmm2,8
+        mov     edi,ecx
+        add     ebx,DWORD[32+rsp]
+        vpaddd  xmm9,xmm11,xmm5
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        vpsrldq xmm8,xmm5,4
+        add     ebx,esi
+        and     edi,edx
+        vpxor   xmm6,xmm6,xmm2
+        xor     edx,ebp
+        add     ebx,ecx
+        vpxor   xmm8,xmm8,xmm4
+        shrd    ecx,ecx,7
+        xor     edi,ebp
+        mov     esi,ebx
+        add     eax,DWORD[36+rsp]
+        vpxor   xmm6,xmm6,xmm8
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vmovdqa XMMWORD[16+rsp],xmm9
+        add     eax,edi
+        and     esi,ecx
+        vpsrld  xmm8,xmm6,31
+        xor     ecx,edx
+        add     eax,ebx
+        shrd    ebx,ebx,7
+        xor     esi,edx
+        vpslldq xmm10,xmm6,12
+        vpaddd  xmm6,xmm6,xmm6
+        mov     edi,eax
+        add     ebp,DWORD[40+rsp]
+        xor     ebx,ecx
+        shld    eax,eax,5
+        vpsrld  xmm9,xmm10,30
+        vpor    xmm6,xmm6,xmm8
+        add     ebp,esi
+        and     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpslld  xmm10,xmm10,2
+        vpxor   xmm6,xmm6,xmm9
+        shrd    eax,eax,7
+        xor     edi,ecx
+        mov     esi,ebp
+        add     edx,DWORD[44+rsp]
+        vpxor   xmm6,xmm6,xmm10
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        and     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     esi,ebx
+        vpalignr        xmm7,xmm4,xmm3,8
+        mov     edi,edx
+        add     ecx,DWORD[48+rsp]
+        vpaddd  xmm9,xmm11,xmm6
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpsrldq xmm8,xmm6,4
+        add     ecx,esi
+        and     edi,ebp
+        vpxor   xmm7,xmm7,xmm3
+        xor     ebp,eax
+        add     ecx,edx
+        vpxor   xmm8,xmm8,xmm5
+        shrd    edx,edx,7
+        xor     edi,eax
+        mov     esi,ecx
+        add     ebx,DWORD[52+rsp]
+        vpxor   xmm7,xmm7,xmm8
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     ebx,edi
+        and     esi,edx
+        vpsrld  xmm8,xmm7,31
+        xor     edx,ebp
+        add     ebx,ecx
+        shrd    ecx,ecx,7
+        xor     esi,ebp
+        vpslldq xmm10,xmm7,12
+        vpaddd  xmm7,xmm7,xmm7
+        mov     edi,ebx
+        add     eax,DWORD[56+rsp]
+        xor     ecx,edx
+        shld    ebx,ebx,5
+        vpsrld  xmm9,xmm10,30
+        vpor    xmm7,xmm7,xmm8
+        add     eax,esi
+        and     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        vpslld  xmm10,xmm10,2
+        vpxor   xmm7,xmm7,xmm9
+        shrd    ebx,ebx,7
+        xor     edi,edx
+        mov     esi,eax
+        add     ebp,DWORD[60+rsp]
+        vpxor   xmm7,xmm7,xmm10
+        xor     ebx,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        and     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpalignr        xmm8,xmm7,xmm6,8
+        vpxor   xmm0,xmm0,xmm4
+        shrd    eax,eax,7
+        xor     esi,ecx
+        mov     edi,ebp
+        add     edx,DWORD[rsp]
+        vpxor   xmm0,xmm0,xmm1
+        xor     eax,ebx
+        shld    ebp,ebp,5
+        vpaddd  xmm9,xmm11,xmm7
+        add     edx,esi
+        and     edi,eax
+        vpxor   xmm0,xmm0,xmm8
+        xor     eax,ebx
+        add     edx,ebp
+        shrd    ebp,ebp,7
+        xor     edi,ebx
+        vpsrld  xmm8,xmm0,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        mov     esi,edx
+        add     ecx,DWORD[4+rsp]
+        xor     ebp,eax
+        shld    edx,edx,5
+        vpslld  xmm0,xmm0,2
+        add     ecx,edi
+        and     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        shrd    edx,edx,7
+        xor     esi,eax
+        mov     edi,ecx
+        add     ebx,DWORD[8+rsp]
+        vpor    xmm0,xmm0,xmm8
+        xor     edx,ebp
+        shld    ecx,ecx,5
+        add     ebx,esi
+        and     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[12+rsp]
+        xor     edi,ebp
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpalignr        xmm8,xmm0,xmm7,8
+        vpxor   xmm1,xmm1,xmm5
+        add     ebp,DWORD[16+rsp]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        vpxor   xmm1,xmm1,xmm2
+        add     ebp,esi
+        xor     edi,ecx
+        vpaddd  xmm9,xmm11,xmm0
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpxor   xmm1,xmm1,xmm8
+        add     edx,DWORD[20+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        vpsrld  xmm8,xmm1,30
+        vmovdqa XMMWORD[rsp],xmm9
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpslld  xmm1,xmm1,2
+        add     ecx,DWORD[24+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpor    xmm1,xmm1,xmm8
+        add     ebx,DWORD[28+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpalignr        xmm8,xmm1,xmm0,8
+        vpxor   xmm2,xmm2,xmm6
+        add     eax,DWORD[32+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        vpxor   xmm2,xmm2,xmm3
+        add     eax,esi
+        xor     edi,edx
+        vpaddd  xmm9,xmm11,xmm1
+        vmovdqa xmm11,XMMWORD[r14]
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpxor   xmm2,xmm2,xmm8
+        add     ebp,DWORD[36+rsp]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        vpsrld  xmm8,xmm2,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpslld  xmm2,xmm2,2
+        add     edx,DWORD[40+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpor    xmm2,xmm2,xmm8
+        add     ecx,DWORD[44+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpalignr        xmm8,xmm2,xmm1,8
+        vpxor   xmm3,xmm3,xmm7
+        add     ebx,DWORD[48+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        vpxor   xmm3,xmm3,xmm4
+        add     ebx,esi
+        xor     edi,ebp
+        vpaddd  xmm9,xmm11,xmm2
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpxor   xmm3,xmm3,xmm8
+        add     eax,DWORD[52+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        vpsrld  xmm8,xmm3,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpslld  xmm3,xmm3,2
+        add     ebp,DWORD[56+rsp]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpor    xmm3,xmm3,xmm8
+        add     edx,DWORD[60+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpalignr        xmm8,xmm3,xmm2,8
+        vpxor   xmm4,xmm4,xmm0
+        add     ecx,DWORD[rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        vpxor   xmm4,xmm4,xmm5
+        add     ecx,esi
+        xor     edi,eax
+        vpaddd  xmm9,xmm11,xmm3
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpxor   xmm4,xmm4,xmm8
+        add     ebx,DWORD[4+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        vpsrld  xmm8,xmm4,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpslld  xmm4,xmm4,2
+        add     eax,DWORD[8+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vpor    xmm4,xmm4,xmm8
+        add     ebp,DWORD[12+rsp]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpalignr        xmm8,xmm4,xmm3,8
+        vpxor   xmm5,xmm5,xmm1
+        add     edx,DWORD[16+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        vpxor   xmm5,xmm5,xmm6
+        add     edx,esi
+        xor     edi,ebx
+        vpaddd  xmm9,xmm11,xmm4
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpxor   xmm5,xmm5,xmm8
+        add     ecx,DWORD[20+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        vpsrld  xmm8,xmm5,30
+        vmovdqa XMMWORD[rsp],xmm9
+        add     ecx,edi
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpslld  xmm5,xmm5,2
+        add     ebx,DWORD[24+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vpor    xmm5,xmm5,xmm8
+        add     eax,DWORD[28+rsp]
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        vpalignr        xmm8,xmm5,xmm4,8
+        vpxor   xmm6,xmm6,xmm2
+        add     ebp,DWORD[32+rsp]
+        and     esi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        vpxor   xmm6,xmm6,xmm7
+        mov     edi,eax
+        xor     esi,ecx
+        vpaddd  xmm9,xmm11,xmm5
+        shld    eax,eax,5
+        add     ebp,esi
+        vpxor   xmm6,xmm6,xmm8
+        xor     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[36+rsp]
+        vpsrld  xmm8,xmm6,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        and     edi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     esi,ebp
+        vpslld  xmm6,xmm6,2
+        xor     edi,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[40+rsp]
+        and     esi,eax
+        vpor    xmm6,xmm6,xmm8
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     edi,edx
+        xor     esi,eax
+        shld    edx,edx,5
+        add     ecx,esi
+        xor     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[44+rsp]
+        and     edi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        mov     esi,ecx
+        xor     edi,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        vpalignr        xmm8,xmm6,xmm5,8
+        vpxor   xmm7,xmm7,xmm3
+        add     eax,DWORD[48+rsp]
+        and     esi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        vpxor   xmm7,xmm7,xmm0
+        mov     edi,ebx
+        xor     esi,edx
+        vpaddd  xmm9,xmm11,xmm6
+        vmovdqa xmm11,XMMWORD[32+r14]
+        shld    ebx,ebx,5
+        add     eax,esi
+        vpxor   xmm7,xmm7,xmm8
+        xor     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[52+rsp]
+        vpsrld  xmm8,xmm7,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        and     edi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     esi,eax
+        vpslld  xmm7,xmm7,2
+        xor     edi,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[56+rsp]
+        and     esi,ebx
+        vpor    xmm7,xmm7,xmm8
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     edi,ebp
+        xor     esi,ebx
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[60+rsp]
+        and     edi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     esi,edx
+        xor     edi,eax
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        vpalignr        xmm8,xmm7,xmm6,8
+        vpxor   xmm0,xmm0,xmm4
+        add     ebx,DWORD[rsp]
+        and     esi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        vpxor   xmm0,xmm0,xmm1
+        mov     edi,ecx
+        xor     esi,ebp
+        vpaddd  xmm9,xmm11,xmm7
+        shld    ecx,ecx,5
+        add     ebx,esi
+        vpxor   xmm0,xmm0,xmm8
+        xor     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[4+rsp]
+        vpsrld  xmm8,xmm0,30
+        vmovdqa XMMWORD[48+rsp],xmm9
+        and     edi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        vpslld  xmm0,xmm0,2
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[8+rsp]
+        and     esi,ecx
+        vpor    xmm0,xmm0,xmm8
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     edi,eax
+        xor     esi,ecx
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        add     edx,DWORD[12+rsp]
+        and     edi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        mov     esi,ebp
+        xor     edi,ebx
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        vpalignr        xmm8,xmm0,xmm7,8
+        vpxor   xmm1,xmm1,xmm5
+        add     ecx,DWORD[16+rsp]
+        and     esi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        vpxor   xmm1,xmm1,xmm2
+        mov     edi,edx
+        xor     esi,eax
+        vpaddd  xmm9,xmm11,xmm0
+        shld    edx,edx,5
+        add     ecx,esi
+        vpxor   xmm1,xmm1,xmm8
+        xor     edi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[20+rsp]
+        vpsrld  xmm8,xmm1,30
+        vmovdqa XMMWORD[rsp],xmm9
+        and     edi,ebp
+        xor     ebp,eax
+        shrd    edx,edx,7
+        mov     esi,ecx
+        vpslld  xmm1,xmm1,2
+        xor     edi,ebp
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[24+rsp]
+        and     esi,edx
+        vpor    xmm1,xmm1,xmm8
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     edi,ebx
+        xor     esi,edx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,ecx
+        xor     ecx,edx
+        add     eax,ebx
+        add     ebp,DWORD[28+rsp]
+        and     edi,ecx
+        xor     ecx,edx
+        shrd    ebx,ebx,7
+        mov     esi,eax
+        xor     edi,ecx
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ebx
+        xor     ebx,ecx
+        add     ebp,eax
+        vpalignr        xmm8,xmm1,xmm0,8
+        vpxor   xmm2,xmm2,xmm6
+        add     edx,DWORD[32+rsp]
+        and     esi,ebx
+        xor     ebx,ecx
+        shrd    eax,eax,7
+        vpxor   xmm2,xmm2,xmm3
+        mov     edi,ebp
+        xor     esi,ebx
+        vpaddd  xmm9,xmm11,xmm1
+        shld    ebp,ebp,5
+        add     edx,esi
+        vpxor   xmm2,xmm2,xmm8
+        xor     edi,eax
+        xor     eax,ebx
+        add     edx,ebp
+        add     ecx,DWORD[36+rsp]
+        vpsrld  xmm8,xmm2,30
+        vmovdqa XMMWORD[16+rsp],xmm9
+        and     edi,eax
+        xor     eax,ebx
+        shrd    ebp,ebp,7
+        mov     esi,edx
+        vpslld  xmm2,xmm2,2
+        xor     edi,eax
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,ebp
+        xor     ebp,eax
+        add     ecx,edx
+        add     ebx,DWORD[40+rsp]
+        and     esi,ebp
+        vpor    xmm2,xmm2,xmm8
+        xor     ebp,eax
+        shrd    edx,edx,7
+        mov     edi,ecx
+        xor     esi,ebp
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,edx
+        xor     edx,ebp
+        add     ebx,ecx
+        add     eax,DWORD[44+rsp]
+        and     edi,edx
+        xor     edx,ebp
+        shrd    ecx,ecx,7
+        mov     esi,ebx
+        xor     edi,edx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        add     eax,ebx
+        vpalignr        xmm8,xmm2,xmm1,8
+        vpxor   xmm3,xmm3,xmm7
+        add     ebp,DWORD[48+rsp]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        vpxor   xmm3,xmm3,xmm4
+        add     ebp,esi
+        xor     edi,ecx
+        vpaddd  xmm9,xmm11,xmm2
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        vpxor   xmm3,xmm3,xmm8
+        add     edx,DWORD[52+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        vpsrld  xmm8,xmm3,30
+        vmovdqa XMMWORD[32+rsp],xmm9
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vpslld  xmm3,xmm3,2
+        add     ecx,DWORD[56+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vpor    xmm3,xmm3,xmm8
+        add     ebx,DWORD[60+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[rsp]
+        vpaddd  xmm9,xmm11,xmm3
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        vmovdqa XMMWORD[48+rsp],xmm9
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[4+rsp]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[8+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[12+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        cmp     r9,r10
+        je      NEAR $L$done_avx
+        vmovdqa xmm6,XMMWORD[64+r14]
+        vmovdqa xmm11,XMMWORD[((-64))+r14]
+        vmovdqu xmm0,XMMWORD[r9]
+        vmovdqu xmm1,XMMWORD[16+r9]
+        vmovdqu xmm2,XMMWORD[32+r9]
+        vmovdqu xmm3,XMMWORD[48+r9]
+        vpshufb xmm0,xmm0,xmm6
+        add     r9,64
+        add     ebx,DWORD[16+rsp]
+        xor     esi,ebp
+        vpshufb xmm1,xmm1,xmm6
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        vpaddd  xmm4,xmm0,xmm11
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        vmovdqa XMMWORD[rsp],xmm4
+        add     eax,DWORD[20+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[24+rsp]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[28+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[32+rsp]
+        xor     esi,eax
+        vpshufb xmm2,xmm2,xmm6
+        mov     edi,edx
+        shld    edx,edx,5
+        vpaddd  xmm5,xmm1,xmm11
+        add     ecx,esi
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        vmovdqa XMMWORD[16+rsp],xmm5
+        add     ebx,DWORD[36+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[40+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[44+rsp]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[48+rsp]
+        xor     esi,ebx
+        vpshufb xmm3,xmm3,xmm6
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        vpaddd  xmm6,xmm2,xmm11
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        vmovdqa XMMWORD[32+rsp],xmm6
+        add     ecx,DWORD[52+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[56+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[60+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     eax,DWORD[r8]
+        add     esi,DWORD[4+r8]
+        add     ecx,DWORD[8+r8]
+        add     edx,DWORD[12+r8]
+        mov     DWORD[r8],eax
+        add     ebp,DWORD[16+r8]
+        mov     DWORD[4+r8],esi
+        mov     ebx,esi
+        mov     DWORD[8+r8],ecx
+        mov     edi,ecx
+        mov     DWORD[12+r8],edx
+        xor     edi,edx
+        mov     DWORD[16+r8],ebp
+        and     esi,edi
+        jmp     NEAR $L$oop_avx
+
+ALIGN   16
+$L$done_avx:
+        add     ebx,DWORD[16+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[20+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        xor     esi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[24+rsp]
+        xor     esi,ecx
+        mov     edi,eax
+        shld    eax,eax,5
+        add     ebp,esi
+        xor     edi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[28+rsp]
+        xor     edi,ebx
+        mov     esi,ebp
+        shld    ebp,ebp,5
+        add     edx,edi
+        xor     esi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[32+rsp]
+        xor     esi,eax
+        mov     edi,edx
+        shld    edx,edx,5
+        add     ecx,esi
+        xor     edi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[36+rsp]
+        xor     edi,ebp
+        mov     esi,ecx
+        shld    ecx,ecx,5
+        add     ebx,edi
+        xor     esi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[40+rsp]
+        xor     esi,edx
+        mov     edi,ebx
+        shld    ebx,ebx,5
+        add     eax,esi
+        xor     edi,edx
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        add     ebp,DWORD[44+rsp]
+        xor     edi,ecx
+        mov     esi,eax
+        shld    eax,eax,5
+        add     ebp,edi
+        xor     esi,ecx
+        shrd    ebx,ebx,7
+        add     ebp,eax
+        add     edx,DWORD[48+rsp]
+        xor     esi,ebx
+        mov     edi,ebp
+        shld    ebp,ebp,5
+        add     edx,esi
+        xor     edi,ebx
+        shrd    eax,eax,7
+        add     edx,ebp
+        add     ecx,DWORD[52+rsp]
+        xor     edi,eax
+        mov     esi,edx
+        shld    edx,edx,5
+        add     ecx,edi
+        xor     esi,eax
+        shrd    ebp,ebp,7
+        add     ecx,edx
+        add     ebx,DWORD[56+rsp]
+        xor     esi,ebp
+        mov     edi,ecx
+        shld    ecx,ecx,5
+        add     ebx,esi
+        xor     edi,ebp
+        shrd    edx,edx,7
+        add     ebx,ecx
+        add     eax,DWORD[60+rsp]
+        xor     edi,edx
+        mov     esi,ebx
+        shld    ebx,ebx,5
+        add     eax,edi
+        shrd    ecx,ecx,7
+        add     eax,ebx
+        vzeroupper
+
+        add     eax,DWORD[r8]
+        add     esi,DWORD[4+r8]
+        add     ecx,DWORD[8+r8]
+        mov     DWORD[r8],eax
+        add     edx,DWORD[12+r8]
+        mov     DWORD[4+r8],esi
+        add     ebp,DWORD[16+r8]
+        mov     DWORD[8+r8],ecx
+        mov     DWORD[12+r8],edx
+        mov     DWORD[16+r8],ebp
+        movaps  xmm6,XMMWORD[((-40-96))+r11]
+        movaps  xmm7,XMMWORD[((-40-80))+r11]
+        movaps  xmm8,XMMWORD[((-40-64))+r11]
+        movaps  xmm9,XMMWORD[((-40-48))+r11]
+        movaps  xmm10,XMMWORD[((-40-32))+r11]
+        movaps  xmm11,XMMWORD[((-40-16))+r11]
+        mov     r14,QWORD[((-40))+r11]
+
+        mov     r13,QWORD[((-32))+r11]
+
+        mov     r12,QWORD[((-24))+r11]
+
+        mov     rbp,QWORD[((-16))+r11]
+
+        mov     rbx,QWORD[((-8))+r11]
+
+        lea     rsp,[r11]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha1_block_data_order_avx:
+
+ALIGN   16
+sha1_block_data_order_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha1_block_data_order_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+_avx2_shortcut:
+
+        mov     r11,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        vzeroupper
+        lea     rsp,[((-96))+rsp]
+        vmovaps XMMWORD[(-40-96)+r11],xmm6
+        vmovaps XMMWORD[(-40-80)+r11],xmm7
+        vmovaps XMMWORD[(-40-64)+r11],xmm8
+        vmovaps XMMWORD[(-40-48)+r11],xmm9
+        vmovaps XMMWORD[(-40-32)+r11],xmm10
+        vmovaps XMMWORD[(-40-16)+r11],xmm11
+$L$prologue_avx2:
+        mov     r8,rdi
+        mov     r9,rsi
+        mov     r10,rdx
+
+        lea     rsp,[((-640))+rsp]
+        shl     r10,6
+        lea     r13,[64+r9]
+        and     rsp,-128
+        add     r10,r9
+        lea     r14,[((K_XX_XX+64))]
+
+        mov     eax,DWORD[r8]
+        cmp     r13,r10
+        cmovae  r13,r9
+        mov     ebp,DWORD[4+r8]
+        mov     ecx,DWORD[8+r8]
+        mov     edx,DWORD[12+r8]
+        mov     esi,DWORD[16+r8]
+        vmovdqu ymm6,YMMWORD[64+r14]
+
+        vmovdqu xmm0,XMMWORD[r9]
+        vmovdqu xmm1,XMMWORD[16+r9]
+        vmovdqu xmm2,XMMWORD[32+r9]
+        vmovdqu xmm3,XMMWORD[48+r9]
+        lea     r9,[64+r9]
+        vinserti128     ymm0,ymm0,XMMWORD[r13],1
+        vinserti128     ymm1,ymm1,XMMWORD[16+r13],1
+        vpshufb ymm0,ymm0,ymm6
+        vinserti128     ymm2,ymm2,XMMWORD[32+r13],1
+        vpshufb ymm1,ymm1,ymm6
+        vinserti128     ymm3,ymm3,XMMWORD[48+r13],1
+        vpshufb ymm2,ymm2,ymm6
+        vmovdqu ymm11,YMMWORD[((-64))+r14]
+        vpshufb ymm3,ymm3,ymm6
+
+        vpaddd  ymm4,ymm0,ymm11
+        vpaddd  ymm5,ymm1,ymm11
+        vmovdqu YMMWORD[rsp],ymm4
+        vpaddd  ymm6,ymm2,ymm11
+        vmovdqu YMMWORD[32+rsp],ymm5
+        vpaddd  ymm7,ymm3,ymm11
+        vmovdqu YMMWORD[64+rsp],ymm6
+        vmovdqu YMMWORD[96+rsp],ymm7
+        vpalignr        ymm4,ymm1,ymm0,8
+        vpsrldq ymm8,ymm3,4
+        vpxor   ymm4,ymm4,ymm0
+        vpxor   ymm8,ymm8,ymm2
+        vpxor   ymm4,ymm4,ymm8
+        vpsrld  ymm8,ymm4,31
+        vpslldq ymm10,ymm4,12
+        vpaddd  ymm4,ymm4,ymm4
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm4,ymm4,ymm8
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm4,ymm4,ymm9
+        vpxor   ymm4,ymm4,ymm10
+        vpaddd  ymm9,ymm4,ymm11
+        vmovdqu YMMWORD[128+rsp],ymm9
+        vpalignr        ymm5,ymm2,ymm1,8
+        vpsrldq ymm8,ymm4,4
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm8,ymm8,ymm3
+        vpxor   ymm5,ymm5,ymm8
+        vpsrld  ymm8,ymm5,31
+        vmovdqu ymm11,YMMWORD[((-32))+r14]
+        vpslldq ymm10,ymm5,12
+        vpaddd  ymm5,ymm5,ymm5
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm5,ymm5,ymm8
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm5,ymm5,ymm9
+        vpxor   ymm5,ymm5,ymm10
+        vpaddd  ymm9,ymm5,ymm11
+        vmovdqu YMMWORD[160+rsp],ymm9
+        vpalignr        ymm6,ymm3,ymm2,8
+        vpsrldq ymm8,ymm5,4
+        vpxor   ymm6,ymm6,ymm2
+        vpxor   ymm8,ymm8,ymm4
+        vpxor   ymm6,ymm6,ymm8
+        vpsrld  ymm8,ymm6,31
+        vpslldq ymm10,ymm6,12
+        vpaddd  ymm6,ymm6,ymm6
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm6,ymm6,ymm8
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm6,ymm6,ymm9
+        vpxor   ymm6,ymm6,ymm10
+        vpaddd  ymm9,ymm6,ymm11
+        vmovdqu YMMWORD[192+rsp],ymm9
+        vpalignr        ymm7,ymm4,ymm3,8
+        vpsrldq ymm8,ymm6,4
+        vpxor   ymm7,ymm7,ymm3
+        vpxor   ymm8,ymm8,ymm5
+        vpxor   ymm7,ymm7,ymm8
+        vpsrld  ymm8,ymm7,31
+        vpslldq ymm10,ymm7,12
+        vpaddd  ymm7,ymm7,ymm7
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm7,ymm7,ymm8
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm7,ymm7,ymm9
+        vpxor   ymm7,ymm7,ymm10
+        vpaddd  ymm9,ymm7,ymm11
+        vmovdqu YMMWORD[224+rsp],ymm9
+        lea     r13,[128+rsp]
+        jmp     NEAR $L$oop_avx2
+ALIGN   32
+$L$oop_avx2:
+        rorx    ebx,ebp,2
+        andn    edi,ebp,edx
+        and     ebp,ecx
+        xor     ebp,edi
+        jmp     NEAR $L$align32_1
+ALIGN   32
+$L$align32_1:
+        vpalignr        ymm8,ymm7,ymm6,8
+        vpxor   ymm0,ymm0,ymm4
+        add     esi,DWORD[((-128))+r13]
+        andn    edi,eax,ecx
+        vpxor   ymm0,ymm0,ymm1
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        vpxor   ymm0,ymm0,ymm8
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        vpsrld  ymm8,ymm0,30
+        vpslld  ymm0,ymm0,2
+        add     edx,DWORD[((-124))+r13]
+        andn    edi,esi,ebx
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        vpor    ymm0,ymm0,ymm8
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-120))+r13]
+        andn    edi,edx,ebp
+        vpaddd  ymm9,ymm0,ymm11
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        vmovdqu YMMWORD[256+rsp],ymm9
+        add     ecx,r12d
+        xor     edx,edi
+        add     ebx,DWORD[((-116))+r13]
+        andn    edi,ecx,eax
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        add     ebp,DWORD[((-96))+r13]
+        andn    edi,ebx,esi
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        and     ebx,edx
+        add     ebp,r12d
+        xor     ebx,edi
+        vpalignr        ymm8,ymm0,ymm7,8
+        vpxor   ymm1,ymm1,ymm5
+        add     eax,DWORD[((-92))+r13]
+        andn    edi,ebp,edx
+        vpxor   ymm1,ymm1,ymm2
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        vpxor   ymm1,ymm1,ymm8
+        and     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edi
+        vpsrld  ymm8,ymm1,30
+        vpslld  ymm1,ymm1,2
+        add     esi,DWORD[((-88))+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        vpor    ymm1,ymm1,ymm8
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[((-84))+r13]
+        andn    edi,esi,ebx
+        vpaddd  ymm9,ymm1,ymm11
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        vmovdqu YMMWORD[288+rsp],ymm9
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-64))+r13]
+        andn    edi,edx,ebp
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        add     ecx,r12d
+        xor     edx,edi
+        add     ebx,DWORD[((-60))+r13]
+        andn    edi,ecx,eax
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        vpalignr        ymm8,ymm1,ymm0,8
+        vpxor   ymm2,ymm2,ymm6
+        add     ebp,DWORD[((-56))+r13]
+        andn    edi,ebx,esi
+        vpxor   ymm2,ymm2,ymm3
+        vmovdqu ymm11,YMMWORD[r14]
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        vpxor   ymm2,ymm2,ymm8
+        and     ebx,edx
+        add     ebp,r12d
+        xor     ebx,edi
+        vpsrld  ymm8,ymm2,30
+        vpslld  ymm2,ymm2,2
+        add     eax,DWORD[((-52))+r13]
+        andn    edi,ebp,edx
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        and     ebp,ecx
+        vpor    ymm2,ymm2,ymm8
+        add     eax,r12d
+        xor     ebp,edi
+        add     esi,DWORD[((-32))+r13]
+        andn    edi,eax,ecx
+        vpaddd  ymm9,ymm2,ymm11
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        vmovdqu YMMWORD[320+rsp],ymm9
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[((-28))+r13]
+        andn    edi,esi,ebx
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-24))+r13]
+        andn    edi,edx,ebp
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        add     ecx,r12d
+        xor     edx,edi
+        vpalignr        ymm8,ymm2,ymm1,8
+        vpxor   ymm3,ymm3,ymm7
+        add     ebx,DWORD[((-20))+r13]
+        andn    edi,ecx,eax
+        vpxor   ymm3,ymm3,ymm4
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        vpxor   ymm3,ymm3,ymm8
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        vpsrld  ymm8,ymm3,30
+        vpslld  ymm3,ymm3,2
+        add     ebp,DWORD[r13]
+        andn    edi,ebx,esi
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        and     ebx,edx
+        vpor    ymm3,ymm3,ymm8
+        add     ebp,r12d
+        xor     ebx,edi
+        add     eax,DWORD[4+r13]
+        andn    edi,ebp,edx
+        vpaddd  ymm9,ymm3,ymm11
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        and     ebp,ecx
+        vmovdqu YMMWORD[352+rsp],ymm9
+        add     eax,r12d
+        xor     ebp,edi
+        add     esi,DWORD[8+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[12+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        vpalignr        ymm8,ymm3,ymm2,8
+        vpxor   ymm4,ymm4,ymm0
+        add     ecx,DWORD[32+r13]
+        lea     ecx,[rsi*1+rcx]
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        vpxor   ymm4,ymm4,ymm8
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[36+r13]
+        vpsrld  ymm8,ymm4,30
+        vpslld  ymm4,ymm4,2
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        vpor    ymm4,ymm4,ymm8
+        add     ebp,DWORD[40+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        vpaddd  ymm9,ymm4,ymm11
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[44+r13]
+        vmovdqu YMMWORD[384+rsp],ymm9
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[64+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        vpalignr        ymm8,ymm4,ymm3,8
+        vpxor   ymm5,ymm5,ymm1
+        add     edx,DWORD[68+r13]
+        lea     edx,[rax*1+rdx]
+        vpxor   ymm5,ymm5,ymm6
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        vpxor   ymm5,ymm5,ymm8
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[72+r13]
+        vpsrld  ymm8,ymm5,30
+        vpslld  ymm5,ymm5,2
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        vpor    ymm5,ymm5,ymm8
+        add     ebx,DWORD[76+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        vpaddd  ymm9,ymm5,ymm11
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[96+r13]
+        vmovdqu YMMWORD[416+rsp],ymm9
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[100+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        vpalignr        ymm8,ymm5,ymm4,8
+        vpxor   ymm6,ymm6,ymm2
+        add     esi,DWORD[104+r13]
+        lea     esi,[rbp*1+rsi]
+        vpxor   ymm6,ymm6,ymm7
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        vpxor   ymm6,ymm6,ymm8
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[108+r13]
+        lea     r13,[256+r13]
+        vpsrld  ymm8,ymm6,30
+        vpslld  ymm6,ymm6,2
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        vpor    ymm6,ymm6,ymm8
+        add     ecx,DWORD[((-128))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        vpaddd  ymm9,ymm6,ymm11
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-124))+r13]
+        vmovdqu YMMWORD[448+rsp],ymm9
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-120))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        vpalignr        ymm8,ymm6,ymm5,8
+        vpxor   ymm7,ymm7,ymm3
+        add     eax,DWORD[((-116))+r13]
+        lea     eax,[rbx*1+rax]
+        vpxor   ymm7,ymm7,ymm0
+        vmovdqu ymm11,YMMWORD[32+r14]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        vpxor   ymm7,ymm7,ymm8
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-96))+r13]
+        vpsrld  ymm8,ymm7,30
+        vpslld  ymm7,ymm7,2
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        vpor    ymm7,ymm7,ymm8
+        add     edx,DWORD[((-92))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        vpaddd  ymm9,ymm7,ymm11
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[((-88))+r13]
+        vmovdqu YMMWORD[480+rsp],ymm9
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-84))+r13]
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        and     ecx,edi
+        jmp     NEAR $L$align32_2
+ALIGN   32
+$L$align32_2:
+        vpalignr        ymm8,ymm7,ymm6,8
+        vpxor   ymm0,ymm0,ymm4
+        add     ebp,DWORD[((-64))+r13]
+        xor     ecx,esi
+        vpxor   ymm0,ymm0,ymm1
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        vpxor   ymm0,ymm0,ymm8
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        vpsrld  ymm8,ymm0,30
+        vpslld  ymm0,ymm0,2
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[((-60))+r13]
+        xor     ebx,edx
+        mov     edi,ecx
+        xor     edi,edx
+        vpor    ymm0,ymm0,ymm8
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        vpaddd  ymm9,ymm0,ymm11
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[((-56))+r13]
+        xor     ebp,ecx
+        vmovdqu YMMWORD[512+rsp],ymm9
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        add     edx,DWORD[((-52))+r13]
+        xor     eax,ebx
+        mov     edi,ebp
+        xor     edi,ebx
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        and     esi,edi
+        add     ecx,DWORD[((-32))+r13]
+        xor     esi,ebp
+        mov     edi,eax
+        xor     edi,ebp
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        and     edx,edi
+        vpalignr        ymm8,ymm0,ymm7,8
+        vpxor   ymm1,ymm1,ymm5
+        add     ebx,DWORD[((-28))+r13]
+        xor     edx,eax
+        vpxor   ymm1,ymm1,ymm2
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        vpxor   ymm1,ymm1,ymm8
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        vpsrld  ymm8,ymm1,30
+        vpslld  ymm1,ymm1,2
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[((-24))+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        vpor    ymm1,ymm1,ymm8
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        vpaddd  ymm9,ymm1,ymm11
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[((-20))+r13]
+        xor     ebx,edx
+        vmovdqu YMMWORD[544+rsp],ymm9
+        mov     edi,ecx
+        xor     edi,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[r13]
+        xor     ebp,ecx
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        add     edx,DWORD[4+r13]
+        xor     eax,ebx
+        mov     edi,ebp
+        xor     edi,ebx
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        and     esi,edi
+        vpalignr        ymm8,ymm1,ymm0,8
+        vpxor   ymm2,ymm2,ymm6
+        add     ecx,DWORD[8+r13]
+        xor     esi,ebp
+        vpxor   ymm2,ymm2,ymm3
+        mov     edi,eax
+        xor     edi,ebp
+        lea     ecx,[rsi*1+rcx]
+        vpxor   ymm2,ymm2,ymm8
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        vpsrld  ymm8,ymm2,30
+        vpslld  ymm2,ymm2,2
+        add     ecx,r12d
+        and     edx,edi
+        add     ebx,DWORD[12+r13]
+        xor     edx,eax
+        mov     edi,esi
+        xor     edi,eax
+        vpor    ymm2,ymm2,ymm8
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        vpaddd  ymm9,ymm2,ymm11
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[32+r13]
+        xor     ecx,esi
+        vmovdqu YMMWORD[576+rsp],ymm9
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[36+r13]
+        xor     ebx,edx
+        mov     edi,ecx
+        xor     edi,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[40+r13]
+        xor     ebp,ecx
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        vpalignr        ymm8,ymm2,ymm1,8
+        vpxor   ymm3,ymm3,ymm7
+        add     edx,DWORD[44+r13]
+        xor     eax,ebx
+        vpxor   ymm3,ymm3,ymm4
+        mov     edi,ebp
+        xor     edi,ebx
+        lea     edx,[rax*1+rdx]
+        vpxor   ymm3,ymm3,ymm8
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        vpsrld  ymm8,ymm3,30
+        vpslld  ymm3,ymm3,2
+        add     edx,r12d
+        and     esi,edi
+        add     ecx,DWORD[64+r13]
+        xor     esi,ebp
+        mov     edi,eax
+        xor     edi,ebp
+        vpor    ymm3,ymm3,ymm8
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        vpaddd  ymm9,ymm3,ymm11
+        add     ecx,r12d
+        and     edx,edi
+        add     ebx,DWORD[68+r13]
+        xor     edx,eax
+        vmovdqu YMMWORD[608+rsp],ymm9
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[72+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[76+r13]
+        xor     ebx,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[96+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[100+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[104+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[108+r13]
+        lea     r13,[256+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-128))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[((-124))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-120))+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[((-116))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[((-96))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-92))+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-88))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[((-84))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-64))+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[((-60))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[((-56))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-52))+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-32))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[((-28))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-24))+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[((-20))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        add     edx,r12d
+        lea     r13,[128+r9]
+        lea     rdi,[128+r9]
+        cmp     r13,r10
+        cmovae  r13,r9
+
+
+        add     edx,DWORD[r8]
+        add     esi,DWORD[4+r8]
+        add     ebp,DWORD[8+r8]
+        mov     DWORD[r8],edx
+        add     ebx,DWORD[12+r8]
+        mov     DWORD[4+r8],esi
+        mov     eax,edx
+        add     ecx,DWORD[16+r8]
+        mov     r12d,ebp
+        mov     DWORD[8+r8],ebp
+        mov     edx,ebx
+
+        mov     DWORD[12+r8],ebx
+        mov     ebp,esi
+        mov     DWORD[16+r8],ecx
+
+        mov     esi,ecx
+        mov     ecx,r12d
+
+
+        cmp     r9,r10
+        je      NEAR $L$done_avx2
+        vmovdqu ymm6,YMMWORD[64+r14]
+        cmp     rdi,r10
+        ja      NEAR $L$ast_avx2
+
+        vmovdqu xmm0,XMMWORD[((-64))+rdi]
+        vmovdqu xmm1,XMMWORD[((-48))+rdi]
+        vmovdqu xmm2,XMMWORD[((-32))+rdi]
+        vmovdqu xmm3,XMMWORD[((-16))+rdi]
+        vinserti128     ymm0,ymm0,XMMWORD[r13],1
+        vinserti128     ymm1,ymm1,XMMWORD[16+r13],1
+        vinserti128     ymm2,ymm2,XMMWORD[32+r13],1
+        vinserti128     ymm3,ymm3,XMMWORD[48+r13],1
+        jmp     NEAR $L$ast_avx2
+
+ALIGN   32
+$L$ast_avx2:
+        lea     r13,[((128+16))+rsp]
+        rorx    ebx,ebp,2
+        andn    edi,ebp,edx
+        and     ebp,ecx
+        xor     ebp,edi
+        sub     r9,-128
+        add     esi,DWORD[((-128))+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[((-124))+r13]
+        andn    edi,esi,ebx
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-120))+r13]
+        andn    edi,edx,ebp
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        add     ecx,r12d
+        xor     edx,edi
+        add     ebx,DWORD[((-116))+r13]
+        andn    edi,ecx,eax
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        add     ebp,DWORD[((-96))+r13]
+        andn    edi,ebx,esi
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        and     ebx,edx
+        add     ebp,r12d
+        xor     ebx,edi
+        add     eax,DWORD[((-92))+r13]
+        andn    edi,ebp,edx
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        and     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edi
+        add     esi,DWORD[((-88))+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[((-84))+r13]
+        andn    edi,esi,ebx
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-64))+r13]
+        andn    edi,edx,ebp
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        add     ecx,r12d
+        xor     edx,edi
+        add     ebx,DWORD[((-60))+r13]
+        andn    edi,ecx,eax
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        add     ebp,DWORD[((-56))+r13]
+        andn    edi,ebx,esi
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        and     ebx,edx
+        add     ebp,r12d
+        xor     ebx,edi
+        add     eax,DWORD[((-52))+r13]
+        andn    edi,ebp,edx
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        and     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edi
+        add     esi,DWORD[((-32))+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[((-28))+r13]
+        andn    edi,esi,ebx
+        add     edx,eax
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        and     esi,ebp
+        add     edx,r12d
+        xor     esi,edi
+        add     ecx,DWORD[((-24))+r13]
+        andn    edi,edx,ebp
+        add     ecx,esi
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        and     edx,eax
+        add     ecx,r12d
+        xor     edx,edi
+        add     ebx,DWORD[((-20))+r13]
+        andn    edi,ecx,eax
+        add     ebx,edx
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        and     ecx,esi
+        add     ebx,r12d
+        xor     ecx,edi
+        add     ebp,DWORD[r13]
+        andn    edi,ebx,esi
+        add     ebp,ecx
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        and     ebx,edx
+        add     ebp,r12d
+        xor     ebx,edi
+        add     eax,DWORD[4+r13]
+        andn    edi,ebp,edx
+        add     eax,ebx
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        and     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edi
+        add     esi,DWORD[8+r13]
+        andn    edi,eax,ecx
+        add     esi,ebp
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        and     eax,ebx
+        add     esi,r12d
+        xor     eax,edi
+        add     edx,DWORD[12+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[32+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[36+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[40+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[44+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[64+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        vmovdqu ymm11,YMMWORD[((-64))+r14]
+        vpshufb ymm0,ymm0,ymm6
+        add     edx,DWORD[68+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[72+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[76+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[96+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[100+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        vpshufb ymm1,ymm1,ymm6
+        vpaddd  ymm8,ymm0,ymm11
+        add     esi,DWORD[104+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[108+r13]
+        lea     r13,[256+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[((-128))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-124))+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-120))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        vmovdqu YMMWORD[rsp],ymm8
+        vpshufb ymm2,ymm2,ymm6
+        vpaddd  ymm9,ymm1,ymm11
+        add     eax,DWORD[((-116))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-96))+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[((-92))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        add     ecx,DWORD[((-88))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-84))+r13]
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        and     ecx,edi
+        vmovdqu YMMWORD[32+rsp],ymm9
+        vpshufb ymm3,ymm3,ymm6
+        vpaddd  ymm6,ymm2,ymm11
+        add     ebp,DWORD[((-64))+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[((-60))+r13]
+        xor     ebx,edx
+        mov     edi,ecx
+        xor     edi,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[((-56))+r13]
+        xor     ebp,ecx
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        add     edx,DWORD[((-52))+r13]
+        xor     eax,ebx
+        mov     edi,ebp
+        xor     edi,ebx
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        and     esi,edi
+        add     ecx,DWORD[((-32))+r13]
+        xor     esi,ebp
+        mov     edi,eax
+        xor     edi,ebp
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        and     edx,edi
+        jmp     NEAR $L$align32_3
+ALIGN   32
+$L$align32_3:
+        vmovdqu YMMWORD[64+rsp],ymm6
+        vpaddd  ymm7,ymm3,ymm11
+        add     ebx,DWORD[((-28))+r13]
+        xor     edx,eax
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[((-24))+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[((-20))+r13]
+        xor     ebx,edx
+        mov     edi,ecx
+        xor     edi,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[r13]
+        xor     ebp,ecx
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        add     edx,DWORD[4+r13]
+        xor     eax,ebx
+        mov     edi,ebp
+        xor     edi,ebx
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        and     esi,edi
+        vmovdqu YMMWORD[96+rsp],ymm7
+        add     ecx,DWORD[8+r13]
+        xor     esi,ebp
+        mov     edi,eax
+        xor     edi,ebp
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        and     edx,edi
+        add     ebx,DWORD[12+r13]
+        xor     edx,eax
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[32+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[36+r13]
+        xor     ebx,edx
+        mov     edi,ecx
+        xor     edi,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        and     ebp,edi
+        add     esi,DWORD[40+r13]
+        xor     ebp,ecx
+        mov     edi,ebx
+        xor     edi,ecx
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        and     eax,edi
+        vpalignr        ymm4,ymm1,ymm0,8
+        add     edx,DWORD[44+r13]
+        xor     eax,ebx
+        mov     edi,ebp
+        xor     edi,ebx
+        vpsrldq ymm8,ymm3,4
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        vpxor   ymm4,ymm4,ymm0
+        vpxor   ymm8,ymm8,ymm2
+        xor     esi,ebp
+        add     edx,r12d
+        vpxor   ymm4,ymm4,ymm8
+        and     esi,edi
+        add     ecx,DWORD[64+r13]
+        xor     esi,ebp
+        mov     edi,eax
+        vpsrld  ymm8,ymm4,31
+        xor     edi,ebp
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        vpslldq ymm10,ymm4,12
+        vpaddd  ymm4,ymm4,ymm4
+        rorx    esi,edx,2
+        xor     edx,eax
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm4,ymm4,ymm8
+        add     ecx,r12d
+        and     edx,edi
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm4,ymm4,ymm9
+        add     ebx,DWORD[68+r13]
+        xor     edx,eax
+        vpxor   ymm4,ymm4,ymm10
+        mov     edi,esi
+        xor     edi,eax
+        lea     ebx,[rdx*1+rbx]
+        vpaddd  ymm9,ymm4,ymm11
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        vmovdqu YMMWORD[128+rsp],ymm9
+        add     ebx,r12d
+        and     ecx,edi
+        add     ebp,DWORD[72+r13]
+        xor     ecx,esi
+        mov     edi,edx
+        xor     edi,esi
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        and     ebx,edi
+        add     eax,DWORD[76+r13]
+        xor     ebx,edx
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        vpalignr        ymm5,ymm2,ymm1,8
+        add     esi,DWORD[96+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        vpsrldq ymm8,ymm4,4
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        vpxor   ymm5,ymm5,ymm1
+        vpxor   ymm8,ymm8,ymm3
+        add     edx,DWORD[100+r13]
+        lea     edx,[rax*1+rdx]
+        vpxor   ymm5,ymm5,ymm8
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        xor     esi,ebp
+        add     edx,r12d
+        vpsrld  ymm8,ymm5,31
+        vmovdqu ymm11,YMMWORD[((-32))+r14]
+        xor     esi,ebx
+        add     ecx,DWORD[104+r13]
+        lea     ecx,[rsi*1+rcx]
+        vpslldq ymm10,ymm5,12
+        vpaddd  ymm5,ymm5,ymm5
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm5,ymm5,ymm8
+        xor     edx,eax
+        add     ecx,r12d
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm5,ymm5,ymm9
+        xor     edx,ebp
+        add     ebx,DWORD[108+r13]
+        lea     r13,[256+r13]
+        vpxor   ymm5,ymm5,ymm10
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        vpaddd  ymm9,ymm5,ymm11
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        vmovdqu YMMWORD[160+rsp],ymm9
+        add     ebp,DWORD[((-128))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        vpalignr        ymm6,ymm3,ymm2,8
+        add     eax,DWORD[((-124))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        vpsrldq ymm8,ymm5,4
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        vpxor   ymm6,ymm6,ymm2
+        vpxor   ymm8,ymm8,ymm4
+        add     esi,DWORD[((-120))+r13]
+        lea     esi,[rbp*1+rsi]
+        vpxor   ymm6,ymm6,ymm8
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        vpsrld  ymm8,ymm6,31
+        xor     eax,ecx
+        add     edx,DWORD[((-116))+r13]
+        lea     edx,[rax*1+rdx]
+        vpslldq ymm10,ymm6,12
+        vpaddd  ymm6,ymm6,ymm6
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm6,ymm6,ymm8
+        xor     esi,ebp
+        add     edx,r12d
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm6,ymm6,ymm9
+        xor     esi,ebx
+        add     ecx,DWORD[((-96))+r13]
+        vpxor   ymm6,ymm6,ymm10
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        vpaddd  ymm9,ymm6,ymm11
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        vmovdqu YMMWORD[192+rsp],ymm9
+        add     ebx,DWORD[((-92))+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        vpalignr        ymm7,ymm4,ymm3,8
+        add     ebp,DWORD[((-88))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        vpsrldq ymm8,ymm6,4
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        vpxor   ymm7,ymm7,ymm3
+        vpxor   ymm8,ymm8,ymm5
+        add     eax,DWORD[((-84))+r13]
+        lea     eax,[rbx*1+rax]
+        vpxor   ymm7,ymm7,ymm8
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        vpsrld  ymm8,ymm7,31
+        xor     ebp,edx
+        add     esi,DWORD[((-64))+r13]
+        lea     esi,[rbp*1+rsi]
+        vpslldq ymm10,ymm7,12
+        vpaddd  ymm7,ymm7,ymm7
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        vpsrld  ymm9,ymm10,30
+        vpor    ymm7,ymm7,ymm8
+        xor     eax,ebx
+        add     esi,r12d
+        vpslld  ymm10,ymm10,2
+        vpxor   ymm7,ymm7,ymm9
+        xor     eax,ecx
+        add     edx,DWORD[((-60))+r13]
+        vpxor   ymm7,ymm7,ymm10
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        rorx    eax,esi,2
+        vpaddd  ymm9,ymm7,ymm11
+        xor     esi,ebp
+        add     edx,r12d
+        xor     esi,ebx
+        vmovdqu YMMWORD[224+rsp],ymm9
+        add     ecx,DWORD[((-56))+r13]
+        lea     ecx,[rsi*1+rcx]
+        rorx    r12d,edx,27
+        rorx    esi,edx,2
+        xor     edx,eax
+        add     ecx,r12d
+        xor     edx,ebp
+        add     ebx,DWORD[((-52))+r13]
+        lea     ebx,[rdx*1+rbx]
+        rorx    r12d,ecx,27
+        rorx    edx,ecx,2
+        xor     ecx,esi
+        add     ebx,r12d
+        xor     ecx,eax
+        add     ebp,DWORD[((-32))+r13]
+        lea     ebp,[rbp*1+rcx]
+        rorx    r12d,ebx,27
+        rorx    ecx,ebx,2
+        xor     ebx,edx
+        add     ebp,r12d
+        xor     ebx,esi
+        add     eax,DWORD[((-28))+r13]
+        lea     eax,[rbx*1+rax]
+        rorx    r12d,ebp,27
+        rorx    ebx,ebp,2
+        xor     ebp,ecx
+        add     eax,r12d
+        xor     ebp,edx
+        add     esi,DWORD[((-24))+r13]
+        lea     esi,[rbp*1+rsi]
+        rorx    r12d,eax,27
+        rorx    ebp,eax,2
+        xor     eax,ebx
+        add     esi,r12d
+        xor     eax,ecx
+        add     edx,DWORD[((-20))+r13]
+        lea     edx,[rax*1+rdx]
+        rorx    r12d,esi,27
+        add     edx,r12d
+        lea     r13,[128+rsp]
+
+
+        add     edx,DWORD[r8]
+        add     esi,DWORD[4+r8]
+        add     ebp,DWORD[8+r8]
+        mov     DWORD[r8],edx
+        add     ebx,DWORD[12+r8]
+        mov     DWORD[4+r8],esi
+        mov     eax,edx
+        add     ecx,DWORD[16+r8]
+        mov     r12d,ebp
+        mov     DWORD[8+r8],ebp
+        mov     edx,ebx
+
+        mov     DWORD[12+r8],ebx
+        mov     ebp,esi
+        mov     DWORD[16+r8],ecx
+
+        mov     esi,ecx
+        mov     ecx,r12d
+
+
+        cmp     r9,r10
+        jbe     NEAR $L$oop_avx2
+
+$L$done_avx2:
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-40-96))+r11]
+        movaps  xmm7,XMMWORD[((-40-80))+r11]
+        movaps  xmm8,XMMWORD[((-40-64))+r11]
+        movaps  xmm9,XMMWORD[((-40-48))+r11]
+        movaps  xmm10,XMMWORD[((-40-32))+r11]
+        movaps  xmm11,XMMWORD[((-40-16))+r11]
+        mov     r14,QWORD[((-40))+r11]
+
+        mov     r13,QWORD[((-32))+r11]
+
+        mov     r12,QWORD[((-24))+r11]
+
+        mov     rbp,QWORD[((-16))+r11]
+
+        mov     rbx,QWORD[((-8))+r11]
+
+        lea     rsp,[r11]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha1_block_data_order_avx2:
 ALIGN   64
 K_XX_XX:
@@ -2861 +5739 @@
         DD      $L$SEH_end_sha1_block_data_order_ssse3 wrt ..imagebase
         DD      $L$SEH_info_sha1_block_data_order_ssse3 wrt ..imagebase
+        DD      $L$SEH_begin_sha1_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_end_sha1_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_info_sha1_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_begin_sha1_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_end_sha1_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_info_sha1_block_data_order_avx2 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -2873 +5757 @@
         DD      ssse3_handler wrt ..imagebase
         DD      $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase
+$L$SEH_info_sha1_block_data_order_avx:
+DB      9,0,0,0
+        DD      ssse3_handler wrt ..imagebase
+        DD      $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_sha1_block_data_order_avx2:
+DB      9,0,0,0
+        DD      ssse3_handler wrt ..imagebase
+        DD      $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/sha256-mb-x86_64.S

@@ -25 +25 @@
         bt      rcx,61
         jc      NEAR _shaext_shortcut
+        test    ecx,268435456
+        jnz     NEAR _avx_shortcut
         mov     rax,rsp
 
@@ -3205 +3207 @@
 
 $L$SEH_end_sha256_multi_block_shaext:
+
+ALIGN   32
+sha256_multi_block_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha256_multi_block_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx_shortcut:
+        shr     rcx,32
+        cmp     edx,2
+        jb      NEAR $L$avx
+        test    ecx,32
+        jnz     NEAR _avx2_shortcut
+        jmp     NEAR $L$avx
+ALIGN   32
+$L$avx:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[(-120)+rax],xmm10
+        movaps  XMMWORD[(-104)+rax],xmm11
+        movaps  XMMWORD[(-88)+rax],xmm12
+        movaps  XMMWORD[(-72)+rax],xmm13
+        movaps  XMMWORD[(-56)+rax],xmm14
+        movaps  XMMWORD[(-40)+rax],xmm15
+        sub     rsp,288
+        and     rsp,-256
+        mov     QWORD[272+rsp],rax
+
+$L$body_avx:
+        lea     rbp,[((K256+128))]
+        lea     rbx,[256+rsp]
+        lea     rdi,[128+rdi]
+
+$L$oop_grande_avx:
+        mov     DWORD[280+rsp],edx
+        xor     edx,edx
+
+        mov     r8,QWORD[rsi]
+
+        mov     ecx,DWORD[8+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[rbx],ecx
+        cmovle  r8,rbp
+
+        mov     r9,QWORD[16+rsi]
+
+        mov     ecx,DWORD[24+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[4+rbx],ecx
+        cmovle  r9,rbp
+
+        mov     r10,QWORD[32+rsi]
+
+        mov     ecx,DWORD[40+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[8+rbx],ecx
+        cmovle  r10,rbp
+
+        mov     r11,QWORD[48+rsi]
+
+        mov     ecx,DWORD[56+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[12+rbx],ecx
+        cmovle  r11,rbp
+        test    edx,edx
+        jz      NEAR $L$done_avx
+
+        vmovdqu xmm8,XMMWORD[((0-128))+rdi]
+        lea     rax,[128+rsp]
+        vmovdqu xmm9,XMMWORD[((32-128))+rdi]
+        vmovdqu xmm10,XMMWORD[((64-128))+rdi]
+        vmovdqu xmm11,XMMWORD[((96-128))+rdi]
+        vmovdqu xmm12,XMMWORD[((128-128))+rdi]
+        vmovdqu xmm13,XMMWORD[((160-128))+rdi]
+        vmovdqu xmm14,XMMWORD[((192-128))+rdi]
+        vmovdqu xmm15,XMMWORD[((224-128))+rdi]
+        vmovdqu xmm6,XMMWORD[$L$pbswap]
+        jmp     NEAR $L$oop_avx
+
+ALIGN   32
+$L$oop_avx:
+        vpxor   xmm4,xmm10,xmm9
+        vmovd   xmm5,DWORD[r8]
+        vmovd   xmm0,DWORD[r9]
+        vpinsrd xmm5,xmm5,DWORD[r10],1
+        vpinsrd xmm0,xmm0,DWORD[r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm12,6
+        vpslld  xmm2,xmm12,26
+        vmovdqu XMMWORD[(0-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm15
+
+        vpsrld  xmm1,xmm12,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm12,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-128))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm12,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,7
+        vpandn  xmm0,xmm12,xmm14
+        vpand   xmm3,xmm12,xmm13
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm15,xmm8,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm8,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm9,xmm8
+
+        vpxor   xmm15,xmm15,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm8,13
+
+        vpslld  xmm2,xmm8,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm15,xmm1
+
+        vpsrld  xmm1,xmm8,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,10
+        vpxor   xmm15,xmm9,xmm4
+        vpaddd  xmm11,xmm11,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm15,xmm15,xmm5
+        vpaddd  xmm15,xmm15,xmm7
+        vmovd   xmm5,DWORD[4+r8]
+        vmovd   xmm0,DWORD[4+r9]
+        vpinsrd xmm5,xmm5,DWORD[4+r10],1
+        vpinsrd xmm0,xmm0,DWORD[4+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm11,6
+        vpslld  xmm2,xmm11,26
+        vmovdqu XMMWORD[(16-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm14
+
+        vpsrld  xmm1,xmm11,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm11,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-96))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm11,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,7
+        vpandn  xmm0,xmm11,xmm13
+        vpand   xmm4,xmm11,xmm12
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm14,xmm15,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm15,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm8,xmm15
+
+        vpxor   xmm14,xmm14,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm15,13
+
+        vpslld  xmm2,xmm15,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm14,xmm1
+
+        vpsrld  xmm1,xmm15,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,10
+        vpxor   xmm14,xmm8,xmm3
+        vpaddd  xmm10,xmm10,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm14,xmm14,xmm5
+        vpaddd  xmm14,xmm14,xmm7
+        vmovd   xmm5,DWORD[8+r8]
+        vmovd   xmm0,DWORD[8+r9]
+        vpinsrd xmm5,xmm5,DWORD[8+r10],1
+        vpinsrd xmm0,xmm0,DWORD[8+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm10,6
+        vpslld  xmm2,xmm10,26
+        vmovdqu XMMWORD[(32-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm13
+
+        vpsrld  xmm1,xmm10,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm10,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-64))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm10,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,7
+        vpandn  xmm0,xmm10,xmm12
+        vpand   xmm3,xmm10,xmm11
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm13,xmm14,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm14,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm15,xmm14
+
+        vpxor   xmm13,xmm13,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm14,13
+
+        vpslld  xmm2,xmm14,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm13,xmm1
+
+        vpsrld  xmm1,xmm14,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,10
+        vpxor   xmm13,xmm15,xmm4
+        vpaddd  xmm9,xmm9,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm13,xmm13,xmm7
+        vmovd   xmm5,DWORD[12+r8]
+        vmovd   xmm0,DWORD[12+r9]
+        vpinsrd xmm5,xmm5,DWORD[12+r10],1
+        vpinsrd xmm0,xmm0,DWORD[12+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm9,6
+        vpslld  xmm2,xmm9,26
+        vmovdqu XMMWORD[(48-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm12
+
+        vpsrld  xmm1,xmm9,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm9,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-32))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm9,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,7
+        vpandn  xmm0,xmm9,xmm11
+        vpand   xmm4,xmm9,xmm10
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm12,xmm13,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm13,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm14,xmm13
+
+        vpxor   xmm12,xmm12,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm13,13
+
+        vpslld  xmm2,xmm13,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm12,xmm1
+
+        vpsrld  xmm1,xmm13,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,10
+        vpxor   xmm12,xmm14,xmm3
+        vpaddd  xmm8,xmm8,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm12,xmm12,xmm5
+        vpaddd  xmm12,xmm12,xmm7
+        vmovd   xmm5,DWORD[16+r8]
+        vmovd   xmm0,DWORD[16+r9]
+        vpinsrd xmm5,xmm5,DWORD[16+r10],1
+        vpinsrd xmm0,xmm0,DWORD[16+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm8,6
+        vpslld  xmm2,xmm8,26
+        vmovdqu XMMWORD[(64-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm11
+
+        vpsrld  xmm1,xmm8,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm8,21
+        vpaddd  xmm5,xmm5,XMMWORD[rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm8,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,7
+        vpandn  xmm0,xmm8,xmm10
+        vpand   xmm3,xmm8,xmm9
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm11,xmm12,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm12,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm13,xmm12
+
+        vpxor   xmm11,xmm11,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm12,13
+
+        vpslld  xmm2,xmm12,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm11,xmm1
+
+        vpsrld  xmm1,xmm12,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,10
+        vpxor   xmm11,xmm13,xmm4
+        vpaddd  xmm15,xmm15,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm11,xmm11,xmm5
+        vpaddd  xmm11,xmm11,xmm7
+        vmovd   xmm5,DWORD[20+r8]
+        vmovd   xmm0,DWORD[20+r9]
+        vpinsrd xmm5,xmm5,DWORD[20+r10],1
+        vpinsrd xmm0,xmm0,DWORD[20+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm15,6
+        vpslld  xmm2,xmm15,26
+        vmovdqu XMMWORD[(80-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm10
+
+        vpsrld  xmm1,xmm15,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm15,21
+        vpaddd  xmm5,xmm5,XMMWORD[32+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm15,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,7
+        vpandn  xmm0,xmm15,xmm9
+        vpand   xmm4,xmm15,xmm8
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm10,xmm11,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm11,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm12,xmm11
+
+        vpxor   xmm10,xmm10,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm11,13
+
+        vpslld  xmm2,xmm11,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm10,xmm1
+
+        vpsrld  xmm1,xmm11,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,10
+        vpxor   xmm10,xmm12,xmm3
+        vpaddd  xmm14,xmm14,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm10,xmm10,xmm5
+        vpaddd  xmm10,xmm10,xmm7
+        vmovd   xmm5,DWORD[24+r8]
+        vmovd   xmm0,DWORD[24+r9]
+        vpinsrd xmm5,xmm5,DWORD[24+r10],1
+        vpinsrd xmm0,xmm0,DWORD[24+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm14,6
+        vpslld  xmm2,xmm14,26
+        vmovdqu XMMWORD[(96-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm9
+
+        vpsrld  xmm1,xmm14,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm14,21
+        vpaddd  xmm5,xmm5,XMMWORD[64+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm14,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,7
+        vpandn  xmm0,xmm14,xmm8
+        vpand   xmm3,xmm14,xmm15
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm9,xmm10,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm10,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm11,xmm10
+
+        vpxor   xmm9,xmm9,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm10,13
+
+        vpslld  xmm2,xmm10,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm9,xmm1
+
+        vpsrld  xmm1,xmm10,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,10
+        vpxor   xmm9,xmm11,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm9,xmm9,xmm5
+        vpaddd  xmm9,xmm9,xmm7
+        vmovd   xmm5,DWORD[28+r8]
+        vmovd   xmm0,DWORD[28+r9]
+        vpinsrd xmm5,xmm5,DWORD[28+r10],1
+        vpinsrd xmm0,xmm0,DWORD[28+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm13,6
+        vpslld  xmm2,xmm13,26
+        vmovdqu XMMWORD[(112-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm8
+
+        vpsrld  xmm1,xmm13,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm13,21
+        vpaddd  xmm5,xmm5,XMMWORD[96+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm13,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,7
+        vpandn  xmm0,xmm13,xmm15
+        vpand   xmm4,xmm13,xmm14
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm8,xmm9,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm9,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm10,xmm9
+
+        vpxor   xmm8,xmm8,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm9,13
+
+        vpslld  xmm2,xmm9,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm8,xmm1
+
+        vpsrld  xmm1,xmm9,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,10
+        vpxor   xmm8,xmm10,xmm3
+        vpaddd  xmm12,xmm12,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm8,xmm8,xmm5
+        vpaddd  xmm8,xmm8,xmm7
+        add     rbp,256
+        vmovd   xmm5,DWORD[32+r8]
+        vmovd   xmm0,DWORD[32+r9]
+        vpinsrd xmm5,xmm5,DWORD[32+r10],1
+        vpinsrd xmm0,xmm0,DWORD[32+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm12,6
+        vpslld  xmm2,xmm12,26
+        vmovdqu XMMWORD[(128-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm15
+
+        vpsrld  xmm1,xmm12,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm12,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-128))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm12,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,7
+        vpandn  xmm0,xmm12,xmm14
+        vpand   xmm3,xmm12,xmm13
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm15,xmm8,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm8,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm9,xmm8
+
+        vpxor   xmm15,xmm15,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm8,13
+
+        vpslld  xmm2,xmm8,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm15,xmm1
+
+        vpsrld  xmm1,xmm8,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,10
+        vpxor   xmm15,xmm9,xmm4
+        vpaddd  xmm11,xmm11,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm15,xmm15,xmm5
+        vpaddd  xmm15,xmm15,xmm7
+        vmovd   xmm5,DWORD[36+r8]
+        vmovd   xmm0,DWORD[36+r9]
+        vpinsrd xmm5,xmm5,DWORD[36+r10],1
+        vpinsrd xmm0,xmm0,DWORD[36+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm11,6
+        vpslld  xmm2,xmm11,26
+        vmovdqu XMMWORD[(144-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm14
+
+        vpsrld  xmm1,xmm11,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm11,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-96))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm11,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,7
+        vpandn  xmm0,xmm11,xmm13
+        vpand   xmm4,xmm11,xmm12
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm14,xmm15,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm15,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm8,xmm15
+
+        vpxor   xmm14,xmm14,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm15,13
+
+        vpslld  xmm2,xmm15,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm14,xmm1
+
+        vpsrld  xmm1,xmm15,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,10
+        vpxor   xmm14,xmm8,xmm3
+        vpaddd  xmm10,xmm10,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm14,xmm14,xmm5
+        vpaddd  xmm14,xmm14,xmm7
+        vmovd   xmm5,DWORD[40+r8]
+        vmovd   xmm0,DWORD[40+r9]
+        vpinsrd xmm5,xmm5,DWORD[40+r10],1
+        vpinsrd xmm0,xmm0,DWORD[40+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm10,6
+        vpslld  xmm2,xmm10,26
+        vmovdqu XMMWORD[(160-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm13
+
+        vpsrld  xmm1,xmm10,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm10,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-64))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm10,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,7
+        vpandn  xmm0,xmm10,xmm12
+        vpand   xmm3,xmm10,xmm11
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm13,xmm14,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm14,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm15,xmm14
+
+        vpxor   xmm13,xmm13,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm14,13
+
+        vpslld  xmm2,xmm14,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm13,xmm1
+
+        vpsrld  xmm1,xmm14,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,10
+        vpxor   xmm13,xmm15,xmm4
+        vpaddd  xmm9,xmm9,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm13,xmm13,xmm7
+        vmovd   xmm5,DWORD[44+r8]
+        vmovd   xmm0,DWORD[44+r9]
+        vpinsrd xmm5,xmm5,DWORD[44+r10],1
+        vpinsrd xmm0,xmm0,DWORD[44+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm9,6
+        vpslld  xmm2,xmm9,26
+        vmovdqu XMMWORD[(176-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm12
+
+        vpsrld  xmm1,xmm9,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm9,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-32))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm9,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,7
+        vpandn  xmm0,xmm9,xmm11
+        vpand   xmm4,xmm9,xmm10
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm12,xmm13,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm13,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm14,xmm13
+
+        vpxor   xmm12,xmm12,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm13,13
+
+        vpslld  xmm2,xmm13,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm12,xmm1
+
+        vpsrld  xmm1,xmm13,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,10
+        vpxor   xmm12,xmm14,xmm3
+        vpaddd  xmm8,xmm8,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm12,xmm12,xmm5
+        vpaddd  xmm12,xmm12,xmm7
+        vmovd   xmm5,DWORD[48+r8]
+        vmovd   xmm0,DWORD[48+r9]
+        vpinsrd xmm5,xmm5,DWORD[48+r10],1
+        vpinsrd xmm0,xmm0,DWORD[48+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm8,6
+        vpslld  xmm2,xmm8,26
+        vmovdqu XMMWORD[(192-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm11
+
+        vpsrld  xmm1,xmm8,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm8,21
+        vpaddd  xmm5,xmm5,XMMWORD[rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm8,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,7
+        vpandn  xmm0,xmm8,xmm10
+        vpand   xmm3,xmm8,xmm9
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm11,xmm12,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm12,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm13,xmm12
+
+        vpxor   xmm11,xmm11,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm12,13
+
+        vpslld  xmm2,xmm12,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm11,xmm1
+
+        vpsrld  xmm1,xmm12,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,10
+        vpxor   xmm11,xmm13,xmm4
+        vpaddd  xmm15,xmm15,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm11,xmm11,xmm5
+        vpaddd  xmm11,xmm11,xmm7
+        vmovd   xmm5,DWORD[52+r8]
+        vmovd   xmm0,DWORD[52+r9]
+        vpinsrd xmm5,xmm5,DWORD[52+r10],1
+        vpinsrd xmm0,xmm0,DWORD[52+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm15,6
+        vpslld  xmm2,xmm15,26
+        vmovdqu XMMWORD[(208-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm10
+
+        vpsrld  xmm1,xmm15,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm15,21
+        vpaddd  xmm5,xmm5,XMMWORD[32+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm15,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,7
+        vpandn  xmm0,xmm15,xmm9
+        vpand   xmm4,xmm15,xmm8
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm10,xmm11,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm11,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm12,xmm11
+
+        vpxor   xmm10,xmm10,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm11,13
+
+        vpslld  xmm2,xmm11,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm10,xmm1
+
+        vpsrld  xmm1,xmm11,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,10
+        vpxor   xmm10,xmm12,xmm3
+        vpaddd  xmm14,xmm14,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm10,xmm10,xmm5
+        vpaddd  xmm10,xmm10,xmm7
+        vmovd   xmm5,DWORD[56+r8]
+        vmovd   xmm0,DWORD[56+r9]
+        vpinsrd xmm5,xmm5,DWORD[56+r10],1
+        vpinsrd xmm0,xmm0,DWORD[56+r11],1
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm14,6
+        vpslld  xmm2,xmm14,26
+        vmovdqu XMMWORD[(224-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm9
+
+        vpsrld  xmm1,xmm14,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm14,21
+        vpaddd  xmm5,xmm5,XMMWORD[64+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm14,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,7
+        vpandn  xmm0,xmm14,xmm8
+        vpand   xmm3,xmm14,xmm15
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm9,xmm10,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm10,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm11,xmm10
+
+        vpxor   xmm9,xmm9,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm10,13
+
+        vpslld  xmm2,xmm10,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm9,xmm1
+
+        vpsrld  xmm1,xmm10,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,10
+        vpxor   xmm9,xmm11,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm9,xmm9,xmm5
+        vpaddd  xmm9,xmm9,xmm7
+        vmovd   xmm5,DWORD[60+r8]
+        lea     r8,[64+r8]
+        vmovd   xmm0,DWORD[60+r9]
+        lea     r9,[64+r9]
+        vpinsrd xmm5,xmm5,DWORD[60+r10],1
+        lea     r10,[64+r10]
+        vpinsrd xmm0,xmm0,DWORD[60+r11],1
+        lea     r11,[64+r11]
+        vpunpckldq      xmm5,xmm5,xmm0
+        vpshufb xmm5,xmm5,xmm6
+        vpsrld  xmm7,xmm13,6
+        vpslld  xmm2,xmm13,26
+        vmovdqu XMMWORD[(240-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm8
+
+        vpsrld  xmm1,xmm13,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm13,21
+        vpaddd  xmm5,xmm5,XMMWORD[96+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm13,25
+        vpxor   xmm7,xmm7,xmm2
+        prefetcht0      [63+r8]
+        vpslld  xmm2,xmm13,7
+        vpandn  xmm0,xmm13,xmm15
+        vpand   xmm4,xmm13,xmm14
+        prefetcht0      [63+r9]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm8,xmm9,2
+        vpxor   xmm7,xmm7,xmm2
+        prefetcht0      [63+r10]
+        vpslld  xmm1,xmm9,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm10,xmm9
+        prefetcht0      [63+r11]
+        vpxor   xmm8,xmm8,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm9,13
+
+        vpslld  xmm2,xmm9,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm8,xmm1
+
+        vpsrld  xmm1,xmm9,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,10
+        vpxor   xmm8,xmm10,xmm3
+        vpaddd  xmm12,xmm12,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm8,xmm8,xmm5
+        vpaddd  xmm8,xmm8,xmm7
+        add     rbp,256
+        vmovdqu xmm5,XMMWORD[((0-128))+rax]
+        mov     ecx,3
+        jmp     NEAR $L$oop_16_xx_avx
+ALIGN   32
+$L$oop_16_xx_avx:
+        vmovdqu xmm6,XMMWORD[((16-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((144-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((224-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm12,6
+        vpslld  xmm2,xmm12,26
+        vmovdqu XMMWORD[(0-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm15
+
+        vpsrld  xmm1,xmm12,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm12,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-128))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm12,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,7
+        vpandn  xmm0,xmm12,xmm14
+        vpand   xmm3,xmm12,xmm13
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm15,xmm8,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm8,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm9,xmm8
+
+        vpxor   xmm15,xmm15,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm8,13
+
+        vpslld  xmm2,xmm8,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm15,xmm1
+
+        vpsrld  xmm1,xmm8,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,10
+        vpxor   xmm15,xmm9,xmm4
+        vpaddd  xmm11,xmm11,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm15,xmm15,xmm5
+        vpaddd  xmm15,xmm15,xmm7
+        vmovdqu xmm5,XMMWORD[((32-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((160-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((240-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm11,6
+        vpslld  xmm2,xmm11,26
+        vmovdqu XMMWORD[(16-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm14
+
+        vpsrld  xmm1,xmm11,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm11,21
+        vpaddd  xmm6,xmm6,XMMWORD[((-96))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm11,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,7
+        vpandn  xmm0,xmm11,xmm13
+        vpand   xmm4,xmm11,xmm12
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm14,xmm15,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm15,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm8,xmm15
+
+        vpxor   xmm14,xmm14,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm15,13
+
+        vpslld  xmm2,xmm15,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm14,xmm1
+
+        vpsrld  xmm1,xmm15,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,10
+        vpxor   xmm14,xmm8,xmm3
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm14,xmm14,xmm6
+        vpaddd  xmm14,xmm14,xmm7
+        vmovdqu xmm6,XMMWORD[((48-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((176-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((0-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm10,6
+        vpslld  xmm2,xmm10,26
+        vmovdqu XMMWORD[(32-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm13
+
+        vpsrld  xmm1,xmm10,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm10,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-64))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm10,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,7
+        vpandn  xmm0,xmm10,xmm12
+        vpand   xmm3,xmm10,xmm11
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm13,xmm14,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm14,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm15,xmm14
+
+        vpxor   xmm13,xmm13,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm14,13
+
+        vpslld  xmm2,xmm14,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm13,xmm1
+
+        vpsrld  xmm1,xmm14,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,10
+        vpxor   xmm13,xmm15,xmm4
+        vpaddd  xmm9,xmm9,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm13,xmm13,xmm7
+        vmovdqu xmm5,XMMWORD[((64-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((192-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((16-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm9,6
+        vpslld  xmm2,xmm9,26
+        vmovdqu XMMWORD[(48-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm12
+
+        vpsrld  xmm1,xmm9,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm9,21
+        vpaddd  xmm6,xmm6,XMMWORD[((-32))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm9,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,7
+        vpandn  xmm0,xmm9,xmm11
+        vpand   xmm4,xmm9,xmm10
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm12,xmm13,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm13,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm14,xmm13
+
+        vpxor   xmm12,xmm12,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm13,13
+
+        vpslld  xmm2,xmm13,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm12,xmm1
+
+        vpsrld  xmm1,xmm13,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,10
+        vpxor   xmm12,xmm14,xmm3
+        vpaddd  xmm8,xmm8,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm12,xmm12,xmm6
+        vpaddd  xmm12,xmm12,xmm7
+        vmovdqu xmm6,XMMWORD[((80-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((208-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((32-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm8,6
+        vpslld  xmm2,xmm8,26
+        vmovdqu XMMWORD[(64-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm11
+
+        vpsrld  xmm1,xmm8,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm8,21
+        vpaddd  xmm5,xmm5,XMMWORD[rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm8,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,7
+        vpandn  xmm0,xmm8,xmm10
+        vpand   xmm3,xmm8,xmm9
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm11,xmm12,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm12,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm13,xmm12
+
+        vpxor   xmm11,xmm11,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm12,13
+
+        vpslld  xmm2,xmm12,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm11,xmm1
+
+        vpsrld  xmm1,xmm12,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,10
+        vpxor   xmm11,xmm13,xmm4
+        vpaddd  xmm15,xmm15,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm11,xmm11,xmm5
+        vpaddd  xmm11,xmm11,xmm7
+        vmovdqu xmm5,XMMWORD[((96-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((224-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((48-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm15,6
+        vpslld  xmm2,xmm15,26
+        vmovdqu XMMWORD[(80-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm10
+
+        vpsrld  xmm1,xmm15,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm15,21
+        vpaddd  xmm6,xmm6,XMMWORD[32+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm15,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,7
+        vpandn  xmm0,xmm15,xmm9
+        vpand   xmm4,xmm15,xmm8
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm10,xmm11,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm11,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm12,xmm11
+
+        vpxor   xmm10,xmm10,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm11,13
+
+        vpslld  xmm2,xmm11,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm10,xmm1
+
+        vpsrld  xmm1,xmm11,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,10
+        vpxor   xmm10,xmm12,xmm3
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm10,xmm10,xmm6
+        vpaddd  xmm10,xmm10,xmm7
+        vmovdqu xmm6,XMMWORD[((112-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((240-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((64-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm14,6
+        vpslld  xmm2,xmm14,26
+        vmovdqu XMMWORD[(96-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm9
+
+        vpsrld  xmm1,xmm14,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm14,21
+        vpaddd  xmm5,xmm5,XMMWORD[64+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm14,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,7
+        vpandn  xmm0,xmm14,xmm8
+        vpand   xmm3,xmm14,xmm15
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm9,xmm10,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm10,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm11,xmm10
+
+        vpxor   xmm9,xmm9,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm10,13
+
+        vpslld  xmm2,xmm10,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm9,xmm1
+
+        vpsrld  xmm1,xmm10,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,10
+        vpxor   xmm9,xmm11,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm9,xmm9,xmm5
+        vpaddd  xmm9,xmm9,xmm7
+        vmovdqu xmm5,XMMWORD[((128-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((0-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((80-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm13,6
+        vpslld  xmm2,xmm13,26
+        vmovdqu XMMWORD[(112-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm8
+
+        vpsrld  xmm1,xmm13,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm13,21
+        vpaddd  xmm6,xmm6,XMMWORD[96+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm13,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,7
+        vpandn  xmm0,xmm13,xmm15
+        vpand   xmm4,xmm13,xmm14
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm8,xmm9,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm9,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm10,xmm9
+
+        vpxor   xmm8,xmm8,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm9,13
+
+        vpslld  xmm2,xmm9,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm8,xmm1
+
+        vpsrld  xmm1,xmm9,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,10
+        vpxor   xmm8,xmm10,xmm3
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm8,xmm8,xmm6
+        vpaddd  xmm8,xmm8,xmm7
+        add     rbp,256
+        vmovdqu xmm6,XMMWORD[((144-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((16-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((96-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm12,6
+        vpslld  xmm2,xmm12,26
+        vmovdqu XMMWORD[(128-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm15
+
+        vpsrld  xmm1,xmm12,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm12,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-128))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm12,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,7
+        vpandn  xmm0,xmm12,xmm14
+        vpand   xmm3,xmm12,xmm13
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm15,xmm8,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm8,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm9,xmm8
+
+        vpxor   xmm15,xmm15,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm8,13
+
+        vpslld  xmm2,xmm8,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm15,xmm1
+
+        vpsrld  xmm1,xmm8,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,10
+        vpxor   xmm15,xmm9,xmm4
+        vpaddd  xmm11,xmm11,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm15,xmm15,xmm5
+        vpaddd  xmm15,xmm15,xmm7
+        vmovdqu xmm5,XMMWORD[((160-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((32-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((112-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm11,6
+        vpslld  xmm2,xmm11,26
+        vmovdqu XMMWORD[(144-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm14
+
+        vpsrld  xmm1,xmm11,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm11,21
+        vpaddd  xmm6,xmm6,XMMWORD[((-96))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm11,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,7
+        vpandn  xmm0,xmm11,xmm13
+        vpand   xmm4,xmm11,xmm12
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm14,xmm15,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm15,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm8,xmm15
+
+        vpxor   xmm14,xmm14,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm15,13
+
+        vpslld  xmm2,xmm15,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm14,xmm1
+
+        vpsrld  xmm1,xmm15,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,10
+        vpxor   xmm14,xmm8,xmm3
+        vpaddd  xmm10,xmm10,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm14,xmm14,xmm6
+        vpaddd  xmm14,xmm14,xmm7
+        vmovdqu xmm6,XMMWORD[((176-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((48-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((128-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm10,6
+        vpslld  xmm2,xmm10,26
+        vmovdqu XMMWORD[(160-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm13
+
+        vpsrld  xmm1,xmm10,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm10,21
+        vpaddd  xmm5,xmm5,XMMWORD[((-64))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm10,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,7
+        vpandn  xmm0,xmm10,xmm12
+        vpand   xmm3,xmm10,xmm11
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm13,xmm14,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm14,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm15,xmm14
+
+        vpxor   xmm13,xmm13,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm14,13
+
+        vpslld  xmm2,xmm14,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm13,xmm1
+
+        vpsrld  xmm1,xmm14,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,10
+        vpxor   xmm13,xmm15,xmm4
+        vpaddd  xmm9,xmm9,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm13,xmm13,xmm5
+        vpaddd  xmm13,xmm13,xmm7
+        vmovdqu xmm5,XMMWORD[((192-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((64-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((144-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm9,6
+        vpslld  xmm2,xmm9,26
+        vmovdqu XMMWORD[(176-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm12
+
+        vpsrld  xmm1,xmm9,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm9,21
+        vpaddd  xmm6,xmm6,XMMWORD[((-32))+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm9,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,7
+        vpandn  xmm0,xmm9,xmm11
+        vpand   xmm4,xmm9,xmm10
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm12,xmm13,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm13,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm14,xmm13
+
+        vpxor   xmm12,xmm12,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm13,13
+
+        vpslld  xmm2,xmm13,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm12,xmm1
+
+        vpsrld  xmm1,xmm13,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,10
+        vpxor   xmm12,xmm14,xmm3
+        vpaddd  xmm8,xmm8,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm12,xmm12,xmm6
+        vpaddd  xmm12,xmm12,xmm7
+        vmovdqu xmm6,XMMWORD[((208-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((80-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((160-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm8,6
+        vpslld  xmm2,xmm8,26
+        vmovdqu XMMWORD[(192-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm11
+
+        vpsrld  xmm1,xmm8,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm8,21
+        vpaddd  xmm5,xmm5,XMMWORD[rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm8,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm8,7
+        vpandn  xmm0,xmm8,xmm10
+        vpand   xmm3,xmm8,xmm9
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm11,xmm12,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm12,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm13,xmm12
+
+        vpxor   xmm11,xmm11,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm12,13
+
+        vpslld  xmm2,xmm12,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm11,xmm1
+
+        vpsrld  xmm1,xmm12,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm12,10
+        vpxor   xmm11,xmm13,xmm4
+        vpaddd  xmm15,xmm15,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm11,xmm11,xmm5
+        vpaddd  xmm11,xmm11,xmm7
+        vmovdqu xmm5,XMMWORD[((224-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((96-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((176-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm15,6
+        vpslld  xmm2,xmm15,26
+        vmovdqu XMMWORD[(208-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm10
+
+        vpsrld  xmm1,xmm15,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm15,21
+        vpaddd  xmm6,xmm6,XMMWORD[32+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm15,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm15,7
+        vpandn  xmm0,xmm15,xmm9
+        vpand   xmm4,xmm15,xmm8
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm10,xmm11,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm11,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm12,xmm11
+
+        vpxor   xmm10,xmm10,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm11,13
+
+        vpslld  xmm2,xmm11,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm10,xmm1
+
+        vpsrld  xmm1,xmm11,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm11,10
+        vpxor   xmm10,xmm12,xmm3
+        vpaddd  xmm14,xmm14,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm10,xmm10,xmm6
+        vpaddd  xmm10,xmm10,xmm7
+        vmovdqu xmm6,XMMWORD[((240-128))+rax]
+        vpaddd  xmm5,xmm5,XMMWORD[((112-128))+rax]
+
+        vpsrld  xmm7,xmm6,3
+        vpsrld  xmm1,xmm6,7
+        vpslld  xmm2,xmm6,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm6,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm6,14
+        vmovdqu xmm0,XMMWORD[((192-128))+rax]
+        vpsrld  xmm3,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm5,xmm5,xmm7
+        vpxor   xmm7,xmm3,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm5,xmm5,xmm7
+        vpsrld  xmm7,xmm14,6
+        vpslld  xmm2,xmm14,26
+        vmovdqu XMMWORD[(224-128)+rax],xmm5
+        vpaddd  xmm5,xmm5,xmm9
+
+        vpsrld  xmm1,xmm14,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm14,21
+        vpaddd  xmm5,xmm5,XMMWORD[64+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm14,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm14,7
+        vpandn  xmm0,xmm14,xmm8
+        vpand   xmm3,xmm14,xmm15
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm9,xmm10,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm10,30
+        vpxor   xmm0,xmm0,xmm3
+        vpxor   xmm3,xmm11,xmm10
+
+        vpxor   xmm9,xmm9,xmm1
+        vpaddd  xmm5,xmm5,xmm7
+
+        vpsrld  xmm1,xmm10,13
+
+        vpslld  xmm2,xmm10,19
+        vpaddd  xmm5,xmm5,xmm0
+        vpand   xmm4,xmm4,xmm3
+
+        vpxor   xmm7,xmm9,xmm1
+
+        vpsrld  xmm1,xmm10,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm10,10
+        vpxor   xmm9,xmm11,xmm4
+        vpaddd  xmm13,xmm13,xmm5
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm9,xmm9,xmm5
+        vpaddd  xmm9,xmm9,xmm7
+        vmovdqu xmm5,XMMWORD[((0-128))+rax]
+        vpaddd  xmm6,xmm6,XMMWORD[((128-128))+rax]
+
+        vpsrld  xmm7,xmm5,3
+        vpsrld  xmm1,xmm5,7
+        vpslld  xmm2,xmm5,25
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm5,18
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm5,14
+        vmovdqu xmm0,XMMWORD[((208-128))+rax]
+        vpsrld  xmm4,xmm0,10
+
+        vpxor   xmm7,xmm7,xmm1
+        vpsrld  xmm1,xmm0,17
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,15
+        vpaddd  xmm6,xmm6,xmm7
+        vpxor   xmm7,xmm4,xmm1
+        vpsrld  xmm1,xmm0,19
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm0,13
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+        vpaddd  xmm6,xmm6,xmm7
+        vpsrld  xmm7,xmm13,6
+        vpslld  xmm2,xmm13,26
+        vmovdqu XMMWORD[(240-128)+rax],xmm6
+        vpaddd  xmm6,xmm6,xmm8
+
+        vpsrld  xmm1,xmm13,11
+        vpxor   xmm7,xmm7,xmm2
+        vpslld  xmm2,xmm13,21
+        vpaddd  xmm6,xmm6,XMMWORD[96+rbp]
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm1,xmm13,25
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm13,7
+        vpandn  xmm0,xmm13,xmm15
+        vpand   xmm4,xmm13,xmm14
+
+        vpxor   xmm7,xmm7,xmm1
+
+        vpsrld  xmm8,xmm9,2
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm1,xmm9,30
+        vpxor   xmm0,xmm0,xmm4
+        vpxor   xmm4,xmm10,xmm9
+
+        vpxor   xmm8,xmm8,xmm1
+        vpaddd  xmm6,xmm6,xmm7
+
+        vpsrld  xmm1,xmm9,13
+
+        vpslld  xmm2,xmm9,19
+        vpaddd  xmm6,xmm6,xmm0
+        vpand   xmm3,xmm3,xmm4
+
+        vpxor   xmm7,xmm8,xmm1
+
+        vpsrld  xmm1,xmm9,22
+        vpxor   xmm7,xmm7,xmm2
+
+        vpslld  xmm2,xmm9,10
+        vpxor   xmm8,xmm10,xmm3
+        vpaddd  xmm12,xmm12,xmm6
+
+        vpxor   xmm7,xmm7,xmm1
+        vpxor   xmm7,xmm7,xmm2
+
+        vpaddd  xmm8,xmm8,xmm6
+        vpaddd  xmm8,xmm8,xmm7
+        add     rbp,256
+        dec     ecx
+        jnz     NEAR $L$oop_16_xx_avx
+
+        mov     ecx,1
+        lea     rbp,[((K256+128))]
+        cmp     ecx,DWORD[rbx]
+        cmovge  r8,rbp
+        cmp     ecx,DWORD[4+rbx]
+        cmovge  r9,rbp
+        cmp     ecx,DWORD[8+rbx]
+        cmovge  r10,rbp
+        cmp     ecx,DWORD[12+rbx]
+        cmovge  r11,rbp
+        vmovdqa xmm7,XMMWORD[rbx]
+        vpxor   xmm0,xmm0,xmm0
+        vmovdqa xmm6,xmm7
+        vpcmpgtd        xmm6,xmm6,xmm0
+        vpaddd  xmm7,xmm7,xmm6
+
+        vmovdqu xmm0,XMMWORD[((0-128))+rdi]
+        vpand   xmm8,xmm8,xmm6
+        vmovdqu xmm1,XMMWORD[((32-128))+rdi]
+        vpand   xmm9,xmm9,xmm6
+        vmovdqu xmm2,XMMWORD[((64-128))+rdi]
+        vpand   xmm10,xmm10,xmm6
+        vmovdqu xmm5,XMMWORD[((96-128))+rdi]
+        vpand   xmm11,xmm11,xmm6
+        vpaddd  xmm8,xmm8,xmm0
+        vmovdqu xmm0,XMMWORD[((128-128))+rdi]
+        vpand   xmm12,xmm12,xmm6
+        vpaddd  xmm9,xmm9,xmm1
+        vmovdqu xmm1,XMMWORD[((160-128))+rdi]
+        vpand   xmm13,xmm13,xmm6
+        vpaddd  xmm10,xmm10,xmm2
+        vmovdqu xmm2,XMMWORD[((192-128))+rdi]
+        vpand   xmm14,xmm14,xmm6
+        vpaddd  xmm11,xmm11,xmm5
+        vmovdqu xmm5,XMMWORD[((224-128))+rdi]
+        vpand   xmm15,xmm15,xmm6
+        vpaddd  xmm12,xmm12,xmm0
+        vpaddd  xmm13,xmm13,xmm1
+        vmovdqu XMMWORD[(0-128)+rdi],xmm8
+        vpaddd  xmm14,xmm14,xmm2
+        vmovdqu XMMWORD[(32-128)+rdi],xmm9
+        vpaddd  xmm15,xmm15,xmm5
+        vmovdqu XMMWORD[(64-128)+rdi],xmm10
+        vmovdqu XMMWORD[(96-128)+rdi],xmm11
+        vmovdqu XMMWORD[(128-128)+rdi],xmm12
+        vmovdqu XMMWORD[(160-128)+rdi],xmm13
+        vmovdqu XMMWORD[(192-128)+rdi],xmm14
+        vmovdqu XMMWORD[(224-128)+rdi],xmm15
+
+        vmovdqu XMMWORD[rbx],xmm7
+        vmovdqu xmm6,XMMWORD[$L$pbswap]
+        dec     edx
+        jnz     NEAR $L$oop_avx
+
+        mov     edx,DWORD[280+rsp]
+        lea     rdi,[16+rdi]
+        lea     rsi,[64+rsi]
+        dec     edx
+        jnz     NEAR $L$oop_grande_avx
+
+$L$done_avx:
+        mov     rax,QWORD[272+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-184))+rax]
+        movaps  xmm7,XMMWORD[((-168))+rax]
+        movaps  xmm8,XMMWORD[((-152))+rax]
+        movaps  xmm9,XMMWORD[((-136))+rax]
+        movaps  xmm10,XMMWORD[((-120))+rax]
+        movaps  xmm11,XMMWORD[((-104))+rax]
+        movaps  xmm12,XMMWORD[((-88))+rax]
+        movaps  xmm13,XMMWORD[((-72))+rax]
+        movaps  xmm14,XMMWORD[((-56))+rax]
+        movaps  xmm15,XMMWORD[((-40))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha256_multi_block_avx:
+
+ALIGN   32
+sha256_multi_block_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha256_multi_block_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+_avx2_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        lea     rsp,[((-168))+rsp]
+        movaps  XMMWORD[rsp],xmm6
+        movaps  XMMWORD[16+rsp],xmm7
+        movaps  XMMWORD[32+rsp],xmm8
+        movaps  XMMWORD[48+rsp],xmm9
+        movaps  XMMWORD[64+rsp],xmm10
+        movaps  XMMWORD[80+rsp],xmm11
+        movaps  XMMWORD[(-120)+rax],xmm12
+        movaps  XMMWORD[(-104)+rax],xmm13
+        movaps  XMMWORD[(-88)+rax],xmm14
+        movaps  XMMWORD[(-72)+rax],xmm15
+        sub     rsp,576
+        and     rsp,-256
+        mov     QWORD[544+rsp],rax
+
+$L$body_avx2:
+        lea     rbp,[((K256+128))]
+        lea     rdi,[128+rdi]
+
+$L$oop_grande_avx2:
+        mov     DWORD[552+rsp],edx
+        xor     edx,edx
+        lea     rbx,[512+rsp]
+
+        mov     r12,QWORD[rsi]
+
+        mov     ecx,DWORD[8+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[rbx],ecx
+        cmovle  r12,rbp
+
+        mov     r13,QWORD[16+rsi]
+
+        mov     ecx,DWORD[24+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[4+rbx],ecx
+        cmovle  r13,rbp
+
+        mov     r14,QWORD[32+rsi]
+
+        mov     ecx,DWORD[40+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[8+rbx],ecx
+        cmovle  r14,rbp
+
+        mov     r15,QWORD[48+rsi]
+
+        mov     ecx,DWORD[56+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[12+rbx],ecx
+        cmovle  r15,rbp
+
+        mov     r8,QWORD[64+rsi]
+
+        mov     ecx,DWORD[72+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[16+rbx],ecx
+        cmovle  r8,rbp
+
+        mov     r9,QWORD[80+rsi]
+
+        mov     ecx,DWORD[88+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[20+rbx],ecx
+        cmovle  r9,rbp
+
+        mov     r10,QWORD[96+rsi]
+
+        mov     ecx,DWORD[104+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[24+rbx],ecx
+        cmovle  r10,rbp
+
+        mov     r11,QWORD[112+rsi]
+
+        mov     ecx,DWORD[120+rsi]
+        cmp     ecx,edx
+        cmovg   edx,ecx
+        test    ecx,ecx
+        mov     DWORD[28+rbx],ecx
+        cmovle  r11,rbp
+        vmovdqu ymm8,YMMWORD[((0-128))+rdi]
+        lea     rax,[128+rsp]
+        vmovdqu ymm9,YMMWORD[((32-128))+rdi]
+        lea     rbx,[((256+128))+rsp]
+        vmovdqu ymm10,YMMWORD[((64-128))+rdi]
+        vmovdqu ymm11,YMMWORD[((96-128))+rdi]
+        vmovdqu ymm12,YMMWORD[((128-128))+rdi]
+        vmovdqu ymm13,YMMWORD[((160-128))+rdi]
+        vmovdqu ymm14,YMMWORD[((192-128))+rdi]
+        vmovdqu ymm15,YMMWORD[((224-128))+rdi]
+        vmovdqu ymm6,YMMWORD[$L$pbswap]
+        jmp     NEAR $L$oop_avx2
+
+ALIGN   32
+$L$oop_avx2:
+        vpxor   ymm4,ymm10,ymm9
+        vmovd   xmm5,DWORD[r12]
+        vmovd   xmm0,DWORD[r8]
+        vmovd   xmm1,DWORD[r13]
+        vmovd   xmm2,DWORD[r9]
+        vpinsrd xmm5,xmm5,DWORD[r14],1
+        vpinsrd xmm0,xmm0,DWORD[r10],1
+        vpinsrd xmm1,xmm1,DWORD[r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm12,6
+        vpslld  ymm2,ymm12,26
+        vmovdqu YMMWORD[(0-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm15
+
+        vpsrld  ymm1,ymm12,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm12,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-128))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm12,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,7
+        vpandn  ymm0,ymm12,ymm14
+        vpand   ymm3,ymm12,ymm13
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm15,ymm8,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm8,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm9,ymm8
+
+        vpxor   ymm15,ymm15,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm8,13
+
+        vpslld  ymm2,ymm8,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm15,ymm1
+
+        vpsrld  ymm1,ymm8,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,10
+        vpxor   ymm15,ymm9,ymm4
+        vpaddd  ymm11,ymm11,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm15,ymm15,ymm5
+        vpaddd  ymm15,ymm15,ymm7
+        vmovd   xmm5,DWORD[4+r12]
+        vmovd   xmm0,DWORD[4+r8]
+        vmovd   xmm1,DWORD[4+r13]
+        vmovd   xmm2,DWORD[4+r9]
+        vpinsrd xmm5,xmm5,DWORD[4+r14],1
+        vpinsrd xmm0,xmm0,DWORD[4+r10],1
+        vpinsrd xmm1,xmm1,DWORD[4+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[4+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm11,6
+        vpslld  ymm2,ymm11,26
+        vmovdqu YMMWORD[(32-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm14
+
+        vpsrld  ymm1,ymm11,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm11,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-96))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm11,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,7
+        vpandn  ymm0,ymm11,ymm13
+        vpand   ymm4,ymm11,ymm12
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm14,ymm15,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm15,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm8,ymm15
+
+        vpxor   ymm14,ymm14,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm15,13
+
+        vpslld  ymm2,ymm15,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm14,ymm1
+
+        vpsrld  ymm1,ymm15,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,10
+        vpxor   ymm14,ymm8,ymm3
+        vpaddd  ymm10,ymm10,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm14,ymm14,ymm5
+        vpaddd  ymm14,ymm14,ymm7
+        vmovd   xmm5,DWORD[8+r12]
+        vmovd   xmm0,DWORD[8+r8]
+        vmovd   xmm1,DWORD[8+r13]
+        vmovd   xmm2,DWORD[8+r9]
+        vpinsrd xmm5,xmm5,DWORD[8+r14],1
+        vpinsrd xmm0,xmm0,DWORD[8+r10],1
+        vpinsrd xmm1,xmm1,DWORD[8+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[8+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm10,6
+        vpslld  ymm2,ymm10,26
+        vmovdqu YMMWORD[(64-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm13
+
+        vpsrld  ymm1,ymm10,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm10,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-64))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm10,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,7
+        vpandn  ymm0,ymm10,ymm12
+        vpand   ymm3,ymm10,ymm11
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm13,ymm14,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm14,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm15,ymm14
+
+        vpxor   ymm13,ymm13,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm14,13
+
+        vpslld  ymm2,ymm14,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm13,ymm1
+
+        vpsrld  ymm1,ymm14,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,10
+        vpxor   ymm13,ymm15,ymm4
+        vpaddd  ymm9,ymm9,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm13,ymm13,ymm5
+        vpaddd  ymm13,ymm13,ymm7
+        vmovd   xmm5,DWORD[12+r12]
+        vmovd   xmm0,DWORD[12+r8]
+        vmovd   xmm1,DWORD[12+r13]
+        vmovd   xmm2,DWORD[12+r9]
+        vpinsrd xmm5,xmm5,DWORD[12+r14],1
+        vpinsrd xmm0,xmm0,DWORD[12+r10],1
+        vpinsrd xmm1,xmm1,DWORD[12+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[12+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm9,6
+        vpslld  ymm2,ymm9,26
+        vmovdqu YMMWORD[(96-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm12
+
+        vpsrld  ymm1,ymm9,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm9,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-32))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm9,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,7
+        vpandn  ymm0,ymm9,ymm11
+        vpand   ymm4,ymm9,ymm10
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm12,ymm13,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm13,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm14,ymm13
+
+        vpxor   ymm12,ymm12,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm13,13
+
+        vpslld  ymm2,ymm13,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm12,ymm1
+
+        vpsrld  ymm1,ymm13,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,10
+        vpxor   ymm12,ymm14,ymm3
+        vpaddd  ymm8,ymm8,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm12,ymm12,ymm5
+        vpaddd  ymm12,ymm12,ymm7
+        vmovd   xmm5,DWORD[16+r12]
+        vmovd   xmm0,DWORD[16+r8]
+        vmovd   xmm1,DWORD[16+r13]
+        vmovd   xmm2,DWORD[16+r9]
+        vpinsrd xmm5,xmm5,DWORD[16+r14],1
+        vpinsrd xmm0,xmm0,DWORD[16+r10],1
+        vpinsrd xmm1,xmm1,DWORD[16+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[16+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm8,6
+        vpslld  ymm2,ymm8,26
+        vmovdqu YMMWORD[(128-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm11
+
+        vpsrld  ymm1,ymm8,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm8,21
+        vpaddd  ymm5,ymm5,YMMWORD[rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm8,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,7
+        vpandn  ymm0,ymm8,ymm10
+        vpand   ymm3,ymm8,ymm9
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm11,ymm12,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm12,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm13,ymm12
+
+        vpxor   ymm11,ymm11,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm12,13
+
+        vpslld  ymm2,ymm12,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm11,ymm1
+
+        vpsrld  ymm1,ymm12,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,10
+        vpxor   ymm11,ymm13,ymm4
+        vpaddd  ymm15,ymm15,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm11,ymm11,ymm5
+        vpaddd  ymm11,ymm11,ymm7
+        vmovd   xmm5,DWORD[20+r12]
+        vmovd   xmm0,DWORD[20+r8]
+        vmovd   xmm1,DWORD[20+r13]
+        vmovd   xmm2,DWORD[20+r9]
+        vpinsrd xmm5,xmm5,DWORD[20+r14],1
+        vpinsrd xmm0,xmm0,DWORD[20+r10],1
+        vpinsrd xmm1,xmm1,DWORD[20+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[20+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm15,6
+        vpslld  ymm2,ymm15,26
+        vmovdqu YMMWORD[(160-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm10
+
+        vpsrld  ymm1,ymm15,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm15,21
+        vpaddd  ymm5,ymm5,YMMWORD[32+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm15,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,7
+        vpandn  ymm0,ymm15,ymm9
+        vpand   ymm4,ymm15,ymm8
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm10,ymm11,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm11,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm12,ymm11
+
+        vpxor   ymm10,ymm10,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm11,13
+
+        vpslld  ymm2,ymm11,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm10,ymm1
+
+        vpsrld  ymm1,ymm11,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,10
+        vpxor   ymm10,ymm12,ymm3
+        vpaddd  ymm14,ymm14,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm10,ymm10,ymm5
+        vpaddd  ymm10,ymm10,ymm7
+        vmovd   xmm5,DWORD[24+r12]
+        vmovd   xmm0,DWORD[24+r8]
+        vmovd   xmm1,DWORD[24+r13]
+        vmovd   xmm2,DWORD[24+r9]
+        vpinsrd xmm5,xmm5,DWORD[24+r14],1
+        vpinsrd xmm0,xmm0,DWORD[24+r10],1
+        vpinsrd xmm1,xmm1,DWORD[24+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[24+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm14,6
+        vpslld  ymm2,ymm14,26
+        vmovdqu YMMWORD[(192-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm9
+
+        vpsrld  ymm1,ymm14,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm14,21
+        vpaddd  ymm5,ymm5,YMMWORD[64+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm14,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,7
+        vpandn  ymm0,ymm14,ymm8
+        vpand   ymm3,ymm14,ymm15
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm9,ymm10,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm10,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm11,ymm10
+
+        vpxor   ymm9,ymm9,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm10,13
+
+        vpslld  ymm2,ymm10,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm9,ymm1
+
+        vpsrld  ymm1,ymm10,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,10
+        vpxor   ymm9,ymm11,ymm4
+        vpaddd  ymm13,ymm13,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm9,ymm9,ymm5
+        vpaddd  ymm9,ymm9,ymm7
+        vmovd   xmm5,DWORD[28+r12]
+        vmovd   xmm0,DWORD[28+r8]
+        vmovd   xmm1,DWORD[28+r13]
+        vmovd   xmm2,DWORD[28+r9]
+        vpinsrd xmm5,xmm5,DWORD[28+r14],1
+        vpinsrd xmm0,xmm0,DWORD[28+r10],1
+        vpinsrd xmm1,xmm1,DWORD[28+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[28+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm13,6
+        vpslld  ymm2,ymm13,26
+        vmovdqu YMMWORD[(224-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm8
+
+        vpsrld  ymm1,ymm13,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm13,21
+        vpaddd  ymm5,ymm5,YMMWORD[96+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm13,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,7
+        vpandn  ymm0,ymm13,ymm15
+        vpand   ymm4,ymm13,ymm14
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm8,ymm9,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm9,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm10,ymm9
+
+        vpxor   ymm8,ymm8,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm9,13
+
+        vpslld  ymm2,ymm9,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm8,ymm1
+
+        vpsrld  ymm1,ymm9,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,10
+        vpxor   ymm8,ymm10,ymm3
+        vpaddd  ymm12,ymm12,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm8,ymm8,ymm5
+        vpaddd  ymm8,ymm8,ymm7
+        add     rbp,256
+        vmovd   xmm5,DWORD[32+r12]
+        vmovd   xmm0,DWORD[32+r8]
+        vmovd   xmm1,DWORD[32+r13]
+        vmovd   xmm2,DWORD[32+r9]
+        vpinsrd xmm5,xmm5,DWORD[32+r14],1
+        vpinsrd xmm0,xmm0,DWORD[32+r10],1
+        vpinsrd xmm1,xmm1,DWORD[32+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[32+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm12,6
+        vpslld  ymm2,ymm12,26
+        vmovdqu YMMWORD[(256-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm15
+
+        vpsrld  ymm1,ymm12,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm12,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-128))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm12,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,7
+        vpandn  ymm0,ymm12,ymm14
+        vpand   ymm3,ymm12,ymm13
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm15,ymm8,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm8,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm9,ymm8
+
+        vpxor   ymm15,ymm15,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm8,13
+
+        vpslld  ymm2,ymm8,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm15,ymm1
+
+        vpsrld  ymm1,ymm8,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,10
+        vpxor   ymm15,ymm9,ymm4
+        vpaddd  ymm11,ymm11,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm15,ymm15,ymm5
+        vpaddd  ymm15,ymm15,ymm7
+        vmovd   xmm5,DWORD[36+r12]
+        vmovd   xmm0,DWORD[36+r8]
+        vmovd   xmm1,DWORD[36+r13]
+        vmovd   xmm2,DWORD[36+r9]
+        vpinsrd xmm5,xmm5,DWORD[36+r14],1
+        vpinsrd xmm0,xmm0,DWORD[36+r10],1
+        vpinsrd xmm1,xmm1,DWORD[36+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[36+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm11,6
+        vpslld  ymm2,ymm11,26
+        vmovdqu YMMWORD[(288-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm14
+
+        vpsrld  ymm1,ymm11,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm11,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-96))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm11,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,7
+        vpandn  ymm0,ymm11,ymm13
+        vpand   ymm4,ymm11,ymm12
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm14,ymm15,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm15,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm8,ymm15
+
+        vpxor   ymm14,ymm14,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm15,13
+
+        vpslld  ymm2,ymm15,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm14,ymm1
+
+        vpsrld  ymm1,ymm15,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,10
+        vpxor   ymm14,ymm8,ymm3
+        vpaddd  ymm10,ymm10,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm14,ymm14,ymm5
+        vpaddd  ymm14,ymm14,ymm7
+        vmovd   xmm5,DWORD[40+r12]
+        vmovd   xmm0,DWORD[40+r8]
+        vmovd   xmm1,DWORD[40+r13]
+        vmovd   xmm2,DWORD[40+r9]
+        vpinsrd xmm5,xmm5,DWORD[40+r14],1
+        vpinsrd xmm0,xmm0,DWORD[40+r10],1
+        vpinsrd xmm1,xmm1,DWORD[40+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[40+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm10,6
+        vpslld  ymm2,ymm10,26
+        vmovdqu YMMWORD[(320-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm13
+
+        vpsrld  ymm1,ymm10,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm10,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-64))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm10,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,7
+        vpandn  ymm0,ymm10,ymm12
+        vpand   ymm3,ymm10,ymm11
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm13,ymm14,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm14,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm15,ymm14
+
+        vpxor   ymm13,ymm13,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm14,13
+
+        vpslld  ymm2,ymm14,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm13,ymm1
+
+        vpsrld  ymm1,ymm14,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,10
+        vpxor   ymm13,ymm15,ymm4
+        vpaddd  ymm9,ymm9,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm13,ymm13,ymm5
+        vpaddd  ymm13,ymm13,ymm7
+        vmovd   xmm5,DWORD[44+r12]
+        vmovd   xmm0,DWORD[44+r8]
+        vmovd   xmm1,DWORD[44+r13]
+        vmovd   xmm2,DWORD[44+r9]
+        vpinsrd xmm5,xmm5,DWORD[44+r14],1
+        vpinsrd xmm0,xmm0,DWORD[44+r10],1
+        vpinsrd xmm1,xmm1,DWORD[44+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[44+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm9,6
+        vpslld  ymm2,ymm9,26
+        vmovdqu YMMWORD[(352-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm12
+
+        vpsrld  ymm1,ymm9,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm9,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-32))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm9,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,7
+        vpandn  ymm0,ymm9,ymm11
+        vpand   ymm4,ymm9,ymm10
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm12,ymm13,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm13,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm14,ymm13
+
+        vpxor   ymm12,ymm12,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm13,13
+
+        vpslld  ymm2,ymm13,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm12,ymm1
+
+        vpsrld  ymm1,ymm13,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,10
+        vpxor   ymm12,ymm14,ymm3
+        vpaddd  ymm8,ymm8,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm12,ymm12,ymm5
+        vpaddd  ymm12,ymm12,ymm7
+        vmovd   xmm5,DWORD[48+r12]
+        vmovd   xmm0,DWORD[48+r8]
+        vmovd   xmm1,DWORD[48+r13]
+        vmovd   xmm2,DWORD[48+r9]
+        vpinsrd xmm5,xmm5,DWORD[48+r14],1
+        vpinsrd xmm0,xmm0,DWORD[48+r10],1
+        vpinsrd xmm1,xmm1,DWORD[48+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[48+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm8,6
+        vpslld  ymm2,ymm8,26
+        vmovdqu YMMWORD[(384-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm11
+
+        vpsrld  ymm1,ymm8,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm8,21
+        vpaddd  ymm5,ymm5,YMMWORD[rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm8,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,7
+        vpandn  ymm0,ymm8,ymm10
+        vpand   ymm3,ymm8,ymm9
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm11,ymm12,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm12,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm13,ymm12
+
+        vpxor   ymm11,ymm11,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm12,13
+
+        vpslld  ymm2,ymm12,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm11,ymm1
+
+        vpsrld  ymm1,ymm12,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,10
+        vpxor   ymm11,ymm13,ymm4
+        vpaddd  ymm15,ymm15,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm11,ymm11,ymm5
+        vpaddd  ymm11,ymm11,ymm7
+        vmovd   xmm5,DWORD[52+r12]
+        vmovd   xmm0,DWORD[52+r8]
+        vmovd   xmm1,DWORD[52+r13]
+        vmovd   xmm2,DWORD[52+r9]
+        vpinsrd xmm5,xmm5,DWORD[52+r14],1
+        vpinsrd xmm0,xmm0,DWORD[52+r10],1
+        vpinsrd xmm1,xmm1,DWORD[52+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[52+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm15,6
+        vpslld  ymm2,ymm15,26
+        vmovdqu YMMWORD[(416-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm10
+
+        vpsrld  ymm1,ymm15,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm15,21
+        vpaddd  ymm5,ymm5,YMMWORD[32+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm15,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,7
+        vpandn  ymm0,ymm15,ymm9
+        vpand   ymm4,ymm15,ymm8
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm10,ymm11,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm11,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm12,ymm11
+
+        vpxor   ymm10,ymm10,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm11,13
+
+        vpslld  ymm2,ymm11,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm10,ymm1
+
+        vpsrld  ymm1,ymm11,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,10
+        vpxor   ymm10,ymm12,ymm3
+        vpaddd  ymm14,ymm14,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm10,ymm10,ymm5
+        vpaddd  ymm10,ymm10,ymm7
+        vmovd   xmm5,DWORD[56+r12]
+        vmovd   xmm0,DWORD[56+r8]
+        vmovd   xmm1,DWORD[56+r13]
+        vmovd   xmm2,DWORD[56+r9]
+        vpinsrd xmm5,xmm5,DWORD[56+r14],1
+        vpinsrd xmm0,xmm0,DWORD[56+r10],1
+        vpinsrd xmm1,xmm1,DWORD[56+r15],1
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[56+r11],1
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm14,6
+        vpslld  ymm2,ymm14,26
+        vmovdqu YMMWORD[(448-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm9
+
+        vpsrld  ymm1,ymm14,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm14,21
+        vpaddd  ymm5,ymm5,YMMWORD[64+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm14,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,7
+        vpandn  ymm0,ymm14,ymm8
+        vpand   ymm3,ymm14,ymm15
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm9,ymm10,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm10,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm11,ymm10
+
+        vpxor   ymm9,ymm9,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm10,13
+
+        vpslld  ymm2,ymm10,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm9,ymm1
+
+        vpsrld  ymm1,ymm10,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,10
+        vpxor   ymm9,ymm11,ymm4
+        vpaddd  ymm13,ymm13,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm9,ymm9,ymm5
+        vpaddd  ymm9,ymm9,ymm7
+        vmovd   xmm5,DWORD[60+r12]
+        lea     r12,[64+r12]
+        vmovd   xmm0,DWORD[60+r8]
+        lea     r8,[64+r8]
+        vmovd   xmm1,DWORD[60+r13]
+        lea     r13,[64+r13]
+        vmovd   xmm2,DWORD[60+r9]
+        lea     r9,[64+r9]
+        vpinsrd xmm5,xmm5,DWORD[60+r14],1
+        lea     r14,[64+r14]
+        vpinsrd xmm0,xmm0,DWORD[60+r10],1
+        lea     r10,[64+r10]
+        vpinsrd xmm1,xmm1,DWORD[60+r15],1
+        lea     r15,[64+r15]
+        vpunpckldq      ymm5,ymm5,ymm1
+        vpinsrd xmm2,xmm2,DWORD[60+r11],1
+        lea     r11,[64+r11]
+        vpunpckldq      ymm0,ymm0,ymm2
+        vinserti128     ymm5,ymm5,xmm0,1
+        vpshufb ymm5,ymm5,ymm6
+        vpsrld  ymm7,ymm13,6
+        vpslld  ymm2,ymm13,26
+        vmovdqu YMMWORD[(480-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm8
+
+        vpsrld  ymm1,ymm13,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm13,21
+        vpaddd  ymm5,ymm5,YMMWORD[96+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm13,25
+        vpxor   ymm7,ymm7,ymm2
+        prefetcht0      [63+r12]
+        vpslld  ymm2,ymm13,7
+        vpandn  ymm0,ymm13,ymm15
+        vpand   ymm4,ymm13,ymm14
+        prefetcht0      [63+r13]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm8,ymm9,2
+        vpxor   ymm7,ymm7,ymm2
+        prefetcht0      [63+r14]
+        vpslld  ymm1,ymm9,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm10,ymm9
+        prefetcht0      [63+r15]
+        vpxor   ymm8,ymm8,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm9,13
+        prefetcht0      [63+r8]
+        vpslld  ymm2,ymm9,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm3,ymm3,ymm4
+        prefetcht0      [63+r9]
+        vpxor   ymm7,ymm8,ymm1
+
+        vpsrld  ymm1,ymm9,22
+        vpxor   ymm7,ymm7,ymm2
+        prefetcht0      [63+r10]
+        vpslld  ymm2,ymm9,10
+        vpxor   ymm8,ymm10,ymm3
+        vpaddd  ymm12,ymm12,ymm5
+        prefetcht0      [63+r11]
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm8,ymm8,ymm5
+        vpaddd  ymm8,ymm8,ymm7
+        add     rbp,256
+        vmovdqu ymm5,YMMWORD[((0-128))+rax]
+        mov     ecx,3
+        jmp     NEAR $L$oop_16_xx_avx2
+ALIGN   32
+$L$oop_16_xx_avx2:
+        vmovdqu ymm6,YMMWORD[((32-128))+rax]
+        vpaddd  ymm5,ymm5,YMMWORD[((288-256-128))+rbx]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((448-256-128))+rbx]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm12,6
+        vpslld  ymm2,ymm12,26
+        vmovdqu YMMWORD[(0-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm15
+
+        vpsrld  ymm1,ymm12,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm12,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-128))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm12,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,7
+        vpandn  ymm0,ymm12,ymm14
+        vpand   ymm3,ymm12,ymm13
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm15,ymm8,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm8,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm9,ymm8
+
+        vpxor   ymm15,ymm15,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm8,13
+
+        vpslld  ymm2,ymm8,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm15,ymm1
+
+        vpsrld  ymm1,ymm8,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,10
+        vpxor   ymm15,ymm9,ymm4
+        vpaddd  ymm11,ymm11,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm15,ymm15,ymm5
+        vpaddd  ymm15,ymm15,ymm7
+        vmovdqu ymm5,YMMWORD[((64-128))+rax]
+        vpaddd  ymm6,ymm6,YMMWORD[((320-256-128))+rbx]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((480-256-128))+rbx]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm11,6
+        vpslld  ymm2,ymm11,26
+        vmovdqu YMMWORD[(32-128)+rax],ymm6
+        vpaddd  ymm6,ymm6,ymm14
+
+        vpsrld  ymm1,ymm11,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm11,21
+        vpaddd  ymm6,ymm6,YMMWORD[((-96))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm11,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,7
+        vpandn  ymm0,ymm11,ymm13
+        vpand   ymm4,ymm11,ymm12
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm14,ymm15,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm15,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm8,ymm15
+
+        vpxor   ymm14,ymm14,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm15,13
+
+        vpslld  ymm2,ymm15,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm14,ymm1
+
+        vpsrld  ymm1,ymm15,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,10
+        vpxor   ymm14,ymm8,ymm3
+        vpaddd  ymm10,ymm10,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm14,ymm14,ymm6
+        vpaddd  ymm14,ymm14,ymm7
+        vmovdqu ymm6,YMMWORD[((96-128))+rax]
+        vpaddd  ymm5,ymm5,YMMWORD[((352-256-128))+rbx]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((0-128))+rax]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm10,6
+        vpslld  ymm2,ymm10,26
+        vmovdqu YMMWORD[(64-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm13
+
+        vpsrld  ymm1,ymm10,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm10,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-64))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm10,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,7
+        vpandn  ymm0,ymm10,ymm12
+        vpand   ymm3,ymm10,ymm11
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm13,ymm14,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm14,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm15,ymm14
+
+        vpxor   ymm13,ymm13,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm14,13
+
+        vpslld  ymm2,ymm14,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm13,ymm1
+
+        vpsrld  ymm1,ymm14,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,10
+        vpxor   ymm13,ymm15,ymm4
+        vpaddd  ymm9,ymm9,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm13,ymm13,ymm5
+        vpaddd  ymm13,ymm13,ymm7
+        vmovdqu ymm5,YMMWORD[((128-128))+rax]
+        vpaddd  ymm6,ymm6,YMMWORD[((384-256-128))+rbx]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((32-128))+rax]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm9,6
+        vpslld  ymm2,ymm9,26
+        vmovdqu YMMWORD[(96-128)+rax],ymm6
+        vpaddd  ymm6,ymm6,ymm12
+
+        vpsrld  ymm1,ymm9,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm9,21
+        vpaddd  ymm6,ymm6,YMMWORD[((-32))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm9,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,7
+        vpandn  ymm0,ymm9,ymm11
+        vpand   ymm4,ymm9,ymm10
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm12,ymm13,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm13,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm14,ymm13
+
+        vpxor   ymm12,ymm12,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm13,13
+
+        vpslld  ymm2,ymm13,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm12,ymm1
+
+        vpsrld  ymm1,ymm13,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,10
+        vpxor   ymm12,ymm14,ymm3
+        vpaddd  ymm8,ymm8,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm12,ymm12,ymm6
+        vpaddd  ymm12,ymm12,ymm7
+        vmovdqu ymm6,YMMWORD[((160-128))+rax]
+        vpaddd  ymm5,ymm5,YMMWORD[((416-256-128))+rbx]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((64-128))+rax]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm8,6
+        vpslld  ymm2,ymm8,26
+        vmovdqu YMMWORD[(128-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm11
+
+        vpsrld  ymm1,ymm8,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm8,21
+        vpaddd  ymm5,ymm5,YMMWORD[rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm8,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,7
+        vpandn  ymm0,ymm8,ymm10
+        vpand   ymm3,ymm8,ymm9
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm11,ymm12,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm12,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm13,ymm12
+
+        vpxor   ymm11,ymm11,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm12,13
+
+        vpslld  ymm2,ymm12,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm11,ymm1
+
+        vpsrld  ymm1,ymm12,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,10
+        vpxor   ymm11,ymm13,ymm4
+        vpaddd  ymm15,ymm15,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm11,ymm11,ymm5
+        vpaddd  ymm11,ymm11,ymm7
+        vmovdqu ymm5,YMMWORD[((192-128))+rax]
+        vpaddd  ymm6,ymm6,YMMWORD[((448-256-128))+rbx]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((96-128))+rax]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm15,6
+        vpslld  ymm2,ymm15,26
+        vmovdqu YMMWORD[(160-128)+rax],ymm6
+        vpaddd  ymm6,ymm6,ymm10
+
+        vpsrld  ymm1,ymm15,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm15,21
+        vpaddd  ymm6,ymm6,YMMWORD[32+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm15,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,7
+        vpandn  ymm0,ymm15,ymm9
+        vpand   ymm4,ymm15,ymm8
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm10,ymm11,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm11,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm12,ymm11
+
+        vpxor   ymm10,ymm10,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm11,13
+
+        vpslld  ymm2,ymm11,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm10,ymm1
+
+        vpsrld  ymm1,ymm11,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,10
+        vpxor   ymm10,ymm12,ymm3
+        vpaddd  ymm14,ymm14,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm10,ymm10,ymm6
+        vpaddd  ymm10,ymm10,ymm7
+        vmovdqu ymm6,YMMWORD[((224-128))+rax]
+        vpaddd  ymm5,ymm5,YMMWORD[((480-256-128))+rbx]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((128-128))+rax]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm14,6
+        vpslld  ymm2,ymm14,26
+        vmovdqu YMMWORD[(192-128)+rax],ymm5
+        vpaddd  ymm5,ymm5,ymm9
+
+        vpsrld  ymm1,ymm14,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm14,21
+        vpaddd  ymm5,ymm5,YMMWORD[64+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm14,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,7
+        vpandn  ymm0,ymm14,ymm8
+        vpand   ymm3,ymm14,ymm15
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm9,ymm10,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm10,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm11,ymm10
+
+        vpxor   ymm9,ymm9,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm10,13
+
+        vpslld  ymm2,ymm10,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm9,ymm1
+
+        vpsrld  ymm1,ymm10,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,10
+        vpxor   ymm9,ymm11,ymm4
+        vpaddd  ymm13,ymm13,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm9,ymm9,ymm5
+        vpaddd  ymm9,ymm9,ymm7
+        vmovdqu ymm5,YMMWORD[((256-256-128))+rbx]
+        vpaddd  ymm6,ymm6,YMMWORD[((0-128))+rax]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((160-128))+rax]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm13,6
+        vpslld  ymm2,ymm13,26
+        vmovdqu YMMWORD[(224-128)+rax],ymm6
+        vpaddd  ymm6,ymm6,ymm8
+
+        vpsrld  ymm1,ymm13,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm13,21
+        vpaddd  ymm6,ymm6,YMMWORD[96+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm13,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,7
+        vpandn  ymm0,ymm13,ymm15
+        vpand   ymm4,ymm13,ymm14
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm8,ymm9,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm9,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm10,ymm9
+
+        vpxor   ymm8,ymm8,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm9,13
+
+        vpslld  ymm2,ymm9,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm8,ymm1
+
+        vpsrld  ymm1,ymm9,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,10
+        vpxor   ymm8,ymm10,ymm3
+        vpaddd  ymm12,ymm12,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm8,ymm8,ymm6
+        vpaddd  ymm8,ymm8,ymm7
+        add     rbp,256
+        vmovdqu ymm6,YMMWORD[((288-256-128))+rbx]
+        vpaddd  ymm5,ymm5,YMMWORD[((32-128))+rax]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((192-128))+rax]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm12,6
+        vpslld  ymm2,ymm12,26
+        vmovdqu YMMWORD[(256-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm15
+
+        vpsrld  ymm1,ymm12,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm12,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-128))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm12,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,7
+        vpandn  ymm0,ymm12,ymm14
+        vpand   ymm3,ymm12,ymm13
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm15,ymm8,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm8,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm9,ymm8
+
+        vpxor   ymm15,ymm15,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm8,13
+
+        vpslld  ymm2,ymm8,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm15,ymm1
+
+        vpsrld  ymm1,ymm8,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,10
+        vpxor   ymm15,ymm9,ymm4
+        vpaddd  ymm11,ymm11,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm15,ymm15,ymm5
+        vpaddd  ymm15,ymm15,ymm7
+        vmovdqu ymm5,YMMWORD[((320-256-128))+rbx]
+        vpaddd  ymm6,ymm6,YMMWORD[((64-128))+rax]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((224-128))+rax]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm11,6
+        vpslld  ymm2,ymm11,26
+        vmovdqu YMMWORD[(288-256-128)+rbx],ymm6
+        vpaddd  ymm6,ymm6,ymm14
+
+        vpsrld  ymm1,ymm11,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm11,21
+        vpaddd  ymm6,ymm6,YMMWORD[((-96))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm11,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,7
+        vpandn  ymm0,ymm11,ymm13
+        vpand   ymm4,ymm11,ymm12
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm14,ymm15,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm15,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm8,ymm15
+
+        vpxor   ymm14,ymm14,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm15,13
+
+        vpslld  ymm2,ymm15,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm14,ymm1
+
+        vpsrld  ymm1,ymm15,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,10
+        vpxor   ymm14,ymm8,ymm3
+        vpaddd  ymm10,ymm10,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm14,ymm14,ymm6
+        vpaddd  ymm14,ymm14,ymm7
+        vmovdqu ymm6,YMMWORD[((352-256-128))+rbx]
+        vpaddd  ymm5,ymm5,YMMWORD[((96-128))+rax]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((256-256-128))+rbx]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm10,6
+        vpslld  ymm2,ymm10,26
+        vmovdqu YMMWORD[(320-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm13
+
+        vpsrld  ymm1,ymm10,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm10,21
+        vpaddd  ymm5,ymm5,YMMWORD[((-64))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm10,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,7
+        vpandn  ymm0,ymm10,ymm12
+        vpand   ymm3,ymm10,ymm11
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm13,ymm14,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm14,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm15,ymm14
+
+        vpxor   ymm13,ymm13,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm14,13
+
+        vpslld  ymm2,ymm14,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm13,ymm1
+
+        vpsrld  ymm1,ymm14,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,10
+        vpxor   ymm13,ymm15,ymm4
+        vpaddd  ymm9,ymm9,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm13,ymm13,ymm5
+        vpaddd  ymm13,ymm13,ymm7
+        vmovdqu ymm5,YMMWORD[((384-256-128))+rbx]
+        vpaddd  ymm6,ymm6,YMMWORD[((128-128))+rax]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((288-256-128))+rbx]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm9,6
+        vpslld  ymm2,ymm9,26
+        vmovdqu YMMWORD[(352-256-128)+rbx],ymm6
+        vpaddd  ymm6,ymm6,ymm12
+
+        vpsrld  ymm1,ymm9,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm9,21
+        vpaddd  ymm6,ymm6,YMMWORD[((-32))+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm9,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,7
+        vpandn  ymm0,ymm9,ymm11
+        vpand   ymm4,ymm9,ymm10
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm12,ymm13,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm13,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm14,ymm13
+
+        vpxor   ymm12,ymm12,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm13,13
+
+        vpslld  ymm2,ymm13,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm12,ymm1
+
+        vpsrld  ymm1,ymm13,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,10
+        vpxor   ymm12,ymm14,ymm3
+        vpaddd  ymm8,ymm8,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm12,ymm12,ymm6
+        vpaddd  ymm12,ymm12,ymm7
+        vmovdqu ymm6,YMMWORD[((416-256-128))+rbx]
+        vpaddd  ymm5,ymm5,YMMWORD[((160-128))+rax]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((320-256-128))+rbx]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm8,6
+        vpslld  ymm2,ymm8,26
+        vmovdqu YMMWORD[(384-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm11
+
+        vpsrld  ymm1,ymm8,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm8,21
+        vpaddd  ymm5,ymm5,YMMWORD[rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm8,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm8,7
+        vpandn  ymm0,ymm8,ymm10
+        vpand   ymm3,ymm8,ymm9
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm11,ymm12,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm12,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm13,ymm12
+
+        vpxor   ymm11,ymm11,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm12,13
+
+        vpslld  ymm2,ymm12,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm11,ymm1
+
+        vpsrld  ymm1,ymm12,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm12,10
+        vpxor   ymm11,ymm13,ymm4
+        vpaddd  ymm15,ymm15,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm11,ymm11,ymm5
+        vpaddd  ymm11,ymm11,ymm7
+        vmovdqu ymm5,YMMWORD[((448-256-128))+rbx]
+        vpaddd  ymm6,ymm6,YMMWORD[((192-128))+rax]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((352-256-128))+rbx]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm15,6
+        vpslld  ymm2,ymm15,26
+        vmovdqu YMMWORD[(416-256-128)+rbx],ymm6
+        vpaddd  ymm6,ymm6,ymm10
+
+        vpsrld  ymm1,ymm15,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm15,21
+        vpaddd  ymm6,ymm6,YMMWORD[32+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm15,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm15,7
+        vpandn  ymm0,ymm15,ymm9
+        vpand   ymm4,ymm15,ymm8
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm10,ymm11,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm11,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm12,ymm11
+
+        vpxor   ymm10,ymm10,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm11,13
+
+        vpslld  ymm2,ymm11,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm10,ymm1
+
+        vpsrld  ymm1,ymm11,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm11,10
+        vpxor   ymm10,ymm12,ymm3
+        vpaddd  ymm14,ymm14,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm10,ymm10,ymm6
+        vpaddd  ymm10,ymm10,ymm7
+        vmovdqu ymm6,YMMWORD[((480-256-128))+rbx]
+        vpaddd  ymm5,ymm5,YMMWORD[((224-128))+rax]
+
+        vpsrld  ymm7,ymm6,3
+        vpsrld  ymm1,ymm6,7
+        vpslld  ymm2,ymm6,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm6,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm6,14
+        vmovdqu ymm0,YMMWORD[((384-256-128))+rbx]
+        vpsrld  ymm3,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm5,ymm5,ymm7
+        vpxor   ymm7,ymm3,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm5,ymm5,ymm7
+        vpsrld  ymm7,ymm14,6
+        vpslld  ymm2,ymm14,26
+        vmovdqu YMMWORD[(448-256-128)+rbx],ymm5
+        vpaddd  ymm5,ymm5,ymm9
+
+        vpsrld  ymm1,ymm14,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm14,21
+        vpaddd  ymm5,ymm5,YMMWORD[64+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm14,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm14,7
+        vpandn  ymm0,ymm14,ymm8
+        vpand   ymm3,ymm14,ymm15
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm9,ymm10,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm10,30
+        vpxor   ymm0,ymm0,ymm3
+        vpxor   ymm3,ymm11,ymm10
+
+        vpxor   ymm9,ymm9,ymm1
+        vpaddd  ymm5,ymm5,ymm7
+
+        vpsrld  ymm1,ymm10,13
+
+        vpslld  ymm2,ymm10,19
+        vpaddd  ymm5,ymm5,ymm0
+        vpand   ymm4,ymm4,ymm3
+
+        vpxor   ymm7,ymm9,ymm1
+
+        vpsrld  ymm1,ymm10,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm10,10
+        vpxor   ymm9,ymm11,ymm4
+        vpaddd  ymm13,ymm13,ymm5
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm9,ymm9,ymm5
+        vpaddd  ymm9,ymm9,ymm7
+        vmovdqu ymm5,YMMWORD[((0-128))+rax]
+        vpaddd  ymm6,ymm6,YMMWORD[((256-256-128))+rbx]
+
+        vpsrld  ymm7,ymm5,3
+        vpsrld  ymm1,ymm5,7
+        vpslld  ymm2,ymm5,25
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm5,18
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm5,14
+        vmovdqu ymm0,YMMWORD[((416-256-128))+rbx]
+        vpsrld  ymm4,ymm0,10
+
+        vpxor   ymm7,ymm7,ymm1
+        vpsrld  ymm1,ymm0,17
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,15
+        vpaddd  ymm6,ymm6,ymm7
+        vpxor   ymm7,ymm4,ymm1
+        vpsrld  ymm1,ymm0,19
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm0,13
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+        vpaddd  ymm6,ymm6,ymm7
+        vpsrld  ymm7,ymm13,6
+        vpslld  ymm2,ymm13,26
+        vmovdqu YMMWORD[(480-256-128)+rbx],ymm6
+        vpaddd  ymm6,ymm6,ymm8
+
+        vpsrld  ymm1,ymm13,11
+        vpxor   ymm7,ymm7,ymm2
+        vpslld  ymm2,ymm13,21
+        vpaddd  ymm6,ymm6,YMMWORD[96+rbp]
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm1,ymm13,25
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm13,7
+        vpandn  ymm0,ymm13,ymm15
+        vpand   ymm4,ymm13,ymm14
+
+        vpxor   ymm7,ymm7,ymm1
+
+        vpsrld  ymm8,ymm9,2
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm1,ymm9,30
+        vpxor   ymm0,ymm0,ymm4
+        vpxor   ymm4,ymm10,ymm9
+
+        vpxor   ymm8,ymm8,ymm1
+        vpaddd  ymm6,ymm6,ymm7
+
+        vpsrld  ymm1,ymm9,13
+
+        vpslld  ymm2,ymm9,19
+        vpaddd  ymm6,ymm6,ymm0
+        vpand   ymm3,ymm3,ymm4
+
+        vpxor   ymm7,ymm8,ymm1
+
+        vpsrld  ymm1,ymm9,22
+        vpxor   ymm7,ymm7,ymm2
+
+        vpslld  ymm2,ymm9,10
+        vpxor   ymm8,ymm10,ymm3
+        vpaddd  ymm12,ymm12,ymm6
+
+        vpxor   ymm7,ymm7,ymm1
+        vpxor   ymm7,ymm7,ymm2
+
+        vpaddd  ymm8,ymm8,ymm6
+        vpaddd  ymm8,ymm8,ymm7
+        add     rbp,256
+        dec     ecx
+        jnz     NEAR $L$oop_16_xx_avx2
+
+        mov     ecx,1
+        lea     rbx,[512+rsp]
+        lea     rbp,[((K256+128))]
+        cmp     ecx,DWORD[rbx]
+        cmovge  r12,rbp
+        cmp     ecx,DWORD[4+rbx]
+        cmovge  r13,rbp
+        cmp     ecx,DWORD[8+rbx]
+        cmovge  r14,rbp
+        cmp     ecx,DWORD[12+rbx]
+        cmovge  r15,rbp
+        cmp     ecx,DWORD[16+rbx]
+        cmovge  r8,rbp
+        cmp     ecx,DWORD[20+rbx]
+        cmovge  r9,rbp
+        cmp     ecx,DWORD[24+rbx]
+        cmovge  r10,rbp
+        cmp     ecx,DWORD[28+rbx]
+        cmovge  r11,rbp
+        vmovdqa ymm7,YMMWORD[rbx]
+        vpxor   ymm0,ymm0,ymm0
+        vmovdqa ymm6,ymm7
+        vpcmpgtd        ymm6,ymm6,ymm0
+        vpaddd  ymm7,ymm7,ymm6
+
+        vmovdqu ymm0,YMMWORD[((0-128))+rdi]
+        vpand   ymm8,ymm8,ymm6
+        vmovdqu ymm1,YMMWORD[((32-128))+rdi]
+        vpand   ymm9,ymm9,ymm6
+        vmovdqu ymm2,YMMWORD[((64-128))+rdi]
+        vpand   ymm10,ymm10,ymm6
+        vmovdqu ymm5,YMMWORD[((96-128))+rdi]
+        vpand   ymm11,ymm11,ymm6
+        vpaddd  ymm8,ymm8,ymm0
+        vmovdqu ymm0,YMMWORD[((128-128))+rdi]
+        vpand   ymm12,ymm12,ymm6
+        vpaddd  ymm9,ymm9,ymm1
+        vmovdqu ymm1,YMMWORD[((160-128))+rdi]
+        vpand   ymm13,ymm13,ymm6
+        vpaddd  ymm10,ymm10,ymm2
+        vmovdqu ymm2,YMMWORD[((192-128))+rdi]
+        vpand   ymm14,ymm14,ymm6
+        vpaddd  ymm11,ymm11,ymm5
+        vmovdqu ymm5,YMMWORD[((224-128))+rdi]
+        vpand   ymm15,ymm15,ymm6
+        vpaddd  ymm12,ymm12,ymm0
+        vpaddd  ymm13,ymm13,ymm1
+        vmovdqu YMMWORD[(0-128)+rdi],ymm8
+        vpaddd  ymm14,ymm14,ymm2
+        vmovdqu YMMWORD[(32-128)+rdi],ymm9
+        vpaddd  ymm15,ymm15,ymm5
+        vmovdqu YMMWORD[(64-128)+rdi],ymm10
+        vmovdqu YMMWORD[(96-128)+rdi],ymm11
+        vmovdqu YMMWORD[(128-128)+rdi],ymm12
+        vmovdqu YMMWORD[(160-128)+rdi],ymm13
+        vmovdqu YMMWORD[(192-128)+rdi],ymm14
+        vmovdqu YMMWORD[(224-128)+rdi],ymm15
+
+        vmovdqu YMMWORD[rbx],ymm7
+        lea     rbx,[((256+128))+rsp]
+        vmovdqu ymm6,YMMWORD[$L$pbswap]
+        dec     edx
+        jnz     NEAR $L$oop_avx2
+
+
+
+
+
+
+
+$L$done_avx2:
+        mov     rax,QWORD[544+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((-216))+rax]
+        movaps  xmm7,XMMWORD[((-200))+rax]
+        movaps  xmm8,XMMWORD[((-184))+rax]
+        movaps  xmm9,XMMWORD[((-168))+rax]
+        movaps  xmm10,XMMWORD[((-152))+rax]
+        movaps  xmm11,XMMWORD[((-136))+rax]
+        movaps  xmm12,XMMWORD[((-120))+rax]
+        movaps  xmm13,XMMWORD[((-104))+rax]
+        movaps  xmm14,XMMWORD[((-88))+rax]
+        movaps  xmm15,XMMWORD[((-72))+rax]
+        mov     r15,QWORD[((-48))+rax]
+
+        mov     r14,QWORD[((-40))+rax]
+
+        mov     r13,QWORD[((-32))+rax]
+
+        mov     r12,QWORD[((-24))+rax]
+
+        mov     rbp,QWORD[((-16))+rax]
+
+        mov     rbx,QWORD[((-8))+rax]
+
+        lea     rsp,[rax]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha256_multi_block_avx2:
 ALIGN   256
 K256:
@@ -3444 +8204 @@
         DB      0F3h,0C3h               ;repret
 
+
+ALIGN   16
+avx2_handler:
+        push    rsi
+        push    rdi
+        push    rbx
+        push    rbp
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        pushfq
+        sub     rsp,64
+
+        mov     rax,QWORD[120+r8]
+        mov     rbx,QWORD[248+r8]
+
+        mov     rsi,QWORD[8+r9]
+        mov     r11,QWORD[56+r9]
+
+        mov     r10d,DWORD[r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jb      NEAR $L$in_prologue
+
+        mov     rax,QWORD[152+r8]
+
+        mov     r10d,DWORD[4+r11]
+        lea     r10,[r10*1+rsi]
+        cmp     rbx,r10
+        jae     NEAR $L$in_prologue
+
+        mov     rax,QWORD[544+r8]
+
+        mov     rbx,QWORD[((-8))+rax]
+        mov     rbp,QWORD[((-16))+rax]
+        mov     r12,QWORD[((-24))+rax]
+        mov     r13,QWORD[((-32))+rax]
+        mov     r14,QWORD[((-40))+rax]
+        mov     r15,QWORD[((-48))+rax]
+        mov     QWORD[144+r8],rbx
+        mov     QWORD[160+r8],rbp
+        mov     QWORD[216+r8],r12
+        mov     QWORD[224+r8],r13
+        mov     QWORD[232+r8],r14
+        mov     QWORD[240+r8],r15
+
+        lea     rsi,[((-56-160))+rax]
+        lea     rdi,[512+r8]
+        mov     ecx,20
+        DD      0xa548f3fc
+
+        jmp     NEAR $L$in_prologue
+
 section .pdata rdata align=4
 ALIGN   4
@@ -3452 +8266 @@
         DD      $L$SEH_end_sha256_multi_block_shaext wrt ..imagebase
         DD      $L$SEH_info_sha256_multi_block_shaext wrt ..imagebase
+        DD      $L$SEH_begin_sha256_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_end_sha256_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_info_sha256_multi_block_avx wrt ..imagebase
+        DD      $L$SEH_begin_sha256_multi_block_avx2 wrt ..imagebase
+        DD      $L$SEH_end_sha256_multi_block_avx2 wrt ..imagebase
+        DD      $L$SEH_info_sha256_multi_block_avx2 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -3462 +8282 @@
         DD      se_handler wrt ..imagebase
         DD      $L$body_shaext wrt ..imagebase,$L$epilogue_shaext wrt ..imagebase
+$L$SEH_info_sha256_multi_block_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$body_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_sha256_multi_block_avx2:
+DB      9,0,0,0
+        DD      avx2_handler wrt ..imagebase
+        DD      $L$body_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/sha256-x86_64.S

@@ -27 +27 @@
         test    r11d,536870912
         jnz     NEAR _shaext_shortcut
+        and     r11d,296
+        cmp     r11d,296
+        je      NEAR $L$avx2_shortcut
+        and     r9d,1073741824
+        and     r10d,268435968
+        or      r10d,r9d
+        cmp     r10d,1342177792
+        je      NEAR $L$avx_shortcut
         test    r10d,512
         jnz     NEAR $L$ssse3_shortcut
@@ -3150 +3158 @@
 
 $L$SEH_end_sha256_block_data_order_ssse3:
+
+ALIGN   64
+sha256_block_data_order_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha256_block_data_order_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$avx_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        shl     rdx,4
+        sub     rsp,160
+        lea     rdx,[rdx*4+rsi]
+        and     rsp,-64
+        mov     QWORD[((64+0))+rsp],rdi
+        mov     QWORD[((64+8))+rsp],rsi
+        mov     QWORD[((64+16))+rsp],rdx
+        mov     QWORD[88+rsp],rax
+
+        movaps  XMMWORD[(64+32)+rsp],xmm6
+        movaps  XMMWORD[(64+48)+rsp],xmm7
+        movaps  XMMWORD[(64+64)+rsp],xmm8
+        movaps  XMMWORD[(64+80)+rsp],xmm9
+$L$prologue_avx:
+
+        vzeroupper
+        mov     eax,DWORD[rdi]
+        mov     ebx,DWORD[4+rdi]
+        mov     ecx,DWORD[8+rdi]
+        mov     edx,DWORD[12+rdi]
+        mov     r8d,DWORD[16+rdi]
+        mov     r9d,DWORD[20+rdi]
+        mov     r10d,DWORD[24+rdi]
+        mov     r11d,DWORD[28+rdi]
+        vmovdqa xmm8,XMMWORD[((K256+512+32))]
+        vmovdqa xmm9,XMMWORD[((K256+512+64))]
+        jmp     NEAR $L$loop_avx
+ALIGN   16
+$L$loop_avx:
+        vmovdqa xmm7,XMMWORD[((K256+512))]
+        vmovdqu xmm0,XMMWORD[rsi]
+        vmovdqu xmm1,XMMWORD[16+rsi]
+        vmovdqu xmm2,XMMWORD[32+rsi]
+        vmovdqu xmm3,XMMWORD[48+rsi]
+        vpshufb xmm0,xmm0,xmm7
+        lea     rbp,[K256]
+        vpshufb xmm1,xmm1,xmm7
+        vpshufb xmm2,xmm2,xmm7
+        vpaddd  xmm4,xmm0,XMMWORD[rbp]
+        vpshufb xmm3,xmm3,xmm7
+        vpaddd  xmm5,xmm1,XMMWORD[32+rbp]
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        vpaddd  xmm7,xmm3,XMMWORD[96+rbp]
+        vmovdqa XMMWORD[rsp],xmm4
+        mov     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm5
+        mov     edi,ebx
+        vmovdqa XMMWORD[32+rsp],xmm6
+        xor     edi,ecx
+        vmovdqa XMMWORD[48+rsp],xmm7
+        mov     r13d,r8d
+        jmp     NEAR $L$avx_00_47
+
+ALIGN   16
+$L$avx_00_47:
+        sub     rbp,-128
+        vpalignr        xmm4,xmm1,xmm0,4
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        vpalignr        xmm7,xmm3,xmm2,4
+        shrd    r14d,r14d,9
+        xor     r13d,r8d
+        xor     r12d,r10d
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpaddd  xmm0,xmm0,xmm7
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+        vpsrld  xmm7,xmm4,3
+        xor     r12d,r10d
+        shrd    r14d,r14d,11
+        xor     r15d,ebx
+        vpslld  xmm5,xmm4,14
+        add     r11d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     edi,ebx
+        vpshufd xmm7,xmm3,250
+        shrd    r14d,r14d,2
+        add     edx,r11d
+        add     r11d,edi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     r11d,r14d
+        mov     r12d,r8d
+        shrd    r14d,r14d,9
+        vpslld  xmm5,xmm5,11
+        xor     r13d,edx
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,r11d
+        and     r12d,edx
+        xor     r13d,edx
+        vpsrld  xmm6,xmm7,10
+        add     r10d,DWORD[4+rsp]
+        mov     edi,r11d
+        xor     r12d,r9d
+        vpxor   xmm4,xmm4,xmm5
+        shrd    r14d,r14d,11
+        xor     edi,eax
+        add     r10d,r12d
+        vpsrlq  xmm7,xmm7,17
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r11d
+        vpaddd  xmm0,xmm0,xmm4
+        add     r10d,r13d
+        xor     r15d,eax
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     ecx,r10d
+        add     r10d,r15d
+        mov     r13d,ecx
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,edx
+        shrd    r14d,r14d,9
+        xor     r13d,ecx
+        vpshufb xmm6,xmm6,xmm8
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        vpaddd  xmm0,xmm0,xmm6
+        and     r12d,ecx
+        xor     r13d,ecx
+        add     r9d,DWORD[8+rsp]
+        vpshufd xmm7,xmm0,80
+        mov     r15d,r10d
+        xor     r12d,r8d
+        shrd    r14d,r14d,11
+        vpsrld  xmm6,xmm7,10
+        xor     r15d,r11d
+        add     r9d,r12d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        and     edi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        vpxor   xmm6,xmm6,xmm7
+        xor     edi,r11d
+        shrd    r14d,r14d,2
+        add     ebx,r9d
+        vpsrlq  xmm7,xmm7,2
+        add     r9d,edi
+        mov     r13d,ebx
+        add     r14d,r9d
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        vpshufb xmm6,xmm6,xmm9
+        shrd    r14d,r14d,9
+        xor     r13d,ebx
+        xor     r12d,edx
+        vpaddd  xmm0,xmm0,xmm6
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vpaddd  xmm6,xmm0,XMMWORD[rbp]
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     edi,r9d
+        xor     r12d,edx
+        shrd    r14d,r14d,11
+        xor     edi,r10d
+        add     r8d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        shrd    r14d,r14d,2
+        add     eax,r8d
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[rsp],xmm6
+        vpalignr        xmm4,xmm2,xmm1,4
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        vpalignr        xmm7,xmm0,xmm3,4
+        shrd    r14d,r14d,9
+        xor     r13d,eax
+        xor     r12d,ecx
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpaddd  xmm1,xmm1,xmm7
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+        vpsrld  xmm7,xmm4,3
+        xor     r12d,ecx
+        shrd    r14d,r14d,11
+        xor     r15d,r9d
+        vpslld  xmm5,xmm4,14
+        add     edx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     edi,r9d
+        vpshufd xmm7,xmm0,250
+        shrd    r14d,r14d,2
+        add     r11d,edx
+        add     edx,edi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     edx,r14d
+        mov     r12d,eax
+        shrd    r14d,r14d,9
+        vpslld  xmm5,xmm5,11
+        xor     r13d,r11d
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,edx
+        and     r12d,r11d
+        xor     r13d,r11d
+        vpsrld  xmm6,xmm7,10
+        add     ecx,DWORD[20+rsp]
+        mov     edi,edx
+        xor     r12d,ebx
+        vpxor   xmm4,xmm4,xmm5
+        shrd    r14d,r14d,11
+        xor     edi,r8d
+        add     ecx,r12d
+        vpsrlq  xmm7,xmm7,17
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,edx
+        vpaddd  xmm1,xmm1,xmm4
+        add     ecx,r13d
+        xor     r15d,r8d
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     r10d,ecx
+        add     ecx,r15d
+        mov     r13d,r10d
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,r11d
+        shrd    r14d,r14d,9
+        xor     r13d,r10d
+        vpshufb xmm6,xmm6,xmm8
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        vpaddd  xmm1,xmm1,xmm6
+        and     r12d,r10d
+        xor     r13d,r10d
+        add     ebx,DWORD[24+rsp]
+        vpshufd xmm7,xmm1,80
+        mov     r15d,ecx
+        xor     r12d,eax
+        shrd    r14d,r14d,11
+        vpsrld  xmm6,xmm7,10
+        xor     r15d,edx
+        add     ebx,r12d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        and     edi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        vpxor   xmm6,xmm6,xmm7
+        xor     edi,edx
+        shrd    r14d,r14d,2
+        add     r9d,ebx
+        vpsrlq  xmm7,xmm7,2
+        add     ebx,edi
+        mov     r13d,r9d
+        add     r14d,ebx
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        vpshufb xmm6,xmm6,xmm9
+        shrd    r14d,r14d,9
+        xor     r13d,r9d
+        xor     r12d,r11d
+        vpaddd  xmm1,xmm1,xmm6
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpaddd  xmm6,xmm1,XMMWORD[32+rbp]
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     edi,ebx
+        xor     r12d,r11d
+        shrd    r14d,r14d,11
+        xor     edi,ecx
+        add     eax,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        shrd    r14d,r14d,2
+        add     r8d,eax
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[16+rsp],xmm6
+        vpalignr        xmm4,xmm3,xmm2,4
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        vpalignr        xmm7,xmm1,xmm0,4
+        shrd    r14d,r14d,9
+        xor     r13d,r8d
+        xor     r12d,r10d
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        vpaddd  xmm2,xmm2,xmm7
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+        vpsrld  xmm7,xmm4,3
+        xor     r12d,r10d
+        shrd    r14d,r14d,11
+        xor     r15d,ebx
+        vpslld  xmm5,xmm4,14
+        add     r11d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     edi,ebx
+        vpshufd xmm7,xmm1,250
+        shrd    r14d,r14d,2
+        add     edx,r11d
+        add     r11d,edi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     r11d,r14d
+        mov     r12d,r8d
+        shrd    r14d,r14d,9
+        vpslld  xmm5,xmm5,11
+        xor     r13d,edx
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,r11d
+        and     r12d,edx
+        xor     r13d,edx
+        vpsrld  xmm6,xmm7,10
+        add     r10d,DWORD[36+rsp]
+        mov     edi,r11d
+        xor     r12d,r9d
+        vpxor   xmm4,xmm4,xmm5
+        shrd    r14d,r14d,11
+        xor     edi,eax
+        add     r10d,r12d
+        vpsrlq  xmm7,xmm7,17
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r11d
+        vpaddd  xmm2,xmm2,xmm4
+        add     r10d,r13d
+        xor     r15d,eax
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     ecx,r10d
+        add     r10d,r15d
+        mov     r13d,ecx
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,edx
+        shrd    r14d,r14d,9
+        xor     r13d,ecx
+        vpshufb xmm6,xmm6,xmm8
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        vpaddd  xmm2,xmm2,xmm6
+        and     r12d,ecx
+        xor     r13d,ecx
+        add     r9d,DWORD[40+rsp]
+        vpshufd xmm7,xmm2,80
+        mov     r15d,r10d
+        xor     r12d,r8d
+        shrd    r14d,r14d,11
+        vpsrld  xmm6,xmm7,10
+        xor     r15d,r11d
+        add     r9d,r12d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        and     edi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        vpxor   xmm6,xmm6,xmm7
+        xor     edi,r11d
+        shrd    r14d,r14d,2
+        add     ebx,r9d
+        vpsrlq  xmm7,xmm7,2
+        add     r9d,edi
+        mov     r13d,ebx
+        add     r14d,r9d
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        vpshufb xmm6,xmm6,xmm9
+        shrd    r14d,r14d,9
+        xor     r13d,ebx
+        xor     r12d,edx
+        vpaddd  xmm2,xmm2,xmm6
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        vpaddd  xmm6,xmm2,XMMWORD[64+rbp]
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     edi,r9d
+        xor     r12d,edx
+        shrd    r14d,r14d,11
+        xor     edi,r10d
+        add     r8d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        shrd    r14d,r14d,2
+        add     eax,r8d
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        vmovdqa XMMWORD[32+rsp],xmm6
+        vpalignr        xmm4,xmm0,xmm3,4
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        vpalignr        xmm7,xmm2,xmm1,4
+        shrd    r14d,r14d,9
+        xor     r13d,eax
+        xor     r12d,ecx
+        vpsrld  xmm6,xmm4,7
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        vpaddd  xmm3,xmm3,xmm7
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+        vpsrld  xmm7,xmm4,3
+        xor     r12d,ecx
+        shrd    r14d,r14d,11
+        xor     r15d,r9d
+        vpslld  xmm5,xmm4,14
+        add     edx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        vpxor   xmm4,xmm7,xmm6
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     edi,r9d
+        vpshufd xmm7,xmm2,250
+        shrd    r14d,r14d,2
+        add     r11d,edx
+        add     edx,edi
+        vpsrld  xmm6,xmm6,11
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        vpxor   xmm4,xmm4,xmm5
+        mov     edx,r14d
+        mov     r12d,eax
+        shrd    r14d,r14d,9
+        vpslld  xmm5,xmm5,11
+        xor     r13d,r11d
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        vpxor   xmm4,xmm4,xmm6
+        xor     r14d,edx
+        and     r12d,r11d
+        xor     r13d,r11d
+        vpsrld  xmm6,xmm7,10
+        add     ecx,DWORD[52+rsp]
+        mov     edi,edx
+        xor     r12d,ebx
+        vpxor   xmm4,xmm4,xmm5
+        shrd    r14d,r14d,11
+        xor     edi,r8d
+        add     ecx,r12d
+        vpsrlq  xmm7,xmm7,17
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,edx
+        vpaddd  xmm3,xmm3,xmm4
+        add     ecx,r13d
+        xor     r15d,r8d
+        shrd    r14d,r14d,2
+        vpxor   xmm6,xmm6,xmm7
+        add     r10d,ecx
+        add     ecx,r15d
+        mov     r13d,r10d
+        vpsrlq  xmm7,xmm7,2
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        vpxor   xmm6,xmm6,xmm7
+        mov     r12d,r11d
+        shrd    r14d,r14d,9
+        xor     r13d,r10d
+        vpshufb xmm6,xmm6,xmm8
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        vpaddd  xmm3,xmm3,xmm6
+        and     r12d,r10d
+        xor     r13d,r10d
+        add     ebx,DWORD[56+rsp]
+        vpshufd xmm7,xmm3,80
+        mov     r15d,ecx
+        xor     r12d,eax
+        shrd    r14d,r14d,11
+        vpsrld  xmm6,xmm7,10
+        xor     r15d,edx
+        add     ebx,r12d
+        shrd    r13d,r13d,6
+        vpsrlq  xmm7,xmm7,17
+        and     edi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        vpxor   xmm6,xmm6,xmm7
+        xor     edi,edx
+        shrd    r14d,r14d,2
+        add     r9d,ebx
+        vpsrlq  xmm7,xmm7,2
+        add     ebx,edi
+        mov     r13d,r9d
+        add     r14d,ebx
+        vpxor   xmm6,xmm6,xmm7
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        vpshufb xmm6,xmm6,xmm9
+        shrd    r14d,r14d,9
+        xor     r13d,r9d
+        xor     r12d,r11d
+        vpaddd  xmm3,xmm3,xmm6
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        vpaddd  xmm6,xmm3,XMMWORD[96+rbp]
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     edi,ebx
+        xor     r12d,r11d
+        shrd    r14d,r14d,11
+        xor     edi,ecx
+        add     eax,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        shrd    r14d,r14d,2
+        add     r8d,eax
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        vmovdqa XMMWORD[48+rsp],xmm6
+        cmp     BYTE[131+rbp],0
+        jne     NEAR $L$avx_00_47
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        shrd    r14d,r14d,9
+        xor     r13d,r8d
+        xor     r12d,r10d
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        xor     r13d,r8d
+        add     r11d,DWORD[rsp]
+        mov     r15d,eax
+        xor     r12d,r10d
+        shrd    r14d,r14d,11
+        xor     r15d,ebx
+        add     r11d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     edi,ebx
+        shrd    r14d,r14d,2
+        add     edx,r11d
+        add     r11d,edi
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        shrd    r14d,r14d,9
+        xor     r13d,edx
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        xor     r13d,edx
+        add     r10d,DWORD[4+rsp]
+        mov     edi,r11d
+        xor     r12d,r9d
+        shrd    r14d,r14d,11
+        xor     edi,eax
+        add     r10d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        shrd    r14d,r14d,2
+        add     ecx,r10d
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        shrd    r14d,r14d,9
+        xor     r13d,ecx
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        xor     r13d,ecx
+        add     r9d,DWORD[8+rsp]
+        mov     r15d,r10d
+        xor     r12d,r8d
+        shrd    r14d,r14d,11
+        xor     r15d,r11d
+        add     r9d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     edi,r11d
+        shrd    r14d,r14d,2
+        add     ebx,r9d
+        add     r9d,edi
+        mov     r13d,ebx
+        add     r14d,r9d
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        shrd    r14d,r14d,9
+        xor     r13d,ebx
+        xor     r12d,edx
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        xor     r13d,ebx
+        add     r8d,DWORD[12+rsp]
+        mov     edi,r9d
+        xor     r12d,edx
+        shrd    r14d,r14d,11
+        xor     edi,r10d
+        add     r8d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        shrd    r14d,r14d,2
+        add     eax,r8d
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        shrd    r14d,r14d,9
+        xor     r13d,eax
+        xor     r12d,ecx
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        xor     r13d,eax
+        add     edx,DWORD[16+rsp]
+        mov     r15d,r8d
+        xor     r12d,ecx
+        shrd    r14d,r14d,11
+        xor     r15d,r9d
+        add     edx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     edi,r9d
+        shrd    r14d,r14d,2
+        add     r11d,edx
+        add     edx,edi
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        shrd    r14d,r14d,9
+        xor     r13d,r11d
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        xor     r13d,r11d
+        add     ecx,DWORD[20+rsp]
+        mov     edi,edx
+        xor     r12d,ebx
+        shrd    r14d,r14d,11
+        xor     edi,r8d
+        add     ecx,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        shrd    r14d,r14d,2
+        add     r10d,ecx
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        shrd    r14d,r14d,9
+        xor     r13d,r10d
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        xor     r13d,r10d
+        add     ebx,DWORD[24+rsp]
+        mov     r15d,ecx
+        xor     r12d,eax
+        shrd    r14d,r14d,11
+        xor     r15d,edx
+        add     ebx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     edi,edx
+        shrd    r14d,r14d,2
+        add     r9d,ebx
+        add     ebx,edi
+        mov     r13d,r9d
+        add     r14d,ebx
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        shrd    r14d,r14d,9
+        xor     r13d,r9d
+        xor     r12d,r11d
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        xor     r13d,r9d
+        add     eax,DWORD[28+rsp]
+        mov     edi,ebx
+        xor     r12d,r11d
+        shrd    r14d,r14d,11
+        xor     edi,ecx
+        add     eax,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        shrd    r14d,r14d,2
+        add     r8d,eax
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        shrd    r13d,r13d,14
+        mov     eax,r14d
+        mov     r12d,r9d
+        shrd    r14d,r14d,9
+        xor     r13d,r8d
+        xor     r12d,r10d
+        shrd    r13d,r13d,5
+        xor     r14d,eax
+        and     r12d,r8d
+        xor     r13d,r8d
+        add     r11d,DWORD[32+rsp]
+        mov     r15d,eax
+        xor     r12d,r10d
+        shrd    r14d,r14d,11
+        xor     r15d,ebx
+        add     r11d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,eax
+        add     r11d,r13d
+        xor     edi,ebx
+        shrd    r14d,r14d,2
+        add     edx,r11d
+        add     r11d,edi
+        mov     r13d,edx
+        add     r14d,r11d
+        shrd    r13d,r13d,14
+        mov     r11d,r14d
+        mov     r12d,r8d
+        shrd    r14d,r14d,9
+        xor     r13d,edx
+        xor     r12d,r9d
+        shrd    r13d,r13d,5
+        xor     r14d,r11d
+        and     r12d,edx
+        xor     r13d,edx
+        add     r10d,DWORD[36+rsp]
+        mov     edi,r11d
+        xor     r12d,r9d
+        shrd    r14d,r14d,11
+        xor     edi,eax
+        add     r10d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r11d
+        add     r10d,r13d
+        xor     r15d,eax
+        shrd    r14d,r14d,2
+        add     ecx,r10d
+        add     r10d,r15d
+        mov     r13d,ecx
+        add     r14d,r10d
+        shrd    r13d,r13d,14
+        mov     r10d,r14d
+        mov     r12d,edx
+        shrd    r14d,r14d,9
+        xor     r13d,ecx
+        xor     r12d,r8d
+        shrd    r13d,r13d,5
+        xor     r14d,r10d
+        and     r12d,ecx
+        xor     r13d,ecx
+        add     r9d,DWORD[40+rsp]
+        mov     r15d,r10d
+        xor     r12d,r8d
+        shrd    r14d,r14d,11
+        xor     r15d,r11d
+        add     r9d,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,r10d
+        add     r9d,r13d
+        xor     edi,r11d
+        shrd    r14d,r14d,2
+        add     ebx,r9d
+        add     r9d,edi
+        mov     r13d,ebx
+        add     r14d,r9d
+        shrd    r13d,r13d,14
+        mov     r9d,r14d
+        mov     r12d,ecx
+        shrd    r14d,r14d,9
+        xor     r13d,ebx
+        xor     r12d,edx
+        shrd    r13d,r13d,5
+        xor     r14d,r9d
+        and     r12d,ebx
+        xor     r13d,ebx
+        add     r8d,DWORD[44+rsp]
+        mov     edi,r9d
+        xor     r12d,edx
+        shrd    r14d,r14d,11
+        xor     edi,r10d
+        add     r8d,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,r9d
+        add     r8d,r13d
+        xor     r15d,r10d
+        shrd    r14d,r14d,2
+        add     eax,r8d
+        add     r8d,r15d
+        mov     r13d,eax
+        add     r14d,r8d
+        shrd    r13d,r13d,14
+        mov     r8d,r14d
+        mov     r12d,ebx
+        shrd    r14d,r14d,9
+        xor     r13d,eax
+        xor     r12d,ecx
+        shrd    r13d,r13d,5
+        xor     r14d,r8d
+        and     r12d,eax
+        xor     r13d,eax
+        add     edx,DWORD[48+rsp]
+        mov     r15d,r8d
+        xor     r12d,ecx
+        shrd    r14d,r14d,11
+        xor     r15d,r9d
+        add     edx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,r8d
+        add     edx,r13d
+        xor     edi,r9d
+        shrd    r14d,r14d,2
+        add     r11d,edx
+        add     edx,edi
+        mov     r13d,r11d
+        add     r14d,edx
+        shrd    r13d,r13d,14
+        mov     edx,r14d
+        mov     r12d,eax
+        shrd    r14d,r14d,9
+        xor     r13d,r11d
+        xor     r12d,ebx
+        shrd    r13d,r13d,5
+        xor     r14d,edx
+        and     r12d,r11d
+        xor     r13d,r11d
+        add     ecx,DWORD[52+rsp]
+        mov     edi,edx
+        xor     r12d,ebx
+        shrd    r14d,r14d,11
+        xor     edi,r8d
+        add     ecx,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,edx
+        add     ecx,r13d
+        xor     r15d,r8d
+        shrd    r14d,r14d,2
+        add     r10d,ecx
+        add     ecx,r15d
+        mov     r13d,r10d
+        add     r14d,ecx
+        shrd    r13d,r13d,14
+        mov     ecx,r14d
+        mov     r12d,r11d
+        shrd    r14d,r14d,9
+        xor     r13d,r10d
+        xor     r12d,eax
+        shrd    r13d,r13d,5
+        xor     r14d,ecx
+        and     r12d,r10d
+        xor     r13d,r10d
+        add     ebx,DWORD[56+rsp]
+        mov     r15d,ecx
+        xor     r12d,eax
+        shrd    r14d,r14d,11
+        xor     r15d,edx
+        add     ebx,r12d
+        shrd    r13d,r13d,6
+        and     edi,r15d
+        xor     r14d,ecx
+        add     ebx,r13d
+        xor     edi,edx
+        shrd    r14d,r14d,2
+        add     r9d,ebx
+        add     ebx,edi
+        mov     r13d,r9d
+        add     r14d,ebx
+        shrd    r13d,r13d,14
+        mov     ebx,r14d
+        mov     r12d,r10d
+        shrd    r14d,r14d,9
+        xor     r13d,r9d
+        xor     r12d,r11d
+        shrd    r13d,r13d,5
+        xor     r14d,ebx
+        and     r12d,r9d
+        xor     r13d,r9d
+        add     eax,DWORD[60+rsp]
+        mov     edi,ebx
+        xor     r12d,r11d
+        shrd    r14d,r14d,11
+        xor     edi,ecx
+        add     eax,r12d
+        shrd    r13d,r13d,6
+        and     r15d,edi
+        xor     r14d,ebx
+        add     eax,r13d
+        xor     r15d,ecx
+        shrd    r14d,r14d,2
+        add     r8d,eax
+        add     eax,r15d
+        mov     r13d,r8d
+        add     r14d,eax
+        mov     rdi,QWORD[((64+0))+rsp]
+        mov     eax,r14d
+
+        add     eax,DWORD[rdi]
+        lea     rsi,[64+rsi]
+        add     ebx,DWORD[4+rdi]
+        add     ecx,DWORD[8+rdi]
+        add     edx,DWORD[12+rdi]
+        add     r8d,DWORD[16+rdi]
+        add     r9d,DWORD[20+rdi]
+        add     r10d,DWORD[24+rdi]
+        add     r11d,DWORD[28+rdi]
+
+        cmp     rsi,QWORD[((64+16))+rsp]
+
+        mov     DWORD[rdi],eax
+        mov     DWORD[4+rdi],ebx
+        mov     DWORD[8+rdi],ecx
+        mov     DWORD[12+rdi],edx
+        mov     DWORD[16+rdi],r8d
+        mov     DWORD[20+rdi],r9d
+        mov     DWORD[24+rdi],r10d
+        mov     DWORD[28+rdi],r11d
+        jb      NEAR $L$loop_avx
+
+        mov     rsi,QWORD[88+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((64+32))+rsp]
+        movaps  xmm7,XMMWORD[((64+48))+rsp]
+        movaps  xmm8,XMMWORD[((64+64))+rsp]
+        movaps  xmm9,XMMWORD[((64+80))+rsp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha256_block_data_order_avx:
+
+ALIGN   64
+sha256_block_data_order_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha256_block_data_order_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$avx2_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,608
+        shl     rdx,4
+        and     rsp,-256*4
+        lea     rdx,[rdx*4+rsi]
+        add     rsp,448
+        mov     QWORD[((64+0))+rsp],rdi
+        mov     QWORD[((64+8))+rsp],rsi
+        mov     QWORD[((64+16))+rsp],rdx
+        mov     QWORD[88+rsp],rax
+
+        movaps  XMMWORD[(64+32)+rsp],xmm6
+        movaps  XMMWORD[(64+48)+rsp],xmm7
+        movaps  XMMWORD[(64+64)+rsp],xmm8
+        movaps  XMMWORD[(64+80)+rsp],xmm9
+$L$prologue_avx2:
+
+        vzeroupper
+        sub     rsi,-16*4
+        mov     eax,DWORD[rdi]
+        mov     r12,rsi
+        mov     ebx,DWORD[4+rdi]
+        cmp     rsi,rdx
+        mov     ecx,DWORD[8+rdi]
+        cmove   r12,rsp
+        mov     edx,DWORD[12+rdi]
+        mov     r8d,DWORD[16+rdi]
+        mov     r9d,DWORD[20+rdi]
+        mov     r10d,DWORD[24+rdi]
+        mov     r11d,DWORD[28+rdi]
+        vmovdqa ymm8,YMMWORD[((K256+512+32))]
+        vmovdqa ymm9,YMMWORD[((K256+512+64))]
+        jmp     NEAR $L$oop_avx2
+ALIGN   16
+$L$oop_avx2:
+        vmovdqa ymm7,YMMWORD[((K256+512))]
+        vmovdqu xmm0,XMMWORD[((-64+0))+rsi]
+        vmovdqu xmm1,XMMWORD[((-64+16))+rsi]
+        vmovdqu xmm2,XMMWORD[((-64+32))+rsi]
+        vmovdqu xmm3,XMMWORD[((-64+48))+rsi]
+
+        vinserti128     ymm0,ymm0,XMMWORD[r12],1
+        vinserti128     ymm1,ymm1,XMMWORD[16+r12],1
+        vpshufb ymm0,ymm0,ymm7
+        vinserti128     ymm2,ymm2,XMMWORD[32+r12],1
+        vpshufb ymm1,ymm1,ymm7
+        vinserti128     ymm3,ymm3,XMMWORD[48+r12],1
+
+        lea     rbp,[K256]
+        vpshufb ymm2,ymm2,ymm7
+        vpaddd  ymm4,ymm0,YMMWORD[rbp]
+        vpshufb ymm3,ymm3,ymm7
+        vpaddd  ymm5,ymm1,YMMWORD[32+rbp]
+        vpaddd  ymm6,ymm2,YMMWORD[64+rbp]
+        vpaddd  ymm7,ymm3,YMMWORD[96+rbp]
+        vmovdqa YMMWORD[rsp],ymm4
+        xor     r14d,r14d
+        vmovdqa YMMWORD[32+rsp],ymm5
+        lea     rsp,[((-64))+rsp]
+        mov     edi,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        xor     edi,ecx
+        vmovdqa YMMWORD[32+rsp],ymm7
+        mov     r12d,r9d
+        sub     rbp,-16*2*4
+        jmp     NEAR $L$avx2_00_47
+
+ALIGN   16
+$L$avx2_00_47:
+        lea     rsp,[((-64))+rsp]
+        vpalignr        ymm4,ymm1,ymm0,4
+        add     r11d,DWORD[((0+128))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        vpalignr        ymm7,ymm3,ymm2,4
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        vpaddd  ymm0,ymm0,ymm7
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        vpxor   ymm4,ymm7,ymm6
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,ebx
+        vpshufd ymm7,ymm3,250
+        xor     r14d,r13d
+        lea     r11d,[rdi*1+r11]
+        mov     r12d,r8d
+        vpsrld  ymm6,ymm6,11
+        add     r10d,DWORD[((4+128))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    edi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,edx,r9d
+        xor     r13d,edi
+        rorx    r14d,edx,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     edi,r11d
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     edi,eax
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,eax
+        vpaddd  ymm0,ymm0,ymm4
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        vpxor   ymm6,ymm6,ymm7
+        add     r9d,DWORD[((8+128))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        vpshufb ymm6,ymm6,ymm8
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        vpaddd  ymm0,ymm0,ymm6
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        vpshufd ymm7,ymm0,80
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        vpsrld  ymm6,ymm7,10
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r11d
+        vpsrlq  ymm7,ymm7,17
+        xor     r14d,r13d
+        lea     r9d,[rdi*1+r9]
+        mov     r12d,ecx
+        vpxor   ymm6,ymm6,ymm7
+        add     r8d,DWORD[((12+128))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    edi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ebx,edx
+        xor     r13d,edi
+        rorx    r14d,ebx,6
+        vpshufb ymm6,ymm6,ymm9
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     edi,r9d
+        vpaddd  ymm0,ymm0,ymm6
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     edi,r10d
+        vpaddd  ymm6,ymm0,YMMWORD[rbp]
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        vpalignr        ymm4,ymm2,ymm1,4
+        add     edx,DWORD[((32+128))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        vpalignr        ymm7,ymm0,ymm3,4
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        vpaddd  ymm1,ymm1,ymm7
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        vpxor   ymm4,ymm7,ymm6
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r9d
+        vpshufd ymm7,ymm0,250
+        xor     r14d,r13d
+        lea     edx,[rdi*1+rdx]
+        mov     r12d,eax
+        vpsrld  ymm6,ymm6,11
+        add     ecx,DWORD[((36+128))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    edi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,r11d,ebx
+        xor     r13d,edi
+        rorx    r14d,r11d,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     edi,edx
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     edi,r8d
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r8d
+        vpaddd  ymm1,ymm1,ymm4
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        vpxor   ymm6,ymm6,ymm7
+        add     ebx,DWORD[((40+128))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        vpshufb ymm6,ymm6,ymm8
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        vpaddd  ymm1,ymm1,ymm6
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        vpshufd ymm7,ymm1,80
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        vpsrld  ymm6,ymm7,10
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,edx
+        vpsrlq  ymm7,ymm7,17
+        xor     r14d,r13d
+        lea     ebx,[rdi*1+rbx]
+        mov     r12d,r10d
+        vpxor   ymm6,ymm6,ymm7
+        add     eax,DWORD[((44+128))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    edi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r9d,r11d
+        xor     r13d,edi
+        rorx    r14d,r9d,6
+        vpshufb ymm6,ymm6,ymm9
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     edi,ebx
+        vpaddd  ymm1,ymm1,ymm6
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     edi,ecx
+        vpaddd  ymm6,ymm1,YMMWORD[32+rbp]
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vmovdqa YMMWORD[32+rsp],ymm6
+        lea     rsp,[((-64))+rsp]
+        vpalignr        ymm4,ymm3,ymm2,4
+        add     r11d,DWORD[((0+128))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        vpalignr        ymm7,ymm1,ymm0,4
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        vpaddd  ymm2,ymm2,ymm7
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        vpxor   ymm4,ymm7,ymm6
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,ebx
+        vpshufd ymm7,ymm1,250
+        xor     r14d,r13d
+        lea     r11d,[rdi*1+r11]
+        mov     r12d,r8d
+        vpsrld  ymm6,ymm6,11
+        add     r10d,DWORD[((4+128))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    edi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,edx,r9d
+        xor     r13d,edi
+        rorx    r14d,edx,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     edi,r11d
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     edi,eax
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,eax
+        vpaddd  ymm2,ymm2,ymm4
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        vpxor   ymm6,ymm6,ymm7
+        add     r9d,DWORD[((8+128))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        vpshufb ymm6,ymm6,ymm8
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        vpaddd  ymm2,ymm2,ymm6
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        vpshufd ymm7,ymm2,80
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        vpsrld  ymm6,ymm7,10
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r11d
+        vpsrlq  ymm7,ymm7,17
+        xor     r14d,r13d
+        lea     r9d,[rdi*1+r9]
+        mov     r12d,ecx
+        vpxor   ymm6,ymm6,ymm7
+        add     r8d,DWORD[((12+128))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    edi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,ebx,edx
+        xor     r13d,edi
+        rorx    r14d,ebx,6
+        vpshufb ymm6,ymm6,ymm9
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     edi,r9d
+        vpaddd  ymm2,ymm2,ymm6
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     edi,r10d
+        vpaddd  ymm6,ymm2,YMMWORD[64+rbp]
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        vmovdqa YMMWORD[rsp],ymm6
+        vpalignr        ymm4,ymm0,ymm3,4
+        add     edx,DWORD[((32+128))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        vpalignr        ymm7,ymm2,ymm1,4
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        vpsrld  ymm6,ymm4,7
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        vpaddd  ymm3,ymm3,ymm7
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        vpsrld  ymm7,ymm4,3
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        vpslld  ymm5,ymm4,14
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        vpxor   ymm4,ymm7,ymm6
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r9d
+        vpshufd ymm7,ymm2,250
+        xor     r14d,r13d
+        lea     edx,[rdi*1+rdx]
+        mov     r12d,eax
+        vpsrld  ymm6,ymm6,11
+        add     ecx,DWORD[((36+128))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        vpxor   ymm4,ymm4,ymm5
+        rorx    edi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        vpslld  ymm5,ymm5,11
+        andn    r12d,r11d,ebx
+        xor     r13d,edi
+        rorx    r14d,r11d,6
+        vpxor   ymm4,ymm4,ymm6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     edi,edx
+        vpsrld  ymm6,ymm7,10
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     edi,r8d
+        vpxor   ymm4,ymm4,ymm5
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        vpsrlq  ymm7,ymm7,17
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r8d
+        vpaddd  ymm3,ymm3,ymm4
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        vpxor   ymm6,ymm6,ymm7
+        add     ebx,DWORD[((40+128))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        vpshufb ymm6,ymm6,ymm8
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        vpaddd  ymm3,ymm3,ymm6
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        vpshufd ymm7,ymm3,80
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        vpsrld  ymm6,ymm7,10
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,edx
+        vpsrlq  ymm7,ymm7,17
+        xor     r14d,r13d
+        lea     ebx,[rdi*1+rbx]
+        mov     r12d,r10d
+        vpxor   ymm6,ymm6,ymm7
+        add     eax,DWORD[((44+128))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        vpsrlq  ymm7,ymm7,2
+        rorx    edi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        vpxor   ymm6,ymm6,ymm7
+        andn    r12d,r9d,r11d
+        xor     r13d,edi
+        rorx    r14d,r9d,6
+        vpshufb ymm6,ymm6,ymm9
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     edi,ebx
+        vpaddd  ymm3,ymm3,ymm6
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     edi,ecx
+        vpaddd  ymm6,ymm3,YMMWORD[96+rbp]
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        vmovdqa YMMWORD[32+rsp],ymm6
+        lea     rbp,[128+rbp]
+        cmp     BYTE[3+rbp],0
+        jne     NEAR $L$avx2_00_47
+        add     r11d,DWORD[((0+64))+rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rdi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[((4+64))+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    edi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,edi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     edi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     edi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[((8+64))+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rdi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[((12+64))+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    edi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,edi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     edi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     edi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[((32+64))+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r9d
+        xor     r14d,r13d
+        lea     edx,[rdi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[((36+64))+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    edi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,edi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     edi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     edi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[((40+64))+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,edx
+        xor     r14d,r13d
+        lea     ebx,[rdi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[((44+64))+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    edi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,edi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     edi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     edi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        add     r11d,DWORD[rsp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rdi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[4+rsp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    edi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,edi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     edi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     edi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[8+rsp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rdi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[12+rsp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    edi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,edi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     edi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     edi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[32+rsp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r9d
+        xor     r14d,r13d
+        lea     edx,[rdi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[36+rsp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    edi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,edi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     edi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     edi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[40+rsp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,edx
+        xor     r14d,r13d
+        lea     ebx,[rdi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[44+rsp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    edi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,edi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     edi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     edi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        mov     rdi,QWORD[512+rsp]
+        add     eax,r14d
+
+        lea     rbp,[448+rsp]
+
+        add     eax,DWORD[rdi]
+        add     ebx,DWORD[4+rdi]
+        add     ecx,DWORD[8+rdi]
+        add     edx,DWORD[12+rdi]
+        add     r8d,DWORD[16+rdi]
+        add     r9d,DWORD[20+rdi]
+        add     r10d,DWORD[24+rdi]
+        add     r11d,DWORD[28+rdi]
+
+        mov     DWORD[rdi],eax
+        mov     DWORD[4+rdi],ebx
+        mov     DWORD[8+rdi],ecx
+        mov     DWORD[12+rdi],edx
+        mov     DWORD[16+rdi],r8d
+        mov     DWORD[20+rdi],r9d
+        mov     DWORD[24+rdi],r10d
+        mov     DWORD[28+rdi],r11d
+
+        cmp     rsi,QWORD[80+rbp]
+        je      NEAR $L$done_avx2
+
+        xor     r14d,r14d
+        mov     edi,ebx
+        xor     edi,ecx
+        mov     r12d,r9d
+        jmp     NEAR $L$ower_avx2
+ALIGN   16
+$L$ower_avx2:
+        add     r11d,DWORD[((0+16))+rbp]
+        and     r12d,r8d
+        rorx    r13d,r8d,25
+        rorx    r15d,r8d,11
+        lea     eax,[r14*1+rax]
+        lea     r11d,[r12*1+r11]
+        andn    r12d,r8d,r10d
+        xor     r13d,r15d
+        rorx    r14d,r8d,6
+        lea     r11d,[r12*1+r11]
+        xor     r13d,r14d
+        mov     r15d,eax
+        rorx    r12d,eax,22
+        lea     r11d,[r13*1+r11]
+        xor     r15d,ebx
+        rorx    r14d,eax,13
+        rorx    r13d,eax,2
+        lea     edx,[r11*1+rdx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,ebx
+        xor     r14d,r13d
+        lea     r11d,[rdi*1+r11]
+        mov     r12d,r8d
+        add     r10d,DWORD[((4+16))+rbp]
+        and     r12d,edx
+        rorx    r13d,edx,25
+        rorx    edi,edx,11
+        lea     r11d,[r14*1+r11]
+        lea     r10d,[r12*1+r10]
+        andn    r12d,edx,r9d
+        xor     r13d,edi
+        rorx    r14d,edx,6
+        lea     r10d,[r12*1+r10]
+        xor     r13d,r14d
+        mov     edi,r11d
+        rorx    r12d,r11d,22
+        lea     r10d,[r13*1+r10]
+        xor     edi,eax
+        rorx    r14d,r11d,13
+        rorx    r13d,r11d,2
+        lea     ecx,[r10*1+rcx]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,eax
+        xor     r14d,r13d
+        lea     r10d,[r15*1+r10]
+        mov     r12d,edx
+        add     r9d,DWORD[((8+16))+rbp]
+        and     r12d,ecx
+        rorx    r13d,ecx,25
+        rorx    r15d,ecx,11
+        lea     r10d,[r14*1+r10]
+        lea     r9d,[r12*1+r9]
+        andn    r12d,ecx,r8d
+        xor     r13d,r15d
+        rorx    r14d,ecx,6
+        lea     r9d,[r12*1+r9]
+        xor     r13d,r14d
+        mov     r15d,r10d
+        rorx    r12d,r10d,22
+        lea     r9d,[r13*1+r9]
+        xor     r15d,r11d
+        rorx    r14d,r10d,13
+        rorx    r13d,r10d,2
+        lea     ebx,[r9*1+rbx]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r11d
+        xor     r14d,r13d
+        lea     r9d,[rdi*1+r9]
+        mov     r12d,ecx
+        add     r8d,DWORD[((12+16))+rbp]
+        and     r12d,ebx
+        rorx    r13d,ebx,25
+        rorx    edi,ebx,11
+        lea     r9d,[r14*1+r9]
+        lea     r8d,[r12*1+r8]
+        andn    r12d,ebx,edx
+        xor     r13d,edi
+        rorx    r14d,ebx,6
+        lea     r8d,[r12*1+r8]
+        xor     r13d,r14d
+        mov     edi,r9d
+        rorx    r12d,r9d,22
+        lea     r8d,[r13*1+r8]
+        xor     edi,r10d
+        rorx    r14d,r9d,13
+        rorx    r13d,r9d,2
+        lea     eax,[r8*1+rax]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r10d
+        xor     r14d,r13d
+        lea     r8d,[r15*1+r8]
+        mov     r12d,ebx
+        add     edx,DWORD[((32+16))+rbp]
+        and     r12d,eax
+        rorx    r13d,eax,25
+        rorx    r15d,eax,11
+        lea     r8d,[r14*1+r8]
+        lea     edx,[r12*1+rdx]
+        andn    r12d,eax,ecx
+        xor     r13d,r15d
+        rorx    r14d,eax,6
+        lea     edx,[r12*1+rdx]
+        xor     r13d,r14d
+        mov     r15d,r8d
+        rorx    r12d,r8d,22
+        lea     edx,[r13*1+rdx]
+        xor     r15d,r9d
+        rorx    r14d,r8d,13
+        rorx    r13d,r8d,2
+        lea     r11d,[rdx*1+r11]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,r9d
+        xor     r14d,r13d
+        lea     edx,[rdi*1+rdx]
+        mov     r12d,eax
+        add     ecx,DWORD[((36+16))+rbp]
+        and     r12d,r11d
+        rorx    r13d,r11d,25
+        rorx    edi,r11d,11
+        lea     edx,[r14*1+rdx]
+        lea     ecx,[r12*1+rcx]
+        andn    r12d,r11d,ebx
+        xor     r13d,edi
+        rorx    r14d,r11d,6
+        lea     ecx,[r12*1+rcx]
+        xor     r13d,r14d
+        mov     edi,edx
+        rorx    r12d,edx,22
+        lea     ecx,[r13*1+rcx]
+        xor     edi,r8d
+        rorx    r14d,edx,13
+        rorx    r13d,edx,2
+        lea     r10d,[rcx*1+r10]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,r8d
+        xor     r14d,r13d
+        lea     ecx,[r15*1+rcx]
+        mov     r12d,r11d
+        add     ebx,DWORD[((40+16))+rbp]
+        and     r12d,r10d
+        rorx    r13d,r10d,25
+        rorx    r15d,r10d,11
+        lea     ecx,[r14*1+rcx]
+        lea     ebx,[r12*1+rbx]
+        andn    r12d,r10d,eax
+        xor     r13d,r15d
+        rorx    r14d,r10d,6
+        lea     ebx,[r12*1+rbx]
+        xor     r13d,r14d
+        mov     r15d,ecx
+        rorx    r12d,ecx,22
+        lea     ebx,[r13*1+rbx]
+        xor     r15d,edx
+        rorx    r14d,ecx,13
+        rorx    r13d,ecx,2
+        lea     r9d,[rbx*1+r9]
+        and     edi,r15d
+        xor     r14d,r12d
+        xor     edi,edx
+        xor     r14d,r13d
+        lea     ebx,[rdi*1+rbx]
+        mov     r12d,r10d
+        add     eax,DWORD[((44+16))+rbp]
+        and     r12d,r9d
+        rorx    r13d,r9d,25
+        rorx    edi,r9d,11
+        lea     ebx,[r14*1+rbx]
+        lea     eax,[r12*1+rax]
+        andn    r12d,r9d,r11d
+        xor     r13d,edi
+        rorx    r14d,r9d,6
+        lea     eax,[r12*1+rax]
+        xor     r13d,r14d
+        mov     edi,ebx
+        rorx    r12d,ebx,22
+        lea     eax,[r13*1+rax]
+        xor     edi,ecx
+        rorx    r14d,ebx,13
+        rorx    r13d,ebx,2
+        lea     r8d,[rax*1+r8]
+        and     r15d,edi
+        xor     r14d,r12d
+        xor     r15d,ecx
+        xor     r14d,r13d
+        lea     eax,[r15*1+rax]
+        mov     r12d,r9d
+        lea     rbp,[((-64))+rbp]
+        cmp     rbp,rsp
+        jae     NEAR $L$ower_avx2
+
+        mov     rdi,QWORD[512+rsp]
+        add     eax,r14d
+
+        lea     rsp,[448+rsp]
+
+
+
+        add     eax,DWORD[rdi]
+        add     ebx,DWORD[4+rdi]
+        add     ecx,DWORD[8+rdi]
+        add     edx,DWORD[12+rdi]
+        add     r8d,DWORD[16+rdi]
+        add     r9d,DWORD[20+rdi]
+        lea     rsi,[128+rsi]
+        add     r10d,DWORD[24+rdi]
+        mov     r12,rsi
+        add     r11d,DWORD[28+rdi]
+        cmp     rsi,QWORD[((64+16))+rsp]
+
+        mov     DWORD[rdi],eax
+        cmove   r12,rsp
+        mov     DWORD[4+rdi],ebx
+        mov     DWORD[8+rdi],ecx
+        mov     DWORD[12+rdi],edx
+        mov     DWORD[16+rdi],r8d
+        mov     DWORD[20+rdi],r9d
+        mov     DWORD[24+rdi],r10d
+        mov     DWORD[28+rdi],r11d
+
+        jbe     NEAR $L$oop_avx2
+        lea     rbp,[rsp]
+
+
+
+
+$L$done_avx2:
+        mov     rsi,QWORD[88+rbp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((64+32))+rbp]
+        movaps  xmm7,XMMWORD[((64+48))+rbp]
+        movaps  xmm8,XMMWORD[((64+64))+rbp]
+        movaps  xmm9,XMMWORD[((64+80))+rbp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha256_block_data_order_avx2:
 EXTERN  __imp_RtlVirtualUnwind
 
@@ -3182 +5569 @@
         cmp     rbx,r10
         jae     NEAR $L$in_prologue
+        lea     r10,[$L$avx2_shortcut]
+        cmp     rbx,r10
+        jb      NEAR $L$not_in_avx2
+
+        and     rax,-256*4
+        add     rax,448
+$L$not_in_avx2:
         mov     rsi,rax
         mov     rax,QWORD[((64+24))+rax]
@@ -3289 +5683 @@
         DD      $L$SEH_end_sha256_block_data_order_ssse3 wrt ..imagebase
         DD      $L$SEH_info_sha256_block_data_order_ssse3 wrt ..imagebase
+        DD      $L$SEH_begin_sha256_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_end_sha256_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_info_sha256_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_begin_sha256_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_end_sha256_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_info_sha256_block_data_order_avx2 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -3302 +5702 @@
         DD      se_handler wrt ..imagebase
         DD      $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase
+$L$SEH_info_sha256_block_data_order_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_sha256_block_data_order_avx2:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/sha512-x86_64.S

@@ -21 +21 @@
 
 
+        lea     r11,[OPENSSL_ia32cap_P]
+        mov     r9d,DWORD[r11]
+        mov     r10d,DWORD[4+r11]
+        mov     r11d,DWORD[8+r11]
+        test    r10d,2048
+        jnz     NEAR $L$xop_shortcut
+        and     r11d,296
+        cmp     r11d,296
+        je      NEAR $L$avx2_shortcut
+        and     r9d,1073741824
+        and     r10d,268435968
+        or      r10d,r9d
+        cmp     r10d,1342177792
+        je      NEAR $L$avx_shortcut
         mov     rax,rsp
 
@@ -1820 +1834 @@
 DB      32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46
 DB      111,114,103,62,0
+
+ALIGN   64
+sha512_block_data_order_xop:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha512_block_data_order_xop:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$xop_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        shl     rdx,4
+        sub     rsp,256
+        lea     rdx,[rdx*8+rsi]
+        and     rsp,-64
+        mov     QWORD[((128+0))+rsp],rdi
+        mov     QWORD[((128+8))+rsp],rsi
+        mov     QWORD[((128+16))+rsp],rdx
+        mov     QWORD[152+rsp],rax
+
+        movaps  XMMWORD[(128+32)+rsp],xmm6
+        movaps  XMMWORD[(128+48)+rsp],xmm7
+        movaps  XMMWORD[(128+64)+rsp],xmm8
+        movaps  XMMWORD[(128+80)+rsp],xmm9
+        movaps  XMMWORD[(128+96)+rsp],xmm10
+        movaps  XMMWORD[(128+112)+rsp],xmm11
+$L$prologue_xop:
+
+        vzeroupper
+        mov     rax,QWORD[rdi]
+        mov     rbx,QWORD[8+rdi]
+        mov     rcx,QWORD[16+rdi]
+        mov     rdx,QWORD[24+rdi]
+        mov     r8,QWORD[32+rdi]
+        mov     r9,QWORD[40+rdi]
+        mov     r10,QWORD[48+rdi]
+        mov     r11,QWORD[56+rdi]
+        jmp     NEAR $L$loop_xop
+ALIGN   16
+$L$loop_xop:
+        vmovdqa xmm11,XMMWORD[((K512+1280))]
+        vmovdqu xmm0,XMMWORD[rsi]
+        lea     rbp,[((K512+128))]
+        vmovdqu xmm1,XMMWORD[16+rsi]
+        vmovdqu xmm2,XMMWORD[32+rsi]
+        vpshufb xmm0,xmm0,xmm11
+        vmovdqu xmm3,XMMWORD[48+rsi]
+        vpshufb xmm1,xmm1,xmm11
+        vmovdqu xmm4,XMMWORD[64+rsi]
+        vpshufb xmm2,xmm2,xmm11
+        vmovdqu xmm5,XMMWORD[80+rsi]
+        vpshufb xmm3,xmm3,xmm11
+        vmovdqu xmm6,XMMWORD[96+rsi]
+        vpshufb xmm4,xmm4,xmm11
+        vmovdqu xmm7,XMMWORD[112+rsi]
+        vpshufb xmm5,xmm5,xmm11
+        vpaddq  xmm8,xmm0,XMMWORD[((-128))+rbp]
+        vpshufb xmm6,xmm6,xmm11
+        vpaddq  xmm9,xmm1,XMMWORD[((-96))+rbp]
+        vpshufb xmm7,xmm7,xmm11
+        vpaddq  xmm10,xmm2,XMMWORD[((-64))+rbp]
+        vpaddq  xmm11,xmm3,XMMWORD[((-32))+rbp]
+        vmovdqa XMMWORD[rsp],xmm8
+        vpaddq  xmm8,xmm4,XMMWORD[rbp]
+        vmovdqa XMMWORD[16+rsp],xmm9
+        vpaddq  xmm9,xmm5,XMMWORD[32+rbp]
+        vmovdqa XMMWORD[32+rsp],xmm10
+        vpaddq  xmm10,xmm6,XMMWORD[64+rbp]
+        vmovdqa XMMWORD[48+rsp],xmm11
+        vpaddq  xmm11,xmm7,XMMWORD[96+rbp]
+        vmovdqa XMMWORD[64+rsp],xmm8
+        mov     r14,rax
+        vmovdqa XMMWORD[80+rsp],xmm9
+        mov     rdi,rbx
+        vmovdqa XMMWORD[96+rsp],xmm10
+        xor     rdi,rcx
+        vmovdqa XMMWORD[112+rsp],xmm11
+        mov     r13,r8
+        jmp     NEAR $L$xop_00_47
+
+ALIGN   16
+$L$xop_00_47:
+        add     rbp,256
+        vpalignr        xmm8,xmm1,xmm0,8
+        ror     r13,23
+        mov     rax,r14
+        vpalignr        xmm11,xmm5,xmm4,8
+        mov     r12,r9
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,r8
+        xor     r12,r10
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,rax
+        vpaddq  xmm0,xmm0,xmm11
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[rsp]
+        mov     r15,rax
+DB      143,72,120,195,209,7
+        xor     r12,r10
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,rbx
+        add     r11,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,223,3
+        xor     r14,rax
+        add     r11,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rbx
+        ror     r14,28
+        vpsrlq  xmm10,xmm7,6
+        add     rdx,r11
+        add     r11,rdi
+        vpaddq  xmm0,xmm0,xmm8
+        mov     r13,rdx
+        add     r14,r11
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     r11,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,r8
+        ror     r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        vpaddq  xmm0,xmm0,xmm11
+        add     r10,QWORD[8+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        ror     r14,6
+        vpaddq  xmm10,xmm0,XMMWORD[((-128))+rbp]
+        xor     rdi,rax
+        add     r10,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        ror     r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        vmovdqa XMMWORD[rsp],xmm10
+        vpalignr        xmm8,xmm2,xmm1,8
+        ror     r13,23
+        mov     r10,r14
+        vpalignr        xmm11,xmm6,xmm5,8
+        mov     r12,rdx
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,rcx
+        xor     r12,r8
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,r10
+        vpaddq  xmm1,xmm1,xmm11
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[16+rsp]
+        mov     r15,r10
+DB      143,72,120,195,209,7
+        xor     r12,r8
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,r11
+        add     r9,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,216,3
+        xor     r14,r10
+        add     r9,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r11
+        ror     r14,28
+        vpsrlq  xmm10,xmm0,6
+        add     rbx,r9
+        add     r9,rdi
+        vpaddq  xmm1,xmm1,xmm8
+        mov     r13,rbx
+        add     r14,r9
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     r9,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,rcx
+        ror     r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        vpaddq  xmm1,xmm1,xmm11
+        add     r8,QWORD[24+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        ror     r14,6
+        vpaddq  xmm10,xmm1,XMMWORD[((-96))+rbp]
+        xor     rdi,r10
+        add     r8,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        ror     r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        vmovdqa XMMWORD[16+rsp],xmm10
+        vpalignr        xmm8,xmm3,xmm2,8
+        ror     r13,23
+        mov     r8,r14
+        vpalignr        xmm11,xmm7,xmm6,8
+        mov     r12,rbx
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,rax
+        xor     r12,rcx
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,r8
+        vpaddq  xmm2,xmm2,xmm11
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[32+rsp]
+        mov     r15,r8
+DB      143,72,120,195,209,7
+        xor     r12,rcx
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,r9
+        add     rdx,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,217,3
+        xor     r14,r8
+        add     rdx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r9
+        ror     r14,28
+        vpsrlq  xmm10,xmm1,6
+        add     r11,rdx
+        add     rdx,rdi
+        vpaddq  xmm2,xmm2,xmm8
+        mov     r13,r11
+        add     r14,rdx
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     rdx,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,rax
+        ror     r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        vpaddq  xmm2,xmm2,xmm11
+        add     rcx,QWORD[40+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        ror     r14,6
+        vpaddq  xmm10,xmm2,XMMWORD[((-64))+rbp]
+        xor     rdi,r8
+        add     rcx,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        ror     r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        vmovdqa XMMWORD[32+rsp],xmm10
+        vpalignr        xmm8,xmm4,xmm3,8
+        ror     r13,23
+        mov     rcx,r14
+        vpalignr        xmm11,xmm0,xmm7,8
+        mov     r12,r11
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,r10
+        xor     r12,rax
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,rcx
+        vpaddq  xmm3,xmm3,xmm11
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[48+rsp]
+        mov     r15,rcx
+DB      143,72,120,195,209,7
+        xor     r12,rax
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,rdx
+        add     rbx,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,218,3
+        xor     r14,rcx
+        add     rbx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rdx
+        ror     r14,28
+        vpsrlq  xmm10,xmm2,6
+        add     r9,rbx
+        add     rbx,rdi
+        vpaddq  xmm3,xmm3,xmm8
+        mov     r13,r9
+        add     r14,rbx
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     rbx,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,r10
+        ror     r14,5
+        xor     r13,r9
+        xor     r12,r11
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        vpaddq  xmm3,xmm3,xmm11
+        add     rax,QWORD[56+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        ror     r14,6
+        vpaddq  xmm10,xmm3,XMMWORD[((-32))+rbp]
+        xor     rdi,rcx
+        add     rax,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        ror     r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        vmovdqa XMMWORD[48+rsp],xmm10
+        vpalignr        xmm8,xmm5,xmm4,8
+        ror     r13,23
+        mov     rax,r14
+        vpalignr        xmm11,xmm1,xmm0,8
+        mov     r12,r9
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,r8
+        xor     r12,r10
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,rax
+        vpaddq  xmm4,xmm4,xmm11
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[64+rsp]
+        mov     r15,rax
+DB      143,72,120,195,209,7
+        xor     r12,r10
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,rbx
+        add     r11,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,219,3
+        xor     r14,rax
+        add     r11,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rbx
+        ror     r14,28
+        vpsrlq  xmm10,xmm3,6
+        add     rdx,r11
+        add     r11,rdi
+        vpaddq  xmm4,xmm4,xmm8
+        mov     r13,rdx
+        add     r14,r11
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     r11,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,r8
+        ror     r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        vpaddq  xmm4,xmm4,xmm11
+        add     r10,QWORD[72+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        ror     r14,6
+        vpaddq  xmm10,xmm4,XMMWORD[rbp]
+        xor     rdi,rax
+        add     r10,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        ror     r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        vmovdqa XMMWORD[64+rsp],xmm10
+        vpalignr        xmm8,xmm6,xmm5,8
+        ror     r13,23
+        mov     r10,r14
+        vpalignr        xmm11,xmm2,xmm1,8
+        mov     r12,rdx
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,rcx
+        xor     r12,r8
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,r10
+        vpaddq  xmm5,xmm5,xmm11
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[80+rsp]
+        mov     r15,r10
+DB      143,72,120,195,209,7
+        xor     r12,r8
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,r11
+        add     r9,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,220,3
+        xor     r14,r10
+        add     r9,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r11
+        ror     r14,28
+        vpsrlq  xmm10,xmm4,6
+        add     rbx,r9
+        add     r9,rdi
+        vpaddq  xmm5,xmm5,xmm8
+        mov     r13,rbx
+        add     r14,r9
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     r9,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,rcx
+        ror     r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        vpaddq  xmm5,xmm5,xmm11
+        add     r8,QWORD[88+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        ror     r14,6
+        vpaddq  xmm10,xmm5,XMMWORD[32+rbp]
+        xor     rdi,r10
+        add     r8,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        ror     r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        vmovdqa XMMWORD[80+rsp],xmm10
+        vpalignr        xmm8,xmm7,xmm6,8
+        ror     r13,23
+        mov     r8,r14
+        vpalignr        xmm11,xmm3,xmm2,8
+        mov     r12,rbx
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,rax
+        xor     r12,rcx
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,r8
+        vpaddq  xmm6,xmm6,xmm11
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[96+rsp]
+        mov     r15,r8
+DB      143,72,120,195,209,7
+        xor     r12,rcx
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,r9
+        add     rdx,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,221,3
+        xor     r14,r8
+        add     rdx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r9
+        ror     r14,28
+        vpsrlq  xmm10,xmm5,6
+        add     r11,rdx
+        add     rdx,rdi
+        vpaddq  xmm6,xmm6,xmm8
+        mov     r13,r11
+        add     r14,rdx
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     rdx,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,rax
+        ror     r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        vpaddq  xmm6,xmm6,xmm11
+        add     rcx,QWORD[104+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        ror     r14,6
+        vpaddq  xmm10,xmm6,XMMWORD[64+rbp]
+        xor     rdi,r8
+        add     rcx,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        ror     r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        vmovdqa XMMWORD[96+rsp],xmm10
+        vpalignr        xmm8,xmm0,xmm7,8
+        ror     r13,23
+        mov     rcx,r14
+        vpalignr        xmm11,xmm4,xmm3,8
+        mov     r12,r11
+        ror     r14,5
+DB      143,72,120,195,200,56
+        xor     r13,r10
+        xor     r12,rax
+        vpsrlq  xmm8,xmm8,7
+        ror     r13,4
+        xor     r14,rcx
+        vpaddq  xmm7,xmm7,xmm11
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[112+rsp]
+        mov     r15,rcx
+DB      143,72,120,195,209,7
+        xor     r12,rax
+        ror     r14,6
+        vpxor   xmm8,xmm8,xmm9
+        xor     r15,rdx
+        add     rbx,r12
+        ror     r13,14
+        and     rdi,r15
+DB      143,104,120,195,222,3
+        xor     r14,rcx
+        add     rbx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rdx
+        ror     r14,28
+        vpsrlq  xmm10,xmm6,6
+        add     r9,rbx
+        add     rbx,rdi
+        vpaddq  xmm7,xmm7,xmm8
+        mov     r13,r9
+        add     r14,rbx
+DB      143,72,120,195,203,42
+        ror     r13,23
+        mov     rbx,r14
+        vpxor   xmm11,xmm11,xmm10
+        mov     r12,r10
+        ror     r14,5
+        xor     r13,r9
+        xor     r12,r11
+        vpxor   xmm11,xmm11,xmm9
+        ror     r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        vpaddq  xmm7,xmm7,xmm11
+        add     rax,QWORD[120+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        ror     r14,6
+        vpaddq  xmm10,xmm7,XMMWORD[96+rbp]
+        xor     rdi,rcx
+        add     rax,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        ror     r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        vmovdqa XMMWORD[112+rsp],xmm10
+        cmp     BYTE[135+rbp],0
+        jne     NEAR $L$xop_00_47
+        ror     r13,23
+        mov     rax,r14
+        mov     r12,r9
+        ror     r14,5
+        xor     r13,r8
+        xor     r12,r10
+        ror     r13,4
+        xor     r14,rax
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[rsp]
+        mov     r15,rax
+        xor     r12,r10
+        ror     r14,6
+        xor     r15,rbx
+        add     r11,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,rax
+        add     r11,r13
+        xor     rdi,rbx
+        ror     r14,28
+        add     rdx,r11
+        add     r11,rdi
+        mov     r13,rdx
+        add     r14,r11
+        ror     r13,23
+        mov     r11,r14
+        mov     r12,r8
+        ror     r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        ror     r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        add     r10,QWORD[8+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        ror     r14,6
+        xor     rdi,rax
+        add     r10,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        ror     r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        ror     r13,23
+        mov     r10,r14
+        mov     r12,rdx
+        ror     r14,5
+        xor     r13,rcx
+        xor     r12,r8
+        ror     r13,4
+        xor     r14,r10
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[16+rsp]
+        mov     r15,r10
+        xor     r12,r8
+        ror     r14,6
+        xor     r15,r11
+        add     r9,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,r10
+        add     r9,r13
+        xor     rdi,r11
+        ror     r14,28
+        add     rbx,r9
+        add     r9,rdi
+        mov     r13,rbx
+        add     r14,r9
+        ror     r13,23
+        mov     r9,r14
+        mov     r12,rcx
+        ror     r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        ror     r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        add     r8,QWORD[24+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        ror     r14,6
+        xor     rdi,r10
+        add     r8,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        ror     r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        ror     r13,23
+        mov     r8,r14
+        mov     r12,rbx
+        ror     r14,5
+        xor     r13,rax
+        xor     r12,rcx
+        ror     r13,4
+        xor     r14,r8
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[32+rsp]
+        mov     r15,r8
+        xor     r12,rcx
+        ror     r14,6
+        xor     r15,r9
+        add     rdx,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,r8
+        add     rdx,r13
+        xor     rdi,r9
+        ror     r14,28
+        add     r11,rdx
+        add     rdx,rdi
+        mov     r13,r11
+        add     r14,rdx
+        ror     r13,23
+        mov     rdx,r14
+        mov     r12,rax
+        ror     r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        ror     r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        add     rcx,QWORD[40+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        ror     r14,6
+        xor     rdi,r8
+        add     rcx,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        ror     r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        ror     r13,23
+        mov     rcx,r14
+        mov     r12,r11
+        ror     r14,5
+        xor     r13,r10
+        xor     r12,rax
+        ror     r13,4
+        xor     r14,rcx
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[48+rsp]
+        mov     r15,rcx
+        xor     r12,rax
+        ror     r14,6
+        xor     r15,rdx
+        add     rbx,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,rcx
+        add     rbx,r13
+        xor     rdi,rdx
+        ror     r14,28
+        add     r9,rbx
+        add     rbx,rdi
+        mov     r13,r9
+        add     r14,rbx
+        ror     r13,23
+        mov     rbx,r14
+        mov     r12,r10
+        ror     r14,5
+        xor     r13,r9
+        xor     r12,r11
+        ror     r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        add     rax,QWORD[56+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        ror     r14,6
+        xor     rdi,rcx
+        add     rax,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        ror     r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        ror     r13,23
+        mov     rax,r14
+        mov     r12,r9
+        ror     r14,5
+        xor     r13,r8
+        xor     r12,r10
+        ror     r13,4
+        xor     r14,rax
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[64+rsp]
+        mov     r15,rax
+        xor     r12,r10
+        ror     r14,6
+        xor     r15,rbx
+        add     r11,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,rax
+        add     r11,r13
+        xor     rdi,rbx
+        ror     r14,28
+        add     rdx,r11
+        add     r11,rdi
+        mov     r13,rdx
+        add     r14,r11
+        ror     r13,23
+        mov     r11,r14
+        mov     r12,r8
+        ror     r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        ror     r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        add     r10,QWORD[72+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        ror     r14,6
+        xor     rdi,rax
+        add     r10,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        ror     r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        ror     r13,23
+        mov     r10,r14
+        mov     r12,rdx
+        ror     r14,5
+        xor     r13,rcx
+        xor     r12,r8
+        ror     r13,4
+        xor     r14,r10
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[80+rsp]
+        mov     r15,r10
+        xor     r12,r8
+        ror     r14,6
+        xor     r15,r11
+        add     r9,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,r10
+        add     r9,r13
+        xor     rdi,r11
+        ror     r14,28
+        add     rbx,r9
+        add     r9,rdi
+        mov     r13,rbx
+        add     r14,r9
+        ror     r13,23
+        mov     r9,r14
+        mov     r12,rcx
+        ror     r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        ror     r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        add     r8,QWORD[88+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        ror     r14,6
+        xor     rdi,r10
+        add     r8,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        ror     r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        ror     r13,23
+        mov     r8,r14
+        mov     r12,rbx
+        ror     r14,5
+        xor     r13,rax
+        xor     r12,rcx
+        ror     r13,4
+        xor     r14,r8
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[96+rsp]
+        mov     r15,r8
+        xor     r12,rcx
+        ror     r14,6
+        xor     r15,r9
+        add     rdx,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,r8
+        add     rdx,r13
+        xor     rdi,r9
+        ror     r14,28
+        add     r11,rdx
+        add     rdx,rdi
+        mov     r13,r11
+        add     r14,rdx
+        ror     r13,23
+        mov     rdx,r14
+        mov     r12,rax
+        ror     r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        ror     r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        add     rcx,QWORD[104+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        ror     r14,6
+        xor     rdi,r8
+        add     rcx,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        ror     r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        ror     r13,23
+        mov     rcx,r14
+        mov     r12,r11
+        ror     r14,5
+        xor     r13,r10
+        xor     r12,rax
+        ror     r13,4
+        xor     r14,rcx
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[112+rsp]
+        mov     r15,rcx
+        xor     r12,rax
+        ror     r14,6
+        xor     r15,rdx
+        add     rbx,r12
+        ror     r13,14
+        and     rdi,r15
+        xor     r14,rcx
+        add     rbx,r13
+        xor     rdi,rdx
+        ror     r14,28
+        add     r9,rbx
+        add     rbx,rdi
+        mov     r13,r9
+        add     r14,rbx
+        ror     r13,23
+        mov     rbx,r14
+        mov     r12,r10
+        ror     r14,5
+        xor     r13,r9
+        xor     r12,r11
+        ror     r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        add     rax,QWORD[120+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        ror     r14,6
+        xor     rdi,rcx
+        add     rax,r12
+        ror     r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        ror     r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        mov     rdi,QWORD[((128+0))+rsp]
+        mov     rax,r14
+
+        add     rax,QWORD[rdi]
+        lea     rsi,[128+rsi]
+        add     rbx,QWORD[8+rdi]
+        add     rcx,QWORD[16+rdi]
+        add     rdx,QWORD[24+rdi]
+        add     r8,QWORD[32+rdi]
+        add     r9,QWORD[40+rdi]
+        add     r10,QWORD[48+rdi]
+        add     r11,QWORD[56+rdi]
+
+        cmp     rsi,QWORD[((128+16))+rsp]
+
+        mov     QWORD[rdi],rax
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rcx
+        mov     QWORD[24+rdi],rdx
+        mov     QWORD[32+rdi],r8
+        mov     QWORD[40+rdi],r9
+        mov     QWORD[48+rdi],r10
+        mov     QWORD[56+rdi],r11
+        jb      NEAR $L$loop_xop
+
+        mov     rsi,QWORD[152+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((128+32))+rsp]
+        movaps  xmm7,XMMWORD[((128+48))+rsp]
+        movaps  xmm8,XMMWORD[((128+64))+rsp]
+        movaps  xmm9,XMMWORD[((128+80))+rsp]
+        movaps  xmm10,XMMWORD[((128+96))+rsp]
+        movaps  xmm11,XMMWORD[((128+112))+rsp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_xop:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha512_block_data_order_xop:
+
+ALIGN   64
+sha512_block_data_order_avx:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha512_block_data_order_avx:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$avx_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        shl     rdx,4
+        sub     rsp,256
+        lea     rdx,[rdx*8+rsi]
+        and     rsp,-64
+        mov     QWORD[((128+0))+rsp],rdi
+        mov     QWORD[((128+8))+rsp],rsi
+        mov     QWORD[((128+16))+rsp],rdx
+        mov     QWORD[152+rsp],rax
+
+        movaps  XMMWORD[(128+32)+rsp],xmm6
+        movaps  XMMWORD[(128+48)+rsp],xmm7
+        movaps  XMMWORD[(128+64)+rsp],xmm8
+        movaps  XMMWORD[(128+80)+rsp],xmm9
+        movaps  XMMWORD[(128+96)+rsp],xmm10
+        movaps  XMMWORD[(128+112)+rsp],xmm11
+$L$prologue_avx:
+
+        vzeroupper
+        mov     rax,QWORD[rdi]
+        mov     rbx,QWORD[8+rdi]
+        mov     rcx,QWORD[16+rdi]
+        mov     rdx,QWORD[24+rdi]
+        mov     r8,QWORD[32+rdi]
+        mov     r9,QWORD[40+rdi]
+        mov     r10,QWORD[48+rdi]
+        mov     r11,QWORD[56+rdi]
+        jmp     NEAR $L$loop_avx
+ALIGN   16
+$L$loop_avx:
+        vmovdqa xmm11,XMMWORD[((K512+1280))]
+        vmovdqu xmm0,XMMWORD[rsi]
+        lea     rbp,[((K512+128))]
+        vmovdqu xmm1,XMMWORD[16+rsi]
+        vmovdqu xmm2,XMMWORD[32+rsi]
+        vpshufb xmm0,xmm0,xmm11
+        vmovdqu xmm3,XMMWORD[48+rsi]
+        vpshufb xmm1,xmm1,xmm11
+        vmovdqu xmm4,XMMWORD[64+rsi]
+        vpshufb xmm2,xmm2,xmm11
+        vmovdqu xmm5,XMMWORD[80+rsi]
+        vpshufb xmm3,xmm3,xmm11
+        vmovdqu xmm6,XMMWORD[96+rsi]
+        vpshufb xmm4,xmm4,xmm11
+        vmovdqu xmm7,XMMWORD[112+rsi]
+        vpshufb xmm5,xmm5,xmm11
+        vpaddq  xmm8,xmm0,XMMWORD[((-128))+rbp]
+        vpshufb xmm6,xmm6,xmm11
+        vpaddq  xmm9,xmm1,XMMWORD[((-96))+rbp]
+        vpshufb xmm7,xmm7,xmm11
+        vpaddq  xmm10,xmm2,XMMWORD[((-64))+rbp]
+        vpaddq  xmm11,xmm3,XMMWORD[((-32))+rbp]
+        vmovdqa XMMWORD[rsp],xmm8
+        vpaddq  xmm8,xmm4,XMMWORD[rbp]
+        vmovdqa XMMWORD[16+rsp],xmm9
+        vpaddq  xmm9,xmm5,XMMWORD[32+rbp]
+        vmovdqa XMMWORD[32+rsp],xmm10
+        vpaddq  xmm10,xmm6,XMMWORD[64+rbp]
+        vmovdqa XMMWORD[48+rsp],xmm11
+        vpaddq  xmm11,xmm7,XMMWORD[96+rbp]
+        vmovdqa XMMWORD[64+rsp],xmm8
+        mov     r14,rax
+        vmovdqa XMMWORD[80+rsp],xmm9
+        mov     rdi,rbx
+        vmovdqa XMMWORD[96+rsp],xmm10
+        xor     rdi,rcx
+        vmovdqa XMMWORD[112+rsp],xmm11
+        mov     r13,r8
+        jmp     NEAR $L$avx_00_47
+
+ALIGN   16
+$L$avx_00_47:
+        add     rbp,256
+        vpalignr        xmm8,xmm1,xmm0,8
+        shrd    r13,r13,23
+        mov     rax,r14
+        vpalignr        xmm11,xmm5,xmm4,8
+        mov     r12,r9
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,r8
+        xor     r12,r10
+        vpaddq  xmm0,xmm0,xmm11
+        shrd    r13,r13,4
+        xor     r14,rax
+        vpsrlq  xmm11,xmm8,7
+        and     r12,r8
+        xor     r13,r8
+        vpsllq  xmm9,xmm8,56
+        add     r11,QWORD[rsp]
+        mov     r15,rax
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,r10
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,rbx
+        add     r11,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,rax
+        add     r11,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rbx
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm7,6
+        add     rdx,r11
+        add     r11,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,rdx
+        add     r14,r11
+        vpsllq  xmm10,xmm7,3
+        shrd    r13,r13,23
+        mov     r11,r14
+        vpaddq  xmm0,xmm0,xmm8
+        mov     r12,r8
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm7,19
+        xor     r13,rdx
+        xor     r12,r9
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,r11
+        vpsllq  xmm10,xmm10,42
+        and     r12,rdx
+        xor     r13,rdx
+        vpxor   xmm11,xmm11,xmm9
+        add     r10,QWORD[8+rsp]
+        mov     rdi,r11
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,r9
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,rax
+        add     r10,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm0,xmm0,xmm11
+        xor     r14,r11
+        add     r10,r13
+        vpaddq  xmm10,xmm0,XMMWORD[((-128))+rbp]
+        xor     r15,rax
+        shrd    r14,r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        vmovdqa XMMWORD[rsp],xmm10
+        vpalignr        xmm8,xmm2,xmm1,8
+        shrd    r13,r13,23
+        mov     r10,r14
+        vpalignr        xmm11,xmm6,xmm5,8
+        mov     r12,rdx
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,rcx
+        xor     r12,r8
+        vpaddq  xmm1,xmm1,xmm11
+        shrd    r13,r13,4
+        xor     r14,r10
+        vpsrlq  xmm11,xmm8,7
+        and     r12,rcx
+        xor     r13,rcx
+        vpsllq  xmm9,xmm8,56
+        add     r9,QWORD[16+rsp]
+        mov     r15,r10
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,r8
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,r11
+        add     r9,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,r10
+        add     r9,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r11
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm0,6
+        add     rbx,r9
+        add     r9,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,rbx
+        add     r14,r9
+        vpsllq  xmm10,xmm0,3
+        shrd    r13,r13,23
+        mov     r9,r14
+        vpaddq  xmm1,xmm1,xmm8
+        mov     r12,rcx
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm0,19
+        xor     r13,rbx
+        xor     r12,rdx
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,r9
+        vpsllq  xmm10,xmm10,42
+        and     r12,rbx
+        xor     r13,rbx
+        vpxor   xmm11,xmm11,xmm9
+        add     r8,QWORD[24+rsp]
+        mov     rdi,r9
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,rdx
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,r10
+        add     r8,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm1,xmm1,xmm11
+        xor     r14,r9
+        add     r8,r13
+        vpaddq  xmm10,xmm1,XMMWORD[((-96))+rbp]
+        xor     r15,r10
+        shrd    r14,r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        vmovdqa XMMWORD[16+rsp],xmm10
+        vpalignr        xmm8,xmm3,xmm2,8
+        shrd    r13,r13,23
+        mov     r8,r14
+        vpalignr        xmm11,xmm7,xmm6,8
+        mov     r12,rbx
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,rax
+        xor     r12,rcx
+        vpaddq  xmm2,xmm2,xmm11
+        shrd    r13,r13,4
+        xor     r14,r8
+        vpsrlq  xmm11,xmm8,7
+        and     r12,rax
+        xor     r13,rax
+        vpsllq  xmm9,xmm8,56
+        add     rdx,QWORD[32+rsp]
+        mov     r15,r8
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,rcx
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,r9
+        add     rdx,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,r8
+        add     rdx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r9
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm1,6
+        add     r11,rdx
+        add     rdx,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,r11
+        add     r14,rdx
+        vpsllq  xmm10,xmm1,3
+        shrd    r13,r13,23
+        mov     rdx,r14
+        vpaddq  xmm2,xmm2,xmm8
+        mov     r12,rax
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm1,19
+        xor     r13,r11
+        xor     r12,rbx
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,rdx
+        vpsllq  xmm10,xmm10,42
+        and     r12,r11
+        xor     r13,r11
+        vpxor   xmm11,xmm11,xmm9
+        add     rcx,QWORD[40+rsp]
+        mov     rdi,rdx
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,rbx
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,r8
+        add     rcx,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm2,xmm2,xmm11
+        xor     r14,rdx
+        add     rcx,r13
+        vpaddq  xmm10,xmm2,XMMWORD[((-64))+rbp]
+        xor     r15,r8
+        shrd    r14,r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        vmovdqa XMMWORD[32+rsp],xmm10
+        vpalignr        xmm8,xmm4,xmm3,8
+        shrd    r13,r13,23
+        mov     rcx,r14
+        vpalignr        xmm11,xmm0,xmm7,8
+        mov     r12,r11
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,r10
+        xor     r12,rax
+        vpaddq  xmm3,xmm3,xmm11
+        shrd    r13,r13,4
+        xor     r14,rcx
+        vpsrlq  xmm11,xmm8,7
+        and     r12,r10
+        xor     r13,r10
+        vpsllq  xmm9,xmm8,56
+        add     rbx,QWORD[48+rsp]
+        mov     r15,rcx
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,rax
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,rdx
+        add     rbx,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,rcx
+        add     rbx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rdx
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm2,6
+        add     r9,rbx
+        add     rbx,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,r9
+        add     r14,rbx
+        vpsllq  xmm10,xmm2,3
+        shrd    r13,r13,23
+        mov     rbx,r14
+        vpaddq  xmm3,xmm3,xmm8
+        mov     r12,r10
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm2,19
+        xor     r13,r9
+        xor     r12,r11
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,rbx
+        vpsllq  xmm10,xmm10,42
+        and     r12,r9
+        xor     r13,r9
+        vpxor   xmm11,xmm11,xmm9
+        add     rax,QWORD[56+rsp]
+        mov     rdi,rbx
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,r11
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,rcx
+        add     rax,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm3,xmm3,xmm11
+        xor     r14,rbx
+        add     rax,r13
+        vpaddq  xmm10,xmm3,XMMWORD[((-32))+rbp]
+        xor     r15,rcx
+        shrd    r14,r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        vmovdqa XMMWORD[48+rsp],xmm10
+        vpalignr        xmm8,xmm5,xmm4,8
+        shrd    r13,r13,23
+        mov     rax,r14
+        vpalignr        xmm11,xmm1,xmm0,8
+        mov     r12,r9
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,r8
+        xor     r12,r10
+        vpaddq  xmm4,xmm4,xmm11
+        shrd    r13,r13,4
+        xor     r14,rax
+        vpsrlq  xmm11,xmm8,7
+        and     r12,r8
+        xor     r13,r8
+        vpsllq  xmm9,xmm8,56
+        add     r11,QWORD[64+rsp]
+        mov     r15,rax
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,r10
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,rbx
+        add     r11,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,rax
+        add     r11,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rbx
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm3,6
+        add     rdx,r11
+        add     r11,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,rdx
+        add     r14,r11
+        vpsllq  xmm10,xmm3,3
+        shrd    r13,r13,23
+        mov     r11,r14
+        vpaddq  xmm4,xmm4,xmm8
+        mov     r12,r8
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm3,19
+        xor     r13,rdx
+        xor     r12,r9
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,r11
+        vpsllq  xmm10,xmm10,42
+        and     r12,rdx
+        xor     r13,rdx
+        vpxor   xmm11,xmm11,xmm9
+        add     r10,QWORD[72+rsp]
+        mov     rdi,r11
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,r9
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,rax
+        add     r10,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm4,xmm4,xmm11
+        xor     r14,r11
+        add     r10,r13
+        vpaddq  xmm10,xmm4,XMMWORD[rbp]
+        xor     r15,rax
+        shrd    r14,r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        vmovdqa XMMWORD[64+rsp],xmm10
+        vpalignr        xmm8,xmm6,xmm5,8
+        shrd    r13,r13,23
+        mov     r10,r14
+        vpalignr        xmm11,xmm2,xmm1,8
+        mov     r12,rdx
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,rcx
+        xor     r12,r8
+        vpaddq  xmm5,xmm5,xmm11
+        shrd    r13,r13,4
+        xor     r14,r10
+        vpsrlq  xmm11,xmm8,7
+        and     r12,rcx
+        xor     r13,rcx
+        vpsllq  xmm9,xmm8,56
+        add     r9,QWORD[80+rsp]
+        mov     r15,r10
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,r8
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,r11
+        add     r9,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,r10
+        add     r9,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r11
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm4,6
+        add     rbx,r9
+        add     r9,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,rbx
+        add     r14,r9
+        vpsllq  xmm10,xmm4,3
+        shrd    r13,r13,23
+        mov     r9,r14
+        vpaddq  xmm5,xmm5,xmm8
+        mov     r12,rcx
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm4,19
+        xor     r13,rbx
+        xor     r12,rdx
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,r9
+        vpsllq  xmm10,xmm10,42
+        and     r12,rbx
+        xor     r13,rbx
+        vpxor   xmm11,xmm11,xmm9
+        add     r8,QWORD[88+rsp]
+        mov     rdi,r9
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,rdx
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,r10
+        add     r8,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm5,xmm5,xmm11
+        xor     r14,r9
+        add     r8,r13
+        vpaddq  xmm10,xmm5,XMMWORD[32+rbp]
+        xor     r15,r10
+        shrd    r14,r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        vmovdqa XMMWORD[80+rsp],xmm10
+        vpalignr        xmm8,xmm7,xmm6,8
+        shrd    r13,r13,23
+        mov     r8,r14
+        vpalignr        xmm11,xmm3,xmm2,8
+        mov     r12,rbx
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,rax
+        xor     r12,rcx
+        vpaddq  xmm6,xmm6,xmm11
+        shrd    r13,r13,4
+        xor     r14,r8
+        vpsrlq  xmm11,xmm8,7
+        and     r12,rax
+        xor     r13,rax
+        vpsllq  xmm9,xmm8,56
+        add     rdx,QWORD[96+rsp]
+        mov     r15,r8
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,rcx
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,r9
+        add     rdx,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,r8
+        add     rdx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,r9
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm5,6
+        add     r11,rdx
+        add     rdx,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,r11
+        add     r14,rdx
+        vpsllq  xmm10,xmm5,3
+        shrd    r13,r13,23
+        mov     rdx,r14
+        vpaddq  xmm6,xmm6,xmm8
+        mov     r12,rax
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm5,19
+        xor     r13,r11
+        xor     r12,rbx
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,rdx
+        vpsllq  xmm10,xmm10,42
+        and     r12,r11
+        xor     r13,r11
+        vpxor   xmm11,xmm11,xmm9
+        add     rcx,QWORD[104+rsp]
+        mov     rdi,rdx
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,rbx
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,r8
+        add     rcx,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm6,xmm6,xmm11
+        xor     r14,rdx
+        add     rcx,r13
+        vpaddq  xmm10,xmm6,XMMWORD[64+rbp]
+        xor     r15,r8
+        shrd    r14,r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        vmovdqa XMMWORD[96+rsp],xmm10
+        vpalignr        xmm8,xmm0,xmm7,8
+        shrd    r13,r13,23
+        mov     rcx,r14
+        vpalignr        xmm11,xmm4,xmm3,8
+        mov     r12,r11
+        shrd    r14,r14,5
+        vpsrlq  xmm10,xmm8,1
+        xor     r13,r10
+        xor     r12,rax
+        vpaddq  xmm7,xmm7,xmm11
+        shrd    r13,r13,4
+        xor     r14,rcx
+        vpsrlq  xmm11,xmm8,7
+        and     r12,r10
+        xor     r13,r10
+        vpsllq  xmm9,xmm8,56
+        add     rbx,QWORD[112+rsp]
+        mov     r15,rcx
+        vpxor   xmm8,xmm11,xmm10
+        xor     r12,rax
+        shrd    r14,r14,6
+        vpsrlq  xmm10,xmm10,7
+        xor     r15,rdx
+        add     rbx,r12
+        vpxor   xmm8,xmm8,xmm9
+        shrd    r13,r13,14
+        and     rdi,r15
+        vpsllq  xmm9,xmm9,7
+        xor     r14,rcx
+        add     rbx,r13
+        vpxor   xmm8,xmm8,xmm10
+        xor     rdi,rdx
+        shrd    r14,r14,28
+        vpsrlq  xmm11,xmm6,6
+        add     r9,rbx
+        add     rbx,rdi
+        vpxor   xmm8,xmm8,xmm9
+        mov     r13,r9
+        add     r14,rbx
+        vpsllq  xmm10,xmm6,3
+        shrd    r13,r13,23
+        mov     rbx,r14
+        vpaddq  xmm7,xmm7,xmm8
+        mov     r12,r10
+        shrd    r14,r14,5
+        vpsrlq  xmm9,xmm6,19
+        xor     r13,r9
+        xor     r12,r11
+        vpxor   xmm11,xmm11,xmm10
+        shrd    r13,r13,4
+        xor     r14,rbx
+        vpsllq  xmm10,xmm10,42
+        and     r12,r9
+        xor     r13,r9
+        vpxor   xmm11,xmm11,xmm9
+        add     rax,QWORD[120+rsp]
+        mov     rdi,rbx
+        vpsrlq  xmm9,xmm9,42
+        xor     r12,r11
+        shrd    r14,r14,6
+        vpxor   xmm11,xmm11,xmm10
+        xor     rdi,rcx
+        add     rax,r12
+        vpxor   xmm11,xmm11,xmm9
+        shrd    r13,r13,14
+        and     r15,rdi
+        vpaddq  xmm7,xmm7,xmm11
+        xor     r14,rbx
+        add     rax,r13
+        vpaddq  xmm10,xmm7,XMMWORD[96+rbp]
+        xor     r15,rcx
+        shrd    r14,r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        vmovdqa XMMWORD[112+rsp],xmm10
+        cmp     BYTE[135+rbp],0
+        jne     NEAR $L$avx_00_47
+        shrd    r13,r13,23
+        mov     rax,r14
+        mov     r12,r9
+        shrd    r14,r14,5
+        xor     r13,r8
+        xor     r12,r10
+        shrd    r13,r13,4
+        xor     r14,rax
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[rsp]
+        mov     r15,rax
+        xor     r12,r10
+        shrd    r14,r14,6
+        xor     r15,rbx
+        add     r11,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,rax
+        add     r11,r13
+        xor     rdi,rbx
+        shrd    r14,r14,28
+        add     rdx,r11
+        add     r11,rdi
+        mov     r13,rdx
+        add     r14,r11
+        shrd    r13,r13,23
+        mov     r11,r14
+        mov     r12,r8
+        shrd    r14,r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        shrd    r13,r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        add     r10,QWORD[8+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        shrd    r14,r14,6
+        xor     rdi,rax
+        add     r10,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        shrd    r14,r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        shrd    r13,r13,23
+        mov     r10,r14
+        mov     r12,rdx
+        shrd    r14,r14,5
+        xor     r13,rcx
+        xor     r12,r8
+        shrd    r13,r13,4
+        xor     r14,r10
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[16+rsp]
+        mov     r15,r10
+        xor     r12,r8
+        shrd    r14,r14,6
+        xor     r15,r11
+        add     r9,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,r10
+        add     r9,r13
+        xor     rdi,r11
+        shrd    r14,r14,28
+        add     rbx,r9
+        add     r9,rdi
+        mov     r13,rbx
+        add     r14,r9
+        shrd    r13,r13,23
+        mov     r9,r14
+        mov     r12,rcx
+        shrd    r14,r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        shrd    r13,r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        add     r8,QWORD[24+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        shrd    r14,r14,6
+        xor     rdi,r10
+        add     r8,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        shrd    r14,r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        shrd    r13,r13,23
+        mov     r8,r14
+        mov     r12,rbx
+        shrd    r14,r14,5
+        xor     r13,rax
+        xor     r12,rcx
+        shrd    r13,r13,4
+        xor     r14,r8
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[32+rsp]
+        mov     r15,r8
+        xor     r12,rcx
+        shrd    r14,r14,6
+        xor     r15,r9
+        add     rdx,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,r8
+        add     rdx,r13
+        xor     rdi,r9
+        shrd    r14,r14,28
+        add     r11,rdx
+        add     rdx,rdi
+        mov     r13,r11
+        add     r14,rdx
+        shrd    r13,r13,23
+        mov     rdx,r14
+        mov     r12,rax
+        shrd    r14,r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        shrd    r13,r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        add     rcx,QWORD[40+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        shrd    r14,r14,6
+        xor     rdi,r8
+        add     rcx,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        shrd    r14,r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        shrd    r13,r13,23
+        mov     rcx,r14
+        mov     r12,r11
+        shrd    r14,r14,5
+        xor     r13,r10
+        xor     r12,rax
+        shrd    r13,r13,4
+        xor     r14,rcx
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[48+rsp]
+        mov     r15,rcx
+        xor     r12,rax
+        shrd    r14,r14,6
+        xor     r15,rdx
+        add     rbx,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,rcx
+        add     rbx,r13
+        xor     rdi,rdx
+        shrd    r14,r14,28
+        add     r9,rbx
+        add     rbx,rdi
+        mov     r13,r9
+        add     r14,rbx
+        shrd    r13,r13,23
+        mov     rbx,r14
+        mov     r12,r10
+        shrd    r14,r14,5
+        xor     r13,r9
+        xor     r12,r11
+        shrd    r13,r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        add     rax,QWORD[56+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        shrd    r14,r14,6
+        xor     rdi,rcx
+        add     rax,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        shrd    r14,r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        shrd    r13,r13,23
+        mov     rax,r14
+        mov     r12,r9
+        shrd    r14,r14,5
+        xor     r13,r8
+        xor     r12,r10
+        shrd    r13,r13,4
+        xor     r14,rax
+        and     r12,r8
+        xor     r13,r8
+        add     r11,QWORD[64+rsp]
+        mov     r15,rax
+        xor     r12,r10
+        shrd    r14,r14,6
+        xor     r15,rbx
+        add     r11,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,rax
+        add     r11,r13
+        xor     rdi,rbx
+        shrd    r14,r14,28
+        add     rdx,r11
+        add     r11,rdi
+        mov     r13,rdx
+        add     r14,r11
+        shrd    r13,r13,23
+        mov     r11,r14
+        mov     r12,r8
+        shrd    r14,r14,5
+        xor     r13,rdx
+        xor     r12,r9
+        shrd    r13,r13,4
+        xor     r14,r11
+        and     r12,rdx
+        xor     r13,rdx
+        add     r10,QWORD[72+rsp]
+        mov     rdi,r11
+        xor     r12,r9
+        shrd    r14,r14,6
+        xor     rdi,rax
+        add     r10,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,r11
+        add     r10,r13
+        xor     r15,rax
+        shrd    r14,r14,28
+        add     rcx,r10
+        add     r10,r15
+        mov     r13,rcx
+        add     r14,r10
+        shrd    r13,r13,23
+        mov     r10,r14
+        mov     r12,rdx
+        shrd    r14,r14,5
+        xor     r13,rcx
+        xor     r12,r8
+        shrd    r13,r13,4
+        xor     r14,r10
+        and     r12,rcx
+        xor     r13,rcx
+        add     r9,QWORD[80+rsp]
+        mov     r15,r10
+        xor     r12,r8
+        shrd    r14,r14,6
+        xor     r15,r11
+        add     r9,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,r10
+        add     r9,r13
+        xor     rdi,r11
+        shrd    r14,r14,28
+        add     rbx,r9
+        add     r9,rdi
+        mov     r13,rbx
+        add     r14,r9
+        shrd    r13,r13,23
+        mov     r9,r14
+        mov     r12,rcx
+        shrd    r14,r14,5
+        xor     r13,rbx
+        xor     r12,rdx
+        shrd    r13,r13,4
+        xor     r14,r9
+        and     r12,rbx
+        xor     r13,rbx
+        add     r8,QWORD[88+rsp]
+        mov     rdi,r9
+        xor     r12,rdx
+        shrd    r14,r14,6
+        xor     rdi,r10
+        add     r8,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,r9
+        add     r8,r13
+        xor     r15,r10
+        shrd    r14,r14,28
+        add     rax,r8
+        add     r8,r15
+        mov     r13,rax
+        add     r14,r8
+        shrd    r13,r13,23
+        mov     r8,r14
+        mov     r12,rbx
+        shrd    r14,r14,5
+        xor     r13,rax
+        xor     r12,rcx
+        shrd    r13,r13,4
+        xor     r14,r8
+        and     r12,rax
+        xor     r13,rax
+        add     rdx,QWORD[96+rsp]
+        mov     r15,r8
+        xor     r12,rcx
+        shrd    r14,r14,6
+        xor     r15,r9
+        add     rdx,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,r8
+        add     rdx,r13
+        xor     rdi,r9
+        shrd    r14,r14,28
+        add     r11,rdx
+        add     rdx,rdi
+        mov     r13,r11
+        add     r14,rdx
+        shrd    r13,r13,23
+        mov     rdx,r14
+        mov     r12,rax
+        shrd    r14,r14,5
+        xor     r13,r11
+        xor     r12,rbx
+        shrd    r13,r13,4
+        xor     r14,rdx
+        and     r12,r11
+        xor     r13,r11
+        add     rcx,QWORD[104+rsp]
+        mov     rdi,rdx
+        xor     r12,rbx
+        shrd    r14,r14,6
+        xor     rdi,r8
+        add     rcx,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,rdx
+        add     rcx,r13
+        xor     r15,r8
+        shrd    r14,r14,28
+        add     r10,rcx
+        add     rcx,r15
+        mov     r13,r10
+        add     r14,rcx
+        shrd    r13,r13,23
+        mov     rcx,r14
+        mov     r12,r11
+        shrd    r14,r14,5
+        xor     r13,r10
+        xor     r12,rax
+        shrd    r13,r13,4
+        xor     r14,rcx
+        and     r12,r10
+        xor     r13,r10
+        add     rbx,QWORD[112+rsp]
+        mov     r15,rcx
+        xor     r12,rax
+        shrd    r14,r14,6
+        xor     r15,rdx
+        add     rbx,r12
+        shrd    r13,r13,14
+        and     rdi,r15
+        xor     r14,rcx
+        add     rbx,r13
+        xor     rdi,rdx
+        shrd    r14,r14,28
+        add     r9,rbx
+        add     rbx,rdi
+        mov     r13,r9
+        add     r14,rbx
+        shrd    r13,r13,23
+        mov     rbx,r14
+        mov     r12,r10
+        shrd    r14,r14,5
+        xor     r13,r9
+        xor     r12,r11
+        shrd    r13,r13,4
+        xor     r14,rbx
+        and     r12,r9
+        xor     r13,r9
+        add     rax,QWORD[120+rsp]
+        mov     rdi,rbx
+        xor     r12,r11
+        shrd    r14,r14,6
+        xor     rdi,rcx
+        add     rax,r12
+        shrd    r13,r13,14
+        and     r15,rdi
+        xor     r14,rbx
+        add     rax,r13
+        xor     r15,rcx
+        shrd    r14,r14,28
+        add     r8,rax
+        add     rax,r15
+        mov     r13,r8
+        add     r14,rax
+        mov     rdi,QWORD[((128+0))+rsp]
+        mov     rax,r14
+
+        add     rax,QWORD[rdi]
+        lea     rsi,[128+rsi]
+        add     rbx,QWORD[8+rdi]
+        add     rcx,QWORD[16+rdi]
+        add     rdx,QWORD[24+rdi]
+        add     r8,QWORD[32+rdi]
+        add     r9,QWORD[40+rdi]
+        add     r10,QWORD[48+rdi]
+        add     r11,QWORD[56+rdi]
+
+        cmp     rsi,QWORD[((128+16))+rsp]
+
+        mov     QWORD[rdi],rax
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rcx
+        mov     QWORD[24+rdi],rdx
+        mov     QWORD[32+rdi],r8
+        mov     QWORD[40+rdi],r9
+        mov     QWORD[48+rdi],r10
+        mov     QWORD[56+rdi],r11
+        jb      NEAR $L$loop_avx
+
+        mov     rsi,QWORD[152+rsp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((128+32))+rsp]
+        movaps  xmm7,XMMWORD[((128+48))+rsp]
+        movaps  xmm8,XMMWORD[((128+64))+rsp]
+        movaps  xmm9,XMMWORD[((128+80))+rsp]
+        movaps  xmm10,XMMWORD[((128+96))+rsp]
+        movaps  xmm11,XMMWORD[((128+112))+rsp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha512_block_data_order_avx:
+
+ALIGN   64
+sha512_block_data_order_avx2:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_sha512_block_data_order_avx2:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+$L$avx2_shortcut:
+        mov     rax,rsp
+
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        sub     rsp,1408
+        shl     rdx,4
+        and     rsp,-256*8
+        lea     rdx,[rdx*8+rsi]
+        add     rsp,1152
+        mov     QWORD[((128+0))+rsp],rdi
+        mov     QWORD[((128+8))+rsp],rsi
+        mov     QWORD[((128+16))+rsp],rdx
+        mov     QWORD[152+rsp],rax
+
+        movaps  XMMWORD[(128+32)+rsp],xmm6
+        movaps  XMMWORD[(128+48)+rsp],xmm7
+        movaps  XMMWORD[(128+64)+rsp],xmm8
+        movaps  XMMWORD[(128+80)+rsp],xmm9
+        movaps  XMMWORD[(128+96)+rsp],xmm10
+        movaps  XMMWORD[(128+112)+rsp],xmm11
+$L$prologue_avx2:
+
+        vzeroupper
+        sub     rsi,-16*8
+        mov     rax,QWORD[rdi]
+        mov     r12,rsi
+        mov     rbx,QWORD[8+rdi]
+        cmp     rsi,rdx
+        mov     rcx,QWORD[16+rdi]
+        cmove   r12,rsp
+        mov     rdx,QWORD[24+rdi]
+        mov     r8,QWORD[32+rdi]
+        mov     r9,QWORD[40+rdi]
+        mov     r10,QWORD[48+rdi]
+        mov     r11,QWORD[56+rdi]
+        jmp     NEAR $L$oop_avx2
+ALIGN   16
+$L$oop_avx2:
+        vmovdqu xmm0,XMMWORD[((-128))+rsi]
+        vmovdqu xmm1,XMMWORD[((-128+16))+rsi]
+        vmovdqu xmm2,XMMWORD[((-128+32))+rsi]
+        lea     rbp,[((K512+128))]
+        vmovdqu xmm3,XMMWORD[((-128+48))+rsi]
+        vmovdqu xmm4,XMMWORD[((-128+64))+rsi]
+        vmovdqu xmm5,XMMWORD[((-128+80))+rsi]
+        vmovdqu xmm6,XMMWORD[((-128+96))+rsi]
+        vmovdqu xmm7,XMMWORD[((-128+112))+rsi]
+
+        vmovdqa ymm10,YMMWORD[1152+rbp]
+        vinserti128     ymm0,ymm0,XMMWORD[r12],1
+        vinserti128     ymm1,ymm1,XMMWORD[16+r12],1
+        vpshufb ymm0,ymm0,ymm10
+        vinserti128     ymm2,ymm2,XMMWORD[32+r12],1
+        vpshufb ymm1,ymm1,ymm10
+        vinserti128     ymm3,ymm3,XMMWORD[48+r12],1
+        vpshufb ymm2,ymm2,ymm10
+        vinserti128     ymm4,ymm4,XMMWORD[64+r12],1
+        vpshufb ymm3,ymm3,ymm10
+        vinserti128     ymm5,ymm5,XMMWORD[80+r12],1
+        vpshufb ymm4,ymm4,ymm10
+        vinserti128     ymm6,ymm6,XMMWORD[96+r12],1
+        vpshufb ymm5,ymm5,ymm10
+        vinserti128     ymm7,ymm7,XMMWORD[112+r12],1
+
+        vpaddq  ymm8,ymm0,YMMWORD[((-128))+rbp]
+        vpshufb ymm6,ymm6,ymm10
+        vpaddq  ymm9,ymm1,YMMWORD[((-96))+rbp]
+        vpshufb ymm7,ymm7,ymm10
+        vpaddq  ymm10,ymm2,YMMWORD[((-64))+rbp]
+        vpaddq  ymm11,ymm3,YMMWORD[((-32))+rbp]
+        vmovdqa YMMWORD[rsp],ymm8
+        vpaddq  ymm8,ymm4,YMMWORD[rbp]
+        vmovdqa YMMWORD[32+rsp],ymm9
+        vpaddq  ymm9,ymm5,YMMWORD[32+rbp]
+        vmovdqa YMMWORD[64+rsp],ymm10
+        vpaddq  ymm10,ymm6,YMMWORD[64+rbp]
+        vmovdqa YMMWORD[96+rsp],ymm11
+        lea     rsp,[((-128))+rsp]
+        vpaddq  ymm11,ymm7,YMMWORD[96+rbp]
+        vmovdqa YMMWORD[rsp],ymm8
+        xor     r14,r14
+        vmovdqa YMMWORD[32+rsp],ymm9
+        mov     rdi,rbx
+        vmovdqa YMMWORD[64+rsp],ymm10
+        xor     rdi,rcx
+        vmovdqa YMMWORD[96+rsp],ymm11
+        mov     r12,r9
+        add     rbp,16*2*8
+        jmp     NEAR $L$avx2_00_47
+
+ALIGN   16
+$L$avx2_00_47:
+        lea     rsp,[((-128))+rsp]
+        vpalignr        ymm8,ymm1,ymm0,8
+        add     r11,QWORD[((0+256))+rsp]
+        and     r12,r8
+        rorx    r13,r8,41
+        vpalignr        ymm11,ymm5,ymm4,8
+        rorx    r15,r8,18
+        lea     rax,[r14*1+rax]
+        lea     r11,[r12*1+r11]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,r8,r10
+        xor     r13,r15
+        rorx    r14,r8,14
+        vpaddq  ymm0,ymm0,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     r11,[r12*1+r11]
+        xor     r13,r14
+        mov     r15,rax
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,rax,39
+        lea     r11,[r13*1+r11]
+        xor     r15,rbx
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,rax,34
+        rorx    r13,rax,28
+        lea     rdx,[r11*1+rdx]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rbx
+        vpsrlq  ymm11,ymm7,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     r11,[rdi*1+r11]
+        mov     r12,r8
+        vpsllq  ymm10,ymm7,3
+        vpaddq  ymm0,ymm0,ymm8
+        add     r10,QWORD[((8+256))+rsp]
+        and     r12,rdx
+        rorx    r13,rdx,41
+        vpsrlq  ymm9,ymm7,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,rdx,18
+        lea     r11,[r14*1+r11]
+        lea     r10,[r12*1+r10]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,rdx,r9
+        xor     r13,rdi
+        rorx    r14,rdx,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     r10,[r12*1+r10]
+        xor     r13,r14
+        mov     rdi,r11
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,r11,39
+        lea     r10,[r13*1+r10]
+        xor     rdi,rax
+        vpaddq  ymm0,ymm0,ymm11
+        rorx    r14,r11,34
+        rorx    r13,r11,28
+        lea     rcx,[r10*1+rcx]
+        vpaddq  ymm10,ymm0,YMMWORD[((-128))+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rax
+        xor     r14,r13
+        lea     r10,[r15*1+r10]
+        mov     r12,rdx
+        vmovdqa YMMWORD[rsp],ymm10
+        vpalignr        ymm8,ymm2,ymm1,8
+        add     r9,QWORD[((32+256))+rsp]
+        and     r12,rcx
+        rorx    r13,rcx,41
+        vpalignr        ymm11,ymm6,ymm5,8
+        rorx    r15,rcx,18
+        lea     r10,[r14*1+r10]
+        lea     r9,[r12*1+r9]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,rcx,r8
+        xor     r13,r15
+        rorx    r14,rcx,14
+        vpaddq  ymm1,ymm1,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     r9,[r12*1+r9]
+        xor     r13,r14
+        mov     r15,r10
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,r10,39
+        lea     r9,[r13*1+r9]
+        xor     r15,r11
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,r10,34
+        rorx    r13,r10,28
+        lea     rbx,[r9*1+rbx]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r11
+        vpsrlq  ymm11,ymm0,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     r9,[rdi*1+r9]
+        mov     r12,rcx
+        vpsllq  ymm10,ymm0,3
+        vpaddq  ymm1,ymm1,ymm8
+        add     r8,QWORD[((40+256))+rsp]
+        and     r12,rbx
+        rorx    r13,rbx,41
+        vpsrlq  ymm9,ymm0,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,rbx,18
+        lea     r9,[r14*1+r9]
+        lea     r8,[r12*1+r8]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,rbx,rdx
+        xor     r13,rdi
+        rorx    r14,rbx,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     r8,[r12*1+r8]
+        xor     r13,r14
+        mov     rdi,r9
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,r9,39
+        lea     r8,[r13*1+r8]
+        xor     rdi,r10
+        vpaddq  ymm1,ymm1,ymm11
+        rorx    r14,r9,34
+        rorx    r13,r9,28
+        lea     rax,[r8*1+rax]
+        vpaddq  ymm10,ymm1,YMMWORD[((-96))+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r10
+        xor     r14,r13
+        lea     r8,[r15*1+r8]
+        mov     r12,rbx
+        vmovdqa YMMWORD[32+rsp],ymm10
+        vpalignr        ymm8,ymm3,ymm2,8
+        add     rdx,QWORD[((64+256))+rsp]
+        and     r12,rax
+        rorx    r13,rax,41
+        vpalignr        ymm11,ymm7,ymm6,8
+        rorx    r15,rax,18
+        lea     r8,[r14*1+r8]
+        lea     rdx,[r12*1+rdx]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,rax,rcx
+        xor     r13,r15
+        rorx    r14,rax,14
+        vpaddq  ymm2,ymm2,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     rdx,[r12*1+rdx]
+        xor     r13,r14
+        mov     r15,r8
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,r8,39
+        lea     rdx,[r13*1+rdx]
+        xor     r15,r9
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,r8,34
+        rorx    r13,r8,28
+        lea     r11,[rdx*1+r11]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r9
+        vpsrlq  ymm11,ymm1,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     rdx,[rdi*1+rdx]
+        mov     r12,rax
+        vpsllq  ymm10,ymm1,3
+        vpaddq  ymm2,ymm2,ymm8
+        add     rcx,QWORD[((72+256))+rsp]
+        and     r12,r11
+        rorx    r13,r11,41
+        vpsrlq  ymm9,ymm1,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,r11,18
+        lea     rdx,[r14*1+rdx]
+        lea     rcx,[r12*1+rcx]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,r11,rbx
+        xor     r13,rdi
+        rorx    r14,r11,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     rcx,[r12*1+rcx]
+        xor     r13,r14
+        mov     rdi,rdx
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,rdx,39
+        lea     rcx,[r13*1+rcx]
+        xor     rdi,r8
+        vpaddq  ymm2,ymm2,ymm11
+        rorx    r14,rdx,34
+        rorx    r13,rdx,28
+        lea     r10,[rcx*1+r10]
+        vpaddq  ymm10,ymm2,YMMWORD[((-64))+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r8
+        xor     r14,r13
+        lea     rcx,[r15*1+rcx]
+        mov     r12,r11
+        vmovdqa YMMWORD[64+rsp],ymm10
+        vpalignr        ymm8,ymm4,ymm3,8
+        add     rbx,QWORD[((96+256))+rsp]
+        and     r12,r10
+        rorx    r13,r10,41
+        vpalignr        ymm11,ymm0,ymm7,8
+        rorx    r15,r10,18
+        lea     rcx,[r14*1+rcx]
+        lea     rbx,[r12*1+rbx]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,r10,rax
+        xor     r13,r15
+        rorx    r14,r10,14
+        vpaddq  ymm3,ymm3,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     rbx,[r12*1+rbx]
+        xor     r13,r14
+        mov     r15,rcx
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,rcx,39
+        lea     rbx,[r13*1+rbx]
+        xor     r15,rdx
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,rcx,34
+        rorx    r13,rcx,28
+        lea     r9,[rbx*1+r9]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rdx
+        vpsrlq  ymm11,ymm2,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     rbx,[rdi*1+rbx]
+        mov     r12,r10
+        vpsllq  ymm10,ymm2,3
+        vpaddq  ymm3,ymm3,ymm8
+        add     rax,QWORD[((104+256))+rsp]
+        and     r12,r9
+        rorx    r13,r9,41
+        vpsrlq  ymm9,ymm2,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,r9,18
+        lea     rbx,[r14*1+rbx]
+        lea     rax,[r12*1+rax]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,r9,r11
+        xor     r13,rdi
+        rorx    r14,r9,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     rax,[r12*1+rax]
+        xor     r13,r14
+        mov     rdi,rbx
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,rbx,39
+        lea     rax,[r13*1+rax]
+        xor     rdi,rcx
+        vpaddq  ymm3,ymm3,ymm11
+        rorx    r14,rbx,34
+        rorx    r13,rbx,28
+        lea     r8,[rax*1+r8]
+        vpaddq  ymm10,ymm3,YMMWORD[((-32))+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rcx
+        xor     r14,r13
+        lea     rax,[r15*1+rax]
+        mov     r12,r9
+        vmovdqa YMMWORD[96+rsp],ymm10
+        lea     rsp,[((-128))+rsp]
+        vpalignr        ymm8,ymm5,ymm4,8
+        add     r11,QWORD[((0+256))+rsp]
+        and     r12,r8
+        rorx    r13,r8,41
+        vpalignr        ymm11,ymm1,ymm0,8
+        rorx    r15,r8,18
+        lea     rax,[r14*1+rax]
+        lea     r11,[r12*1+r11]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,r8,r10
+        xor     r13,r15
+        rorx    r14,r8,14
+        vpaddq  ymm4,ymm4,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     r11,[r12*1+r11]
+        xor     r13,r14
+        mov     r15,rax
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,rax,39
+        lea     r11,[r13*1+r11]
+        xor     r15,rbx
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,rax,34
+        rorx    r13,rax,28
+        lea     rdx,[r11*1+rdx]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rbx
+        vpsrlq  ymm11,ymm3,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     r11,[rdi*1+r11]
+        mov     r12,r8
+        vpsllq  ymm10,ymm3,3
+        vpaddq  ymm4,ymm4,ymm8
+        add     r10,QWORD[((8+256))+rsp]
+        and     r12,rdx
+        rorx    r13,rdx,41
+        vpsrlq  ymm9,ymm3,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,rdx,18
+        lea     r11,[r14*1+r11]
+        lea     r10,[r12*1+r10]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,rdx,r9
+        xor     r13,rdi
+        rorx    r14,rdx,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     r10,[r12*1+r10]
+        xor     r13,r14
+        mov     rdi,r11
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,r11,39
+        lea     r10,[r13*1+r10]
+        xor     rdi,rax
+        vpaddq  ymm4,ymm4,ymm11
+        rorx    r14,r11,34
+        rorx    r13,r11,28
+        lea     rcx,[r10*1+rcx]
+        vpaddq  ymm10,ymm4,YMMWORD[rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rax
+        xor     r14,r13
+        lea     r10,[r15*1+r10]
+        mov     r12,rdx
+        vmovdqa YMMWORD[rsp],ymm10
+        vpalignr        ymm8,ymm6,ymm5,8
+        add     r9,QWORD[((32+256))+rsp]
+        and     r12,rcx
+        rorx    r13,rcx,41
+        vpalignr        ymm11,ymm2,ymm1,8
+        rorx    r15,rcx,18
+        lea     r10,[r14*1+r10]
+        lea     r9,[r12*1+r9]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,rcx,r8
+        xor     r13,r15
+        rorx    r14,rcx,14
+        vpaddq  ymm5,ymm5,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     r9,[r12*1+r9]
+        xor     r13,r14
+        mov     r15,r10
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,r10,39
+        lea     r9,[r13*1+r9]
+        xor     r15,r11
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,r10,34
+        rorx    r13,r10,28
+        lea     rbx,[r9*1+rbx]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r11
+        vpsrlq  ymm11,ymm4,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     r9,[rdi*1+r9]
+        mov     r12,rcx
+        vpsllq  ymm10,ymm4,3
+        vpaddq  ymm5,ymm5,ymm8
+        add     r8,QWORD[((40+256))+rsp]
+        and     r12,rbx
+        rorx    r13,rbx,41
+        vpsrlq  ymm9,ymm4,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,rbx,18
+        lea     r9,[r14*1+r9]
+        lea     r8,[r12*1+r8]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,rbx,rdx
+        xor     r13,rdi
+        rorx    r14,rbx,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     r8,[r12*1+r8]
+        xor     r13,r14
+        mov     rdi,r9
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,r9,39
+        lea     r8,[r13*1+r8]
+        xor     rdi,r10
+        vpaddq  ymm5,ymm5,ymm11
+        rorx    r14,r9,34
+        rorx    r13,r9,28
+        lea     rax,[r8*1+rax]
+        vpaddq  ymm10,ymm5,YMMWORD[32+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r10
+        xor     r14,r13
+        lea     r8,[r15*1+r8]
+        mov     r12,rbx
+        vmovdqa YMMWORD[32+rsp],ymm10
+        vpalignr        ymm8,ymm7,ymm6,8
+        add     rdx,QWORD[((64+256))+rsp]
+        and     r12,rax
+        rorx    r13,rax,41
+        vpalignr        ymm11,ymm3,ymm2,8
+        rorx    r15,rax,18
+        lea     r8,[r14*1+r8]
+        lea     rdx,[r12*1+rdx]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,rax,rcx
+        xor     r13,r15
+        rorx    r14,rax,14
+        vpaddq  ymm6,ymm6,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     rdx,[r12*1+rdx]
+        xor     r13,r14
+        mov     r15,r8
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,r8,39
+        lea     rdx,[r13*1+rdx]
+        xor     r15,r9
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,r8,34
+        rorx    r13,r8,28
+        lea     r11,[rdx*1+r11]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r9
+        vpsrlq  ymm11,ymm5,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     rdx,[rdi*1+rdx]
+        mov     r12,rax
+        vpsllq  ymm10,ymm5,3
+        vpaddq  ymm6,ymm6,ymm8
+        add     rcx,QWORD[((72+256))+rsp]
+        and     r12,r11
+        rorx    r13,r11,41
+        vpsrlq  ymm9,ymm5,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,r11,18
+        lea     rdx,[r14*1+rdx]
+        lea     rcx,[r12*1+rcx]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,r11,rbx
+        xor     r13,rdi
+        rorx    r14,r11,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     rcx,[r12*1+rcx]
+        xor     r13,r14
+        mov     rdi,rdx
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,rdx,39
+        lea     rcx,[r13*1+rcx]
+        xor     rdi,r8
+        vpaddq  ymm6,ymm6,ymm11
+        rorx    r14,rdx,34
+        rorx    r13,rdx,28
+        lea     r10,[rcx*1+r10]
+        vpaddq  ymm10,ymm6,YMMWORD[64+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r8
+        xor     r14,r13
+        lea     rcx,[r15*1+rcx]
+        mov     r12,r11
+        vmovdqa YMMWORD[64+rsp],ymm10
+        vpalignr        ymm8,ymm0,ymm7,8
+        add     rbx,QWORD[((96+256))+rsp]
+        and     r12,r10
+        rorx    r13,r10,41
+        vpalignr        ymm11,ymm4,ymm3,8
+        rorx    r15,r10,18
+        lea     rcx,[r14*1+rcx]
+        lea     rbx,[r12*1+rbx]
+        vpsrlq  ymm10,ymm8,1
+        andn    r12,r10,rax
+        xor     r13,r15
+        rorx    r14,r10,14
+        vpaddq  ymm7,ymm7,ymm11
+        vpsrlq  ymm11,ymm8,7
+        lea     rbx,[r12*1+rbx]
+        xor     r13,r14
+        mov     r15,rcx
+        vpsllq  ymm9,ymm8,56
+        vpxor   ymm8,ymm11,ymm10
+        rorx    r12,rcx,39
+        lea     rbx,[r13*1+rbx]
+        xor     r15,rdx
+        vpsrlq  ymm10,ymm10,7
+        vpxor   ymm8,ymm8,ymm9
+        rorx    r14,rcx,34
+        rorx    r13,rcx,28
+        lea     r9,[rbx*1+r9]
+        vpsllq  ymm9,ymm9,7
+        vpxor   ymm8,ymm8,ymm10
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rdx
+        vpsrlq  ymm11,ymm6,6
+        vpxor   ymm8,ymm8,ymm9
+        xor     r14,r13
+        lea     rbx,[rdi*1+rbx]
+        mov     r12,r10
+        vpsllq  ymm10,ymm6,3
+        vpaddq  ymm7,ymm7,ymm8
+        add     rax,QWORD[((104+256))+rsp]
+        and     r12,r9
+        rorx    r13,r9,41
+        vpsrlq  ymm9,ymm6,19
+        vpxor   ymm11,ymm11,ymm10
+        rorx    rdi,r9,18
+        lea     rbx,[r14*1+rbx]
+        lea     rax,[r12*1+rax]
+        vpsllq  ymm10,ymm10,42
+        vpxor   ymm11,ymm11,ymm9
+        andn    r12,r9,r11
+        xor     r13,rdi
+        rorx    r14,r9,14
+        vpsrlq  ymm9,ymm9,42
+        vpxor   ymm11,ymm11,ymm10
+        lea     rax,[r12*1+rax]
+        xor     r13,r14
+        mov     rdi,rbx
+        vpxor   ymm11,ymm11,ymm9
+        rorx    r12,rbx,39
+        lea     rax,[r13*1+rax]
+        xor     rdi,rcx
+        vpaddq  ymm7,ymm7,ymm11
+        rorx    r14,rbx,34
+        rorx    r13,rbx,28
+        lea     r8,[rax*1+r8]
+        vpaddq  ymm10,ymm7,YMMWORD[96+rbp]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rcx
+        xor     r14,r13
+        lea     rax,[r15*1+rax]
+        mov     r12,r9
+        vmovdqa YMMWORD[96+rsp],ymm10
+        lea     rbp,[256+rbp]
+        cmp     BYTE[((-121))+rbp],0
+        jne     NEAR $L$avx2_00_47
+        add     r11,QWORD[((0+128))+rsp]
+        and     r12,r8
+        rorx    r13,r8,41
+        rorx    r15,r8,18
+        lea     rax,[r14*1+rax]
+        lea     r11,[r12*1+r11]
+        andn    r12,r8,r10
+        xor     r13,r15
+        rorx    r14,r8,14
+        lea     r11,[r12*1+r11]
+        xor     r13,r14
+        mov     r15,rax
+        rorx    r12,rax,39
+        lea     r11,[r13*1+r11]
+        xor     r15,rbx
+        rorx    r14,rax,34
+        rorx    r13,rax,28
+        lea     rdx,[r11*1+rdx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rbx
+        xor     r14,r13
+        lea     r11,[rdi*1+r11]
+        mov     r12,r8
+        add     r10,QWORD[((8+128))+rsp]
+        and     r12,rdx
+        rorx    r13,rdx,41
+        rorx    rdi,rdx,18
+        lea     r11,[r14*1+r11]
+        lea     r10,[r12*1+r10]
+        andn    r12,rdx,r9
+        xor     r13,rdi
+        rorx    r14,rdx,14
+        lea     r10,[r12*1+r10]
+        xor     r13,r14
+        mov     rdi,r11
+        rorx    r12,r11,39
+        lea     r10,[r13*1+r10]
+        xor     rdi,rax
+        rorx    r14,r11,34
+        rorx    r13,r11,28
+        lea     rcx,[r10*1+rcx]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rax
+        xor     r14,r13
+        lea     r10,[r15*1+r10]
+        mov     r12,rdx
+        add     r9,QWORD[((32+128))+rsp]
+        and     r12,rcx
+        rorx    r13,rcx,41
+        rorx    r15,rcx,18
+        lea     r10,[r14*1+r10]
+        lea     r9,[r12*1+r9]
+        andn    r12,rcx,r8
+        xor     r13,r15
+        rorx    r14,rcx,14
+        lea     r9,[r12*1+r9]
+        xor     r13,r14
+        mov     r15,r10
+        rorx    r12,r10,39
+        lea     r9,[r13*1+r9]
+        xor     r15,r11
+        rorx    r14,r10,34
+        rorx    r13,r10,28
+        lea     rbx,[r9*1+rbx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r11
+        xor     r14,r13
+        lea     r9,[rdi*1+r9]
+        mov     r12,rcx
+        add     r8,QWORD[((40+128))+rsp]
+        and     r12,rbx
+        rorx    r13,rbx,41
+        rorx    rdi,rbx,18
+        lea     r9,[r14*1+r9]
+        lea     r8,[r12*1+r8]
+        andn    r12,rbx,rdx
+        xor     r13,rdi
+        rorx    r14,rbx,14
+        lea     r8,[r12*1+r8]
+        xor     r13,r14
+        mov     rdi,r9
+        rorx    r12,r9,39
+        lea     r8,[r13*1+r8]
+        xor     rdi,r10
+        rorx    r14,r9,34
+        rorx    r13,r9,28
+        lea     rax,[r8*1+rax]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r10
+        xor     r14,r13
+        lea     r8,[r15*1+r8]
+        mov     r12,rbx
+        add     rdx,QWORD[((64+128))+rsp]
+        and     r12,rax
+        rorx    r13,rax,41
+        rorx    r15,rax,18
+        lea     r8,[r14*1+r8]
+        lea     rdx,[r12*1+rdx]
+        andn    r12,rax,rcx
+        xor     r13,r15
+        rorx    r14,rax,14
+        lea     rdx,[r12*1+rdx]
+        xor     r13,r14
+        mov     r15,r8
+        rorx    r12,r8,39
+        lea     rdx,[r13*1+rdx]
+        xor     r15,r9
+        rorx    r14,r8,34
+        rorx    r13,r8,28
+        lea     r11,[rdx*1+r11]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r9
+        xor     r14,r13
+        lea     rdx,[rdi*1+rdx]
+        mov     r12,rax
+        add     rcx,QWORD[((72+128))+rsp]
+        and     r12,r11
+        rorx    r13,r11,41
+        rorx    rdi,r11,18
+        lea     rdx,[r14*1+rdx]
+        lea     rcx,[r12*1+rcx]
+        andn    r12,r11,rbx
+        xor     r13,rdi
+        rorx    r14,r11,14
+        lea     rcx,[r12*1+rcx]
+        xor     r13,r14
+        mov     rdi,rdx
+        rorx    r12,rdx,39
+        lea     rcx,[r13*1+rcx]
+        xor     rdi,r8
+        rorx    r14,rdx,34
+        rorx    r13,rdx,28
+        lea     r10,[rcx*1+r10]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r8
+        xor     r14,r13
+        lea     rcx,[r15*1+rcx]
+        mov     r12,r11
+        add     rbx,QWORD[((96+128))+rsp]
+        and     r12,r10
+        rorx    r13,r10,41
+        rorx    r15,r10,18
+        lea     rcx,[r14*1+rcx]
+        lea     rbx,[r12*1+rbx]
+        andn    r12,r10,rax
+        xor     r13,r15
+        rorx    r14,r10,14
+        lea     rbx,[r12*1+rbx]
+        xor     r13,r14
+        mov     r15,rcx
+        rorx    r12,rcx,39
+        lea     rbx,[r13*1+rbx]
+        xor     r15,rdx
+        rorx    r14,rcx,34
+        rorx    r13,rcx,28
+        lea     r9,[rbx*1+r9]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rdx
+        xor     r14,r13
+        lea     rbx,[rdi*1+rbx]
+        mov     r12,r10
+        add     rax,QWORD[((104+128))+rsp]
+        and     r12,r9
+        rorx    r13,r9,41
+        rorx    rdi,r9,18
+        lea     rbx,[r14*1+rbx]
+        lea     rax,[r12*1+rax]
+        andn    r12,r9,r11
+        xor     r13,rdi
+        rorx    r14,r9,14
+        lea     rax,[r12*1+rax]
+        xor     r13,r14
+        mov     rdi,rbx
+        rorx    r12,rbx,39
+        lea     rax,[r13*1+rax]
+        xor     rdi,rcx
+        rorx    r14,rbx,34
+        rorx    r13,rbx,28
+        lea     r8,[rax*1+r8]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rcx
+        xor     r14,r13
+        lea     rax,[r15*1+rax]
+        mov     r12,r9
+        add     r11,QWORD[rsp]
+        and     r12,r8
+        rorx    r13,r8,41
+        rorx    r15,r8,18
+        lea     rax,[r14*1+rax]
+        lea     r11,[r12*1+r11]
+        andn    r12,r8,r10
+        xor     r13,r15
+        rorx    r14,r8,14
+        lea     r11,[r12*1+r11]
+        xor     r13,r14
+        mov     r15,rax
+        rorx    r12,rax,39
+        lea     r11,[r13*1+r11]
+        xor     r15,rbx
+        rorx    r14,rax,34
+        rorx    r13,rax,28
+        lea     rdx,[r11*1+rdx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rbx
+        xor     r14,r13
+        lea     r11,[rdi*1+r11]
+        mov     r12,r8
+        add     r10,QWORD[8+rsp]
+        and     r12,rdx
+        rorx    r13,rdx,41
+        rorx    rdi,rdx,18
+        lea     r11,[r14*1+r11]
+        lea     r10,[r12*1+r10]
+        andn    r12,rdx,r9
+        xor     r13,rdi
+        rorx    r14,rdx,14
+        lea     r10,[r12*1+r10]
+        xor     r13,r14
+        mov     rdi,r11
+        rorx    r12,r11,39
+        lea     r10,[r13*1+r10]
+        xor     rdi,rax
+        rorx    r14,r11,34
+        rorx    r13,r11,28
+        lea     rcx,[r10*1+rcx]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rax
+        xor     r14,r13
+        lea     r10,[r15*1+r10]
+        mov     r12,rdx
+        add     r9,QWORD[32+rsp]
+        and     r12,rcx
+        rorx    r13,rcx,41
+        rorx    r15,rcx,18
+        lea     r10,[r14*1+r10]
+        lea     r9,[r12*1+r9]
+        andn    r12,rcx,r8
+        xor     r13,r15
+        rorx    r14,rcx,14
+        lea     r9,[r12*1+r9]
+        xor     r13,r14
+        mov     r15,r10
+        rorx    r12,r10,39
+        lea     r9,[r13*1+r9]
+        xor     r15,r11
+        rorx    r14,r10,34
+        rorx    r13,r10,28
+        lea     rbx,[r9*1+rbx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r11
+        xor     r14,r13
+        lea     r9,[rdi*1+r9]
+        mov     r12,rcx
+        add     r8,QWORD[40+rsp]
+        and     r12,rbx
+        rorx    r13,rbx,41
+        rorx    rdi,rbx,18
+        lea     r9,[r14*1+r9]
+        lea     r8,[r12*1+r8]
+        andn    r12,rbx,rdx
+        xor     r13,rdi
+        rorx    r14,rbx,14
+        lea     r8,[r12*1+r8]
+        xor     r13,r14
+        mov     rdi,r9
+        rorx    r12,r9,39
+        lea     r8,[r13*1+r8]
+        xor     rdi,r10
+        rorx    r14,r9,34
+        rorx    r13,r9,28
+        lea     rax,[r8*1+rax]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r10
+        xor     r14,r13
+        lea     r8,[r15*1+r8]
+        mov     r12,rbx
+        add     rdx,QWORD[64+rsp]
+        and     r12,rax
+        rorx    r13,rax,41
+        rorx    r15,rax,18
+        lea     r8,[r14*1+r8]
+        lea     rdx,[r12*1+rdx]
+        andn    r12,rax,rcx
+        xor     r13,r15
+        rorx    r14,rax,14
+        lea     rdx,[r12*1+rdx]
+        xor     r13,r14
+        mov     r15,r8
+        rorx    r12,r8,39
+        lea     rdx,[r13*1+rdx]
+        xor     r15,r9
+        rorx    r14,r8,34
+        rorx    r13,r8,28
+        lea     r11,[rdx*1+r11]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r9
+        xor     r14,r13
+        lea     rdx,[rdi*1+rdx]
+        mov     r12,rax
+        add     rcx,QWORD[72+rsp]
+        and     r12,r11
+        rorx    r13,r11,41
+        rorx    rdi,r11,18
+        lea     rdx,[r14*1+rdx]
+        lea     rcx,[r12*1+rcx]
+        andn    r12,r11,rbx
+        xor     r13,rdi
+        rorx    r14,r11,14
+        lea     rcx,[r12*1+rcx]
+        xor     r13,r14
+        mov     rdi,rdx
+        rorx    r12,rdx,39
+        lea     rcx,[r13*1+rcx]
+        xor     rdi,r8
+        rorx    r14,rdx,34
+        rorx    r13,rdx,28
+        lea     r10,[rcx*1+r10]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r8
+        xor     r14,r13
+        lea     rcx,[r15*1+rcx]
+        mov     r12,r11
+        add     rbx,QWORD[96+rsp]
+        and     r12,r10
+        rorx    r13,r10,41
+        rorx    r15,r10,18
+        lea     rcx,[r14*1+rcx]
+        lea     rbx,[r12*1+rbx]
+        andn    r12,r10,rax
+        xor     r13,r15
+        rorx    r14,r10,14
+        lea     rbx,[r12*1+rbx]
+        xor     r13,r14
+        mov     r15,rcx
+        rorx    r12,rcx,39
+        lea     rbx,[r13*1+rbx]
+        xor     r15,rdx
+        rorx    r14,rcx,34
+        rorx    r13,rcx,28
+        lea     r9,[rbx*1+r9]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rdx
+        xor     r14,r13
+        lea     rbx,[rdi*1+rbx]
+        mov     r12,r10
+        add     rax,QWORD[104+rsp]
+        and     r12,r9
+        rorx    r13,r9,41
+        rorx    rdi,r9,18
+        lea     rbx,[r14*1+rbx]
+        lea     rax,[r12*1+rax]
+        andn    r12,r9,r11
+        xor     r13,rdi
+        rorx    r14,r9,14
+        lea     rax,[r12*1+rax]
+        xor     r13,r14
+        mov     rdi,rbx
+        rorx    r12,rbx,39
+        lea     rax,[r13*1+rax]
+        xor     rdi,rcx
+        rorx    r14,rbx,34
+        rorx    r13,rbx,28
+        lea     r8,[rax*1+r8]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rcx
+        xor     r14,r13
+        lea     rax,[r15*1+rax]
+        mov     r12,r9
+        mov     rdi,QWORD[1280+rsp]
+        add     rax,r14
+
+        lea     rbp,[1152+rsp]
+
+        add     rax,QWORD[rdi]
+        add     rbx,QWORD[8+rdi]
+        add     rcx,QWORD[16+rdi]
+        add     rdx,QWORD[24+rdi]
+        add     r8,QWORD[32+rdi]
+        add     r9,QWORD[40+rdi]
+        add     r10,QWORD[48+rdi]
+        add     r11,QWORD[56+rdi]
+
+        mov     QWORD[rdi],rax
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rcx
+        mov     QWORD[24+rdi],rdx
+        mov     QWORD[32+rdi],r8
+        mov     QWORD[40+rdi],r9
+        mov     QWORD[48+rdi],r10
+        mov     QWORD[56+rdi],r11
+
+        cmp     rsi,QWORD[144+rbp]
+        je      NEAR $L$done_avx2
+
+        xor     r14,r14
+        mov     rdi,rbx
+        xor     rdi,rcx
+        mov     r12,r9
+        jmp     NEAR $L$ower_avx2
+ALIGN   16
+$L$ower_avx2:
+        add     r11,QWORD[((0+16))+rbp]
+        and     r12,r8
+        rorx    r13,r8,41
+        rorx    r15,r8,18
+        lea     rax,[r14*1+rax]
+        lea     r11,[r12*1+r11]
+        andn    r12,r8,r10
+        xor     r13,r15
+        rorx    r14,r8,14
+        lea     r11,[r12*1+r11]
+        xor     r13,r14
+        mov     r15,rax
+        rorx    r12,rax,39
+        lea     r11,[r13*1+r11]
+        xor     r15,rbx
+        rorx    r14,rax,34
+        rorx    r13,rax,28
+        lea     rdx,[r11*1+rdx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rbx
+        xor     r14,r13
+        lea     r11,[rdi*1+r11]
+        mov     r12,r8
+        add     r10,QWORD[((8+16))+rbp]
+        and     r12,rdx
+        rorx    r13,rdx,41
+        rorx    rdi,rdx,18
+        lea     r11,[r14*1+r11]
+        lea     r10,[r12*1+r10]
+        andn    r12,rdx,r9
+        xor     r13,rdi
+        rorx    r14,rdx,14
+        lea     r10,[r12*1+r10]
+        xor     r13,r14
+        mov     rdi,r11
+        rorx    r12,r11,39
+        lea     r10,[r13*1+r10]
+        xor     rdi,rax
+        rorx    r14,r11,34
+        rorx    r13,r11,28
+        lea     rcx,[r10*1+rcx]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rax
+        xor     r14,r13
+        lea     r10,[r15*1+r10]
+        mov     r12,rdx
+        add     r9,QWORD[((32+16))+rbp]
+        and     r12,rcx
+        rorx    r13,rcx,41
+        rorx    r15,rcx,18
+        lea     r10,[r14*1+r10]
+        lea     r9,[r12*1+r9]
+        andn    r12,rcx,r8
+        xor     r13,r15
+        rorx    r14,rcx,14
+        lea     r9,[r12*1+r9]
+        xor     r13,r14
+        mov     r15,r10
+        rorx    r12,r10,39
+        lea     r9,[r13*1+r9]
+        xor     r15,r11
+        rorx    r14,r10,34
+        rorx    r13,r10,28
+        lea     rbx,[r9*1+rbx]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r11
+        xor     r14,r13
+        lea     r9,[rdi*1+r9]
+        mov     r12,rcx
+        add     r8,QWORD[((40+16))+rbp]
+        and     r12,rbx
+        rorx    r13,rbx,41
+        rorx    rdi,rbx,18
+        lea     r9,[r14*1+r9]
+        lea     r8,[r12*1+r8]
+        andn    r12,rbx,rdx
+        xor     r13,rdi
+        rorx    r14,rbx,14
+        lea     r8,[r12*1+r8]
+        xor     r13,r14
+        mov     rdi,r9
+        rorx    r12,r9,39
+        lea     r8,[r13*1+r8]
+        xor     rdi,r10
+        rorx    r14,r9,34
+        rorx    r13,r9,28
+        lea     rax,[r8*1+rax]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r10
+        xor     r14,r13
+        lea     r8,[r15*1+r8]
+        mov     r12,rbx
+        add     rdx,QWORD[((64+16))+rbp]
+        and     r12,rax
+        rorx    r13,rax,41
+        rorx    r15,rax,18
+        lea     r8,[r14*1+r8]
+        lea     rdx,[r12*1+rdx]
+        andn    r12,rax,rcx
+        xor     r13,r15
+        rorx    r14,rax,14
+        lea     rdx,[r12*1+rdx]
+        xor     r13,r14
+        mov     r15,r8
+        rorx    r12,r8,39
+        lea     rdx,[r13*1+rdx]
+        xor     r15,r9
+        rorx    r14,r8,34
+        rorx    r13,r8,28
+        lea     r11,[rdx*1+r11]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,r9
+        xor     r14,r13
+        lea     rdx,[rdi*1+rdx]
+        mov     r12,rax
+        add     rcx,QWORD[((72+16))+rbp]
+        and     r12,r11
+        rorx    r13,r11,41
+        rorx    rdi,r11,18
+        lea     rdx,[r14*1+rdx]
+        lea     rcx,[r12*1+rcx]
+        andn    r12,r11,rbx
+        xor     r13,rdi
+        rorx    r14,r11,14
+        lea     rcx,[r12*1+rcx]
+        xor     r13,r14
+        mov     rdi,rdx
+        rorx    r12,rdx,39
+        lea     rcx,[r13*1+rcx]
+        xor     rdi,r8
+        rorx    r14,rdx,34
+        rorx    r13,rdx,28
+        lea     r10,[rcx*1+r10]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,r8
+        xor     r14,r13
+        lea     rcx,[r15*1+rcx]
+        mov     r12,r11
+        add     rbx,QWORD[((96+16))+rbp]
+        and     r12,r10
+        rorx    r13,r10,41
+        rorx    r15,r10,18
+        lea     rcx,[r14*1+rcx]
+        lea     rbx,[r12*1+rbx]
+        andn    r12,r10,rax
+        xor     r13,r15
+        rorx    r14,r10,14
+        lea     rbx,[r12*1+rbx]
+        xor     r13,r14
+        mov     r15,rcx
+        rorx    r12,rcx,39
+        lea     rbx,[r13*1+rbx]
+        xor     r15,rdx
+        rorx    r14,rcx,34
+        rorx    r13,rcx,28
+        lea     r9,[rbx*1+r9]
+        and     rdi,r15
+        xor     r14,r12
+        xor     rdi,rdx
+        xor     r14,r13
+        lea     rbx,[rdi*1+rbx]
+        mov     r12,r10
+        add     rax,QWORD[((104+16))+rbp]
+        and     r12,r9
+        rorx    r13,r9,41
+        rorx    rdi,r9,18
+        lea     rbx,[r14*1+rbx]
+        lea     rax,[r12*1+rax]
+        andn    r12,r9,r11
+        xor     r13,rdi
+        rorx    r14,r9,14
+        lea     rax,[r12*1+rax]
+        xor     r13,r14
+        mov     rdi,rbx
+        rorx    r12,rbx,39
+        lea     rax,[r13*1+rax]
+        xor     rdi,rcx
+        rorx    r14,rbx,34
+        rorx    r13,rbx,28
+        lea     r8,[rax*1+r8]
+        and     r15,rdi
+        xor     r14,r12
+        xor     r15,rcx
+        xor     r14,r13
+        lea     rax,[r15*1+rax]
+        mov     r12,r9
+        lea     rbp,[((-128))+rbp]
+        cmp     rbp,rsp
+        jae     NEAR $L$ower_avx2
+
+        mov     rdi,QWORD[1280+rsp]
+        add     rax,r14
+
+        lea     rsp,[1152+rsp]
+
+
+
+        add     rax,QWORD[rdi]
+        add     rbx,QWORD[8+rdi]
+        add     rcx,QWORD[16+rdi]
+        add     rdx,QWORD[24+rdi]
+        add     r8,QWORD[32+rdi]
+        add     r9,QWORD[40+rdi]
+        lea     rsi,[256+rsi]
+        add     r10,QWORD[48+rdi]
+        mov     r12,rsi
+        add     r11,QWORD[56+rdi]
+        cmp     rsi,QWORD[((128+16))+rsp]
+
+        mov     QWORD[rdi],rax
+        cmove   r12,rsp
+        mov     QWORD[8+rdi],rbx
+        mov     QWORD[16+rdi],rcx
+        mov     QWORD[24+rdi],rdx
+        mov     QWORD[32+rdi],r8
+        mov     QWORD[40+rdi],r9
+        mov     QWORD[48+rdi],r10
+        mov     QWORD[56+rdi],r11
+
+        jbe     NEAR $L$oop_avx2
+        lea     rbp,[rsp]
+
+
+
+
+$L$done_avx2:
+        mov     rsi,QWORD[152+rbp]
+
+        vzeroupper
+        movaps  xmm6,XMMWORD[((128+32))+rbp]
+        movaps  xmm7,XMMWORD[((128+48))+rbp]
+        movaps  xmm8,XMMWORD[((128+64))+rbp]
+        movaps  xmm9,XMMWORD[((128+80))+rbp]
+        movaps  xmm10,XMMWORD[((128+96))+rbp]
+        movaps  xmm11,XMMWORD[((128+112))+rbp]
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$epilogue_avx2:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_sha512_block_data_order_avx2:
 EXTERN  __imp_RtlVirtualUnwind
 
@@ -1852 +5561 @@
         cmp     rbx,r10
         jae     NEAR $L$in_prologue
+        lea     r10,[$L$avx2_shortcut]
+        cmp     rbx,r10
+        jb      NEAR $L$not_in_avx2
+
+        and     rax,-256*8
+        add     rax,1152
+$L$not_in_avx2:
         mov     rsi,rax
         mov     rax,QWORD[((128+24))+rax]
@@ -1921 +5637 @@
         DD      $L$SEH_end_sha512_block_data_order wrt ..imagebase
         DD      $L$SEH_info_sha512_block_data_order wrt ..imagebase
+        DD      $L$SEH_begin_sha512_block_data_order_xop wrt ..imagebase
+        DD      $L$SEH_end_sha512_block_data_order_xop wrt ..imagebase
+        DD      $L$SEH_info_sha512_block_data_order_xop wrt ..imagebase
+        DD      $L$SEH_begin_sha512_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_end_sha512_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_info_sha512_block_data_order_avx wrt ..imagebase
+        DD      $L$SEH_begin_sha512_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_end_sha512_block_data_order_avx2 wrt ..imagebase
+        DD      $L$SEH_info_sha512_block_data_order_avx2 wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -1927 +5652 @@
         DD      se_handler wrt ..imagebase
         DD      $L$prologue wrt ..imagebase,$L$epilogue wrt ..imagebase
+$L$SEH_info_sha512_block_data_order_xop:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_xop wrt ..imagebase,$L$epilogue_xop wrt ..imagebase
+$L$SEH_info_sha512_block_data_order_avx:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase
+$L$SEH_info_sha512_block_data_order_avx2:
+DB      9,0,0,0
+        DD      se_handler wrt ..imagebase
+        DD      $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/x25519-x86_64.S

@@ -410 +410 @@
 
 $L$SEH_end_x25519_fe51_mul121666:
+EXTERN  OPENSSL_ia32cap_P
 global  x25519_fe64_eligible
 
@@ -415 +416 @@
 x25519_fe64_eligible:
 
+        mov     ecx,DWORD[((OPENSSL_ia32cap_P+8))]
         xor     eax,eax
+        and     ecx,0x80100
+        cmp     ecx,0x80100
+        cmove   eax,ecx
         DB      0F3h,0C3h               ;repret
 
@@ -422 +427 @@
 global  x25519_fe64_mul
 
+ALIGN   32
+x25519_fe64_mul:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_mul:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        push    rdi
+
+        lea     rsp,[((-16))+rsp]
+
+$L$fe64_mul_body:
+
+        mov     rax,rdx
+        mov     rbp,QWORD[rdx]
+        mov     rdx,QWORD[rsi]
+        mov     rcx,QWORD[8+rax]
+        mov     r14,QWORD[16+rax]
+        mov     r15,QWORD[24+rax]
+
+        mulx    rax,r8,rbp
+        xor     edi,edi
+        mulx    rbx,r9,rcx
+        adcx    r9,rax
+        mulx    rax,r10,r14
+        adcx    r10,rbx
+        mulx    r12,r11,r15
+        mov     rdx,QWORD[8+rsi]
+        adcx    r11,rax
+        mov     QWORD[rsp],r14
+        adcx    r12,rdi
+
+        mulx    rbx,rax,rbp
+        adox    r9,rax
+        adcx    r10,rbx
+        mulx    rbx,rax,rcx
+        adox    r10,rax
+        adcx    r11,rbx
+        mulx    rbx,rax,r14
+        adox    r11,rax
+        adcx    r12,rbx
+        mulx    r13,rax,r15
+        mov     rdx,QWORD[16+rsi]
+        adox    r12,rax
+        adcx    r13,rdi
+        adox    r13,rdi
+
+        mulx    rbx,rax,rbp
+        adcx    r10,rax
+        adox    r11,rbx
+        mulx    rbx,rax,rcx
+        adcx    r11,rax
+        adox    r12,rbx
+        mulx    rbx,rax,r14
+        adcx    r12,rax
+        adox    r13,rbx
+        mulx    r14,rax,r15
+        mov     rdx,QWORD[24+rsi]
+        adcx    r13,rax
+        adox    r14,rdi
+        adcx    r14,rdi
+
+        mulx    rbx,rax,rbp
+        adox    r11,rax
+        adcx    r12,rbx
+        mulx    rbx,rax,rcx
+        adox    r12,rax
+        adcx    r13,rbx
+        mulx    rbx,rax,QWORD[rsp]
+        adox    r13,rax
+        adcx    r14,rbx
+        mulx    r15,rax,r15
+        mov     edx,38
+        adox    r14,rax
+        adcx    r15,rdi
+        adox    r15,rdi
+
+        jmp     NEAR $L$reduce64
+$L$fe64_mul_epilogue:
+
+$L$SEH_end_x25519_fe64_mul:
+
 global  x25519_fe64_sqr
+
+ALIGN   32
+x25519_fe64_sqr:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_sqr:
+        mov     rdi,rcx
+        mov     rsi,rdx
+
+
+
+        push    rbp
+
+        push    rbx
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+        push    rdi
+
+        lea     rsp,[((-16))+rsp]
+
+$L$fe64_sqr_body:
+
+        mov     rdx,QWORD[rsi]
+        mov     rcx,QWORD[8+rsi]
+        mov     rbp,QWORD[16+rsi]
+        mov     rsi,QWORD[24+rsi]
+
+
+        mulx    r15,r8,rdx
+        mulx    rax,r9,rcx
+        xor     edi,edi
+        mulx    rbx,r10,rbp
+        adcx    r10,rax
+        mulx    r12,r11,rsi
+        mov     rdx,rcx
+        adcx    r11,rbx
+        adcx    r12,rdi
+
+
+        mulx    rbx,rax,rbp
+        adox    r11,rax
+        adcx    r12,rbx
+        mulx    r13,rax,rsi
+        mov     rdx,rbp
+        adox    r12,rax
+        adcx    r13,rdi
+
+
+        mulx    r14,rax,rsi
+        mov     rdx,rcx
+        adox    r13,rax
+        adcx    r14,rdi
+        adox    r14,rdi
+
+        adcx    r9,r9
+        adox    r9,r15
+        adcx    r10,r10
+        mulx    rbx,rax,rdx
+        mov     rdx,rbp
+        adcx    r11,r11
+        adox    r10,rax
+        adcx    r12,r12
+        adox    r11,rbx
+        mulx    rbx,rax,rdx
+        mov     rdx,rsi
+        adcx    r13,r13
+        adox    r12,rax
+        adcx    r14,r14
+        adox    r13,rbx
+        mulx    r15,rax,rdx
+        mov     edx,38
+        adox    r14,rax
+        adcx    r15,rdi
+        adox    r15,rdi
+        jmp     NEAR $L$reduce64
+
+ALIGN   32
+$L$reduce64:
+        mulx    rbx,rax,r12
+        adcx    r8,rax
+        adox    r9,rbx
+        mulx    rbx,rax,r13
+        adcx    r9,rax
+        adox    r10,rbx
+        mulx    rbx,rax,r14
+        adcx    r10,rax
+        adox    r11,rbx
+        mulx    r12,rax,r15
+        adcx    r11,rax
+        adox    r12,rdi
+        adcx    r12,rdi
+
+        mov     rdi,QWORD[16+rsp]
+        imul    r12,rdx
+
+        add     r8,r12
+        adc     r9,0
+        adc     r10,0
+        adc     r11,0
+
+        sbb     rax,rax
+        and     rax,38
+
+        add     r8,rax
+        mov     QWORD[8+rdi],r9
+        mov     QWORD[16+rdi],r10
+        mov     QWORD[24+rdi],r11
+        mov     QWORD[rdi],r8
+
+        mov     r15,QWORD[24+rsp]
+
+        mov     r14,QWORD[32+rsp]
+
+        mov     r13,QWORD[40+rsp]
+
+        mov     r12,QWORD[48+rsp]
+
+        mov     rbx,QWORD[56+rsp]
+
+        mov     rbp,QWORD[64+rsp]
+
+        lea     rsp,[72+rsp]
+
+$L$fe64_sqr_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_x25519_fe64_sqr:
+
 global  x25519_fe64_mul121666
+
+ALIGN   32
+x25519_fe64_mul121666:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_mul121666:
+        mov     rdi,rcx
+        mov     rsi,rdx
+
+
+$L$fe64_mul121666_body:
+
+        mov     edx,121666
+        mulx    rcx,r8,QWORD[rsi]
+        mulx    rax,r9,QWORD[8+rsi]
+        add     r9,rcx
+        mulx    rcx,r10,QWORD[16+rsi]
+        adc     r10,rax
+        mulx    rax,r11,QWORD[24+rsi]
+        adc     r11,rcx
+        adc     rax,0
+
+        imul    rax,rax,38
+
+        add     r8,rax
+        adc     r9,0
+        adc     r10,0
+        adc     r11,0
+
+        sbb     rax,rax
+        and     rax,38
+
+        add     r8,rax
+        mov     QWORD[8+rdi],r9
+        mov     QWORD[16+rdi],r10
+        mov     QWORD[24+rdi],r11
+        mov     QWORD[rdi],r8
+
+$L$fe64_mul121666_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_x25519_fe64_mul121666:
+
 global  x25519_fe64_add
+
+ALIGN   32
+x25519_fe64_add:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_add:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+$L$fe64_add_body:
+
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[8+rsi]
+        mov     r10,QWORD[16+rsi]
+        mov     r11,QWORD[24+rsi]
+
+        add     r8,QWORD[rdx]
+        adc     r9,QWORD[8+rdx]
+        adc     r10,QWORD[16+rdx]
+        adc     r11,QWORD[24+rdx]
+
+        sbb     rax,rax
+        and     rax,38
+
+        add     r8,rax
+        adc     r9,0
+        adc     r10,0
+        mov     QWORD[8+rdi],r9
+        adc     r11,0
+        mov     QWORD[16+rdi],r10
+        sbb     rax,rax
+        mov     QWORD[24+rdi],r11
+        and     rax,38
+
+        add     r8,rax
+        mov     QWORD[rdi],r8
+
+$L$fe64_add_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_x25519_fe64_add:
+
 global  x25519_fe64_sub
+
+ALIGN   32
+x25519_fe64_sub:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_sub:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+
+
+$L$fe64_sub_body:
+
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[8+rsi]
+        mov     r10,QWORD[16+rsi]
+        mov     r11,QWORD[24+rsi]
+
+        sub     r8,QWORD[rdx]
+        sbb     r9,QWORD[8+rdx]
+        sbb     r10,QWORD[16+rdx]
+        sbb     r11,QWORD[24+rdx]
+
+        sbb     rax,rax
+        and     rax,38
+
+        sub     r8,rax
+        sbb     r9,0
+        sbb     r10,0
+        mov     QWORD[8+rdi],r9
+        sbb     r11,0
+        mov     QWORD[16+rdi],r10
+        sbb     rax,rax
+        mov     QWORD[24+rdi],r11
+        and     rax,38
+
+        sub     r8,rax
+        mov     QWORD[rdi],r8
+
+$L$fe64_sub_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_x25519_fe64_sub:
+
 global  x25519_fe64_tobytes
-x25519_fe64_mul:
-x25519_fe64_sqr:
-x25519_fe64_mul121666:
-x25519_fe64_add:
-x25519_fe64_sub:
+
+ALIGN   32
 x25519_fe64_tobytes:
-
-DB      0x0f,0x0b
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_x25519_fe64_tobytes:
+        mov     rdi,rcx
+        mov     rsi,rdx
+
+
+$L$fe64_to_body:
+
+        mov     r8,QWORD[rsi]
+        mov     r9,QWORD[8+rsi]
+        mov     r10,QWORD[16+rsi]
+        mov     r11,QWORD[24+rsi]
+
+
+        lea     rax,[r11*1+r11]
+        sar     r11,63
+        shr     rax,1
+        and     r11,19
+        add     r11,19
+
+        add     r8,r11
+        adc     r9,0
+        adc     r10,0
+        adc     rax,0
+
+        lea     r11,[rax*1+rax]
+        sar     rax,63
+        shr     r11,1
+        not     rax
+        and     rax,19
+
+        sub     r8,rax
+        sbb     r9,0
+        sbb     r10,0
+        sbb     r11,0
+
+        mov     QWORD[rdi],r8
+        mov     QWORD[8+rdi],r9
+        mov     QWORD[16+rdi],r10
+        mov     QWORD[24+rdi],r11
+
+$L$fe64_to_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
         DB      0F3h,0C3h               ;repret
 
-
+$L$SEH_end_x25519_fe64_tobytes:
 DB      88,50,53,53,49,57,32,112,114,105,109,105,116,105,118,101
 DB      115,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82
@@ -574 +997 @@
         DD      $L$SEH_end_x25519_fe51_mul121666 wrt ..imagebase
         DD      $L$SEH_info_x25519_fe51_mul121666 wrt ..imagebase
+        DD      $L$SEH_begin_x25519_fe64_mul wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_mul wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_mul wrt ..imagebase
+
+        DD      $L$SEH_begin_x25519_fe64_sqr wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_sqr wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_sqr wrt ..imagebase
+
+        DD      $L$SEH_begin_x25519_fe64_mul121666 wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_mul121666 wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_mul121666 wrt ..imagebase
+
+        DD      $L$SEH_begin_x25519_fe64_add wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_add wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_add wrt ..imagebase
+
+        DD      $L$SEH_begin_x25519_fe64_sub wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_sub wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_sub wrt ..imagebase
+
+        DD      $L$SEH_begin_x25519_fe64_tobytes wrt ..imagebase
+        DD      $L$SEH_end_x25519_fe64_tobytes wrt ..imagebase
+        DD      $L$SEH_info_x25519_fe64_tobytes wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -591 +1037 @@
         DD      $L$fe51_mul121666_body wrt ..imagebase,$L$fe51_mul121666_epilogue wrt ..imagebase
         DD      88,0
+$L$SEH_info_x25519_fe64_mul:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$fe64_mul_body wrt ..imagebase,$L$fe64_mul_epilogue wrt ..imagebase
+        DD      72,0
+$L$SEH_info_x25519_fe64_sqr:
+DB      9,0,0,0
+        DD      full_handler wrt ..imagebase
+        DD      $L$fe64_sqr_body wrt ..imagebase,$L$fe64_sqr_epilogue wrt ..imagebase
+        DD      72,0
+$L$SEH_info_x25519_fe64_mul121666:
+DB      9,0,0,0
+        DD      short_handler wrt ..imagebase
+        DD      $L$fe64_mul121666_body wrt ..imagebase,$L$fe64_mul121666_epilogue wrt ..imagebase
+$L$SEH_info_x25519_fe64_add:
+DB      9,0,0,0
+        DD      short_handler wrt ..imagebase
+        DD      $L$fe64_add_body wrt ..imagebase,$L$fe64_add_epilogue wrt ..imagebase
+$L$SEH_info_x25519_fe64_sub:
+DB      9,0,0,0
+        DD      short_handler wrt ..imagebase
+        DD      $L$fe64_sub_body wrt ..imagebase,$L$fe64_sub_epilogue wrt ..imagebase
+$L$SEH_info_x25519_fe64_tobytes:
+DB      9,0,0,0
+        DD      short_handler wrt ..imagebase
+        DD      $L$fe64_to_body wrt ..imagebase,$L$fe64_to_epilogue wrt ..imagebase

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/x86_64-mont.S

@@ -32 +32 @@
         cmp     r9d,8
         jb      NEAR $L$mul_enter
+        mov     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
         cmp     rdx,rsi
         jne     NEAR $L$mul4x_enter
@@ -294 +295 @@
 
 $L$mul4x_enter:
+        and     r11d,0x80100
+        cmp     r11d,0x80100
+        je      NEAR $L$mulx4x_enter
         push    rbx
 
@@ -719 +723 @@
 
 $L$SEH_end_bn_mul4x_mont:
+EXTERN  bn_sqrx8x_internal
 EXTERN  bn_sqr8x_internal
 
@@ -814 +819 @@
 DB      102,72,15,110,207
 DB      102,73,15,110,218
+        mov     eax,DWORD[((OPENSSL_ia32cap_P+8))]
+        and     eax,0x80100
+        cmp     eax,0x80100
+        jne     NEAR $L$sqr8x_nox
+
+        call    bn_sqrx8x_internal
+
+
+
+
+        lea     rbx,[rcx*1+r8]
+        mov     r9,rcx
+        mov     rdx,rcx
+DB      102,72,15,126,207
+        sar     rcx,3+2
+        jmp     NEAR $L$sqr8x_sub
+
+ALIGN   32
+$L$sqr8x_nox:
         call    bn_sqr8x_internal
 
@@ -903 +927 @@
 
 $L$SEH_end_bn_sqr8x_mont:
+
+ALIGN   32
+bn_mulx4x_mont:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_bn_mulx4x_mont:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        mov     rax,rsp
+
+$L$mulx4x_enter:
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$mulx4x_prologue:
+
+        shl     r9d,3
+        xor     r10,r10
+        sub     r10,r9
+        mov     r8,QWORD[r8]
+        lea     rbp,[((-72))+r10*1+rsp]
+        and     rbp,-128
+        mov     r11,rsp
+        sub     r11,rbp
+        and     r11,-4096
+        lea     rsp,[rbp*1+r11]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$mulx4x_page_walk
+        jmp     NEAR $L$mulx4x_page_walk_done
+
+ALIGN   16
+$L$mulx4x_page_walk:
+        lea     rsp,[((-4096))+rsp]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$mulx4x_page_walk
+$L$mulx4x_page_walk_done:
+
+        lea     r10,[r9*1+rdx]
+
+
+
+
+
+
+
+
+
+
+
+
+        mov     QWORD[rsp],r9
+        shr     r9,5
+        mov     QWORD[16+rsp],r10
+        sub     r9,1
+        mov     QWORD[24+rsp],r8
+        mov     QWORD[32+rsp],rdi
+        mov     QWORD[40+rsp],rax
+
+        mov     QWORD[48+rsp],r9
+        jmp     NEAR $L$mulx4x_body
+
+ALIGN   32
+$L$mulx4x_body:
+        lea     rdi,[8+rdx]
+        mov     rdx,QWORD[rdx]
+        lea     rbx,[((64+32))+rsp]
+        mov     r9,rdx
+
+        mulx    rax,r8,QWORD[rsi]
+        mulx    r14,r11,QWORD[8+rsi]
+        add     r11,rax
+        mov     QWORD[8+rsp],rdi
+        mulx    r13,r12,QWORD[16+rsi]
+        adc     r12,r14
+        adc     r13,0
+
+        mov     rdi,r8
+        imul    r8,QWORD[24+rsp]
+        xor     rbp,rbp
+
+        mulx    r14,rax,QWORD[24+rsi]
+        mov     rdx,r8
+        lea     rsi,[32+rsi]
+        adcx    r13,rax
+        adcx    r14,rbp
+
+        mulx    r10,rax,QWORD[rcx]
+        adcx    rdi,rax
+        adox    r10,r11
+        mulx    r11,rax,QWORD[8+rcx]
+        adcx    r10,rax
+        adox    r11,r12
+DB      0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00
+        mov     rdi,QWORD[48+rsp]
+        mov     QWORD[((-32))+rbx],r10
+        adcx    r11,rax
+        adox    r12,r13
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-24))+rbx],r11
+        adcx    r12,rax
+        adox    r15,rbp
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-16))+rbx],r12
+
+        jmp     NEAR $L$mulx4x_1st
+
+ALIGN   32
+$L$mulx4x_1st:
+        adcx    r15,rbp
+        mulx    rax,r10,QWORD[rsi]
+        adcx    r10,r14
+        mulx    r14,r11,QWORD[8+rsi]
+        adcx    r11,rax
+        mulx    rax,r12,QWORD[16+rsi]
+        adcx    r12,r14
+        mulx    r14,r13,QWORD[24+rsi]
+DB      0x67,0x67
+        mov     rdx,r8
+        adcx    r13,rax
+        adcx    r14,rbp
+        lea     rsi,[32+rsi]
+        lea     rbx,[32+rbx]
+
+        adox    r10,r15
+        mulx    r15,rax,QWORD[rcx]
+        adcx    r10,rax
+        adox    r11,r15
+        mulx    r15,rax,QWORD[8+rcx]
+        adcx    r11,rax
+        adox    r12,r15
+        mulx    r15,rax,QWORD[16+rcx]
+        mov     QWORD[((-40))+rbx],r10
+        adcx    r12,rax
+        mov     QWORD[((-32))+rbx],r11
+        adox    r13,r15
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-24))+rbx],r12
+        adcx    r13,rax
+        adox    r15,rbp
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-16))+rbx],r13
+
+        dec     rdi
+        jnz     NEAR $L$mulx4x_1st
+
+        mov     rax,QWORD[rsp]
+        mov     rdi,QWORD[8+rsp]
+        adc     r15,rbp
+        add     r14,r15
+        sbb     r15,r15
+        mov     QWORD[((-8))+rbx],r14
+        jmp     NEAR $L$mulx4x_outer
+
+ALIGN   32
+$L$mulx4x_outer:
+        mov     rdx,QWORD[rdi]
+        lea     rdi,[8+rdi]
+        sub     rsi,rax
+        mov     QWORD[rbx],r15
+        lea     rbx,[((64+32))+rsp]
+        sub     rcx,rax
+
+        mulx    r11,r8,QWORD[rsi]
+        xor     ebp,ebp
+        mov     r9,rdx
+        mulx    r12,r14,QWORD[8+rsi]
+        adox    r8,QWORD[((-32))+rbx]
+        adcx    r11,r14
+        mulx    r13,r15,QWORD[16+rsi]
+        adox    r11,QWORD[((-24))+rbx]
+        adcx    r12,r15
+        adox    r12,QWORD[((-16))+rbx]
+        adcx    r13,rbp
+        adox    r13,rbp
+
+        mov     QWORD[8+rsp],rdi
+        mov     r15,r8
+        imul    r8,QWORD[24+rsp]
+        xor     ebp,ebp
+
+        mulx    r14,rax,QWORD[24+rsi]
+        mov     rdx,r8
+        adcx    r13,rax
+        adox    r13,QWORD[((-8))+rbx]
+        adcx    r14,rbp
+        lea     rsi,[32+rsi]
+        adox    r14,rbp
+
+        mulx    r10,rax,QWORD[rcx]
+        adcx    r15,rax
+        adox    r10,r11
+        mulx    r11,rax,QWORD[8+rcx]
+        adcx    r10,rax
+        adox    r11,r12
+        mulx    r12,rax,QWORD[16+rcx]
+        mov     QWORD[((-32))+rbx],r10
+        adcx    r11,rax
+        adox    r12,r13
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-24))+rbx],r11
+        lea     rcx,[32+rcx]
+        adcx    r12,rax
+        adox    r15,rbp
+        mov     rdi,QWORD[48+rsp]
+        mov     QWORD[((-16))+rbx],r12
+
+        jmp     NEAR $L$mulx4x_inner
+
+ALIGN   32
+$L$mulx4x_inner:
+        mulx    rax,r10,QWORD[rsi]
+        adcx    r15,rbp
+        adox    r10,r14
+        mulx    r14,r11,QWORD[8+rsi]
+        adcx    r10,QWORD[rbx]
+        adox    r11,rax
+        mulx    rax,r12,QWORD[16+rsi]
+        adcx    r11,QWORD[8+rbx]
+        adox    r12,r14
+        mulx    r14,r13,QWORD[24+rsi]
+        mov     rdx,r8
+        adcx    r12,QWORD[16+rbx]
+        adox    r13,rax
+        adcx    r13,QWORD[24+rbx]
+        adox    r14,rbp
+        lea     rsi,[32+rsi]
+        lea     rbx,[32+rbx]
+        adcx    r14,rbp
+
+        adox    r10,r15
+        mulx    r15,rax,QWORD[rcx]
+        adcx    r10,rax
+        adox    r11,r15
+        mulx    r15,rax,QWORD[8+rcx]
+        adcx    r11,rax
+        adox    r12,r15
+        mulx    r15,rax,QWORD[16+rcx]
+        mov     QWORD[((-40))+rbx],r10
+        adcx    r12,rax
+        adox    r13,r15
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-32))+rbx],r11
+        mov     QWORD[((-24))+rbx],r12
+        adcx    r13,rax
+        adox    r15,rbp
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-16))+rbx],r13
+
+        dec     rdi
+        jnz     NEAR $L$mulx4x_inner
+
+        mov     rax,QWORD[rsp]
+        mov     rdi,QWORD[8+rsp]
+        adc     r15,rbp
+        sub     rbp,QWORD[rbx]
+        adc     r14,r15
+        sbb     r15,r15
+        mov     QWORD[((-8))+rbx],r14
+
+        cmp     rdi,QWORD[16+rsp]
+        jne     NEAR $L$mulx4x_outer
+
+        lea     rbx,[64+rsp]
+        sub     rcx,rax
+        neg     r15
+        mov     rdx,rax
+        shr     rax,3+2
+        mov     rdi,QWORD[32+rsp]
+        jmp     NEAR $L$mulx4x_sub
+
+ALIGN   32
+$L$mulx4x_sub:
+        mov     r11,QWORD[rbx]
+        mov     r12,QWORD[8+rbx]
+        mov     r13,QWORD[16+rbx]
+        mov     r14,QWORD[24+rbx]
+        lea     rbx,[32+rbx]
+        sbb     r11,QWORD[rcx]
+        sbb     r12,QWORD[8+rcx]
+        sbb     r13,QWORD[16+rcx]
+        sbb     r14,QWORD[24+rcx]
+        lea     rcx,[32+rcx]
+        mov     QWORD[rdi],r11
+        mov     QWORD[8+rdi],r12
+        mov     QWORD[16+rdi],r13
+        mov     QWORD[24+rdi],r14
+        lea     rdi,[32+rdi]
+        dec     rax
+        jnz     NEAR $L$mulx4x_sub
+
+        sbb     r15,0
+        lea     rbx,[64+rsp]
+        sub     rdi,rdx
+
+DB      102,73,15,110,207
+        pxor    xmm0,xmm0
+        pshufd  xmm1,xmm1,0
+        mov     rsi,QWORD[40+rsp]
+
+        jmp     NEAR $L$mulx4x_cond_copy
+
+ALIGN   32
+$L$mulx4x_cond_copy:
+        movdqa  xmm2,XMMWORD[rbx]
+        movdqa  xmm3,XMMWORD[16+rbx]
+        lea     rbx,[32+rbx]
+        movdqu  xmm4,XMMWORD[rdi]
+        movdqu  xmm5,XMMWORD[16+rdi]
+        lea     rdi,[32+rdi]
+        movdqa  XMMWORD[(-32)+rbx],xmm0
+        movdqa  XMMWORD[(-16)+rbx],xmm0
+        pcmpeqd xmm0,xmm1
+        pand    xmm2,xmm1
+        pand    xmm3,xmm1
+        pand    xmm4,xmm0
+        pand    xmm5,xmm0
+        pxor    xmm0,xmm0
+        por     xmm4,xmm2
+        por     xmm5,xmm3
+        movdqu  XMMWORD[(-32)+rdi],xmm4
+        movdqu  XMMWORD[(-16)+rdi],xmm5
+        sub     rdx,32
+        jnz     NEAR $L$mulx4x_cond_copy
+
+        mov     QWORD[rbx],rdx
+
+        mov     rax,1
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$mulx4x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_bn_mulx4x_mont:
 DB      77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105
 DB      112,108,105,99,97,116,105,111,110,32,102,111,114,32,120,56
@@ -1054 +1448 @@
         DD      $L$SEH_end_bn_sqr8x_mont wrt ..imagebase
         DD      $L$SEH_info_bn_sqr8x_mont wrt ..imagebase
+        DD      $L$SEH_begin_bn_mulx4x_mont wrt ..imagebase
+        DD      $L$SEH_end_bn_mulx4x_mont wrt ..imagebase
+        DD      $L$SEH_info_bn_mulx4x_mont wrt ..imagebase
 section .xdata rdata align=8
 ALIGN   8
@@ -1069 +1466 @@
         DD      $L$sqr8x_prologue wrt ..imagebase,$L$sqr8x_body wrt ..imagebase,$L$sqr8x_epilogue wrt ..imagebase
 ALIGN   8
+$L$SEH_info_bn_mulx4x_mont:
+DB      9,0,0,0
+        DD      sqr_handler wrt ..imagebase
+        DD      $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase
+ALIGN   8

trunk/src/libs/openssl-3.1.0/crypto/genasm-nasm/x86_64-mont5.S

@@ -30 +30 @@
         test    r9d,7
         jnz     NEAR $L$mul_enter
+        mov     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
         jmp     NEAR $L$mul4x_enter
 
@@ -480 +481 @@
 
 $L$mul4x_enter:
+        and     r11d,0x80108
+        cmp     r11d,0x80108
+        je      NEAR $L$mulx4x_enter
         push    rbx
 
@@ -1123 +1127 @@
         mov     rax,rsp
 
+        mov     r11d,DWORD[((OPENSSL_ia32cap_P+8))]
+        and     r11d,0x80108
+        cmp     r11d,0x80108
+        je      NEAR $L$powerx5_enter
         push    rbx
 
@@ -2085 +2093 @@
         mov     r10,r9
         neg     r9
+        DB      0F3h,0C3h               ;repret
+
+
+
+ALIGN   32
+bn_mulx4x_mont_gather5:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_bn_mulx4x_mont_gather5:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        mov     rax,rsp
+
+$L$mulx4x_enter:
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$mulx4x_prologue:
+
+        shl     r9d,3
+        lea     r10,[r9*2+r9]
+        neg     r9
+        mov     r8,QWORD[r8]
+
+
+
+
+
+
+
+
+
+
+        lea     r11,[((-320))+r9*2+rsp]
+        mov     rbp,rsp
+        sub     r11,rdi
+        and     r11,4095
+        cmp     r10,r11
+        jb      NEAR $L$mulx4xsp_alt
+        sub     rbp,r11
+        lea     rbp,[((-320))+r9*2+rbp]
+        jmp     NEAR $L$mulx4xsp_done
+
+$L$mulx4xsp_alt:
+        lea     r10,[((4096-320))+r9*2]
+        lea     rbp,[((-320))+r9*2+rbp]
+        sub     r11,r10
+        mov     r10,0
+        cmovc   r11,r10
+        sub     rbp,r11
+$L$mulx4xsp_done:
+        and     rbp,-64
+        mov     r11,rsp
+        sub     r11,rbp
+        and     r11,-4096
+        lea     rsp,[rbp*1+r11]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$mulx4x_page_walk
+        jmp     NEAR $L$mulx4x_page_walk_done
+
+$L$mulx4x_page_walk:
+        lea     rsp,[((-4096))+rsp]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$mulx4x_page_walk
+$L$mulx4x_page_walk_done:
+
+
+
+
+
+
+
+
+
+
+
+
+
+        mov     QWORD[32+rsp],r8
+        mov     QWORD[40+rsp],rax
+
+$L$mulx4x_body:
+        call    mulx4x_internal
+
+        mov     rsi,QWORD[40+rsp]
+
+        mov     rax,1
+
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$mulx4x_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_bn_mulx4x_mont_gather5:
+
+
+ALIGN   32
+mulx4x_internal:
+
+        mov     QWORD[8+rsp],r9
+        mov     r10,r9
+        neg     r9
+        shl     r9,5
+        neg     r10
+        lea     r13,[128+r9*1+rdx]
+        shr     r9,5+5
+        movd    xmm5,DWORD[56+rax]
+        sub     r9,1
+        lea     rax,[$L$inc]
+        mov     QWORD[((16+8))+rsp],r13
+        mov     QWORD[((24+8))+rsp],r9
+        mov     QWORD[((56+8))+rsp],rdi
+        movdqa  xmm0,XMMWORD[rax]
+        movdqa  xmm1,XMMWORD[16+rax]
+        lea     r10,[((88-112))+r10*1+rsp]
+        lea     rdi,[128+rdx]
+
+        pshufd  xmm5,xmm5,0
+        movdqa  xmm4,xmm1
+DB      0x67
+        movdqa  xmm2,xmm1
+DB      0x67
+        paddd   xmm1,xmm0
+        pcmpeqd xmm0,xmm5
+        movdqa  xmm3,xmm4
+        paddd   xmm2,xmm1
+        pcmpeqd xmm1,xmm5
+        movdqa  XMMWORD[112+r10],xmm0
+        movdqa  xmm0,xmm4
+
+        paddd   xmm3,xmm2
+        pcmpeqd xmm2,xmm5
+        movdqa  XMMWORD[128+r10],xmm1
+        movdqa  xmm1,xmm4
+
+        paddd   xmm0,xmm3
+        pcmpeqd xmm3,xmm5
+        movdqa  XMMWORD[144+r10],xmm2
+        movdqa  xmm2,xmm4
+
+        paddd   xmm1,xmm0
+        pcmpeqd xmm0,xmm5
+        movdqa  XMMWORD[160+r10],xmm3
+        movdqa  xmm3,xmm4
+        paddd   xmm2,xmm1
+        pcmpeqd xmm1,xmm5
+        movdqa  XMMWORD[176+r10],xmm0
+        movdqa  xmm0,xmm4
+
+        paddd   xmm3,xmm2
+        pcmpeqd xmm2,xmm5
+        movdqa  XMMWORD[192+r10],xmm1
+        movdqa  xmm1,xmm4
+
+        paddd   xmm0,xmm3
+        pcmpeqd xmm3,xmm5
+        movdqa  XMMWORD[208+r10],xmm2
+        movdqa  xmm2,xmm4
+
+        paddd   xmm1,xmm0
+        pcmpeqd xmm0,xmm5
+        movdqa  XMMWORD[224+r10],xmm3
+        movdqa  xmm3,xmm4
+        paddd   xmm2,xmm1
+        pcmpeqd xmm1,xmm5
+        movdqa  XMMWORD[240+r10],xmm0
+        movdqa  xmm0,xmm4
+
+        paddd   xmm3,xmm2
+        pcmpeqd xmm2,xmm5
+        movdqa  XMMWORD[256+r10],xmm1
+        movdqa  xmm1,xmm4
+
+        paddd   xmm0,xmm3
+        pcmpeqd xmm3,xmm5
+        movdqa  XMMWORD[272+r10],xmm2
+        movdqa  xmm2,xmm4
+
+        paddd   xmm1,xmm0
+        pcmpeqd xmm0,xmm5
+        movdqa  XMMWORD[288+r10],xmm3
+        movdqa  xmm3,xmm4
+DB      0x67
+        paddd   xmm2,xmm1
+        pcmpeqd xmm1,xmm5
+        movdqa  XMMWORD[304+r10],xmm0
+
+        paddd   xmm3,xmm2
+        pcmpeqd xmm2,xmm5
+        movdqa  XMMWORD[320+r10],xmm1
+
+        pcmpeqd xmm3,xmm5
+        movdqa  XMMWORD[336+r10],xmm2
+
+        pand    xmm0,XMMWORD[64+rdi]
+        pand    xmm1,XMMWORD[80+rdi]
+        pand    xmm2,XMMWORD[96+rdi]
+        movdqa  XMMWORD[352+r10],xmm3
+        pand    xmm3,XMMWORD[112+rdi]
+        por     xmm0,xmm2
+        por     xmm1,xmm3
+        movdqa  xmm4,XMMWORD[((-128))+rdi]
+        movdqa  xmm5,XMMWORD[((-112))+rdi]
+        movdqa  xmm2,XMMWORD[((-96))+rdi]
+        pand    xmm4,XMMWORD[112+r10]
+        movdqa  xmm3,XMMWORD[((-80))+rdi]
+        pand    xmm5,XMMWORD[128+r10]
+        por     xmm0,xmm4
+        pand    xmm2,XMMWORD[144+r10]
+        por     xmm1,xmm5
+        pand    xmm3,XMMWORD[160+r10]
+        por     xmm0,xmm2
+        por     xmm1,xmm3
+        movdqa  xmm4,XMMWORD[((-64))+rdi]
+        movdqa  xmm5,XMMWORD[((-48))+rdi]
+        movdqa  xmm2,XMMWORD[((-32))+rdi]
+        pand    xmm4,XMMWORD[176+r10]
+        movdqa  xmm3,XMMWORD[((-16))+rdi]
+        pand    xmm5,XMMWORD[192+r10]
+        por     xmm0,xmm4
+        pand    xmm2,XMMWORD[208+r10]
+        por     xmm1,xmm5
+        pand    xmm3,XMMWORD[224+r10]
+        por     xmm0,xmm2
+        por     xmm1,xmm3
+        movdqa  xmm4,XMMWORD[rdi]
+        movdqa  xmm5,XMMWORD[16+rdi]
+        movdqa  xmm2,XMMWORD[32+rdi]
+        pand    xmm4,XMMWORD[240+r10]
+        movdqa  xmm3,XMMWORD[48+rdi]
+        pand    xmm5,XMMWORD[256+r10]
+        por     xmm0,xmm4
+        pand    xmm2,XMMWORD[272+r10]
+        por     xmm1,xmm5
+        pand    xmm3,XMMWORD[288+r10]
+        por     xmm0,xmm2
+        por     xmm1,xmm3
+        pxor    xmm0,xmm1
+        pshufd  xmm1,xmm0,0x4e
+        por     xmm0,xmm1
+        lea     rdi,[256+rdi]
+DB      102,72,15,126,194
+        lea     rbx,[((64+32+8))+rsp]
+
+        mov     r9,rdx
+        mulx    rax,r8,QWORD[rsi]
+        mulx    r12,r11,QWORD[8+rsi]
+        add     r11,rax
+        mulx    r13,rax,QWORD[16+rsi]
+        adc     r12,rax
+        adc     r13,0
+        mulx    r14,rax,QWORD[24+rsi]
+
+        mov     r15,r8
+        imul    r8,QWORD[((32+8))+rsp]
+        xor     rbp,rbp
+        mov     rdx,r8
+
+        mov     QWORD[((8+8))+rsp],rdi
+
+        lea     rsi,[32+rsi]
+        adcx    r13,rax
+        adcx    r14,rbp
+
+        mulx    r10,rax,QWORD[rcx]
+        adcx    r15,rax
+        adox    r10,r11
+        mulx    r11,rax,QWORD[8+rcx]
+        adcx    r10,rax
+        adox    r11,r12
+        mulx    r12,rax,QWORD[16+rcx]
+        mov     rdi,QWORD[((24+8))+rsp]
+        mov     QWORD[((-32))+rbx],r10
+        adcx    r11,rax
+        adox    r12,r13
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-24))+rbx],r11
+        adcx    r12,rax
+        adox    r15,rbp
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-16))+rbx],r12
+        jmp     NEAR $L$mulx4x_1st
+
+ALIGN   32
+$L$mulx4x_1st:
+        adcx    r15,rbp
+        mulx    rax,r10,QWORD[rsi]
+        adcx    r10,r14
+        mulx    r14,r11,QWORD[8+rsi]
+        adcx    r11,rax
+        mulx    rax,r12,QWORD[16+rsi]
+        adcx    r12,r14
+        mulx    r14,r13,QWORD[24+rsi]
+DB      0x67,0x67
+        mov     rdx,r8
+        adcx    r13,rax
+        adcx    r14,rbp
+        lea     rsi,[32+rsi]
+        lea     rbx,[32+rbx]
+
+        adox    r10,r15
+        mulx    r15,rax,QWORD[rcx]
+        adcx    r10,rax
+        adox    r11,r15
+        mulx    r15,rax,QWORD[8+rcx]
+        adcx    r11,rax
+        adox    r12,r15
+        mulx    r15,rax,QWORD[16+rcx]
+        mov     QWORD[((-40))+rbx],r10
+        adcx    r12,rax
+        mov     QWORD[((-32))+rbx],r11
+        adox    r13,r15
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     QWORD[((-24))+rbx],r12
+        adcx    r13,rax
+        adox    r15,rbp
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-16))+rbx],r13
+
+        dec     rdi
+        jnz     NEAR $L$mulx4x_1st
+
+        mov     rax,QWORD[8+rsp]
+        adc     r15,rbp
+        lea     rsi,[rax*1+rsi]
+        add     r14,r15
+        mov     rdi,QWORD[((8+8))+rsp]
+        adc     rbp,rbp
+        mov     QWORD[((-8))+rbx],r14
+        jmp     NEAR $L$mulx4x_outer
+
+ALIGN   32
+$L$mulx4x_outer:
+        lea     r10,[((16-256))+rbx]
+        pxor    xmm4,xmm4
+DB      0x67,0x67
+        pxor    xmm5,xmm5
+        movdqa  xmm0,XMMWORD[((-128))+rdi]
+        movdqa  xmm1,XMMWORD[((-112))+rdi]
+        movdqa  xmm2,XMMWORD[((-96))+rdi]
+        pand    xmm0,XMMWORD[256+r10]
+        movdqa  xmm3,XMMWORD[((-80))+rdi]
+        pand    xmm1,XMMWORD[272+r10]
+        por     xmm4,xmm0
+        pand    xmm2,XMMWORD[288+r10]
+        por     xmm5,xmm1
+        pand    xmm3,XMMWORD[304+r10]
+        por     xmm4,xmm2
+        por     xmm5,xmm3
+        movdqa  xmm0,XMMWORD[((-64))+rdi]
+        movdqa  xmm1,XMMWORD[((-48))+rdi]
+        movdqa  xmm2,XMMWORD[((-32))+rdi]
+        pand    xmm0,XMMWORD[320+r10]
+        movdqa  xmm3,XMMWORD[((-16))+rdi]
+        pand    xmm1,XMMWORD[336+r10]
+        por     xmm4,xmm0
+        pand    xmm2,XMMWORD[352+r10]
+        por     xmm5,xmm1
+        pand    xmm3,XMMWORD[368+r10]
+        por     xmm4,xmm2
+        por     xmm5,xmm3
+        movdqa  xmm0,XMMWORD[rdi]
+        movdqa  xmm1,XMMWORD[16+rdi]
+        movdqa  xmm2,XMMWORD[32+rdi]
+        pand    xmm0,XMMWORD[384+r10]
+        movdqa  xmm3,XMMWORD[48+rdi]
+        pand    xmm1,XMMWORD[400+r10]
+        por     xmm4,xmm0
+        pand    xmm2,XMMWORD[416+r10]
+        por     xmm5,xmm1
+        pand    xmm3,XMMWORD[432+r10]
+        por     xmm4,xmm2
+        por     xmm5,xmm3
+        movdqa  xmm0,XMMWORD[64+rdi]
+        movdqa  xmm1,XMMWORD[80+rdi]
+        movdqa  xmm2,XMMWORD[96+rdi]
+        pand    xmm0,XMMWORD[448+r10]
+        movdqa  xmm3,XMMWORD[112+rdi]
+        pand    xmm1,XMMWORD[464+r10]
+        por     xmm4,xmm0
+        pand    xmm2,XMMWORD[480+r10]
+        por     xmm5,xmm1
+        pand    xmm3,XMMWORD[496+r10]
+        por     xmm4,xmm2
+        por     xmm5,xmm3
+        por     xmm4,xmm5
+        pshufd  xmm0,xmm4,0x4e
+        por     xmm0,xmm4
+        lea     rdi,[256+rdi]
+DB      102,72,15,126,194
+
+        mov     QWORD[rbx],rbp
+        lea     rbx,[32+rax*1+rbx]
+        mulx    r11,r8,QWORD[rsi]
+        xor     rbp,rbp
+        mov     r9,rdx
+        mulx    r12,r14,QWORD[8+rsi]
+        adox    r8,QWORD[((-32))+rbx]
+        adcx    r11,r14
+        mulx    r13,r15,QWORD[16+rsi]
+        adox    r11,QWORD[((-24))+rbx]
+        adcx    r12,r15
+        mulx    r14,rdx,QWORD[24+rsi]
+        adox    r12,QWORD[((-16))+rbx]
+        adcx    r13,rdx
+        lea     rcx,[rax*1+rcx]
+        lea     rsi,[32+rsi]
+        adox    r13,QWORD[((-8))+rbx]
+        adcx    r14,rbp
+        adox    r14,rbp
+
+        mov     r15,r8
+        imul    r8,QWORD[((32+8))+rsp]
+
+        mov     rdx,r8
+        xor     rbp,rbp
+        mov     QWORD[((8+8))+rsp],rdi
+
+        mulx    r10,rax,QWORD[rcx]
+        adcx    r15,rax
+        adox    r10,r11
+        mulx    r11,rax,QWORD[8+rcx]
+        adcx    r10,rax
+        adox    r11,r12
+        mulx    r12,rax,QWORD[16+rcx]
+        adcx    r11,rax
+        adox    r12,r13
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        mov     rdi,QWORD[((24+8))+rsp]
+        mov     QWORD[((-32))+rbx],r10
+        adcx    r12,rax
+        mov     QWORD[((-24))+rbx],r11
+        adox    r15,rbp
+        mov     QWORD[((-16))+rbx],r12
+        lea     rcx,[32+rcx]
+        jmp     NEAR $L$mulx4x_inner
+
+ALIGN   32
+$L$mulx4x_inner:
+        mulx    rax,r10,QWORD[rsi]
+        adcx    r15,rbp
+        adox    r10,r14
+        mulx    r14,r11,QWORD[8+rsi]
+        adcx    r10,QWORD[rbx]
+        adox    r11,rax
+        mulx    rax,r12,QWORD[16+rsi]
+        adcx    r11,QWORD[8+rbx]
+        adox    r12,r14
+        mulx    r14,r13,QWORD[24+rsi]
+        mov     rdx,r8
+        adcx    r12,QWORD[16+rbx]
+        adox    r13,rax
+        adcx    r13,QWORD[24+rbx]
+        adox    r14,rbp
+        lea     rsi,[32+rsi]
+        lea     rbx,[32+rbx]
+        adcx    r14,rbp
+
+        adox    r10,r15
+        mulx    r15,rax,QWORD[rcx]
+        adcx    r10,rax
+        adox    r11,r15
+        mulx    r15,rax,QWORD[8+rcx]
+        adcx    r11,rax
+        adox    r12,r15
+        mulx    r15,rax,QWORD[16+rcx]
+        mov     QWORD[((-40))+rbx],r10
+        adcx    r12,rax
+        adox    r13,r15
+        mov     QWORD[((-32))+rbx],r11
+        mulx    r15,rax,QWORD[24+rcx]
+        mov     rdx,r9
+        lea     rcx,[32+rcx]
+        mov     QWORD[((-24))+rbx],r12
+        adcx    r13,rax
+        adox    r15,rbp
+        mov     QWORD[((-16))+rbx],r13
+
+        dec     rdi
+        jnz     NEAR $L$mulx4x_inner
+
+        mov     rax,QWORD[((0+8))+rsp]
+        adc     r15,rbp
+        sub     rdi,QWORD[rbx]
+        mov     rdi,QWORD[((8+8))+rsp]
+        mov     r10,QWORD[((16+8))+rsp]
+        adc     r14,r15
+        lea     rsi,[rax*1+rsi]
+        adc     rbp,rbp
+        mov     QWORD[((-8))+rbx],r14
+
+        cmp     rdi,r10
+        jb      NEAR $L$mulx4x_outer
+
+        mov     r10,QWORD[((-8))+rcx]
+        mov     r8,rbp
+        mov     r12,QWORD[rax*1+rcx]
+        lea     rbp,[rax*1+rcx]
+        mov     rcx,rax
+        lea     rdi,[rax*1+rbx]
+        xor     eax,eax
+        xor     r15,r15
+        sub     r10,r14
+        adc     r15,r15
+        or      r8,r15
+        sar     rcx,3+2
+        sub     rax,r8
+        mov     rdx,QWORD[((56+8))+rsp]
+        dec     r12
+        mov     r13,QWORD[8+rbp]
+        xor     r8,r8
+        mov     r14,QWORD[16+rbp]
+        mov     r15,QWORD[24+rbp]
+        jmp     NEAR $L$sqrx4x_sub_entry
+
+
+
+ALIGN   32
+bn_powerx5:
+        mov     QWORD[8+rsp],rdi        ;WIN64 prologue
+        mov     QWORD[16+rsp],rsi
+        mov     rax,rsp
+$L$SEH_begin_bn_powerx5:
+        mov     rdi,rcx
+        mov     rsi,rdx
+        mov     rdx,r8
+        mov     rcx,r9
+        mov     r8,QWORD[40+rsp]
+        mov     r9,QWORD[48+rsp]
+
+
+
+        mov     rax,rsp
+
+$L$powerx5_enter:
+        push    rbx
+
+        push    rbp
+
+        push    r12
+
+        push    r13
+
+        push    r14
+
+        push    r15
+
+$L$powerx5_prologue:
+
+        shl     r9d,3
+        lea     r10,[r9*2+r9]
+        neg     r9
+        mov     r8,QWORD[r8]
+
+
+
+
+
+
+
+
+        lea     r11,[((-320))+r9*2+rsp]
+        mov     rbp,rsp
+        sub     r11,rdi
+        and     r11,4095
+        cmp     r10,r11
+        jb      NEAR $L$pwrx_sp_alt
+        sub     rbp,r11
+        lea     rbp,[((-320))+r9*2+rbp]
+        jmp     NEAR $L$pwrx_sp_done
+
+ALIGN   32
+$L$pwrx_sp_alt:
+        lea     r10,[((4096-320))+r9*2]
+        lea     rbp,[((-320))+r9*2+rbp]
+        sub     r11,r10
+        mov     r10,0
+        cmovc   r11,r10
+        sub     rbp,r11
+$L$pwrx_sp_done:
+        and     rbp,-64
+        mov     r11,rsp
+        sub     r11,rbp
+        and     r11,-4096
+        lea     rsp,[rbp*1+r11]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$pwrx_page_walk
+        jmp     NEAR $L$pwrx_page_walk_done
+
+$L$pwrx_page_walk:
+        lea     rsp,[((-4096))+rsp]
+        mov     r10,QWORD[rsp]
+        cmp     rsp,rbp
+        ja      NEAR $L$pwrx_page_walk
+$L$pwrx_page_walk_done:
+
+        mov     r10,r9
+        neg     r9
+
+
+
+
+
+
+
+
+
+
+
+
+        pxor    xmm0,xmm0
+DB      102,72,15,110,207
+DB      102,72,15,110,209
+DB      102,73,15,110,218
+DB      102,72,15,110,226
+        mov     QWORD[32+rsp],r8
+        mov     QWORD[40+rsp],rax
+
+$L$powerx5_body:
+
+        call    __bn_sqrx8x_internal
+        call    __bn_postx4x_internal
+        call    __bn_sqrx8x_internal
+        call    __bn_postx4x_internal
+        call    __bn_sqrx8x_internal
+        call    __bn_postx4x_internal
+        call    __bn_sqrx8x_internal
+        call    __bn_postx4x_internal
+        call    __bn_sqrx8x_internal
+        call    __bn_postx4x_internal
+
+        mov     r9,r10
+        mov     rdi,rsi
+DB      102,72,15,126,209
+DB      102,72,15,126,226
+        mov     rax,QWORD[40+rsp]
+
+        call    mulx4x_internal
+
+        mov     rsi,QWORD[40+rsp]
+
+        mov     rax,1
+
+        mov     r15,QWORD[((-48))+rsi]
+
+        mov     r14,QWORD[((-40))+rsi]
+
+        mov     r13,QWORD[((-32))+rsi]
+
+        mov     r12,QWORD[((-24))+rsi]
+
+        mov     rbp,QWORD[((-16))+rsi]
+
+        mov     rbx,QWORD[((-8))+rsi]
+
+        lea     rsp,[rsi]
+
+$L$powerx5_epilogue:
+        mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
+        mov     rsi,QWORD[16+rsp]
+        DB      0F3h,0C3h               ;repret
+
+$L$SEH_end_bn_powerx5:
+
+global  bn_sqrx8x_internal
+
+
+ALIGN   32
+bn_sqrx8x_internal:
+__bn_sqrx8x_internal:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        lea     rdi,[((48+8))+rsp]
+        lea     rbp,[r9*1+rsi]
+        mov     QWORD[((0+8))+rsp],r9
+        mov     QWORD[((8+8))+rsp],rbp
+        jmp     NEAR $L$sqr8x_zero_start
+
+ALIGN   32
+DB      0x66,0x66,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00
+$L$sqrx8x_zero:
+DB      0x3e
+        movdqa  XMMWORD[rdi],xmm0
+        movdqa  XMMWORD[16+rdi],xmm0
+        movdqa  XMMWORD[32+rdi],xmm0
+        movdqa  XMMWORD[48+rdi],xmm0
+$L$sqr8x_zero_start:
+        movdqa  XMMWORD[64+rdi],xmm0
+        movdqa  XMMWORD[80+rdi],xmm0
+        movdqa  XMMWORD[96+rdi],xmm0
+        movdqa  XMMWORD[112+rdi],xmm0
+        lea     rdi,[128+rdi]
+        sub     r9,64
+        jnz     NEAR $L$sqrx8x_zero
+
+        mov     rdx,QWORD[rsi]
+
+        xor     r10,r10
+        xor     r11,r11
+        xor     r12,r12
+        xor     r13,r13
+        xor     r14,r14
+        xor     r15,r15
+        lea     rdi,[((48+8))+rsp]
+        xor     rbp,rbp
+        jmp     NEAR $L$sqrx8x_outer_loop
+
+ALIGN   32
+$L$sqrx8x_outer_loop:
+        mulx    rax,r8,QWORD[8+rsi]
+        adcx    r8,r9
+        adox    r10,rax
+        mulx    rax,r9,QWORD[16+rsi]
+        adcx    r9,r10
+        adox    r11,rax
+DB      0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00
+        adcx    r10,r11
+        adox    r12,rax
+DB      0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00
+        adcx    r11,r12
+        adox    r13,rax
+        mulx    rax,r12,QWORD[40+rsi]
+        adcx    r12,r13
+        adox    r14,rax
+        mulx    rax,r13,QWORD[48+rsi]
+        adcx    r13,r14
+        adox    rax,r15
+        mulx    r15,r14,QWORD[56+rsi]
+        mov     rdx,QWORD[8+rsi]
+        adcx    r14,rax
+        adox    r15,rbp
+        adc     r15,QWORD[64+rdi]
+        mov     QWORD[8+rdi],r8
+        mov     QWORD[16+rdi],r9
+        sbb     rcx,rcx
+        xor     rbp,rbp
+
+
+        mulx    rbx,r8,QWORD[16+rsi]
+        mulx    rax,r9,QWORD[24+rsi]
+        adcx    r8,r10
+        adox    r9,rbx
+        mulx    rbx,r10,QWORD[32+rsi]
+        adcx    r9,r11
+        adox    r10,rax
+DB      0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00
+        adcx    r10,r12
+        adox    r11,rbx
+DB      0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00
+        adcx    r11,r13
+        adox    r12,r14
+DB      0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00
+        mov     rdx,QWORD[16+rsi]
+        adcx    r12,rax
+        adox    r13,rbx
+        adcx    r13,r15
+        adox    r14,rbp
+        adcx    r14,rbp
+
+        mov     QWORD[24+rdi],r8
+        mov     QWORD[32+rdi],r9
+
+        mulx    rbx,r8,QWORD[24+rsi]
+        mulx    rax,r9,QWORD[32+rsi]
+        adcx    r8,r10
+        adox    r9,rbx
+        mulx    rbx,r10,QWORD[40+rsi]
+        adcx    r9,r11
+        adox    r10,rax
+DB      0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00
+        adcx    r10,r12
+        adox    r11,r13
+DB      0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00
+DB      0x3e
+        mov     rdx,QWORD[24+rsi]
+        adcx    r11,rbx
+        adox    r12,rax
+        adcx    r12,r14
+        mov     QWORD[40+rdi],r8
+        mov     QWORD[48+rdi],r9
+        mulx    rax,r8,QWORD[32+rsi]
+        adox    r13,rbp
+        adcx    r13,rbp
+
+        mulx    rbx,r9,QWORD[40+rsi]
+        adcx    r8,r10
+        adox    r9,rax
+        mulx    rax,r10,QWORD[48+rsi]
+        adcx    r9,r11
+        adox    r10,r12
+        mulx    r12,r11,QWORD[56+rsi]
+        mov     rdx,QWORD[32+rsi]
+        mov     r14,QWORD[40+rsi]
+        adcx    r10,rbx
+        adox    r11,rax
+        mov     r15,QWORD[48+rsi]
+        adcx    r11,r13
+        adox    r12,rbp
+        adcx    r12,rbp
+
+        mov     QWORD[56+rdi],r8
+        mov     QWORD[64+rdi],r9
+
+        mulx    rax,r9,r14
+        mov     r8,QWORD[56+rsi]
+        adcx    r9,r10
+        mulx    rbx,r10,r15
+        adox    r10,rax
+        adcx    r10,r11
+        mulx    rax,r11,r8
+        mov     rdx,r14
+        adox    r11,rbx
+        adcx    r11,r12
+
+        adcx    rax,rbp
+
+        mulx    rbx,r14,r15
+        mulx    r13,r12,r8
+        mov     rdx,r15
+        lea     rsi,[64+rsi]
+        adcx    r11,r14
+        adox    r12,rbx
+        adcx    r12,rax
+        adox    r13,rbp
+
+DB      0x67,0x67
+        mulx    r14,r8,r8
+        adcx    r13,r8
+        adcx    r14,rbp
+
+        cmp     rsi,QWORD[((8+8))+rsp]
+        je      NEAR $L$sqrx8x_outer_break
+
+        neg     rcx
+        mov     rcx,-8
+        mov     r15,rbp
+        mov     r8,QWORD[64+rdi]
+        adcx    r9,QWORD[72+rdi]
+        adcx    r10,QWORD[80+rdi]
+        adcx    r11,QWORD[88+rdi]
+        adc     r12,QWORD[96+rdi]
+        adc     r13,QWORD[104+rdi]
+        adc     r14,QWORD[112+rdi]
+        adc     r15,QWORD[120+rdi]
+        lea     rbp,[rsi]
+        lea     rdi,[128+rdi]
+        sbb     rax,rax
+
+        mov     rdx,QWORD[((-64))+rsi]
+        mov     QWORD[((16+8))+rsp],rax
+        mov     QWORD[((24+8))+rsp],rdi
+
+
+        xor     eax,eax
+        jmp     NEAR $L$sqrx8x_loop
+
+ALIGN   32
+$L$sqrx8x_loop:
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rbp]
+        adcx    rbx,rax
+        adox    r8,r9
+
+        mulx    r9,rax,QWORD[8+rbp]
+        adcx    r8,rax
+        adox    r9,r10
+
+        mulx    r10,rax,QWORD[16+rbp]
+        adcx    r9,rax
+        adox    r10,r11
+
+        mulx    r11,rax,QWORD[24+rbp]
+        adcx    r10,rax
+        adox    r11,r12
+
+DB      0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00
+        adcx    r11,rax
+        adox    r12,r13
+
+        mulx    r13,rax,QWORD[40+rbp]
+        adcx    r12,rax
+        adox    r13,r14
+
+        mulx    r14,rax,QWORD[48+rbp]
+        mov     QWORD[rcx*8+rdi],rbx
+        mov     ebx,0
+        adcx    r13,rax
+        adox    r14,r15
+
+DB      0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00
+        mov     rdx,QWORD[8+rcx*8+rsi]
+        adcx    r14,rax
+        adox    r15,rbx
+        adcx    r15,rbx
+
+DB      0x67
+        inc     rcx
+        jnz     NEAR $L$sqrx8x_loop
+
+        lea     rbp,[64+rbp]
+        mov     rcx,-8
+        cmp     rbp,QWORD[((8+8))+rsp]
+        je      NEAR $L$sqrx8x_break
+
+        sub     rbx,QWORD[((16+8))+rsp]
+DB      0x66
+        mov     rdx,QWORD[((-64))+rsi]
+        adcx    r8,QWORD[rdi]
+        adcx    r9,QWORD[8+rdi]
+        adc     r10,QWORD[16+rdi]
+        adc     r11,QWORD[24+rdi]
+        adc     r12,QWORD[32+rdi]
+        adc     r13,QWORD[40+rdi]
+        adc     r14,QWORD[48+rdi]
+        adc     r15,QWORD[56+rdi]
+        lea     rdi,[64+rdi]
+DB      0x67
+        sbb     rax,rax
+        xor     ebx,ebx
+        mov     QWORD[((16+8))+rsp],rax
+        jmp     NEAR $L$sqrx8x_loop
+
+ALIGN   32
+$L$sqrx8x_break:
+        xor     rbp,rbp
+        sub     rbx,QWORD[((16+8))+rsp]
+        adcx    r8,rbp
+        mov     rcx,QWORD[((24+8))+rsp]
+        adcx    r9,rbp
+        mov     rdx,QWORD[rsi]
+        adc     r10,0
+        mov     QWORD[rdi],r8
+        adc     r11,0
+        adc     r12,0
+        adc     r13,0
+        adc     r14,0
+        adc     r15,0
+        cmp     rdi,rcx
+        je      NEAR $L$sqrx8x_outer_loop
+
+        mov     QWORD[8+rdi],r9
+        mov     r9,QWORD[8+rcx]
+        mov     QWORD[16+rdi],r10
+        mov     r10,QWORD[16+rcx]
+        mov     QWORD[24+rdi],r11
+        mov     r11,QWORD[24+rcx]
+        mov     QWORD[32+rdi],r12
+        mov     r12,QWORD[32+rcx]
+        mov     QWORD[40+rdi],r13
+        mov     r13,QWORD[40+rcx]
+        mov     QWORD[48+rdi],r14
+        mov     r14,QWORD[48+rcx]
+        mov     QWORD[56+rdi],r15
+        mov     r15,QWORD[56+rcx]
+        mov     rdi,rcx
+        jmp     NEAR $L$sqrx8x_outer_loop
+
+ALIGN   32
+$L$sqrx8x_outer_break:
+        mov     QWORD[72+rdi],r9
+DB      102,72,15,126,217
+        mov     QWORD[80+rdi],r10
+        mov     QWORD[88+rdi],r11
+        mov     QWORD[96+rdi],r12
+        mov     QWORD[104+rdi],r13
+        mov     QWORD[112+rdi],r14
+        lea     rdi,[((48+8))+rsp]
+        mov     rdx,QWORD[rcx*1+rsi]
+
+        mov     r11,QWORD[8+rdi]
+        xor     r10,r10
+        mov     r9,QWORD[((0+8))+rsp]
+        adox    r11,r11
+        mov     r12,QWORD[16+rdi]
+        mov     r13,QWORD[24+rdi]
+
+
+ALIGN   32
+$L$sqrx4x_shift_n_add:
+        mulx    rbx,rax,rdx
+        adox    r12,r12
+        adcx    rax,r10
+DB      0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00
+DB      0x4c,0x8b,0x97,0x20,0x00,0x00,0x00
+        adox    r13,r13
+        adcx    rbx,r11
+        mov     r11,QWORD[40+rdi]
+        mov     QWORD[rdi],rax
+        mov     QWORD[8+rdi],rbx
+
+        mulx    rbx,rax,rdx
+        adox    r10,r10
+        adcx    rax,r12
+        mov     rdx,QWORD[16+rcx*1+rsi]
+        mov     r12,QWORD[48+rdi]
+        adox    r11,r11
+        adcx    rbx,r13
+        mov     r13,QWORD[56+rdi]
+        mov     QWORD[16+rdi],rax
+        mov     QWORD[24+rdi],rbx
+
+        mulx    rbx,rax,rdx
+        adox    r12,r12
+        adcx    rax,r10
+        mov     rdx,QWORD[24+rcx*1+rsi]
+        lea     rcx,[32+rcx]
+        mov     r10,QWORD[64+rdi]
+        adox    r13,r13
+        adcx    rbx,r11
+        mov     r11,QWORD[72+rdi]
+        mov     QWORD[32+rdi],rax
+        mov     QWORD[40+rdi],rbx
+
+        mulx    rbx,rax,rdx
+        adox    r10,r10
+        adcx    rax,r12
+        jrcxz   $L$sqrx4x_shift_n_add_break
+DB      0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00
+        adox    r11,r11
+        adcx    rbx,r13
+        mov     r12,QWORD[80+rdi]
+        mov     r13,QWORD[88+rdi]
+        mov     QWORD[48+rdi],rax
+        mov     QWORD[56+rdi],rbx
+        lea     rdi,[64+rdi]
+        nop
+        jmp     NEAR $L$sqrx4x_shift_n_add
+
+ALIGN   32
+$L$sqrx4x_shift_n_add_break:
+        adcx    rbx,r13
+        mov     QWORD[48+rdi],rax
+        mov     QWORD[56+rdi],rbx
+        lea     rdi,[64+rdi]
+DB      102,72,15,126,213
+__bn_sqrx8x_reduction:
+        xor     eax,eax
+        mov     rbx,QWORD[((32+8))+rsp]
+        mov     rdx,QWORD[((48+8))+rsp]
+        lea     rcx,[((-64))+r9*1+rbp]
+
+        mov     QWORD[((0+8))+rsp],rcx
+        mov     QWORD[((8+8))+rsp],rdi
+
+        lea     rdi,[((48+8))+rsp]
+        jmp     NEAR $L$sqrx8x_reduction_loop
+
+ALIGN   32
+$L$sqrx8x_reduction_loop:
+        mov     r9,QWORD[8+rdi]
+        mov     r10,QWORD[16+rdi]
+        mov     r11,QWORD[24+rdi]
+        mov     r12,QWORD[32+rdi]
+        mov     r8,rdx
+        imul    rdx,rbx
+        mov     r13,QWORD[40+rdi]
+        mov     r14,QWORD[48+rdi]
+        mov     r15,QWORD[56+rdi]
+        mov     QWORD[((24+8))+rsp],rax
+
+        lea     rdi,[64+rdi]
+        xor     rsi,rsi
+        mov     rcx,-8
+        jmp     NEAR $L$sqrx8x_reduce
+
+ALIGN   32
+$L$sqrx8x_reduce:
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rbp]
+        adcx    rax,rbx
+        adox    r8,r9
+
+        mulx    r9,rbx,QWORD[8+rbp]
+        adcx    r8,rbx
+        adox    r9,r10
+
+        mulx    r10,rbx,QWORD[16+rbp]
+        adcx    r9,rbx
+        adox    r10,r11
+
+        mulx    r11,rbx,QWORD[24+rbp]
+        adcx    r10,rbx
+        adox    r11,r12
+
+DB      0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00
+        mov     rax,rdx
+        mov     rdx,r8
+        adcx    r11,rbx
+        adox    r12,r13
+
+        mulx    rdx,rbx,QWORD[((32+8))+rsp]
+        mov     rdx,rax
+        mov     QWORD[((64+48+8))+rcx*8+rsp],rax
+
+        mulx    r13,rax,QWORD[40+rbp]
+        adcx    r12,rax
+        adox    r13,r14
+
+        mulx    r14,rax,QWORD[48+rbp]
+        adcx    r13,rax
+        adox    r14,r15
+
+        mulx    r15,rax,QWORD[56+rbp]
+        mov     rdx,rbx
+        adcx    r14,rax
+        adox    r15,rsi
+        adcx    r15,rsi
+
+DB      0x67,0x67,0x67
+        inc     rcx
+        jnz     NEAR $L$sqrx8x_reduce
+
+        mov     rax,rsi
+        cmp     rbp,QWORD[((0+8))+rsp]
+        jae     NEAR $L$sqrx8x_no_tail
+
+        mov     rdx,QWORD[((48+8))+rsp]
+        add     r8,QWORD[rdi]
+        lea     rbp,[64+rbp]
+        mov     rcx,-8
+        adcx    r9,QWORD[8+rdi]
+        adcx    r10,QWORD[16+rdi]
+        adc     r11,QWORD[24+rdi]
+        adc     r12,QWORD[32+rdi]
+        adc     r13,QWORD[40+rdi]
+        adc     r14,QWORD[48+rdi]
+        adc     r15,QWORD[56+rdi]
+        lea     rdi,[64+rdi]
+        sbb     rax,rax
+
+        xor     rsi,rsi
+        mov     QWORD[((16+8))+rsp],rax
+        jmp     NEAR $L$sqrx8x_tail
+
+ALIGN   32
+$L$sqrx8x_tail:
+        mov     rbx,r8
+        mulx    r8,rax,QWORD[rbp]
+        adcx    rbx,rax
+        adox    r8,r9
+
+        mulx    r9,rax,QWORD[8+rbp]
+        adcx    r8,rax
+        adox    r9,r10
+
+        mulx    r10,rax,QWORD[16+rbp]
+        adcx    r9,rax
+        adox    r10,r11
+
+        mulx    r11,rax,QWORD[24+rbp]
+        adcx    r10,rax
+        adox    r11,r12
+
+DB      0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00
+        adcx    r11,rax
+        adox    r12,r13
+
+        mulx    r13,rax,QWORD[40+rbp]
+        adcx    r12,rax
+        adox    r13,r14
+
+        mulx    r14,rax,QWORD[48+rbp]
+        adcx    r13,rax
+        adox    r14,r15
+
+        mulx    r15,rax,QWORD[56+rbp]
+        mov     rdx,QWORD[((72+48+8))+rcx*8+rsp]
+        adcx    r14,rax
+        adox    r15,rsi
+        mov     QWORD[rcx*8+rdi],rbx
+        mov     rbx,r8
+        adcx    r15,rsi
+
+        inc     rcx
+        jnz     NEAR $L$sqrx8x_tail
+
+        cmp     rbp,QWORD[((0+8))+rsp]
+        jae     NEAR $L$sqrx8x_tail_done
+
+        sub     rsi,QWORD[((16+8))+rsp]
+        mov     rdx,QWORD[((48+8))+rsp]
+        lea     rbp,[64+rbp]
+        adc     r8,QWORD[rdi]
+        adc     r9,QWORD[8+rdi]
+        adc     r10,QWORD[16+rdi]
+        adc     r11,QWORD[24+rdi]
+        adc     r12,QWORD[32+rdi]
+        adc     r13,QWORD[40+rdi]
+        adc     r14,QWORD[48+rdi]
+        adc     r15,QWORD[56+rdi]
+        lea     rdi,[64+rdi]
+        sbb     rax,rax
+        sub     rcx,8
+
+        xor     rsi,rsi
+        mov     QWORD[((16+8))+rsp],rax
+        jmp     NEAR $L$sqrx8x_tail
+
+ALIGN   32
+$L$sqrx8x_tail_done:
+        xor     rax,rax
+        add     r8,QWORD[((24+8))+rsp]
+        adc     r9,0
+        adc     r10,0
+        adc     r11,0
+        adc     r12,0
+        adc     r13,0
+        adc     r14,0
+        adc     r15,0
+        adc     rax,0
+
+        sub     rsi,QWORD[((16+8))+rsp]
+$L$sqrx8x_no_tail:
+        adc     r8,QWORD[rdi]
+DB      102,72,15,126,217
+        adc     r9,QWORD[8+rdi]
+        mov     rsi,QWORD[56+rbp]
+DB      102,72,15,126,213
+        adc     r10,QWORD[16+rdi]
+        adc     r11,QWORD[24+rdi]
+        adc     r12,QWORD[32+rdi]
+        adc     r13,QWORD[40+rdi]
+        adc     r14,QWORD[48+rdi]
+        adc     r15,QWORD[56+rdi]
+        adc     rax,0
+
+        mov     rbx,QWORD[((32+8))+rsp]
+        mov     rdx,QWORD[64+rcx*1+rdi]
+
+        mov     QWORD[rdi],r8
+        lea     r8,[64+rdi]
+        mov     QWORD[8+rdi],r9
+        mov     QWORD[16+rdi],r10
+        mov     QWORD[24+rdi],r11
+        mov     QWORD[32+rdi],r12
+        mov     QWORD[40+rdi],r13
+        mov     QWORD[48+rdi],r14
+        mov     QWORD[56+rdi],r15
+
+        lea     rdi,[64+rcx*1+rdi]
+        cmp     r8,QWORD[((8+8))+rsp]
+        jb      NEAR $L$sqrx8x_reduction_loop
+        DB      0F3h,0C3h               ;repret
+
+
+ALIGN   32
+__bn_postx4x_internal:
+
+        mov     r12,QWORD[rbp]
+        mov     r10,rcx
+        mov     r9,rcx
+        neg     rax
+        sar     rcx,3+2
+
+DB      102,72,15,126,202
+DB      102,72,15,126,206
+        dec     r12
+        mov     r13,QWORD[8+rbp]
+        xor     r8,r8
+        mov     r14,QWORD[16+rbp]
+        mov     r15,QWORD[24+rbp]
+        jmp     NEAR $L$sqrx4x_sub_entry
+
+ALIGN   16
+$L$sqrx4x_sub:
+        mov     r12,QWORD[rbp]
+        mov     r13,QWORD[8+rbp]
+        mov     r14,QWORD[16+rbp]
+        mov     r15,QWORD[24+rbp]
+$L$sqrx4x_sub_entry:
+        andn    r12,r12,rax
+        lea     rbp,[32+rbp]
+        andn    r13,r13,rax
+        andn    r14,r14,rax
+        andn    r15,r15,rax
+
+        neg     r8
+        adc     r12,QWORD[rdi]
+        adc     r13,QWORD[8+rdi]
+        adc     r14,QWORD[16+rdi]
+        adc     r15,QWORD[24+rdi]
+        mov     QWORD[rdx],r12
+        lea     rdi,[32+rdi]
+        mov     QWORD[8+rdx],r13
+        sbb     r8,r8
+        mov     QWORD[16+rdx],r14
+        mov     QWORD[24+rdx],r15
+        lea     rdx,[32+rdx]
+
+        inc     rcx
+        jnz     NEAR $L$sqrx4x_sub
+
+        neg     r9
+
         DB      0F3h,0C3h               ;repret
 
@@ -2420 +3798 @@
         DD      $L$SEH_end_bn_power5 wrt ..imagebase
         DD      $L$SEH_info_bn_power5 wrt ..imagebase
+        DD      $L$SEH_begin_bn_mulx4x_mont_gather5 wrt ..imagebase
+        DD      $L$SEH_end_bn_mulx4x_mont_gather5 wrt ..imagebase
+        DD      $L$SEH_info_bn_mulx4x_mont_gather5 wrt ..imagebase
+
+        DD      $L$SEH_begin_bn_powerx5 wrt ..imagebase
+        DD      $L$SEH_end_bn_powerx5 wrt ..imagebase
+        DD      $L$SEH_info_bn_powerx5 wrt ..imagebase
         DD      $L$SEH_begin_bn_gather5 wrt ..imagebase
         DD      $L$SEH_end_bn_gather5 wrt ..imagebase
@@ -2441 +3826 @@
         DD      $L$power5_prologue wrt ..imagebase,$L$power5_body wrt ..imagebase,$L$power5_epilogue wrt ..imagebase
 ALIGN   8
+$L$SEH_info_bn_mulx4x_mont_gather5:
+DB      9,0,0,0
+        DD      mul_handler wrt ..imagebase
+        DD      $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase
+ALIGN   8
+$L$SEH_info_bn_powerx5:
+DB      9,0,0,0
+        DD      mul_handler wrt ..imagebase
+        DD      $L$powerx5_prologue wrt ..imagebase,$L$powerx5_body wrt ..imagebase,$L$powerx5_epilogue wrt ..imagebase
+ALIGN   8
 $L$SEH_info_bn_gather5:
 DB      0x01,0x0b,0x03,0x0a

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/asn1.h

@@ -3 +3 @@
  * Generated by Makefile from asn1.h.in
  *
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -22 +22 @@
 # endif
 
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 # include <time.h>
 # include <openssl/e_os2.h>

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/cmp.h

@@ -3 +3 @@
  * Generated by Makefile from cmp.h.in
  *
- * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2007-2022 The OpenSSL Project Authors. All Rights Reserved.
  * Copyright Nokia 2007-2019
  * Copyright Siemens AG 2015-2019
@@ -194 +194 @@
  *   }
  */
-#  define OSSL_CMP_PKISTATUS_accepted 0
-#  define OSSL_CMP_PKISTATUS_grantedWithMods 1
-#  define OSSL_CMP_PKISTATUS_rejection 2
-#  define OSSL_CMP_PKISTATUS_waiting 3
-#  define OSSL_CMP_PKISTATUS_revocationWarning 4
+#  define OSSL_CMP_PKISTATUS_request                -3
+#  define OSSL_CMP_PKISTATUS_trans                  -2
+#  define OSSL_CMP_PKISTATUS_unspecified            -1
+#  define OSSL_CMP_PKISTATUS_accepted               0
+#  define OSSL_CMP_PKISTATUS_grantedWithMods        1
+#  define OSSL_CMP_PKISTATUS_rejection              2
+#  define OSSL_CMP_PKISTATUS_waiting                3
+#  define OSSL_CMP_PKISTATUS_revocationWarning      4
 #  define OSSL_CMP_PKISTATUS_revocationNotification 5
-#  define OSSL_CMP_PKISTATUS_keyUpdateWarning 6
+#  define OSSL_CMP_PKISTATUS_keyUpdateWarning       6
 
 typedef ASN1_INTEGER OSSL_CMP_PKISTATUS;
@@ -445 +448 @@
 int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name);
 int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav);
+int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx);
 int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx,
                                     STACK_OF(X509) *extraCertsOut);

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/conf.h

@@ -3 +3 @@
  * Generated by Makefile from conf.h.in
  *
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -28 +28 @@
 # include <openssl/types.h>
 # include <openssl/conferr.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef  __cplusplus

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/crypto.h

@@ -342 +342 @@
                               CRYPTO_free_fn *free_fn);
 
-void *CRYPTO_malloc(size_t num, const char *file, int line);
-void *CRYPTO_zalloc(size_t num, const char *file, int line);
-void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line);
-char *CRYPTO_strdup(const char *str, const char *file, int line);
-char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line);
+OSSL_CRYPTO_ALLOC void *CRYPTO_malloc(size_t num, const char *file, int line);
+OSSL_CRYPTO_ALLOC void *CRYPTO_zalloc(size_t num, const char *file, int line);
+OSSL_CRYPTO_ALLOC void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line);
+OSSL_CRYPTO_ALLOC char *CRYPTO_strdup(const char *str, const char *file, int line);
+OSSL_CRYPTO_ALLOC char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line);
 void CRYPTO_free(void *ptr, const char *file, int line);
 void CRYPTO_clear_free(void *ptr, size_t num, const char *file, int line);
@@ -355 +355 @@
 int CRYPTO_secure_malloc_init(size_t sz, size_t minsize);
 int CRYPTO_secure_malloc_done(void);
-void *CRYPTO_secure_malloc(size_t num, const char *file, int line);
-void *CRYPTO_secure_zalloc(size_t num, const char *file, int line);
+OSSL_CRYPTO_ALLOC void *CRYPTO_secure_malloc(size_t num, const char *file, int line);
+OSSL_CRYPTO_ALLOC void *CRYPTO_secure_zalloc(size_t num, const char *file, int line);
 void CRYPTO_secure_free(void *ptr, const char *file, int line);
 void CRYPTO_secure_clear_free(void *ptr, size_t num,

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/lhash.h

@@ -1 +1 @@
 /*
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -25 +25 @@
 # include <openssl/e_os2.h>
 # include <openssl/bio.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef  __cplusplus
@@ -93 +96 @@
 
 # ifndef OPENSSL_NO_STDIO
-void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp);
-void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp);
-void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp);
-# endif
-void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
-void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
-void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
+#  ifndef OPENSSL_NO_DEPRECATED_3_1
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp);
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp);
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp);
+#  endif
+# endif
+# ifndef OPENSSL_NO_DEPRECATED_3_1
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
+OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out);
+# endif
 
 # ifndef OPENSSL_NO_DEPRECATED_1_1_0
@@ -130 +137 @@
 /* Helper macro for internal use */
 # define DEFINE_LHASH_OF_INTERNAL(type) \
-    LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \
+    LHASH_OF(type) { \
+        union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \
+    }; \
     typedef int (*lh_##type##_compfunc)(const type *a, const type *b); \
     typedef unsigned long (*lh_##type##_hashfunc)(const type *a); \
     typedef void (*lh_##type##_doallfunc)(type *a); \
-    static ossl_unused ossl_inline type *ossl_check_##type##_lh_plain_type(type *ptr) \
+    static ossl_unused ossl_inline type *\
+    ossl_check_##type##_lh_plain_type(type *ptr) \
     { \
         return ptr; \
     } \
-    static ossl_unused ossl_inline const type *ossl_check_const_##type##_lh_plain_type(const type *ptr) \
+    static ossl_unused ossl_inline const type * \
+    ossl_check_const_##type##_lh_plain_type(const type *ptr) \
     { \
         return ptr; \
     } \
-    static ossl_unused ossl_inline const OPENSSL_LHASH *ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline const OPENSSL_LHASH * \
+    ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \
     { \
         return (const OPENSSL_LHASH *)lh; \
     } \
-    static ossl_unused ossl_inline OPENSSL_LHASH *ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline OPENSSL_LHASH * \
+    ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \
     { \
         return (OPENSSL_LHASH *)lh; \
     } \
-    static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \
+    static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC \
+    ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \
     { \
         return (OPENSSL_LH_COMPFUNC)cmp; \
     } \
-    static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \
+    static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC \
+    ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \
     { \
         return (OPENSSL_LH_HASHFUNC)hfn; \
     } \
-    static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \
+    static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC \
+    ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \
     { \
         return (OPENSSL_LH_DOALL_FUNC)dfn; \
@@ -164 +180 @@
     LHASH_OF(type)
 
-# define DEFINE_LHASH_OF(type) \
-    LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \
-    static ossl_unused ossl_inline LHASH_OF(type) *lh_##type##_new(unsigned long (*hfn)(const type *), \
-                                                                   int (*cfn)(const type *, const type *)) \
+# ifndef OPENSSL_NO_DEPRECATED_3_1
+#  define DEFINE_LHASH_OF_DEPRECATED(type) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
+    { \
+        OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \
+    } \
+    static ossl_unused ossl_inline void \
+    lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
+    { \
+        OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \
+    } \
+    static ossl_unused ossl_inline void \
+    lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
+    { \
+        OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \
+    }
+# else
+#  define DEFINE_LHASH_OF_DEPRECATED(type)
+# endif
+
+# define DEFINE_LHASH_OF_EX(type) \
+    LHASH_OF(type) { \
+        union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \
+    }; \
+    static ossl_unused ossl_inline LHASH_OF(type) * \
+    lh_##type##_new(unsigned long (*hfn)(const type *), \
+                    int (*cfn)(const type *, const type *)) \
     { \
         return (LHASH_OF(type) *) \
             OPENSSL_LH_new((OPENSSL_LH_HASHFUNC)hfn, (OPENSSL_LH_COMPFUNC)cfn); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_free(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_free(LHASH_OF(type) *lh) \
     { \
         OPENSSL_LH_free((OPENSSL_LHASH *)lh); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_flush(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_flush(LHASH_OF(type) *lh) \
     { \
         OPENSSL_LH_flush((OPENSSL_LHASH *)lh); \
     } \
-    static ossl_unused ossl_inline type *lh_##type##_insert(LHASH_OF(type) *lh, type *d) \
+    static ossl_unused ossl_inline type * \
+    lh_##type##_insert(LHASH_OF(type) *lh, type *d) \
     { \
         return (type *)OPENSSL_LH_insert((OPENSSL_LHASH *)lh, d); \
     } \
-    static ossl_unused ossl_inline type *lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \
+    static ossl_unused ossl_inline type * \
+    lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \
     { \
         return (type *)OPENSSL_LH_delete((OPENSSL_LHASH *)lh, d); \
     } \
-    static ossl_unused ossl_inline type *lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \
+    static ossl_unused ossl_inline type * \
+    lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \
     { \
         return (type *)OPENSSL_LH_retrieve((OPENSSL_LHASH *)lh, d); \
     } \
-    static ossl_unused ossl_inline int lh_##type##_error(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline int \
+    lh_##type##_error(LHASH_OF(type) *lh) \
     { \
         return OPENSSL_LH_error((OPENSSL_LHASH *)lh); \
     } \
-    static ossl_unused ossl_inline unsigned long lh_##type##_num_items(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline unsigned long \
+    lh_##type##_num_items(LHASH_OF(type) *lh) \
     { \
         return OPENSSL_LH_num_items((OPENSSL_LHASH *)lh); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
-    { \
-        OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \
-    } \
-    static ossl_unused ossl_inline void lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
-    { \
-        OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \
-    } \
-    static ossl_unused ossl_inline void lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \
-    { \
-        OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \
-    } \
-    static ossl_unused ossl_inline unsigned long lh_##type##_get_down_load(LHASH_OF(type) *lh) \
+    static ossl_unused ossl_inline unsigned long \
+    lh_##type##_get_down_load(LHASH_OF(type) *lh) \
     { \
         return OPENSSL_LH_get_down_load((OPENSSL_LHASH *)lh); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \
     { \
         OPENSSL_LH_set_down_load((OPENSSL_LHASH *)lh, dl); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_doall(LHASH_OF(type) *lh, \
-                                                          void (*doall)(type *)) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_doall(LHASH_OF(type) *lh, void (*doall)(type *)) \
     { \
         OPENSSL_LH_doall((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNC)doall); \
     } \
-    static ossl_unused ossl_inline void lh_##type##_doall_arg(LHASH_OF(type) *lh, \
-                                                              void (*doallarg)(type *, void *), \
-                                                              void *arg) \
+    static ossl_unused ossl_inline void \
+    lh_##type##_doall_arg(LHASH_OF(type) *lh, \
+                          void (*doallarg)(type *, void *), void *arg) \
     { \
         OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, \
                              (OPENSSL_LH_DOALL_FUNCARG)doallarg, arg); \
     } \
+    LHASH_OF(type)
+
+# define DEFINE_LHASH_OF(type) \
+    DEFINE_LHASH_OF_EX(type); \
+    DEFINE_LHASH_OF_DEPRECATED(type) \
     LHASH_OF(type)
 
@@ -246 +288 @@
                                    argtype *arg) \
     { \
-        OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNCARG)fn, (void *)arg); \
+        OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, \
+                             (OPENSSL_LH_DOALL_FUNCARG)fn, (void *)arg); \
     } \
     LHASH_OF(type)

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/pkcs12.h

@@ -3 +3 @@
  * Generated by Makefile from pkcs12.h.in
  *
- * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -26 +26 @@
 # include <openssl/x509.h>
 # include <openssl/pkcs12err.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef __cplusplus

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/pkcs7.h

@@ -3 +3 @@
  * Generated by Makefile from pkcs7.h.in
  *
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -29 +29 @@
 # include <openssl/types.h>
 # include <openssl/pkcs7err.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef  __cplusplus

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/ssl.h

@@ -43 +43 @@
 # include <openssl/sslerr.h>
 # include <openssl/prov_ssl.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef  __cplusplus

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/x509.h

@@ -41 +41 @@
 # include <openssl/sha.h>
 # include <openssl/x509err.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef  __cplusplus

trunk/src/libs/openssl-3.1.0/gen-includes/openssl/x509v3.h

@@ -3 +3 @@
  * Generated by Makefile from x509v3.h.in
  *
- * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -26 +26 @@
 # include <openssl/conf.h>
 # include <openssl/x509v3err.h>
+# ifndef OPENSSL_NO_STDIO
+#  include <stdio.h>
+# endif
 
 #ifdef __cplusplus
@@ -178 +181 @@
         ASN1_IA5STRING *rfc822Name;
         ASN1_IA5STRING *dNSName;
-        ASN1_TYPE *x400Address;
+        ASN1_STRING *x400Address;
         X509_NAME *directoryName;
         EDIPARTYNAME *ediPartyName;