1#! /usr/bin/env perl 2# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the OpenSSL license (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9 10# ==================================================================== 11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 12# project. The module is, however, dual licensed under OpenSSL and 13# CRYPTOGAMS licenses depending on where you obtain it. For further 14# details see http://www.openssl.org/~appro/cryptogams/. 15# ==================================================================== 16 17# Multi-buffer AES-NI procedures process several independent buffers 18# in parallel by interleaving independent instructions. 19# 20# Cycles per byte for interleave factor 4: 21# 22# asymptotic measured 23# --------------------------- 24# Westmere 5.00/4=1.25 5.13/4=1.28 25# Atom 15.0/4=3.75 ?15.7/4=3.93 26# Sandy Bridge 5.06/4=1.27 5.18/4=1.29 27# Ivy Bridge 5.06/4=1.27 5.14/4=1.29 28# Haswell 4.44/4=1.11 4.44/4=1.11 29# Bulldozer 5.75/4=1.44 5.76/4=1.44 30# 31# Cycles per byte for interleave factor 8 (not implemented for 32# pre-AVX processors, where higher interleave factor incidentally 33# doesn't result in improvement): 34# 35# asymptotic measured 36# --------------------------- 37# Sandy Bridge 5.06/8=0.64 7.10/8=0.89(*) 38# Ivy Bridge 5.06/8=0.64 7.14/8=0.89(*) 39# Haswell 5.00/8=0.63 5.00/8=0.63 40# Bulldozer 5.75/8=0.72 5.77/8=0.72 41# 42# (*) Sandy/Ivy Bridge are known to handle high interleave factors 43# suboptimally; 44 45$flavour = shift; 46$output = shift; 47if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 48 49$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 50 51$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 52( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 53( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 54die "can't locate x86_64-xlate.pl"; 55 56$avx=0; 57 58if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` 59 =~ /GNU assembler version ([2-9]\.[0-9]+)/) { 60 $avx = ($1>=2.19) + ($1>=2.22); 61} 62 63if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && 64 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { 65 $avx = ($1>=2.09) + ($1>=2.10); 66} 67 68if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && 69 `ml64 2>&1` =~ /Version ([0-9]+)\./) { 70 $avx = ($1>=10) + ($1>=11); 71} 72 73if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { 74 $avx = ($2>=3.0) + ($2>3.0); 75} 76 77open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; 78*STDOUT=*OUT; 79 80# void aesni_multi_cbc_encrypt ( 81# struct { void *inp,*out; int blocks; double iv[2]; } inp[8]; 82# const AES_KEY *key, 83# int num); /* 1 or 2 */ 84# 85$inp="%rdi"; # 1st arg 86$key="%rsi"; # 2nd arg 87$num="%edx"; 88 89@inptr=map("%r$_",(8..11)); 90@outptr=map("%r$_",(12..15)); 91 92($rndkey0,$rndkey1)=("%xmm0","%xmm1"); 93@out=map("%xmm$_",(2..5)); 94@inp=map("%xmm$_",(6..9)); 95($counters,$mask,$zero)=map("%xmm$_",(10..12)); 96 97($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx"); 98 99$code.=<<___; 100.text 101 102.extern OPENSSL_ia32cap_P 103 104.globl aesni_multi_cbc_encrypt 105.type aesni_multi_cbc_encrypt,\@function,3 106.align 32 107aesni_multi_cbc_encrypt: 108.cfi_startproc 109___ 110$code.=<<___ if ($avx); 111 cmp \$2,$num 112 jb .Lenc_non_avx 113 mov OPENSSL_ia32cap_P+4(%rip),%ecx 114 test \$`1<<28`,%ecx # AVX bit 115 jnz _avx_cbc_enc_shortcut 116 jmp .Lenc_non_avx 117.align 16 118.Lenc_non_avx: 119___ 120$code.=<<___; 121 mov %rsp,%rax 122.cfi_def_cfa_register %rax 123 push %rbx 124.cfi_push %rbx 125 push %rbp 126.cfi_push %rbp 127 push %r12 128.cfi_push %r12 129 push %r13 130.cfi_push %r13 131 push %r14 132.cfi_push %r14 133 push %r15 134.cfi_push %r15 135___ 136$code.=<<___ if ($win64); 137 lea -0xa8(%rsp),%rsp 138 movaps %xmm6,(%rsp) 139 movaps %xmm7,0x10(%rsp) 140 movaps %xmm8,0x20(%rsp) 141 movaps %xmm9,0x30(%rsp) 142 movaps %xmm10,0x40(%rsp) 143 movaps %xmm11,0x50(%rsp) 144 movaps %xmm12,0x60(%rsp) 145 movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler 146 movaps %xmm14,-0x58(%rax) 147 movaps %xmm15,-0x48(%rax) 148___ 149$code.=<<___; 150 # stack layout 151 # 152 # +0 output sink 153 # +16 input sink [original %rsp and $num] 154 # +32 counters 155 156 sub \$48,%rsp 157 and \$-64,%rsp 158 mov %rax,16(%rsp) # original %rsp 159.cfi_cfa_expression %rsp+16,deref,+8 160 161.Lenc4x_body: 162 movdqu ($key),$zero # 0-round key 163 lea 0x78($key),$key # size optimization 164 lea 40*2($inp),$inp 165 166.Lenc4x_loop_grande: 167 mov $num,24(%rsp) # original $num 168 xor $num,$num 169___ 170for($i=0;$i<4;$i++) { 171 $code.=<<___; 172 mov `40*$i+16-40*2`($inp),$one # borrow $one for number of blocks 173 mov `40*$i+0-40*2`($inp),@inptr[$i] 174 cmp $num,$one 175 mov `40*$i+8-40*2`($inp),@outptr[$i] 176 cmovg $one,$num # find maximum 177 test $one,$one 178 movdqu `40*$i+24-40*2`($inp),@out[$i] # load IV 179 mov $one,`32+4*$i`(%rsp) # initialize counters 180 cmovle %rsp,@inptr[$i] # cancel input 181___ 182} 183$code.=<<___; 184 test $num,$num 185 jz .Lenc4x_done 186 187 movups 0x10-0x78($key),$rndkey1 188 pxor $zero,@out[0] 189 movups 0x20-0x78($key),$rndkey0 190 pxor $zero,@out[1] 191 mov 0xf0-0x78($key),$rounds 192 pxor $zero,@out[2] 193 movdqu (@inptr[0]),@inp[0] # load inputs 194 pxor $zero,@out[3] 195 movdqu (@inptr[1]),@inp[1] 196 pxor @inp[0],@out[0] 197 movdqu (@inptr[2]),@inp[2] 198 pxor @inp[1],@out[1] 199 movdqu (@inptr[3]),@inp[3] 200 pxor @inp[2],@out[2] 201 pxor @inp[3],@out[3] 202 movdqa 32(%rsp),$counters # load counters 203 xor $offset,$offset 204 jmp .Loop_enc4x 205 206.align 32 207.Loop_enc4x: 208 add \$16,$offset 209 lea 16(%rsp),$sink # sink pointer 210 mov \$1,$one # constant of 1 211 sub $offset,$sink 212 213 aesenc $rndkey1,@out[0] 214 prefetcht0 31(@inptr[0],$offset) # prefetch input 215 prefetcht0 31(@inptr[1],$offset) 216 aesenc $rndkey1,@out[1] 217 prefetcht0 31(@inptr[2],$offset) 218 prefetcht0 31(@inptr[2],$offset) 219 aesenc $rndkey1,@out[2] 220 aesenc $rndkey1,@out[3] 221 movups 0x30-0x78($key),$rndkey1 222___ 223for($i=0;$i<4;$i++) { 224my $rndkey = ($i&1) ? $rndkey1 : $rndkey0; 225$code.=<<___; 226 cmp `32+4*$i`(%rsp),$one 227 aesenc $rndkey,@out[0] 228 aesenc $rndkey,@out[1] 229 aesenc $rndkey,@out[2] 230 cmovge $sink,@inptr[$i] # cancel input 231 cmovg $sink,@outptr[$i] # sink output 232 aesenc $rndkey,@out[3] 233 movups `0x40+16*$i-0x78`($key),$rndkey 234___ 235} 236$code.=<<___; 237 movdqa $counters,$mask 238 aesenc $rndkey0,@out[0] 239 prefetcht0 15(@outptr[0],$offset) # prefetch output 240 prefetcht0 15(@outptr[1],$offset) 241 aesenc $rndkey0,@out[1] 242 prefetcht0 15(@outptr[2],$offset) 243 prefetcht0 15(@outptr[3],$offset) 244 aesenc $rndkey0,@out[2] 245 aesenc $rndkey0,@out[3] 246 movups 0x80-0x78($key),$rndkey0 247 pxor $zero,$zero 248 249 aesenc $rndkey1,@out[0] 250 pcmpgtd $zero,$mask 251 movdqu -0x78($key),$zero # reload 0-round key 252 aesenc $rndkey1,@out[1] 253 paddd $mask,$counters # decrement counters 254 movdqa $counters,32(%rsp) # update counters 255 aesenc $rndkey1,@out[2] 256 aesenc $rndkey1,@out[3] 257 movups 0x90-0x78($key),$rndkey1 258 259 cmp \$11,$rounds 260 261 aesenc $rndkey0,@out[0] 262 aesenc $rndkey0,@out[1] 263 aesenc $rndkey0,@out[2] 264 aesenc $rndkey0,@out[3] 265 movups 0xa0-0x78($key),$rndkey0 266 267 jb .Lenc4x_tail 268 269 aesenc $rndkey1,@out[0] 270 aesenc $rndkey1,@out[1] 271 aesenc $rndkey1,@out[2] 272 aesenc $rndkey1,@out[3] 273 movups 0xb0-0x78($key),$rndkey1 274 275 aesenc $rndkey0,@out[0] 276 aesenc $rndkey0,@out[1] 277 aesenc $rndkey0,@out[2] 278 aesenc $rndkey0,@out[3] 279 movups 0xc0-0x78($key),$rndkey0 280 281 je .Lenc4x_tail 282 283 aesenc $rndkey1,@out[0] 284 aesenc $rndkey1,@out[1] 285 aesenc $rndkey1,@out[2] 286 aesenc $rndkey1,@out[3] 287 movups 0xd0-0x78($key),$rndkey1 288 289 aesenc $rndkey0,@out[0] 290 aesenc $rndkey0,@out[1] 291 aesenc $rndkey0,@out[2] 292 aesenc $rndkey0,@out[3] 293 movups 0xe0-0x78($key),$rndkey0 294 jmp .Lenc4x_tail 295 296.align 32 297.Lenc4x_tail: 298 aesenc $rndkey1,@out[0] 299 aesenc $rndkey1,@out[1] 300 aesenc $rndkey1,@out[2] 301 aesenc $rndkey1,@out[3] 302 movdqu (@inptr[0],$offset),@inp[0] 303 movdqu 0x10-0x78($key),$rndkey1 304 305 aesenclast $rndkey0,@out[0] 306 movdqu (@inptr[1],$offset),@inp[1] 307 pxor $zero,@inp[0] 308 aesenclast $rndkey0,@out[1] 309 movdqu (@inptr[2],$offset),@inp[2] 310 pxor $zero,@inp[1] 311 aesenclast $rndkey0,@out[2] 312 movdqu (@inptr[3],$offset),@inp[3] 313 pxor $zero,@inp[2] 314 aesenclast $rndkey0,@out[3] 315 movdqu 0x20-0x78($key),$rndkey0 316 pxor $zero,@inp[3] 317 318 movups @out[0],-16(@outptr[0],$offset) 319 pxor @inp[0],@out[0] 320 movups @out[1],-16(@outptr[1],$offset) 321 pxor @inp[1],@out[1] 322 movups @out[2],-16(@outptr[2],$offset) 323 pxor @inp[2],@out[2] 324 movups @out[3],-16(@outptr[3],$offset) 325 pxor @inp[3],@out[3] 326 327 dec $num 328 jnz .Loop_enc4x 329 330 mov 16(%rsp),%rax # original %rsp 331.cfi_def_cfa %rax,8 332 mov 24(%rsp),$num 333 334 #pxor @inp[0],@out[0] 335 #pxor @inp[1],@out[1] 336 #movdqu @out[0],`40*0+24-40*2`($inp) # output iv FIX ME! 337 #pxor @inp[2],@out[2] 338 #movdqu @out[1],`40*1+24-40*2`($inp) 339 #pxor @inp[3],@out[3] 340 #movdqu @out[2],`40*2+24-40*2`($inp) # won't fix, let caller 341 #movdqu @out[3],`40*3+24-40*2`($inp) # figure this out... 342 343 lea `40*4`($inp),$inp 344 dec $num 345 jnz .Lenc4x_loop_grande 346 347.Lenc4x_done: 348___ 349$code.=<<___ if ($win64); 350 movaps -0xd8(%rax),%xmm6 351 movaps -0xc8(%rax),%xmm7 352 movaps -0xb8(%rax),%xmm8 353 movaps -0xa8(%rax),%xmm9 354 movaps -0x98(%rax),%xmm10 355 movaps -0x88(%rax),%xmm11 356 movaps -0x78(%rax),%xmm12 357 #movaps -0x68(%rax),%xmm13 358 #movaps -0x58(%rax),%xmm14 359 #movaps -0x48(%rax),%xmm15 360___ 361$code.=<<___; 362 mov -48(%rax),%r15 363.cfi_restore %r15 364 mov -40(%rax),%r14 365.cfi_restore %r14 366 mov -32(%rax),%r13 367.cfi_restore %r13 368 mov -24(%rax),%r12 369.cfi_restore %r12 370 mov -16(%rax),%rbp 371.cfi_restore %rbp 372 mov -8(%rax),%rbx 373.cfi_restore %rbx 374 lea (%rax),%rsp 375.cfi_def_cfa_register %rsp 376.Lenc4x_epilogue: 377 ret 378.cfi_endproc 379.size aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt 380 381.globl aesni_multi_cbc_decrypt 382.type aesni_multi_cbc_decrypt,\@function,3 383.align 32 384aesni_multi_cbc_decrypt: 385.cfi_startproc 386___ 387$code.=<<___ if ($avx); 388 cmp \$2,$num 389 jb .Ldec_non_avx 390 mov OPENSSL_ia32cap_P+4(%rip),%ecx 391 test \$`1<<28`,%ecx # AVX bit 392 jnz _avx_cbc_dec_shortcut 393 jmp .Ldec_non_avx 394.align 16 395.Ldec_non_avx: 396___ 397$code.=<<___; 398 mov %rsp,%rax 399.cfi_def_cfa_register %rax 400 push %rbx 401.cfi_push %rbx 402 push %rbp 403.cfi_push %rbp 404 push %r12 405.cfi_push %r12 406 push %r13 407.cfi_push %r13 408 push %r14 409.cfi_push %r14 410 push %r15 411.cfi_push %r15 412___ 413$code.=<<___ if ($win64); 414 lea -0xa8(%rsp),%rsp 415 movaps %xmm6,(%rsp) 416 movaps %xmm7,0x10(%rsp) 417 movaps %xmm8,0x20(%rsp) 418 movaps %xmm9,0x30(%rsp) 419 movaps %xmm10,0x40(%rsp) 420 movaps %xmm11,0x50(%rsp) 421 movaps %xmm12,0x60(%rsp) 422 movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler 423 movaps %xmm14,-0x58(%rax) 424 movaps %xmm15,-0x48(%rax) 425___ 426$code.=<<___; 427 # stack layout 428 # 429 # +0 output sink 430 # +16 input sink [original %rsp and $num] 431 # +32 counters 432 433 sub \$48,%rsp 434 and \$-64,%rsp 435 mov %rax,16(%rsp) # original %rsp 436.cfi_cfa_expression %rsp+16,deref,+8 437 438.Ldec4x_body: 439 movdqu ($key),$zero # 0-round key 440 lea 0x78($key),$key # size optimization 441 lea 40*2($inp),$inp 442 443.Ldec4x_loop_grande: 444 mov $num,24(%rsp) # original $num 445 xor $num,$num 446___ 447for($i=0;$i<4;$i++) { 448 $code.=<<___; 449 mov `40*$i+16-40*2`($inp),$one # borrow $one for number of blocks 450 mov `40*$i+0-40*2`($inp),@inptr[$i] 451 cmp $num,$one 452 mov `40*$i+8-40*2`($inp),@outptr[$i] 453 cmovg $one,$num # find maximum 454 test $one,$one 455 movdqu `40*$i+24-40*2`($inp),@inp[$i] # load IV 456 mov $one,`32+4*$i`(%rsp) # initialize counters 457 cmovle %rsp,@inptr[$i] # cancel input 458___ 459} 460$code.=<<___; 461 test $num,$num 462 jz .Ldec4x_done 463 464 movups 0x10-0x78($key),$rndkey1 465 movups 0x20-0x78($key),$rndkey0 466 mov 0xf0-0x78($key),$rounds 467 movdqu (@inptr[0]),@out[0] # load inputs 468 movdqu (@inptr[1]),@out[1] 469 pxor $zero,@out[0] 470 movdqu (@inptr[2]),@out[2] 471 pxor $zero,@out[1] 472 movdqu (@inptr[3]),@out[3] 473 pxor $zero,@out[2] 474 pxor $zero,@out[3] 475 movdqa 32(%rsp),$counters # load counters 476 xor $offset,$offset 477 jmp .Loop_dec4x 478 479.align 32 480.Loop_dec4x: 481 add \$16,$offset 482 lea 16(%rsp),$sink # sink pointer 483 mov \$1,$one # constant of 1 484 sub $offset,$sink 485 486 aesdec $rndkey1,@out[0] 487 prefetcht0 31(@inptr[0],$offset) # prefetch input 488 prefetcht0 31(@inptr[1],$offset) 489 aesdec $rndkey1,@out[1] 490 prefetcht0 31(@inptr[2],$offset) 491 prefetcht0 31(@inptr[3],$offset) 492 aesdec $rndkey1,@out[2] 493 aesdec $rndkey1,@out[3] 494 movups 0x30-0x78($key),$rndkey1 495___ 496for($i=0;$i<4;$i++) { 497my $rndkey = ($i&1) ? $rndkey1 : $rndkey0; 498$code.=<<___; 499 cmp `32+4*$i`(%rsp),$one 500 aesdec $rndkey,@out[0] 501 aesdec $rndkey,@out[1] 502 aesdec $rndkey,@out[2] 503 cmovge $sink,@inptr[$i] # cancel input 504 cmovg $sink,@outptr[$i] # sink output 505 aesdec $rndkey,@out[3] 506 movups `0x40+16*$i-0x78`($key),$rndkey 507___ 508} 509$code.=<<___; 510 movdqa $counters,$mask 511 aesdec $rndkey0,@out[0] 512 prefetcht0 15(@outptr[0],$offset) # prefetch output 513 prefetcht0 15(@outptr[1],$offset) 514 aesdec $rndkey0,@out[1] 515 prefetcht0 15(@outptr[2],$offset) 516 prefetcht0 15(@outptr[3],$offset) 517 aesdec $rndkey0,@out[2] 518 aesdec $rndkey0,@out[3] 519 movups 0x80-0x78($key),$rndkey0 520 pxor $zero,$zero 521 522 aesdec $rndkey1,@out[0] 523 pcmpgtd $zero,$mask 524 movdqu -0x78($key),$zero # reload 0-round key 525 aesdec $rndkey1,@out[1] 526 paddd $mask,$counters # decrement counters 527 movdqa $counters,32(%rsp) # update counters 528 aesdec $rndkey1,@out[2] 529 aesdec $rndkey1,@out[3] 530 movups 0x90-0x78($key),$rndkey1 531 532 cmp \$11,$rounds 533 534 aesdec $rndkey0,@out[0] 535 aesdec $rndkey0,@out[1] 536 aesdec $rndkey0,@out[2] 537 aesdec $rndkey0,@out[3] 538 movups 0xa0-0x78($key),$rndkey0 539 540 jb .Ldec4x_tail 541 542 aesdec $rndkey1,@out[0] 543 aesdec $rndkey1,@out[1] 544 aesdec $rndkey1,@out[2] 545 aesdec $rndkey1,@out[3] 546 movups 0xb0-0x78($key),$rndkey1 547 548 aesdec $rndkey0,@out[0] 549 aesdec $rndkey0,@out[1] 550 aesdec $rndkey0,@out[2] 551 aesdec $rndkey0,@out[3] 552 movups 0xc0-0x78($key),$rndkey0 553 554 je .Ldec4x_tail 555 556 aesdec $rndkey1,@out[0] 557 aesdec $rndkey1,@out[1] 558 aesdec $rndkey1,@out[2] 559 aesdec $rndkey1,@out[3] 560 movups 0xd0-0x78($key),$rndkey1 561 562 aesdec $rndkey0,@out[0] 563 aesdec $rndkey0,@out[1] 564 aesdec $rndkey0,@out[2] 565 aesdec $rndkey0,@out[3] 566 movups 0xe0-0x78($key),$rndkey0 567 jmp .Ldec4x_tail 568 569.align 32 570.Ldec4x_tail: 571 aesdec $rndkey1,@out[0] 572 aesdec $rndkey1,@out[1] 573 aesdec $rndkey1,@out[2] 574 pxor $rndkey0,@inp[0] 575 pxor $rndkey0,@inp[1] 576 aesdec $rndkey1,@out[3] 577 movdqu 0x10-0x78($key),$rndkey1 578 pxor $rndkey0,@inp[2] 579 pxor $rndkey0,@inp[3] 580 movdqu 0x20-0x78($key),$rndkey0 581 582 aesdeclast @inp[0],@out[0] 583 aesdeclast @inp[1],@out[1] 584 movdqu -16(@inptr[0],$offset),@inp[0] # load next IV 585 movdqu -16(@inptr[1],$offset),@inp[1] 586 aesdeclast @inp[2],@out[2] 587 aesdeclast @inp[3],@out[3] 588 movdqu -16(@inptr[2],$offset),@inp[2] 589 movdqu -16(@inptr[3],$offset),@inp[3] 590 591 movups @out[0],-16(@outptr[0],$offset) 592 movdqu (@inptr[0],$offset),@out[0] 593 movups @out[1],-16(@outptr[1],$offset) 594 movdqu (@inptr[1],$offset),@out[1] 595 pxor $zero,@out[0] 596 movups @out[2],-16(@outptr[2],$offset) 597 movdqu (@inptr[2],$offset),@out[2] 598 pxor $zero,@out[1] 599 movups @out[3],-16(@outptr[3],$offset) 600 movdqu (@inptr[3],$offset),@out[3] 601 pxor $zero,@out[2] 602 pxor $zero,@out[3] 603 604 dec $num 605 jnz .Loop_dec4x 606 607 mov 16(%rsp),%rax # original %rsp 608.cfi_def_cfa %rax,8 609 mov 24(%rsp),$num 610 611 lea `40*4`($inp),$inp 612 dec $num 613 jnz .Ldec4x_loop_grande 614 615.Ldec4x_done: 616___ 617$code.=<<___ if ($win64); 618 movaps -0xd8(%rax),%xmm6 619 movaps -0xc8(%rax),%xmm7 620 movaps -0xb8(%rax),%xmm8 621 movaps -0xa8(%rax),%xmm9 622 movaps -0x98(%rax),%xmm10 623 movaps -0x88(%rax),%xmm11 624 movaps -0x78(%rax),%xmm12 625 #movaps -0x68(%rax),%xmm13 626 #movaps -0x58(%rax),%xmm14 627 #movaps -0x48(%rax),%xmm15 628___ 629$code.=<<___; 630 mov -48(%rax),%r15 631.cfi_restore %r15 632 mov -40(%rax),%r14 633.cfi_restore %r14 634 mov -32(%rax),%r13 635.cfi_restore %r13 636 mov -24(%rax),%r12 637.cfi_restore %r12 638 mov -16(%rax),%rbp 639.cfi_restore %rbp 640 mov -8(%rax),%rbx 641.cfi_restore %rbx 642 lea (%rax),%rsp 643.cfi_def_cfa_register %rsp 644.Ldec4x_epilogue: 645 ret 646.cfi_endproc 647.size aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt 648___ 649 650 if ($avx) {{{ 651my @ptr=map("%r$_",(8..15)); 652my $offload=$sink; 653 654my @out=map("%xmm$_",(2..9)); 655my @inp=map("%xmm$_",(10..13)); 656my ($counters,$zero)=("%xmm14","%xmm15"); 657 658$code.=<<___; 659.type aesni_multi_cbc_encrypt_avx,\@function,3 660.align 32 661aesni_multi_cbc_encrypt_avx: 662.cfi_startproc 663_avx_cbc_enc_shortcut: 664 mov %rsp,%rax 665.cfi_def_cfa_register %rax 666 push %rbx 667.cfi_push %rbx 668 push %rbp 669.cfi_push %rbp 670 push %r12 671.cfi_push %r12 672 push %r13 673.cfi_push %r13 674 push %r14 675.cfi_push %r14 676 push %r15 677.cfi_push %r15 678___ 679$code.=<<___ if ($win64); 680 lea -0xa8(%rsp),%rsp 681 movaps %xmm6,(%rsp) 682 movaps %xmm7,0x10(%rsp) 683 movaps %xmm8,0x20(%rsp) 684 movaps %xmm9,0x30(%rsp) 685 movaps %xmm10,0x40(%rsp) 686 movaps %xmm11,0x50(%rsp) 687 movaps %xmm12,-0x78(%rax) 688 movaps %xmm13,-0x68(%rax) 689 movaps %xmm14,-0x58(%rax) 690 movaps %xmm15,-0x48(%rax) 691___ 692$code.=<<___; 693 # stack layout 694 # 695 # +0 output sink 696 # +16 input sink [original %rsp and $num] 697 # +32 counters 698 # +64 distances between inputs and outputs 699 # +128 off-load area for @inp[0..3] 700 701 sub \$192,%rsp 702 and \$-128,%rsp 703 mov %rax,16(%rsp) # original %rsp 704.cfi_cfa_expression %rsp+16,deref,+8 705 706.Lenc8x_body: 707 vzeroupper 708 vmovdqu ($key),$zero # 0-round key 709 lea 0x78($key),$key # size optimization 710 lea 40*4($inp),$inp 711 shr \$1,$num 712 713.Lenc8x_loop_grande: 714 #mov $num,24(%rsp) # original $num 715 xor $num,$num 716___ 717for($i=0;$i<8;$i++) { 718 my $temp = $i ? $offload : $offset; 719 $code.=<<___; 720 mov `40*$i+16-40*4`($inp),$one # borrow $one for number of blocks 721 mov `40*$i+0-40*4`($inp),@ptr[$i] # input pointer 722 cmp $num,$one 723 mov `40*$i+8-40*4`($inp),$temp # output pointer 724 cmovg $one,$num # find maximum 725 test $one,$one 726 vmovdqu `40*$i+24-40*4`($inp),@out[$i] # load IV 727 mov $one,`32+4*$i`(%rsp) # initialize counters 728 cmovle %rsp,@ptr[$i] # cancel input 729 sub @ptr[$i],$temp # distance between input and output 730 mov $temp,`64+8*$i`(%rsp) # initialize distances 731___ 732} 733$code.=<<___; 734 test $num,$num 735 jz .Lenc8x_done 736 737 vmovups 0x10-0x78($key),$rndkey1 738 vmovups 0x20-0x78($key),$rndkey0 739 mov 0xf0-0x78($key),$rounds 740 741 vpxor (@ptr[0]),$zero,@inp[0] # load inputs and xor with 0-round 742 lea 128(%rsp),$offload # offload area 743 vpxor (@ptr[1]),$zero,@inp[1] 744 vpxor (@ptr[2]),$zero,@inp[2] 745 vpxor (@ptr[3]),$zero,@inp[3] 746 vpxor @inp[0],@out[0],@out[0] 747 vpxor (@ptr[4]),$zero,@inp[0] 748 vpxor @inp[1],@out[1],@out[1] 749 vpxor (@ptr[5]),$zero,@inp[1] 750 vpxor @inp[2],@out[2],@out[2] 751 vpxor (@ptr[6]),$zero,@inp[2] 752 vpxor @inp[3],@out[3],@out[3] 753 vpxor (@ptr[7]),$zero,@inp[3] 754 vpxor @inp[0],@out[4],@out[4] 755 mov \$1,$one # constant of 1 756 vpxor @inp[1],@out[5],@out[5] 757 vpxor @inp[2],@out[6],@out[6] 758 vpxor @inp[3],@out[7],@out[7] 759 jmp .Loop_enc8x 760 761.align 32 762.Loop_enc8x: 763___ 764for($i=0;$i<8;$i++) { 765my $rndkey=($i&1)?$rndkey0:$rndkey1; 766$code.=<<___; 767 vaesenc $rndkey,@out[0],@out[0] 768 cmp 32+4*$i(%rsp),$one 769___ 770$code.=<<___ if ($i); 771 mov 64+8*$i(%rsp),$offset 772___ 773$code.=<<___; 774 vaesenc $rndkey,@out[1],@out[1] 775 prefetcht0 31(@ptr[$i]) # prefetch input 776 vaesenc $rndkey,@out[2],@out[2] 777___ 778$code.=<<___ if ($i>1); 779 prefetcht0 15(@ptr[$i-2]) # prefetch output 780___ 781$code.=<<___; 782 vaesenc $rndkey,@out[3],@out[3] 783 lea (@ptr[$i],$offset),$offset 784 cmovge %rsp,@ptr[$i] # cancel input 785 vaesenc $rndkey,@out[4],@out[4] 786 cmovg %rsp,$offset # sink output 787 vaesenc $rndkey,@out[5],@out[5] 788 sub @ptr[$i],$offset 789 vaesenc $rndkey,@out[6],@out[6] 790 vpxor 16(@ptr[$i]),$zero,@inp[$i%4] # load input and xor with 0-round 791 mov $offset,64+8*$i(%rsp) 792 vaesenc $rndkey,@out[7],@out[7] 793 vmovups `16*(3+$i)-0x78`($key),$rndkey 794 lea 16(@ptr[$i],$offset),@ptr[$i] # switch to output 795___ 796$code.=<<___ if ($i<4) 797 vmovdqu @inp[$i%4],`16*$i`($offload) # off-load 798___ 799} 800$code.=<<___; 801 vmovdqu 32(%rsp),$counters 802 prefetcht0 15(@ptr[$i-2]) # prefetch output 803 prefetcht0 15(@ptr[$i-1]) 804 cmp \$11,$rounds 805 jb .Lenc8x_tail 806 807 vaesenc $rndkey1,@out[0],@out[0] 808 vaesenc $rndkey1,@out[1],@out[1] 809 vaesenc $rndkey1,@out[2],@out[2] 810 vaesenc $rndkey1,@out[3],@out[3] 811 vaesenc $rndkey1,@out[4],@out[4] 812 vaesenc $rndkey1,@out[5],@out[5] 813 vaesenc $rndkey1,@out[6],@out[6] 814 vaesenc $rndkey1,@out[7],@out[7] 815 vmovups 0xb0-0x78($key),$rndkey1 816 817 vaesenc $rndkey0,@out[0],@out[0] 818 vaesenc $rndkey0,@out[1],@out[1] 819 vaesenc $rndkey0,@out[2],@out[2] 820 vaesenc $rndkey0,@out[3],@out[3] 821 vaesenc $rndkey0,@out[4],@out[4] 822 vaesenc $rndkey0,@out[5],@out[5] 823 vaesenc $rndkey0,@out[6],@out[6] 824 vaesenc $rndkey0,@out[7],@out[7] 825 vmovups 0xc0-0x78($key),$rndkey0 826 je .Lenc8x_tail 827 828 vaesenc $rndkey1,@out[0],@out[0] 829 vaesenc $rndkey1,@out[1],@out[1] 830 vaesenc $rndkey1,@out[2],@out[2] 831 vaesenc $rndkey1,@out[3],@out[3] 832 vaesenc $rndkey1,@out[4],@out[4] 833 vaesenc $rndkey1,@out[5],@out[5] 834 vaesenc $rndkey1,@out[6],@out[6] 835 vaesenc $rndkey1,@out[7],@out[7] 836 vmovups 0xd0-0x78($key),$rndkey1 837 838 vaesenc $rndkey0,@out[0],@out[0] 839 vaesenc $rndkey0,@out[1],@out[1] 840 vaesenc $rndkey0,@out[2],@out[2] 841 vaesenc $rndkey0,@out[3],@out[3] 842 vaesenc $rndkey0,@out[4],@out[4] 843 vaesenc $rndkey0,@out[5],@out[5] 844 vaesenc $rndkey0,@out[6],@out[6] 845 vaesenc $rndkey0,@out[7],@out[7] 846 vmovups 0xe0-0x78($key),$rndkey0 847 848.Lenc8x_tail: 849 vaesenc $rndkey1,@out[0],@out[0] 850 vpxor $zero,$zero,$zero 851 vaesenc $rndkey1,@out[1],@out[1] 852 vaesenc $rndkey1,@out[2],@out[2] 853 vpcmpgtd $zero,$counters,$zero 854 vaesenc $rndkey1,@out[3],@out[3] 855 vaesenc $rndkey1,@out[4],@out[4] 856 vpaddd $counters,$zero,$zero # decrement counters 857 vmovdqu 48(%rsp),$counters 858 vaesenc $rndkey1,@out[5],@out[5] 859 mov 64(%rsp),$offset # pre-load 1st offset 860 vaesenc $rndkey1,@out[6],@out[6] 861 vaesenc $rndkey1,@out[7],@out[7] 862 vmovups 0x10-0x78($key),$rndkey1 863 864 vaesenclast $rndkey0,@out[0],@out[0] 865 vmovdqa $zero,32(%rsp) # update counters 866 vpxor $zero,$zero,$zero 867 vaesenclast $rndkey0,@out[1],@out[1] 868 vaesenclast $rndkey0,@out[2],@out[2] 869 vpcmpgtd $zero,$counters,$zero 870 vaesenclast $rndkey0,@out[3],@out[3] 871 vaesenclast $rndkey0,@out[4],@out[4] 872 vpaddd $zero,$counters,$counters # decrement counters 873 vmovdqu -0x78($key),$zero # 0-round 874 vaesenclast $rndkey0,@out[5],@out[5] 875 vaesenclast $rndkey0,@out[6],@out[6] 876 vmovdqa $counters,48(%rsp) # update counters 877 vaesenclast $rndkey0,@out[7],@out[7] 878 vmovups 0x20-0x78($key),$rndkey0 879 880 vmovups @out[0],-16(@ptr[0]) # write output 881 sub $offset,@ptr[0] # switch to input 882 vpxor 0x00($offload),@out[0],@out[0] 883 vmovups @out[1],-16(@ptr[1]) 884 sub `64+1*8`(%rsp),@ptr[1] 885 vpxor 0x10($offload),@out[1],@out[1] 886 vmovups @out[2],-16(@ptr[2]) 887 sub `64+2*8`(%rsp),@ptr[2] 888 vpxor 0x20($offload),@out[2],@out[2] 889 vmovups @out[3],-16(@ptr[3]) 890 sub `64+3*8`(%rsp),@ptr[3] 891 vpxor 0x30($offload),@out[3],@out[3] 892 vmovups @out[4],-16(@ptr[4]) 893 sub `64+4*8`(%rsp),@ptr[4] 894 vpxor @inp[0],@out[4],@out[4] 895 vmovups @out[5],-16(@ptr[5]) 896 sub `64+5*8`(%rsp),@ptr[5] 897 vpxor @inp[1],@out[5],@out[5] 898 vmovups @out[6],-16(@ptr[6]) 899 sub `64+6*8`(%rsp),@ptr[6] 900 vpxor @inp[2],@out[6],@out[6] 901 vmovups @out[7],-16(@ptr[7]) 902 sub `64+7*8`(%rsp),@ptr[7] 903 vpxor @inp[3],@out[7],@out[7] 904 905 dec $num 906 jnz .Loop_enc8x 907 908 mov 16(%rsp),%rax # original %rsp 909.cfi_def_cfa %rax,8 910 #mov 24(%rsp),$num 911 #lea `40*8`($inp),$inp 912 #dec $num 913 #jnz .Lenc8x_loop_grande 914 915.Lenc8x_done: 916 vzeroupper 917___ 918$code.=<<___ if ($win64); 919 movaps -0xd8(%rax),%xmm6 920 movaps -0xc8(%rax),%xmm7 921 movaps -0xb8(%rax),%xmm8 922 movaps -0xa8(%rax),%xmm9 923 movaps -0x98(%rax),%xmm10 924 movaps -0x88(%rax),%xmm11 925 movaps -0x78(%rax),%xmm12 926 movaps -0x68(%rax),%xmm13 927 movaps -0x58(%rax),%xmm14 928 movaps -0x48(%rax),%xmm15 929___ 930$code.=<<___; 931 mov -48(%rax),%r15 932.cfi_restore %r15 933 mov -40(%rax),%r14 934.cfi_restore %r14 935 mov -32(%rax),%r13 936.cfi_restore %r13 937 mov -24(%rax),%r12 938.cfi_restore %r12 939 mov -16(%rax),%rbp 940.cfi_restore %rbp 941 mov -8(%rax),%rbx 942.cfi_restore %rbx 943 lea (%rax),%rsp 944.cfi_def_cfa_register %rsp 945.Lenc8x_epilogue: 946 ret 947.cfi_endproc 948.size aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx 949 950.type aesni_multi_cbc_decrypt_avx,\@function,3 951.align 32 952aesni_multi_cbc_decrypt_avx: 953.cfi_startproc 954_avx_cbc_dec_shortcut: 955 mov %rsp,%rax 956.cfi_def_cfa_register %rax 957 push %rbx 958.cfi_push %rbx 959 push %rbp 960.cfi_push %rbp 961 push %r12 962.cfi_push %r12 963 push %r13 964.cfi_push %r13 965 push %r14 966.cfi_push %r14 967 push %r15 968.cfi_push %r15 969___ 970$code.=<<___ if ($win64); 971 lea -0xa8(%rsp),%rsp 972 movaps %xmm6,(%rsp) 973 movaps %xmm7,0x10(%rsp) 974 movaps %xmm8,0x20(%rsp) 975 movaps %xmm9,0x30(%rsp) 976 movaps %xmm10,0x40(%rsp) 977 movaps %xmm11,0x50(%rsp) 978 movaps %xmm12,-0x78(%rax) 979 movaps %xmm13,-0x68(%rax) 980 movaps %xmm14,-0x58(%rax) 981 movaps %xmm15,-0x48(%rax) 982___ 983$code.=<<___; 984 # stack layout 985 # 986 # +0 output sink 987 # +16 input sink [original %rsp and $num] 988 # +32 counters 989 # +64 distances between inputs and outputs 990 # +128 off-load area for @inp[0..3] 991 # +192 IV/input offload 992 993 sub \$256,%rsp 994 and \$-256,%rsp 995 sub \$192,%rsp 996 mov %rax,16(%rsp) # original %rsp 997.cfi_cfa_expression %rsp+16,deref,+8 998 999.Ldec8x_body: 1000 vzeroupper 1001 vmovdqu ($key),$zero # 0-round key 1002 lea 0x78($key),$key # size optimization 1003 lea 40*4($inp),$inp 1004 shr \$1,$num 1005 1006.Ldec8x_loop_grande: 1007 #mov $num,24(%rsp) # original $num 1008 xor $num,$num 1009___ 1010for($i=0;$i<8;$i++) { 1011 my $temp = $i ? $offload : $offset; 1012 $code.=<<___; 1013 mov `40*$i+16-40*4`($inp),$one # borrow $one for number of blocks 1014 mov `40*$i+0-40*4`($inp),@ptr[$i] # input pointer 1015 cmp $num,$one 1016 mov `40*$i+8-40*4`($inp),$temp # output pointer 1017 cmovg $one,$num # find maximum 1018 test $one,$one 1019 vmovdqu `40*$i+24-40*4`($inp),@out[$i] # load IV 1020 mov $one,`32+4*$i`(%rsp) # initialize counters 1021 cmovle %rsp,@ptr[$i] # cancel input 1022 sub @ptr[$i],$temp # distance between input and output 1023 mov $temp,`64+8*$i`(%rsp) # initialize distances 1024 vmovdqu @out[$i],`192+16*$i`(%rsp) # offload IV 1025___ 1026} 1027$code.=<<___; 1028 test $num,$num 1029 jz .Ldec8x_done 1030 1031 vmovups 0x10-0x78($key),$rndkey1 1032 vmovups 0x20-0x78($key),$rndkey0 1033 mov 0xf0-0x78($key),$rounds 1034 lea 192+128(%rsp),$offload # offload area 1035 1036 vmovdqu (@ptr[0]),@out[0] # load inputs 1037 vmovdqu (@ptr[1]),@out[1] 1038 vmovdqu (@ptr[2]),@out[2] 1039 vmovdqu (@ptr[3]),@out[3] 1040 vmovdqu (@ptr[4]),@out[4] 1041 vmovdqu (@ptr[5]),@out[5] 1042 vmovdqu (@ptr[6]),@out[6] 1043 vmovdqu (@ptr[7]),@out[7] 1044 vmovdqu @out[0],0x00($offload) # offload inputs 1045 vpxor $zero,@out[0],@out[0] # xor inputs with 0-round 1046 vmovdqu @out[1],0x10($offload) 1047 vpxor $zero,@out[1],@out[1] 1048 vmovdqu @out[2],0x20($offload) 1049 vpxor $zero,@out[2],@out[2] 1050 vmovdqu @out[3],0x30($offload) 1051 vpxor $zero,@out[3],@out[3] 1052 vmovdqu @out[4],0x40($offload) 1053 vpxor $zero,@out[4],@out[4] 1054 vmovdqu @out[5],0x50($offload) 1055 vpxor $zero,@out[5],@out[5] 1056 vmovdqu @out[6],0x60($offload) 1057 vpxor $zero,@out[6],@out[6] 1058 vmovdqu @out[7],0x70($offload) 1059 vpxor $zero,@out[7],@out[7] 1060 xor \$0x80,$offload 1061 mov \$1,$one # constant of 1 1062 jmp .Loop_dec8x 1063 1064.align 32 1065.Loop_dec8x: 1066___ 1067for($i=0;$i<8;$i++) { 1068my $rndkey=($i&1)?$rndkey0:$rndkey1; 1069$code.=<<___; 1070 vaesdec $rndkey,@out[0],@out[0] 1071 cmp 32+4*$i(%rsp),$one 1072___ 1073$code.=<<___ if ($i); 1074 mov 64+8*$i(%rsp),$offset 1075___ 1076$code.=<<___; 1077 vaesdec $rndkey,@out[1],@out[1] 1078 prefetcht0 31(@ptr[$i]) # prefetch input 1079 vaesdec $rndkey,@out[2],@out[2] 1080___ 1081$code.=<<___ if ($i>1); 1082 prefetcht0 15(@ptr[$i-2]) # prefetch output 1083___ 1084$code.=<<___; 1085 vaesdec $rndkey,@out[3],@out[3] 1086 lea (@ptr[$i],$offset),$offset 1087 cmovge %rsp,@ptr[$i] # cancel input 1088 vaesdec $rndkey,@out[4],@out[4] 1089 cmovg %rsp,$offset # sink output 1090 vaesdec $rndkey,@out[5],@out[5] 1091 sub @ptr[$i],$offset 1092 vaesdec $rndkey,@out[6],@out[6] 1093 vmovdqu 16(@ptr[$i]),@inp[$i%4] # load input 1094 mov $offset,64+8*$i(%rsp) 1095 vaesdec $rndkey,@out[7],@out[7] 1096 vmovups `16*(3+$i)-0x78`($key),$rndkey 1097 lea 16(@ptr[$i],$offset),@ptr[$i] # switch to output 1098___ 1099$code.=<<___ if ($i<4); 1100 vmovdqu @inp[$i%4],`128+16*$i`(%rsp) # off-load 1101___ 1102} 1103$code.=<<___; 1104 vmovdqu 32(%rsp),$counters 1105 prefetcht0 15(@ptr[$i-2]) # prefetch output 1106 prefetcht0 15(@ptr[$i-1]) 1107 cmp \$11,$rounds 1108 jb .Ldec8x_tail 1109 1110 vaesdec $rndkey1,@out[0],@out[0] 1111 vaesdec $rndkey1,@out[1],@out[1] 1112 vaesdec $rndkey1,@out[2],@out[2] 1113 vaesdec $rndkey1,@out[3],@out[3] 1114 vaesdec $rndkey1,@out[4],@out[4] 1115 vaesdec $rndkey1,@out[5],@out[5] 1116 vaesdec $rndkey1,@out[6],@out[6] 1117 vaesdec $rndkey1,@out[7],@out[7] 1118 vmovups 0xb0-0x78($key),$rndkey1 1119 1120 vaesdec $rndkey0,@out[0],@out[0] 1121 vaesdec $rndkey0,@out[1],@out[1] 1122 vaesdec $rndkey0,@out[2],@out[2] 1123 vaesdec $rndkey0,@out[3],@out[3] 1124 vaesdec $rndkey0,@out[4],@out[4] 1125 vaesdec $rndkey0,@out[5],@out[5] 1126 vaesdec $rndkey0,@out[6],@out[6] 1127 vaesdec $rndkey0,@out[7],@out[7] 1128 vmovups 0xc0-0x78($key),$rndkey0 1129 je .Ldec8x_tail 1130 1131 vaesdec $rndkey1,@out[0],@out[0] 1132 vaesdec $rndkey1,@out[1],@out[1] 1133 vaesdec $rndkey1,@out[2],@out[2] 1134 vaesdec $rndkey1,@out[3],@out[3] 1135 vaesdec $rndkey1,@out[4],@out[4] 1136 vaesdec $rndkey1,@out[5],@out[5] 1137 vaesdec $rndkey1,@out[6],@out[6] 1138 vaesdec $rndkey1,@out[7],@out[7] 1139 vmovups 0xd0-0x78($key),$rndkey1 1140 1141 vaesdec $rndkey0,@out[0],@out[0] 1142 vaesdec $rndkey0,@out[1],@out[1] 1143 vaesdec $rndkey0,@out[2],@out[2] 1144 vaesdec $rndkey0,@out[3],@out[3] 1145 vaesdec $rndkey0,@out[4],@out[4] 1146 vaesdec $rndkey0,@out[5],@out[5] 1147 vaesdec $rndkey0,@out[6],@out[6] 1148 vaesdec $rndkey0,@out[7],@out[7] 1149 vmovups 0xe0-0x78($key),$rndkey0 1150 1151.Ldec8x_tail: 1152 vaesdec $rndkey1,@out[0],@out[0] 1153 vpxor $zero,$zero,$zero 1154 vaesdec $rndkey1,@out[1],@out[1] 1155 vaesdec $rndkey1,@out[2],@out[2] 1156 vpcmpgtd $zero,$counters,$zero 1157 vaesdec $rndkey1,@out[3],@out[3] 1158 vaesdec $rndkey1,@out[4],@out[4] 1159 vpaddd $counters,$zero,$zero # decrement counters 1160 vmovdqu 48(%rsp),$counters 1161 vaesdec $rndkey1,@out[5],@out[5] 1162 mov 64(%rsp),$offset # pre-load 1st offset 1163 vaesdec $rndkey1,@out[6],@out[6] 1164 vaesdec $rndkey1,@out[7],@out[7] 1165 vmovups 0x10-0x78($key),$rndkey1 1166 1167 vaesdeclast $rndkey0,@out[0],@out[0] 1168 vmovdqa $zero,32(%rsp) # update counters 1169 vpxor $zero,$zero,$zero 1170 vaesdeclast $rndkey0,@out[1],@out[1] 1171 vpxor 0x00($offload),@out[0],@out[0] # xor with IV 1172 vaesdeclast $rndkey0,@out[2],@out[2] 1173 vpxor 0x10($offload),@out[1],@out[1] 1174 vpcmpgtd $zero,$counters,$zero 1175 vaesdeclast $rndkey0,@out[3],@out[3] 1176 vpxor 0x20($offload),@out[2],@out[2] 1177 vaesdeclast $rndkey0,@out[4],@out[4] 1178 vpxor 0x30($offload),@out[3],@out[3] 1179 vpaddd $zero,$counters,$counters # decrement counters 1180 vmovdqu -0x78($key),$zero # 0-round 1181 vaesdeclast $rndkey0,@out[5],@out[5] 1182 vpxor 0x40($offload),@out[4],@out[4] 1183 vaesdeclast $rndkey0,@out[6],@out[6] 1184 vpxor 0x50($offload),@out[5],@out[5] 1185 vmovdqa $counters,48(%rsp) # update counters 1186 vaesdeclast $rndkey0,@out[7],@out[7] 1187 vpxor 0x60($offload),@out[6],@out[6] 1188 vmovups 0x20-0x78($key),$rndkey0 1189 1190 vmovups @out[0],-16(@ptr[0]) # write output 1191 sub $offset,@ptr[0] # switch to input 1192 vmovdqu 128+0(%rsp),@out[0] 1193 vpxor 0x70($offload),@out[7],@out[7] 1194 vmovups @out[1],-16(@ptr[1]) 1195 sub `64+1*8`(%rsp),@ptr[1] 1196 vmovdqu @out[0],0x00($offload) 1197 vpxor $zero,@out[0],@out[0] 1198 vmovdqu 128+16(%rsp),@out[1] 1199 vmovups @out[2],-16(@ptr[2]) 1200 sub `64+2*8`(%rsp),@ptr[2] 1201 vmovdqu @out[1],0x10($offload) 1202 vpxor $zero,@out[1],@out[1] 1203 vmovdqu 128+32(%rsp),@out[2] 1204 vmovups @out[3],-16(@ptr[3]) 1205 sub `64+3*8`(%rsp),@ptr[3] 1206 vmovdqu @out[2],0x20($offload) 1207 vpxor $zero,@out[2],@out[2] 1208 vmovdqu 128+48(%rsp),@out[3] 1209 vmovups @out[4],-16(@ptr[4]) 1210 sub `64+4*8`(%rsp),@ptr[4] 1211 vmovdqu @out[3],0x30($offload) 1212 vpxor $zero,@out[3],@out[3] 1213 vmovdqu @inp[0],0x40($offload) 1214 vpxor @inp[0],$zero,@out[4] 1215 vmovups @out[5],-16(@ptr[5]) 1216 sub `64+5*8`(%rsp),@ptr[5] 1217 vmovdqu @inp[1],0x50($offload) 1218 vpxor @inp[1],$zero,@out[5] 1219 vmovups @out[6],-16(@ptr[6]) 1220 sub `64+6*8`(%rsp),@ptr[6] 1221 vmovdqu @inp[2],0x60($offload) 1222 vpxor @inp[2],$zero,@out[6] 1223 vmovups @out[7],-16(@ptr[7]) 1224 sub `64+7*8`(%rsp),@ptr[7] 1225 vmovdqu @inp[3],0x70($offload) 1226 vpxor @inp[3],$zero,@out[7] 1227 1228 xor \$128,$offload 1229 dec $num 1230 jnz .Loop_dec8x 1231 1232 mov 16(%rsp),%rax # original %rsp 1233.cfi_def_cfa %rax,8 1234 #mov 24(%rsp),$num 1235 #lea `40*8`($inp),$inp 1236 #dec $num 1237 #jnz .Ldec8x_loop_grande 1238 1239.Ldec8x_done: 1240 vzeroupper 1241___ 1242$code.=<<___ if ($win64); 1243 movaps -0xd8(%rax),%xmm6 1244 movaps -0xc8(%rax),%xmm7 1245 movaps -0xb8(%rax),%xmm8 1246 movaps -0xa8(%rax),%xmm9 1247 movaps -0x98(%rax),%xmm10 1248 movaps -0x88(%rax),%xmm11 1249 movaps -0x78(%rax),%xmm12 1250 movaps -0x68(%rax),%xmm13 1251 movaps -0x58(%rax),%xmm14 1252 movaps -0x48(%rax),%xmm15 1253___ 1254$code.=<<___; 1255 mov -48(%rax),%r15 1256.cfi_restore %r15 1257 mov -40(%rax),%r14 1258.cfi_restore %r14 1259 mov -32(%rax),%r13 1260.cfi_restore %r13 1261 mov -24(%rax),%r12 1262.cfi_restore %r12 1263 mov -16(%rax),%rbp 1264.cfi_restore %rbp 1265 mov -8(%rax),%rbx 1266.cfi_restore %rbx 1267 lea (%rax),%rsp 1268.cfi_def_cfa_register %rsp 1269.Ldec8x_epilogue: 1270 ret 1271.cfi_endproc 1272.size aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx 1273___ 1274 }}} 1275 1276if ($win64) { 1277# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 1278# CONTEXT *context,DISPATCHER_CONTEXT *disp) 1279$rec="%rcx"; 1280$frame="%rdx"; 1281$context="%r8"; 1282$disp="%r9"; 1283 1284$code.=<<___; 1285.extern __imp_RtlVirtualUnwind 1286.type se_handler,\@abi-omnipotent 1287.align 16 1288se_handler: 1289 push %rsi 1290 push %rdi 1291 push %rbx 1292 push %rbp 1293 push %r12 1294 push %r13 1295 push %r14 1296 push %r15 1297 pushfq 1298 sub \$64,%rsp 1299 1300 mov 120($context),%rax # pull context->Rax 1301 mov 248($context),%rbx # pull context->Rip 1302 1303 mov 8($disp),%rsi # disp->ImageBase 1304 mov 56($disp),%r11 # disp->HandlerData 1305 1306 mov 0(%r11),%r10d # HandlerData[0] 1307 lea (%rsi,%r10),%r10 # prologue label 1308 cmp %r10,%rbx # context->Rip<.Lprologue 1309 jb .Lin_prologue 1310 1311 mov 152($context),%rax # pull context->Rsp 1312 1313 mov 4(%r11),%r10d # HandlerData[1] 1314 lea (%rsi,%r10),%r10 # epilogue label 1315 cmp %r10,%rbx # context->Rip>=.Lepilogue 1316 jae .Lin_prologue 1317 1318 mov 16(%rax),%rax # pull saved stack pointer 1319 1320 mov -8(%rax),%rbx 1321 mov -16(%rax),%rbp 1322 mov -24(%rax),%r12 1323 mov -32(%rax),%r13 1324 mov -40(%rax),%r14 1325 mov -48(%rax),%r15 1326 mov %rbx,144($context) # restore context->Rbx 1327 mov %rbp,160($context) # restore context->Rbp 1328 mov %r12,216($context) # restore context->R12 1329 mov %r13,224($context) # restore context->R13 1330 mov %r14,232($context) # restore context->R14 1331 mov %r15,240($context) # restore context->R15 1332 1333 lea -56-10*16(%rax),%rsi 1334 lea 512($context),%rdi # &context.Xmm6 1335 mov \$20,%ecx 1336 .long 0xa548f3fc # cld; rep movsq 1337 1338.Lin_prologue: 1339 mov 8(%rax),%rdi 1340 mov 16(%rax),%rsi 1341 mov %rax,152($context) # restore context->Rsp 1342 mov %rsi,168($context) # restore context->Rsi 1343 mov %rdi,176($context) # restore context->Rdi 1344 1345 mov 40($disp),%rdi # disp->ContextRecord 1346 mov $context,%rsi # context 1347 mov \$154,%ecx # sizeof(CONTEXT) 1348 .long 0xa548f3fc # cld; rep movsq 1349 1350 mov $disp,%rsi 1351 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1352 mov 8(%rsi),%rdx # arg2, disp->ImageBase 1353 mov 0(%rsi),%r8 # arg3, disp->ControlPc 1354 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1355 mov 40(%rsi),%r10 # disp->ContextRecord 1356 lea 56(%rsi),%r11 # &disp->HandlerData 1357 lea 24(%rsi),%r12 # &disp->EstablisherFrame 1358 mov %r10,32(%rsp) # arg5 1359 mov %r11,40(%rsp) # arg6 1360 mov %r12,48(%rsp) # arg7 1361 mov %rcx,56(%rsp) # arg8, (NULL) 1362 call *__imp_RtlVirtualUnwind(%rip) 1363 1364 mov \$1,%eax # ExceptionContinueSearch 1365 add \$64,%rsp 1366 popfq 1367 pop %r15 1368 pop %r14 1369 pop %r13 1370 pop %r12 1371 pop %rbp 1372 pop %rbx 1373 pop %rdi 1374 pop %rsi 1375 ret 1376.size se_handler,.-se_handler 1377 1378.section .pdata 1379.align 4 1380 .rva .LSEH_begin_aesni_multi_cbc_encrypt 1381 .rva .LSEH_end_aesni_multi_cbc_encrypt 1382 .rva .LSEH_info_aesni_multi_cbc_encrypt 1383 .rva .LSEH_begin_aesni_multi_cbc_decrypt 1384 .rva .LSEH_end_aesni_multi_cbc_decrypt 1385 .rva .LSEH_info_aesni_multi_cbc_decrypt 1386___ 1387$code.=<<___ if ($avx); 1388 .rva .LSEH_begin_aesni_multi_cbc_encrypt_avx 1389 .rva .LSEH_end_aesni_multi_cbc_encrypt_avx 1390 .rva .LSEH_info_aesni_multi_cbc_encrypt_avx 1391 .rva .LSEH_begin_aesni_multi_cbc_decrypt_avx 1392 .rva .LSEH_end_aesni_multi_cbc_decrypt_avx 1393 .rva .LSEH_info_aesni_multi_cbc_decrypt_avx 1394___ 1395$code.=<<___; 1396.section .xdata 1397.align 8 1398.LSEH_info_aesni_multi_cbc_encrypt: 1399 .byte 9,0,0,0 1400 .rva se_handler 1401 .rva .Lenc4x_body,.Lenc4x_epilogue # HandlerData[] 1402.LSEH_info_aesni_multi_cbc_decrypt: 1403 .byte 9,0,0,0 1404 .rva se_handler 1405 .rva .Ldec4x_body,.Ldec4x_epilogue # HandlerData[] 1406___ 1407$code.=<<___ if ($avx); 1408.LSEH_info_aesni_multi_cbc_encrypt_avx: 1409 .byte 9,0,0,0 1410 .rva se_handler 1411 .rva .Lenc8x_body,.Lenc8x_epilogue # HandlerData[] 1412.LSEH_info_aesni_multi_cbc_decrypt_avx: 1413 .byte 9,0,0,0 1414 .rva se_handler 1415 .rva .Ldec8x_body,.Ldec8x_epilogue # HandlerData[] 1416___ 1417} 1418#################################################################### 1419 1420sub rex { 1421 local *opcode=shift; 1422 my ($dst,$src)=@_; 1423 my $rex=0; 1424 1425 $rex|=0x04 if($dst>=8); 1426 $rex|=0x01 if($src>=8); 1427 push @opcode,$rex|0x40 if($rex); 1428} 1429 1430sub aesni { 1431 my $line=shift; 1432 my @opcode=(0x66); 1433 1434 if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) { 1435 rex(\@opcode,$4,$3); 1436 push @opcode,0x0f,0x3a,0xdf; 1437 push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M 1438 my $c=$2; 1439 push @opcode,$c=~/^0/?oct($c):$c; 1440 return ".byte\t".join(',',@opcode); 1441 } 1442 elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) { 1443 my %opcodelet = ( 1444 "aesimc" => 0xdb, 1445 "aesenc" => 0xdc, "aesenclast" => 0xdd, 1446 "aesdec" => 0xde, "aesdeclast" => 0xdf 1447 ); 1448 return undef if (!defined($opcodelet{$1})); 1449 rex(\@opcode,$3,$2); 1450 push @opcode,0x0f,0x38,$opcodelet{$1}; 1451 push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M 1452 return ".byte\t".join(',',@opcode); 1453 } 1454 elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) { 1455 my %opcodelet = ( 1456 "aesenc" => 0xdc, "aesenclast" => 0xdd, 1457 "aesdec" => 0xde, "aesdeclast" => 0xdf 1458 ); 1459 return undef if (!defined($opcodelet{$1})); 1460 my $off = $2; 1461 push @opcode,0x44 if ($3>=8); 1462 push @opcode,0x0f,0x38,$opcodelet{$1}; 1463 push @opcode,0x44|(($3&7)<<3),0x24; # ModR/M 1464 push @opcode,($off=~/^0/?oct($off):$off)&0xff; 1465 return ".byte\t".join(',',@opcode); 1466 } 1467 return $line; 1468} 1469 1470$code =~ s/\`([^\`]*)\`/eval($1)/gem; 1471$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem; 1472 1473print $code; 1474close STDOUT or die "error closing STDOUT: $!"; 1475