1/* 2 * ==================================================================== 3 * Written by Intel Corporation for the OpenSSL project to add support 4 * for Intel AES-NI instructions. Rights for redistribution and usage 5 * in source and binary forms are granted according to the OpenSSL 6 * license. 7 * 8 * Author: Huang Ying <ying.huang at intel dot com> 9 * Vinodh Gopal <vinodh.gopal at intel dot com> 10 * Kahraman Akdemir 11 * 12 * Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD) 13 * instructions that are going to be introduced in the next generation 14 * of Intel processor, as of 2009. These instructions enable fast and 15 * secure data encryption and decryption, using the Advanced Encryption 16 * Standard (AES), defined by FIPS Publication number 197. The 17 * architecture introduces six instructions that offer full hardware 18 * support for AES. Four of them support high performance data 19 * encryption and decryption, and the other two instructions support 20 * the AES key expansion procedure. 21 * ==================================================================== 22 */ 23 24/* 25 * ==================================================================== 26 * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. 27 * 28 * Redistribution and use in source and binary forms, with or without 29 * modification, are permitted provided that the following conditions 30 * are met: 31 * 32 * 1. Redistributions of source code must retain the above copyright 33 * notice, this list of conditions and the following disclaimer. 34 * 35 * 2. Redistributions in binary form must reproduce the above copyright 36 * notice, this list of conditions and the following disclaimer in 37 * the documentation and/or other materials provided with the 38 * distribution. 39 * 40 * 3. All advertising materials mentioning features or use of this 41 * software must display the following acknowledgment: 42 * "This product includes software developed by the OpenSSL Project 43 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" 44 * 45 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 46 * endorse or promote products derived from this software without 47 * prior written permission. For written permission, please contact 48 * openssl-core@openssl.org. 49 * 50 * 5. Products derived from this software may not be called "OpenSSL" 51 * nor may "OpenSSL" appear in their names without prior written 52 * permission of the OpenSSL Project. 53 * 54 * 6. Redistributions of any form whatsoever must retain the following 55 * acknowledgment: 56 * "This product includes software developed by the OpenSSL Project 57 * for use in the OpenSSL Toolkit (http://www.openssl.org/)" 58 * 59 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 60 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 62 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 63 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 64 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 65 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 66 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 68 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 69 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 70 * OF THE POSSIBILITY OF SUCH DAMAGE. 71 * ==================================================================== 72 */ 73 74/* 75 * ==================================================================== 76 * OpenSolaris OS modifications 77 * 78 * This source originates as files aes-intel.S and eng_aesni_asm.pl, in 79 * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by 80 * Huang Ying of Intel to the openssl-dev mailing list under the subject 81 * of "Add support to Intel AES-NI instruction set for x86_64 platform". 82 * 83 * This OpenSolaris version has these major changes from the original source: 84 * 85 * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from 86 * /usr/include/sys/asm_linkage.h, lint(1B) guards, EXPORT DELETE START 87 * and EXPORT DELETE END markers, and dummy C function definitions for lint. 88 * 89 * 2. Formatted code, added comments, and added #includes and #defines. 90 * 91 * 3. If bit CR0.TS is set, clear and set the TS bit, after and before 92 * calling kpreempt_disable() and kpreempt_enable(). 93 * If the TS bit is not set, Save and restore %xmm registers at the beginning 94 * and end of function calls (%xmm* registers are not saved and restored by 95 * during kernel thread preemption). 96 * 97 * 4. Renamed functions, reordered parameters, and changed return value 98 * to match OpenSolaris: 99 * 100 * OpenSSL interface: 101 * int intel_AES_set_encrypt_key(const unsigned char *userKey, 102 * const int bits, AES_KEY *key); 103 * int intel_AES_set_decrypt_key(const unsigned char *userKey, 104 * const int bits, AES_KEY *key); 105 * Return values for above are non-zero on error, 0 on success. 106 * 107 * void intel_AES_encrypt(const unsigned char *in, unsigned char *out, 108 * const AES_KEY *key); 109 * void intel_AES_decrypt(const unsigned char *in, unsigned char *out, 110 * const AES_KEY *key); 111 * typedef struct aes_key_st { 112 * unsigned int rd_key[4 *(AES_MAXNR + 1)]; 113 * int rounds; 114 * unsigned int pad[3]; 115 * } AES_KEY; 116 * Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules 117 * (ks32) instead of 64-bit (ks64). 118 * Number of rounds (aka round count) is at offset 240 of AES_KEY. 119 * 120 * OpenSolaris OS interface (#ifdefs removed for readability): 121 * int rijndael_key_setup_dec_intel(uint32_t rk[], 122 * const uint32_t cipherKey[], uint64_t keyBits); 123 * int rijndael_key_setup_enc_intel(uint32_t rk[], 124 * const uint32_t cipherKey[], uint64_t keyBits); 125 * Return values for above are 0 on error, number of rounds on success. 126 * 127 * void aes_encrypt_intel(const aes_ks_t *ks, int Nr, 128 * const uint32_t pt[4], uint32_t ct[4]); 129 * void aes_decrypt_intel(const aes_ks_t *ks, int Nr, 130 * const uint32_t pt[4], uint32_t ct[4]); 131 * typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4]; 132 * uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t; 133 * 134 * typedef union { 135 * uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)]; 136 * } aes_ks_t; 137 * typedef struct aes_key { 138 * aes_ks_t encr_ks, decr_ks; 139 * long double align128; 140 * int flags, nr, type; 141 * } aes_key_t; 142 * 143 * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text, 144 * ct is crypto text, and MAX_AES_NR is 14. 145 * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64. 146 * 147 * Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary. 148 * 149 * ==================================================================== 150 */ 151 152#if defined(lint) || defined(__lint) 153 154#include <sys/types.h> 155 156/* ARGSUSED */ 157void 158aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4], 159 uint32_t ct[4]) { 160} 161/* ARGSUSED */ 162void 163aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4], 164 uint32_t pt[4]) { 165} 166/* ARGSUSED */ 167int 168rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[], 169 uint64_t keyBits) { 170 return (0); 171} 172/* ARGSUSED */ 173int 174rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], 175 uint64_t keyBits) { 176 return (0); 177} 178 179 180#else /* lint */ 181 182#include <sys/asm_linkage.h> 183#include <sys/controlregs.h> 184#ifdef _KERNEL 185#include <sys/machprivregs.h> 186#endif 187 188#ifdef _KERNEL 189 /* 190 * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is, 191 * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it 192 * uses it to pass P2 to syscall. 193 * This also occurs with the STTS macro, but we don't care if 194 * P2 (%rsi) is modified just before function exit. 195 * The CLTS and STTS macros push and pop P1 (%rdi) already. 196 */ 197#ifdef __xpv 198#define PROTECTED_CLTS \ 199 push %rsi; \ 200 CLTS; \ 201 pop %rsi 202#else 203#define PROTECTED_CLTS \ 204 CLTS 205#endif /* __xpv */ 206 207#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) \ 208 push %rbp; \ 209 mov %rsp, %rbp; \ 210 movq %cr0, tmpreg; \ 211 testq $CR0_TS, tmpreg; \ 212 jnz 1f; \ 213 and $-XMM_ALIGN, %rsp; \ 214 sub $[XMM_SIZE * 2], %rsp; \ 215 movaps %xmm0, 16(%rsp); \ 216 movaps %xmm1, (%rsp); \ 217 jmp 2f; \ 2181: \ 219 PROTECTED_CLTS; \ 2202: 221 222 /* 223 * If CR0_TS was not set above, pop %xmm0 and %xmm1 off stack, 224 * otherwise set CR0_TS. 225 */ 226#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) \ 227 testq $CR0_TS, tmpreg; \ 228 jnz 1f; \ 229 movaps (%rsp), %xmm1; \ 230 movaps 16(%rsp), %xmm0; \ 231 jmp 2f; \ 2321: \ 233 STTS(tmpreg); \ 2342: \ 235 mov %rbp, %rsp; \ 236 pop %rbp 237 238 /* 239 * If CR0_TS is not set, align stack (with push %rbp) and push 240 * %xmm0 - %xmm6 on stack, otherwise clear CR0_TS 241 */ 242#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) \ 243 push %rbp; \ 244 mov %rsp, %rbp; \ 245 movq %cr0, tmpreg; \ 246 testq $CR0_TS, tmpreg; \ 247 jnz 1f; \ 248 and $-XMM_ALIGN, %rsp; \ 249 sub $[XMM_SIZE * 7], %rsp; \ 250 movaps %xmm0, 96(%rsp); \ 251 movaps %xmm1, 80(%rsp); \ 252 movaps %xmm2, 64(%rsp); \ 253 movaps %xmm3, 48(%rsp); \ 254 movaps %xmm4, 32(%rsp); \ 255 movaps %xmm5, 16(%rsp); \ 256 movaps %xmm6, (%rsp); \ 257 jmp 2f; \ 2581: \ 259 PROTECTED_CLTS; \ 2602: 261 262 263 /* 264 * If CR0_TS was not set above, pop %xmm0 - %xmm6 off stack, 265 * otherwise set CR0_TS. 266 */ 267#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) \ 268 testq $CR0_TS, tmpreg; \ 269 jnz 1f; \ 270 movaps (%rsp), %xmm6; \ 271 movaps 16(%rsp), %xmm5; \ 272 movaps 32(%rsp), %xmm4; \ 273 movaps 48(%rsp), %xmm3; \ 274 movaps 64(%rsp), %xmm2; \ 275 movaps 80(%rsp), %xmm1; \ 276 movaps 96(%rsp), %xmm0; \ 277 jmp 2f; \ 2781: \ 279 STTS(tmpreg); \ 2802: \ 281 mov %rbp, %rsp; \ 282 pop %rbp 283 284 285#else 286#define PROTECTED_CLTS 287#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) 288#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) 289#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) 290#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) 291#endif /* _KERNEL */ 292 293 294/* 295 * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(), 296 * _key_expansion_256a(), _key_expansion_256b() 297 * 298 * Helper functions called by rijndael_key_setup_inc_intel(). 299 * Also used indirectly by rijndael_key_setup_dec_intel(). 300 * 301 * Input: 302 * %xmm0 User-provided cipher key 303 * %xmm1 Round constant 304 * Output: 305 * (%rcx) AES key 306 */ 307 308 /* EXPORT DELETE START */ 309.align 16 310_key_expansion_128: 311_key_expansion_256a: 312 pshufd $0b11111111, %xmm1, %xmm1 313 shufps $0b00010000, %xmm0, %xmm4 314 pxor %xmm4, %xmm0 315 shufps $0b10001100, %xmm0, %xmm4 316 pxor %xmm4, %xmm0 317 pxor %xmm1, %xmm0 318 movaps %xmm0, (%rcx) 319 add $0x10, %rcx 320 ret 321 SET_SIZE(_key_expansion_128) 322 SET_SIZE(_key_expansion_256a) 323 324.align 16 325_key_expansion_192a: 326 pshufd $0b01010101, %xmm1, %xmm1 327 shufps $0b00010000, %xmm0, %xmm4 328 pxor %xmm4, %xmm0 329 shufps $0b10001100, %xmm0, %xmm4 330 pxor %xmm4, %xmm0 331 pxor %xmm1, %xmm0 332 333 movaps %xmm2, %xmm5 334 movaps %xmm2, %xmm6 335 pslldq $4, %xmm5 336 pshufd $0b11111111, %xmm0, %xmm3 337 pxor %xmm3, %xmm2 338 pxor %xmm5, %xmm2 339 340 movaps %xmm0, %xmm1 341 shufps $0b01000100, %xmm0, %xmm6 342 movaps %xmm6, (%rcx) 343 shufps $0b01001110, %xmm2, %xmm1 344 movaps %xmm1, 0x10(%rcx) 345 add $0x20, %rcx 346 ret 347 SET_SIZE(_key_expansion_192a) 348 349.align 16 350_key_expansion_192b: 351 pshufd $0b01010101, %xmm1, %xmm1 352 shufps $0b00010000, %xmm0, %xmm4 353 pxor %xmm4, %xmm0 354 shufps $0b10001100, %xmm0, %xmm4 355 pxor %xmm4, %xmm0 356 pxor %xmm1, %xmm0 357 358 movaps %xmm2, %xmm5 359 pslldq $4, %xmm5 360 pshufd $0b11111111, %xmm0, %xmm3 361 pxor %xmm3, %xmm2 362 pxor %xmm5, %xmm2 363 364 movaps %xmm0, (%rcx) 365 add $0x10, %rcx 366 ret 367 SET_SIZE(_key_expansion_192b) 368 369.align 16 370_key_expansion_256b: 371 pshufd $0b10101010, %xmm1, %xmm1 372 shufps $0b00010000, %xmm2, %xmm4 373 pxor %xmm4, %xmm2 374 shufps $0b10001100, %xmm2, %xmm4 375 pxor %xmm4, %xmm2 376 pxor %xmm1, %xmm2 377 movaps %xmm2, (%rcx) 378 add $0x10, %rcx 379 ret 380 SET_SIZE(_key_expansion_256b) 381 /* EXPORT DELETE END */ 382 383 384/* 385 * rijndael_key_setup_enc_intel() 386 * Expand the cipher key into the encryption key schedule. 387 * 388 * For kernel code, caller is responsible for ensuring kpreempt_disable() 389 * has been called. This is because %xmm registers are not saved/restored. 390 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set 391 * on entry. Otherwise, if TS is not set, save and restore %xmm registers 392 * on the stack. 393 * 394 * OpenSolaris interface: 395 * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[], 396 * uint64_t keyBits); 397 * Return value is 0 on error, number of rounds on success. 398 * 399 * Original Intel OpenSSL interface: 400 * int intel_AES_set_encrypt_key(const unsigned char *userKey, 401 * const int bits, AES_KEY *key); 402 * Return value is non-zero on error, 0 on success. 403 */ 404 405#ifdef OPENSSL_INTERFACE 406#define rijndael_key_setup_enc_intel intel_AES_set_encrypt_key 407#define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key 408 409#define USERCIPHERKEY rdi /* P1, 64 bits */ 410#define KEYSIZE32 esi /* P2, 32 bits */ 411#define KEYSIZE64 rsi /* P2, 64 bits */ 412#define AESKEY rdx /* P3, 64 bits */ 413 414#else /* OpenSolaris Interface */ 415#define AESKEY rdi /* P1, 64 bits */ 416#define USERCIPHERKEY rsi /* P2, 64 bits */ 417#define KEYSIZE32 edx /* P3, 32 bits */ 418#define KEYSIZE64 rdx /* P3, 64 bits */ 419#endif /* OPENSSL_INTERFACE */ 420 421#define ROUNDS32 KEYSIZE32 /* temp */ 422#define ROUNDS64 KEYSIZE64 /* temp */ 423#define ENDAESKEY USERCIPHERKEY /* temp */ 424 425 426ENTRY_NP(rijndael_key_setup_enc_intel) 427 /* EXPORT DELETE START */ 428 CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10) 429 430 / NULL pointer sanity check 431 test %USERCIPHERKEY, %USERCIPHERKEY 432 jz .Lenc_key_invalid_param 433 test %AESKEY, %AESKEY 434 jz .Lenc_key_invalid_param 435 436 movups (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes) 437 movaps %xmm0, (%AESKEY) 438 lea 0x10(%AESKEY), %rcx / key addr 439 pxor %xmm4, %xmm4 / xmm4 is assumed 0 in _key_expansion_x 440 441 cmp $256, %KEYSIZE32 442 jnz .Lenc_key192 443 444 / AES 256: 14 rounds in encryption key schedule 445#ifdef OPENSSL_INTERFACE 446 mov $14, %ROUNDS32 447 movl %ROUNDS32, 240(%AESKEY) / key.rounds = 14 448#endif /* OPENSSL_INTERFACE */ 449 450 movups 0x10(%USERCIPHERKEY), %xmm2 / other user key (2nd 16 bytes) 451 movaps %xmm2, (%rcx) 452 add $0x10, %rcx 453 454 aeskeygenassist $0x1, %xmm2, %xmm1 / expand the key 455 call _key_expansion_256a 456 aeskeygenassist $0x1, %xmm0, %xmm1 457 call _key_expansion_256b 458 aeskeygenassist $0x2, %xmm2, %xmm1 / expand the key 459 call _key_expansion_256a 460 aeskeygenassist $0x2, %xmm0, %xmm1 461 call _key_expansion_256b 462 aeskeygenassist $0x4, %xmm2, %xmm1 / expand the key 463 call _key_expansion_256a 464 aeskeygenassist $0x4, %xmm0, %xmm1 465 call _key_expansion_256b 466 aeskeygenassist $0x8, %xmm2, %xmm1 / expand the key 467 call _key_expansion_256a 468 aeskeygenassist $0x8, %xmm0, %xmm1 469 call _key_expansion_256b 470 aeskeygenassist $0x10, %xmm2, %xmm1 / expand the key 471 call _key_expansion_256a 472 aeskeygenassist $0x10, %xmm0, %xmm1 473 call _key_expansion_256b 474 aeskeygenassist $0x20, %xmm2, %xmm1 / expand the key 475 call _key_expansion_256a 476 aeskeygenassist $0x20, %xmm0, %xmm1 477 call _key_expansion_256b 478 aeskeygenassist $0x40, %xmm2, %xmm1 / expand the key 479 call _key_expansion_256a 480 481 SET_TS_OR_POP_XMM0_TO_XMM6(%r10) 482#ifdef OPENSSL_INTERFACE 483 xor %rax, %rax / return 0 (OK) 484#else /* Open Solaris Interface */ 485 mov $14, %rax / return # rounds = 14 486#endif 487 ret 488 489.align 4 490.Lenc_key192: 491 cmp $192, %KEYSIZE32 492 jnz .Lenc_key128 493 494 / AES 192: 12 rounds in encryption key schedule 495#ifdef OPENSSL_INTERFACE 496 mov $12, %ROUNDS32 497 movl %ROUNDS32, 240(%AESKEY) / key.rounds = 12 498#endif /* OPENSSL_INTERFACE */ 499 500 movq 0x10(%USERCIPHERKEY), %xmm2 / other user key 501 aeskeygenassist $0x1, %xmm2, %xmm1 / expand the key 502 call _key_expansion_192a 503 aeskeygenassist $0x2, %xmm2, %xmm1 / expand the key 504 call _key_expansion_192b 505 aeskeygenassist $0x4, %xmm2, %xmm1 / expand the key 506 call _key_expansion_192a 507 aeskeygenassist $0x8, %xmm2, %xmm1 / expand the key 508 call _key_expansion_192b 509 aeskeygenassist $0x10, %xmm2, %xmm1 / expand the key 510 call _key_expansion_192a 511 aeskeygenassist $0x20, %xmm2, %xmm1 / expand the key 512 call _key_expansion_192b 513 aeskeygenassist $0x40, %xmm2, %xmm1 / expand the key 514 call _key_expansion_192a 515 aeskeygenassist $0x80, %xmm2, %xmm1 / expand the key 516 call _key_expansion_192b 517 518 SET_TS_OR_POP_XMM0_TO_XMM6(%r10) 519#ifdef OPENSSL_INTERFACE 520 xor %rax, %rax / return 0 (OK) 521#else /* OpenSolaris Interface */ 522 mov $12, %rax / return # rounds = 12 523#endif 524 ret 525 526.align 4 527.Lenc_key128: 528 cmp $128, %KEYSIZE32 529 jnz .Lenc_key_invalid_key_bits 530 531 / AES 128: 10 rounds in encryption key schedule 532#ifdef OPENSSL_INTERFACE 533 mov $10, %ROUNDS32 534 movl %ROUNDS32, 240(%AESKEY) / key.rounds = 10 535#endif /* OPENSSL_INTERFACE */ 536 537 aeskeygenassist $0x1, %xmm0, %xmm1 / expand the key 538 call _key_expansion_128 539 aeskeygenassist $0x2, %xmm0, %xmm1 / expand the key 540 call _key_expansion_128 541 aeskeygenassist $0x4, %xmm0, %xmm1 / expand the key 542 call _key_expansion_128 543 aeskeygenassist $0x8, %xmm0, %xmm1 / expand the key 544 call _key_expansion_128 545 aeskeygenassist $0x10, %xmm0, %xmm1 / expand the key 546 call _key_expansion_128 547 aeskeygenassist $0x20, %xmm0, %xmm1 / expand the key 548 call _key_expansion_128 549 aeskeygenassist $0x40, %xmm0, %xmm1 / expand the key 550 call _key_expansion_128 551 aeskeygenassist $0x80, %xmm0, %xmm1 / expand the key 552 call _key_expansion_128 553 aeskeygenassist $0x1b, %xmm0, %xmm1 / expand the key 554 call _key_expansion_128 555 aeskeygenassist $0x36, %xmm0, %xmm1 / expand the key 556 call _key_expansion_128 557 558 SET_TS_OR_POP_XMM0_TO_XMM6(%r10) 559#ifdef OPENSSL_INTERFACE 560 xor %rax, %rax / return 0 (OK) 561#else /* OpenSolaris Interface */ 562 mov $10, %rax / return # rounds = 10 563#endif 564 ret 565 566.Lenc_key_invalid_param: 567#ifdef OPENSSL_INTERFACE 568 SET_TS_OR_POP_XMM0_TO_XMM6(%r10) 569 mov $-1, %rax / user key or AES key pointer is NULL 570 ret 571#else 572 /* FALLTHROUGH */ 573#endif /* OPENSSL_INTERFACE */ 574 575.Lenc_key_invalid_key_bits: 576 SET_TS_OR_POP_XMM0_TO_XMM6(%r10) 577#ifdef OPENSSL_INTERFACE 578 mov $-2, %rax / keysize is invalid 579#else /* Open Solaris Interface */ 580 xor %rax, %rax / a key pointer is NULL or invalid keysize 581#endif /* OPENSSL_INTERFACE */ 582 583 /* EXPORT DELETE END */ 584 ret 585 SET_SIZE(rijndael_key_setup_enc_intel) 586 587 588/* 589 * rijndael_key_setup_dec_intel() 590 * Expand the cipher key into the decryption key schedule. 591 * 592 * For kernel code, caller is responsible for ensuring kpreempt_disable() 593 * has been called. This is because %xmm registers are not saved/restored. 594 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set 595 * on entry. Otherwise, if TS is not set, save and restore %xmm registers 596 * on the stack. 597 * 598 * OpenSolaris interface: 599 * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], 600 * uint64_t keyBits); 601 * Return value is 0 on error, number of rounds on success. 602 * P1->P2, P2->P3, P3->P1 603 * 604 * Original Intel OpenSSL interface: 605 * int intel_AES_set_decrypt_key(const unsigned char *userKey, 606 * const int bits, AES_KEY *key); 607 * Return value is non-zero on error, 0 on success. 608 */ 609ENTRY_NP(rijndael_key_setup_dec_intel) 610 /* EXPORT DELETE START */ 611 / Generate round keys used for encryption 612 call rijndael_key_setup_enc_intel 613 test %rax, %rax 614#ifdef OPENSSL_INTERFACE 615 jnz .Ldec_key_exit / Failed if returned non-0 616#else /* OpenSolaris Interface */ 617 jz .Ldec_key_exit / Failed if returned 0 618#endif /* OPENSSL_INTERFACE */ 619 620 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) 621 622 /* 623 * Convert round keys used for encryption 624 * to a form usable for decryption 625 */ 626#ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */ 627 mov %rax, %ROUNDS64 / set # rounds (10, 12, or 14) 628 / (already set for OpenSSL) 629#endif 630 631 lea 0x10(%AESKEY), %rcx / key addr 632 shl $4, %ROUNDS32 633 add %AESKEY, %ROUNDS64 634 mov %ROUNDS64, %ENDAESKEY 635 636.align 4 637.Ldec_key_reorder_loop: 638 movaps (%AESKEY), %xmm0 639 movaps (%ROUNDS64), %xmm1 640 movaps %xmm0, (%ROUNDS64) 641 movaps %xmm1, (%AESKEY) 642 lea 0x10(%AESKEY), %AESKEY 643 lea -0x10(%ROUNDS64), %ROUNDS64 644 cmp %AESKEY, %ROUNDS64 645 ja .Ldec_key_reorder_loop 646 647.align 4 648.Ldec_key_inv_loop: 649 movaps (%rcx), %xmm0 650 / Convert an encryption round key to a form usable for decryption 651 / with the "AES Inverse Mix Columns" instruction 652 aesimc %xmm0, %xmm1 653 movaps %xmm1, (%rcx) 654 lea 0x10(%rcx), %rcx 655 cmp %ENDAESKEY, %rcx 656 jnz .Ldec_key_inv_loop 657 658 SET_TS_OR_POP_XMM0_XMM1(%r10) 659 660.Ldec_key_exit: 661 / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error 662 / OpenSSL: rax = 0 for OK, or non-zero for error 663 /* EXPORT DELETE END */ 664 ret 665 SET_SIZE(rijndael_key_setup_dec_intel) 666 667 668/* 669 * aes_encrypt_intel() 670 * Encrypt a single block (in and out can overlap). 671 * 672 * For kernel code, caller is responsible for ensuring kpreempt_disable() 673 * has been called. This is because %xmm registers are not saved/restored. 674 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set 675 * on entry. Otherwise, if TS is not set, save and restore %xmm registers 676 * on the stack. 677 * 678 * Temporary register usage: 679 * %xmm0 State 680 * %xmm1 Key 681 * 682 * Original OpenSolaris Interface: 683 * void aes_encrypt_intel(const aes_ks_t *ks, int Nr, 684 * const uint32_t pt[4], uint32_t ct[4]) 685 * 686 * Original Intel OpenSSL Interface: 687 * void intel_AES_encrypt(const unsigned char *in, unsigned char *out, 688 * const AES_KEY *key) 689 */ 690 691#ifdef OPENSSL_INTERFACE 692#define aes_encrypt_intel intel_AES_encrypt 693#define aes_decrypt_intel intel_AES_decrypt 694 695#define INP rdi /* P1, 64 bits */ 696#define OUTP rsi /* P2, 64 bits */ 697#define KEYP rdx /* P3, 64 bits */ 698 699/* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */ 700#define NROUNDS32 ecx /* temporary, 32 bits */ 701#define NROUNDS cl /* temporary, 8 bits */ 702 703#else /* OpenSolaris Interface */ 704#define KEYP rdi /* P1, 64 bits */ 705#define NROUNDS esi /* P2, 32 bits */ 706#define INP rdx /* P3, 64 bits */ 707#define OUTP rcx /* P4, 64 bits */ 708#endif /* OPENSSL_INTERFACE */ 709 710#define STATE xmm0 /* temporary, 128 bits */ 711#define KEY xmm1 /* temporary, 128 bits */ 712 713ENTRY_NP(aes_encrypt_intel) 714 /* EXPORT DELETE START */ 715 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) 716 717 movups (%INP), %STATE / input 718 movaps (%KEYP), %KEY / key 719#ifdef OPENSSL_INTERFACE 720 mov 240(%KEYP), %NROUNDS32 / round count 721#else /* OpenSolaris Interface */ 722 /* Round count is already present as P2 in %rsi/%esi */ 723#endif /* OPENSSL_INTERFACE */ 724 725 pxor %KEY, %STATE / round 0 726 lea 0x30(%KEYP), %KEYP 727 cmp $12, %NROUNDS 728 jb .Lenc128 729 lea 0x20(%KEYP), %KEYP 730 je .Lenc192 731 732 / AES 256 733 lea 0x20(%KEYP), %KEYP 734 movaps -0x60(%KEYP), %KEY 735 aesenc %KEY, %STATE 736 movaps -0x50(%KEYP), %KEY 737 aesenc %KEY, %STATE 738 739.align 4 740.Lenc192: 741 / AES 192 and 256 742 movaps -0x40(%KEYP), %KEY 743 aesenc %KEY, %STATE 744 movaps -0x30(%KEYP), %KEY 745 aesenc %KEY, %STATE 746 747.align 4 748.Lenc128: 749 / AES 128, 192, and 256 750 movaps -0x20(%KEYP), %KEY 751 aesenc %KEY, %STATE 752 movaps -0x10(%KEYP), %KEY 753 aesenc %KEY, %STATE 754 movaps (%KEYP), %KEY 755 aesenc %KEY, %STATE 756 movaps 0x10(%KEYP), %KEY 757 aesenc %KEY, %STATE 758 movaps 0x20(%KEYP), %KEY 759 aesenc %KEY, %STATE 760 movaps 0x30(%KEYP), %KEY 761 aesenc %KEY, %STATE 762 movaps 0x40(%KEYP), %KEY 763 aesenc %KEY, %STATE 764 movaps 0x50(%KEYP), %KEY 765 aesenc %KEY, %STATE 766 movaps 0x60(%KEYP), %KEY 767 aesenc %KEY, %STATE 768 movaps 0x70(%KEYP), %KEY 769 aesenclast %KEY, %STATE / last round 770 movups %STATE, (%OUTP) / output 771 772 SET_TS_OR_POP_XMM0_XMM1(%r10) 773 /* EXPORT DELETE END */ 774 ret 775 SET_SIZE(aes_encrypt_intel) 776 777 778/* 779 * aes_decrypt_intel() 780 * Decrypt a single block (in and out can overlap). 781 * 782 * For kernel code, caller is responsible for ensuring kpreempt_disable() 783 * has been called. This is because %xmm registers are not saved/restored. 784 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set 785 * on entry. Otherwise, if TS is not set, save and restore %xmm registers 786 * on the stack. 787 * 788 * Temporary register usage: 789 * %xmm0 State 790 * %xmm1 Key 791 * 792 * Original OpenSolaris Interface: 793 * void aes_decrypt_intel(const aes_ks_t *ks, int Nr, 794 * const uint32_t pt[4], uint32_t ct[4])/ 795 * 796 * Original Intel OpenSSL Interface: 797 * void intel_AES_decrypt(const unsigned char *in, unsigned char *out, 798 * const AES_KEY *key); 799 */ 800ENTRY_NP(aes_decrypt_intel) 801 /* EXPORT DELETE START */ 802 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) 803 804 movups (%INP), %STATE / input 805 movaps (%KEYP), %KEY / key 806#ifdef OPENSSL_INTERFACE 807 mov 240(%KEYP), %NROUNDS32 / round count 808#else /* OpenSolaris Interface */ 809 /* Round count is already present as P2 in %rsi/%esi */ 810#endif /* OPENSSL_INTERFACE */ 811 812 pxor %KEY, %STATE / round 0 813 lea 0x30(%KEYP), %KEYP 814 cmp $12, %NROUNDS 815 jb .Ldec128 816 lea 0x20(%KEYP), %KEYP 817 je .Ldec192 818 819 / AES 256 820 lea 0x20(%KEYP), %KEYP 821 movaps -0x60(%KEYP), %KEY 822 aesdec %KEY, %STATE 823 movaps -0x50(%KEYP), %KEY 824 aesdec %KEY, %STATE 825 826.align 4 827.Ldec192: 828 / AES 192 and 256 829 movaps -0x40(%KEYP), %KEY 830 aesdec %KEY, %STATE 831 movaps -0x30(%KEYP), %KEY 832 aesdec %KEY, %STATE 833 834.align 4 835.Ldec128: 836 / AES 128, 192, and 256 837 movaps -0x20(%KEYP), %KEY 838 aesdec %KEY, %STATE 839 movaps -0x10(%KEYP), %KEY 840 aesdec %KEY, %STATE 841 movaps (%KEYP), %KEY 842 aesdec %KEY, %STATE 843 movaps 0x10(%KEYP), %KEY 844 aesdec %KEY, %STATE 845 movaps 0x20(%KEYP), %KEY 846 aesdec %KEY, %STATE 847 movaps 0x30(%KEYP), %KEY 848 aesdec %KEY, %STATE 849 movaps 0x40(%KEYP), %KEY 850 aesdec %KEY, %STATE 851 movaps 0x50(%KEYP), %KEY 852 aesdec %KEY, %STATE 853 movaps 0x60(%KEYP), %KEY 854 aesdec %KEY, %STATE 855 movaps 0x70(%KEYP), %KEY 856 aesdeclast %KEY, %STATE / last round 857 movups %STATE, (%OUTP) / output 858 859 SET_TS_OR_POP_XMM0_XMM1(%r10) 860 ret 861 /* EXPORT DELETE END */ 862 SET_SIZE(aes_decrypt_intel) 863 864#endif /* lint || __lint */ 865