1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 26 #ifndef _KERNEL 27 #include <strings.h> 28 #include <limits.h> 29 #include <assert.h> 30 #include <security/cryptoki.h> 31 #endif /* _KERNEL */ 32 33 34 #include <sys/types.h> 35 #include <sys/kmem.h> 36 #include <modes/modes.h> 37 #include <sys/crypto/common.h> 38 #include <sys/crypto/impl.h> 39 #include <sys/byteorder.h> 40 41 #ifdef __amd64 42 43 #ifdef _KERNEL 44 #include <sys/cpuvar.h> /* cpu_t, CPU */ 45 #include <sys/x86_archext.h> /* x86_featureset, X86FSET_*, CPUID_* */ 46 #include <sys/disp.h> /* kpreempt_disable(), kpreempt_enable */ 47 /* Workaround for no XMM kernel thread save/restore */ 48 #define KPREEMPT_DISABLE kpreempt_disable() 49 #define KPREEMPT_ENABLE kpreempt_enable() 50 51 #else 52 #include <sys/auxv.h> /* getisax() */ 53 #include <sys/auxv_386.h> /* AV_386_PCLMULQDQ bit */ 54 #define KPREEMPT_DISABLE 55 #define KPREEMPT_ENABLE 56 #endif /* _KERNEL */ 57 58 extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res); 59 static int intel_pclmulqdq_instruction_present(void); 60 #endif /* __amd64 */ 61 62 struct aes_block { 63 uint64_t a; 64 uint64_t b; 65 }; 66 67 68 /* 69 * gcm_mul() 70 * Perform a carry-less multiplication (that is, use XOR instead of the 71 * multiply operator) on *x_in and *y and place the result in *res. 72 * 73 * Byte swap the input (*x_in and *y) and the output (*res). 74 * 75 * Note: x_in, y, and res all point to 16-byte numbers (an array of two 76 * 64-bit integers). 77 */ 78 void 79 gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res) 80 { 81 #ifdef __amd64 82 if (intel_pclmulqdq_instruction_present()) { 83 KPREEMPT_DISABLE; 84 gcm_mul_pclmulqdq(x_in, y, res); 85 KPREEMPT_ENABLE; 86 } else 87 #endif /* __amd64 */ 88 { 89 static const uint64_t R = 0xe100000000000000ULL; 90 struct aes_block z = {0, 0}; 91 struct aes_block v; 92 uint64_t x; 93 int i, j; 94 95 v.a = ntohll(y[0]); 96 v.b = ntohll(y[1]); 97 98 for (j = 0; j < 2; j++) { 99 x = ntohll(x_in[j]); 100 for (i = 0; i < 64; i++, x <<= 1) { 101 if (x & 0x8000000000000000ULL) { 102 z.a ^= v.a; 103 z.b ^= v.b; 104 } 105 if (v.b & 1ULL) { 106 v.b = (v.a << 63)|(v.b >> 1); 107 v.a = (v.a >> 1) ^ R; 108 } else { 109 v.b = (v.a << 63)|(v.b >> 1); 110 v.a = v.a >> 1; 111 } 112 } 113 } 114 res[0] = htonll(z.a); 115 res[1] = htonll(z.b); 116 } 117 } 118 119 120 #define GHASH(c, d, t) \ 121 xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \ 122 gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \ 123 (uint64_t *)(void *)(t)); 124 125 126 /* 127 * Encrypt multiple blocks of data in GCM mode. Decrypt for GCM mode 128 * is done in another function. 129 */ 130 int 131 gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, 132 crypto_data_t *out, size_t block_size, 133 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 134 void (*copy_block)(uint8_t *, uint8_t *), 135 void (*xor_block)(uint8_t *, uint8_t *)) 136 { 137 size_t remainder = length; 138 size_t need; 139 uint8_t *datap = (uint8_t *)data; 140 uint8_t *blockp; 141 uint8_t *lastp; 142 void *iov_or_mp; 143 offset_t offset; 144 uint8_t *out_data_1; 145 uint8_t *out_data_2; 146 size_t out_data_1_len; 147 uint64_t counter; 148 uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 149 150 if (length + ctx->gcm_remainder_len < block_size) { 151 /* accumulate bytes here and return */ 152 bcopy(datap, 153 (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len, 154 length); 155 ctx->gcm_remainder_len += length; 156 ctx->gcm_copy_to = datap; 157 return (CRYPTO_SUCCESS); 158 } 159 160 lastp = (uint8_t *)ctx->gcm_cb; 161 if (out != NULL) 162 crypto_init_ptrs(out, &iov_or_mp, &offset); 163 164 do { 165 /* Unprocessed data from last call. */ 166 if (ctx->gcm_remainder_len > 0) { 167 need = block_size - ctx->gcm_remainder_len; 168 169 if (need > remainder) 170 return (CRYPTO_DATA_LEN_RANGE); 171 172 bcopy(datap, &((uint8_t *)ctx->gcm_remainder) 173 [ctx->gcm_remainder_len], need); 174 175 blockp = (uint8_t *)ctx->gcm_remainder; 176 } else { 177 blockp = datap; 178 } 179 180 /* 181 * Increment counter. Counter bits are confined 182 * to the bottom 32 bits of the counter block. 183 */ 184 counter = ntohll(ctx->gcm_cb[1] & counter_mask); 185 counter = htonll(counter + 1); 186 counter &= counter_mask; 187 ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 188 189 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, 190 (uint8_t *)ctx->gcm_tmp); 191 xor_block(blockp, (uint8_t *)ctx->gcm_tmp); 192 193 lastp = (uint8_t *)ctx->gcm_tmp; 194 195 ctx->gcm_processed_data_len += block_size; 196 197 if (out == NULL) { 198 if (ctx->gcm_remainder_len > 0) { 199 bcopy(blockp, ctx->gcm_copy_to, 200 ctx->gcm_remainder_len); 201 bcopy(blockp + ctx->gcm_remainder_len, datap, 202 need); 203 } 204 } else { 205 crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, 206 &out_data_1_len, &out_data_2, block_size); 207 208 /* copy block to where it belongs */ 209 if (out_data_1_len == block_size) { 210 copy_block(lastp, out_data_1); 211 } else { 212 bcopy(lastp, out_data_1, out_data_1_len); 213 if (out_data_2 != NULL) { 214 bcopy(lastp + out_data_1_len, 215 out_data_2, 216 block_size - out_data_1_len); 217 } 218 } 219 /* update offset */ 220 out->cd_offset += block_size; 221 } 222 223 /* add ciphertext to the hash */ 224 GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); 225 226 /* Update pointer to next block of data to be processed. */ 227 if (ctx->gcm_remainder_len != 0) { 228 datap += need; 229 ctx->gcm_remainder_len = 0; 230 } else { 231 datap += block_size; 232 } 233 234 remainder = (size_t)&data[length] - (size_t)datap; 235 236 /* Incomplete last block. */ 237 if (remainder > 0 && remainder < block_size) { 238 bcopy(datap, ctx->gcm_remainder, remainder); 239 ctx->gcm_remainder_len = remainder; 240 ctx->gcm_copy_to = datap; 241 goto out; 242 } 243 ctx->gcm_copy_to = NULL; 244 245 } while (remainder > 0); 246 out: 247 return (CRYPTO_SUCCESS); 248 } 249 250 /* ARGSUSED */ 251 int 252 gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, 253 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 254 void (*copy_block)(uint8_t *, uint8_t *), 255 void (*xor_block)(uint8_t *, uint8_t *)) 256 { 257 uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 258 uint8_t *ghash, *macp; 259 int i, rv; 260 261 if (out->cd_length < 262 (ctx->gcm_remainder_len + ctx->gcm_tag_len)) { 263 return (CRYPTO_DATA_LEN_RANGE); 264 } 265 266 ghash = (uint8_t *)ctx->gcm_ghash; 267 268 if (ctx->gcm_remainder_len > 0) { 269 uint64_t counter; 270 uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp; 271 272 /* 273 * Here is where we deal with data that is not a 274 * multiple of the block size. 275 */ 276 277 /* 278 * Increment counter. 279 */ 280 counter = ntohll(ctx->gcm_cb[1] & counter_mask); 281 counter = htonll(counter + 1); 282 counter &= counter_mask; 283 ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 284 285 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, 286 (uint8_t *)ctx->gcm_tmp); 287 288 macp = (uint8_t *)ctx->gcm_remainder; 289 bzero(macp + ctx->gcm_remainder_len, 290 block_size - ctx->gcm_remainder_len); 291 292 /* XOR with counter block */ 293 for (i = 0; i < ctx->gcm_remainder_len; i++) { 294 macp[i] ^= tmpp[i]; 295 } 296 297 /* add ciphertext to the hash */ 298 GHASH(ctx, macp, ghash); 299 300 ctx->gcm_processed_data_len += ctx->gcm_remainder_len; 301 } 302 303 ctx->gcm_len_a_len_c[1] = 304 htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len)); 305 GHASH(ctx, ctx->gcm_len_a_len_c, ghash); 306 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, 307 (uint8_t *)ctx->gcm_J0); 308 xor_block((uint8_t *)ctx->gcm_J0, ghash); 309 310 if (ctx->gcm_remainder_len > 0) { 311 rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len); 312 if (rv != CRYPTO_SUCCESS) 313 return (rv); 314 } 315 out->cd_offset += ctx->gcm_remainder_len; 316 ctx->gcm_remainder_len = 0; 317 rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len); 318 if (rv != CRYPTO_SUCCESS) 319 return (rv); 320 out->cd_offset += ctx->gcm_tag_len; 321 322 return (CRYPTO_SUCCESS); 323 } 324 325 /* 326 * This will only deal with decrypting the last block of the input that 327 * might not be a multiple of block length. 328 */ 329 static void 330 gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index, 331 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 332 void (*xor_block)(uint8_t *, uint8_t *)) 333 { 334 uint8_t *datap, *outp, *counterp; 335 uint64_t counter; 336 uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 337 int i; 338 339 /* 340 * Increment counter. 341 * Counter bits are confined to the bottom 32 bits 342 */ 343 counter = ntohll(ctx->gcm_cb[1] & counter_mask); 344 counter = htonll(counter + 1); 345 counter &= counter_mask; 346 ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 347 348 datap = (uint8_t *)ctx->gcm_remainder; 349 outp = &((ctx->gcm_pt_buf)[index]); 350 counterp = (uint8_t *)ctx->gcm_tmp; 351 352 /* authentication tag */ 353 bzero((uint8_t *)ctx->gcm_tmp, block_size); 354 bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len); 355 356 /* add ciphertext to the hash */ 357 GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); 358 359 /* decrypt remaining ciphertext */ 360 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp); 361 362 /* XOR with counter block */ 363 for (i = 0; i < ctx->gcm_remainder_len; i++) { 364 outp[i] = datap[i] ^ counterp[i]; 365 } 366 } 367 368 /* ARGSUSED */ 369 int 370 gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, 371 crypto_data_t *out, size_t block_size, 372 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 373 void (*copy_block)(uint8_t *, uint8_t *), 374 void (*xor_block)(uint8_t *, uint8_t *)) 375 { 376 size_t new_len; 377 uint8_t *new; 378 379 /* 380 * Copy contiguous ciphertext input blocks to plaintext buffer. 381 * Ciphertext will be decrypted in the final. 382 */ 383 if (length > 0) { 384 new_len = ctx->gcm_pt_buf_len + length; 385 #ifdef _KERNEL 386 new = kmem_alloc(new_len, ctx->gcm_kmflag); 387 bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); 388 kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len); 389 #else 390 new = malloc(new_len); 391 bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); 392 free(ctx->gcm_pt_buf); 393 #endif 394 if (new == NULL) 395 return (CRYPTO_HOST_MEMORY); 396 397 ctx->gcm_pt_buf = new; 398 ctx->gcm_pt_buf_len = new_len; 399 bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len], 400 length); 401 ctx->gcm_processed_data_len += length; 402 } 403 404 ctx->gcm_remainder_len = 0; 405 return (CRYPTO_SUCCESS); 406 } 407 408 int 409 gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, 410 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 411 void (*xor_block)(uint8_t *, uint8_t *)) 412 { 413 size_t pt_len; 414 size_t remainder; 415 uint8_t *ghash; 416 uint8_t *blockp; 417 uint8_t *cbp; 418 uint64_t counter; 419 uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 420 int processed = 0, rv; 421 422 ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len); 423 424 pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; 425 ghash = (uint8_t *)ctx->gcm_ghash; 426 blockp = ctx->gcm_pt_buf; 427 remainder = pt_len; 428 while (remainder > 0) { 429 /* Incomplete last block */ 430 if (remainder < block_size) { 431 bcopy(blockp, ctx->gcm_remainder, remainder); 432 ctx->gcm_remainder_len = remainder; 433 /* 434 * not expecting anymore ciphertext, just 435 * compute plaintext for the remaining input 436 */ 437 gcm_decrypt_incomplete_block(ctx, block_size, 438 processed, encrypt_block, xor_block); 439 ctx->gcm_remainder_len = 0; 440 goto out; 441 } 442 /* add ciphertext to the hash */ 443 GHASH(ctx, blockp, ghash); 444 445 /* 446 * Increment counter. 447 * Counter bits are confined to the bottom 32 bits 448 */ 449 counter = ntohll(ctx->gcm_cb[1] & counter_mask); 450 counter = htonll(counter + 1); 451 counter &= counter_mask; 452 ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 453 454 cbp = (uint8_t *)ctx->gcm_tmp; 455 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp); 456 457 /* XOR with ciphertext */ 458 xor_block(cbp, blockp); 459 460 processed += block_size; 461 blockp += block_size; 462 remainder -= block_size; 463 } 464 out: 465 ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len)); 466 GHASH(ctx, ctx->gcm_len_a_len_c, ghash); 467 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, 468 (uint8_t *)ctx->gcm_J0); 469 xor_block((uint8_t *)ctx->gcm_J0, ghash); 470 471 /* compare the input authentication tag with what we calculated */ 472 if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) { 473 /* They don't match */ 474 return (CRYPTO_INVALID_MAC); 475 } else { 476 rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len); 477 if (rv != CRYPTO_SUCCESS) 478 return (rv); 479 out->cd_offset += pt_len; 480 } 481 return (CRYPTO_SUCCESS); 482 } 483 484 static int 485 gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param) 486 { 487 size_t tag_len; 488 489 /* 490 * Check the length of the authentication tag (in bits). 491 */ 492 tag_len = gcm_param->ulTagBits; 493 switch (tag_len) { 494 case 32: 495 case 64: 496 case 96: 497 case 104: 498 case 112: 499 case 120: 500 case 128: 501 break; 502 default: 503 return (CRYPTO_MECHANISM_PARAM_INVALID); 504 } 505 506 if (gcm_param->ulIvLen == 0) 507 return (CRYPTO_MECHANISM_PARAM_INVALID); 508 509 return (CRYPTO_SUCCESS); 510 } 511 512 static void 513 gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len, 514 gcm_ctx_t *ctx, size_t block_size, 515 void (*copy_block)(uint8_t *, uint8_t *), 516 void (*xor_block)(uint8_t *, uint8_t *)) 517 { 518 uint8_t *cb; 519 ulong_t remainder = iv_len; 520 ulong_t processed = 0; 521 uint8_t *datap, *ghash; 522 uint64_t len_a_len_c[2]; 523 524 ghash = (uint8_t *)ctx->gcm_ghash; 525 cb = (uint8_t *)ctx->gcm_cb; 526 if (iv_len == 12) { 527 bcopy(iv, cb, 12); 528 cb[12] = 0; 529 cb[13] = 0; 530 cb[14] = 0; 531 cb[15] = 1; 532 /* J0 will be used again in the final */ 533 copy_block(cb, (uint8_t *)ctx->gcm_J0); 534 } else { 535 /* GHASH the IV */ 536 do { 537 if (remainder < block_size) { 538 bzero(cb, block_size); 539 bcopy(&(iv[processed]), cb, remainder); 540 datap = (uint8_t *)cb; 541 remainder = 0; 542 } else { 543 datap = (uint8_t *)(&(iv[processed])); 544 processed += block_size; 545 remainder -= block_size; 546 } 547 GHASH(ctx, datap, ghash); 548 } while (remainder > 0); 549 550 len_a_len_c[0] = 0; 551 len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len)); 552 GHASH(ctx, len_a_len_c, ctx->gcm_J0); 553 554 /* J0 will be used again in the final */ 555 copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb); 556 } 557 } 558 559 /* 560 * The following function is called at encrypt or decrypt init time 561 * for AES GCM mode. 562 */ 563 int 564 gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len, 565 unsigned char *auth_data, size_t auth_data_len, size_t block_size, 566 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 567 void (*copy_block)(uint8_t *, uint8_t *), 568 void (*xor_block)(uint8_t *, uint8_t *)) 569 { 570 uint8_t *ghash, *datap, *authp; 571 size_t remainder, processed; 572 573 /* encrypt zero block to get subkey H */ 574 bzero(ctx->gcm_H, sizeof (ctx->gcm_H)); 575 encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H, 576 (uint8_t *)ctx->gcm_H); 577 578 gcm_format_initial_blocks(iv, iv_len, ctx, block_size, 579 copy_block, xor_block); 580 581 authp = (uint8_t *)ctx->gcm_tmp; 582 ghash = (uint8_t *)ctx->gcm_ghash; 583 bzero(authp, block_size); 584 bzero(ghash, block_size); 585 586 processed = 0; 587 remainder = auth_data_len; 588 do { 589 if (remainder < block_size) { 590 /* 591 * There's not a block full of data, pad rest of 592 * buffer with zero 593 */ 594 bzero(authp, block_size); 595 bcopy(&(auth_data[processed]), authp, remainder); 596 datap = (uint8_t *)authp; 597 remainder = 0; 598 } else { 599 datap = (uint8_t *)(&(auth_data[processed])); 600 processed += block_size; 601 remainder -= block_size; 602 } 603 604 /* add auth data to the hash */ 605 GHASH(ctx, datap, ghash); 606 607 } while (remainder > 0); 608 609 return (CRYPTO_SUCCESS); 610 } 611 612 int 613 gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, 614 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 615 void (*copy_block)(uint8_t *, uint8_t *), 616 void (*xor_block)(uint8_t *, uint8_t *)) 617 { 618 int rv; 619 CK_AES_GCM_PARAMS *gcm_param; 620 621 if (param != NULL) { 622 gcm_param = (CK_AES_GCM_PARAMS *)(void *)param; 623 624 if ((rv = gcm_validate_args(gcm_param)) != 0) { 625 return (rv); 626 } 627 628 gcm_ctx->gcm_tag_len = gcm_param->ulTagBits; 629 gcm_ctx->gcm_tag_len >>= 3; 630 gcm_ctx->gcm_processed_data_len = 0; 631 632 /* these values are in bits */ 633 gcm_ctx->gcm_len_a_len_c[0] 634 = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen)); 635 636 rv = CRYPTO_SUCCESS; 637 gcm_ctx->gcm_flags |= GCM_MODE; 638 } else { 639 rv = CRYPTO_MECHANISM_PARAM_INVALID; 640 goto out; 641 } 642 643 if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen, 644 gcm_param->pAAD, gcm_param->ulAADLen, block_size, 645 encrypt_block, copy_block, xor_block) != 0) { 646 rv = CRYPTO_MECHANISM_PARAM_INVALID; 647 } 648 out: 649 return (rv); 650 } 651 652 int 653 gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, 654 int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 655 void (*copy_block)(uint8_t *, uint8_t *), 656 void (*xor_block)(uint8_t *, uint8_t *)) 657 { 658 int rv; 659 CK_AES_GMAC_PARAMS *gmac_param; 660 661 if (param != NULL) { 662 gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param; 663 664 gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS); 665 gcm_ctx->gcm_processed_data_len = 0; 666 667 /* these values are in bits */ 668 gcm_ctx->gcm_len_a_len_c[0] 669 = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen)); 670 671 rv = CRYPTO_SUCCESS; 672 gcm_ctx->gcm_flags |= GMAC_MODE; 673 } else { 674 rv = CRYPTO_MECHANISM_PARAM_INVALID; 675 goto out; 676 } 677 678 if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN, 679 gmac_param->pAAD, gmac_param->ulAADLen, block_size, 680 encrypt_block, copy_block, xor_block) != 0) { 681 rv = CRYPTO_MECHANISM_PARAM_INVALID; 682 } 683 out: 684 return (rv); 685 } 686 687 void * 688 gcm_alloc_ctx(int kmflag) 689 { 690 gcm_ctx_t *gcm_ctx; 691 692 #ifdef _KERNEL 693 if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) 694 #else 695 if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL) 696 #endif 697 return (NULL); 698 699 gcm_ctx->gcm_flags = GCM_MODE; 700 return (gcm_ctx); 701 } 702 703 void * 704 gmac_alloc_ctx(int kmflag) 705 { 706 gcm_ctx_t *gcm_ctx; 707 708 #ifdef _KERNEL 709 if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) 710 #else 711 if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL) 712 #endif 713 return (NULL); 714 715 gcm_ctx->gcm_flags = GMAC_MODE; 716 return (gcm_ctx); 717 } 718 719 void 720 gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag) 721 { 722 ctx->gcm_kmflag = kmflag; 723 } 724 725 726 #ifdef __amd64 727 /* 728 * Return 1 if executing on Intel with PCLMULQDQ instructions, 729 * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64). 730 * Cache the result, as the CPU can't change. 731 * 732 * Note: the userland version uses getisax(). The kernel version uses 733 * is_x86_featureset(). 734 */ 735 static int 736 intel_pclmulqdq_instruction_present(void) 737 { 738 static int cached_result = -1; 739 740 if (cached_result == -1) { /* first time */ 741 #ifdef _KERNEL 742 cached_result = 743 is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ); 744 #else 745 uint_t ui = 0; 746 747 (void) getisax(&ui, 1); 748 cached_result = (ui & AV_386_PCLMULQDQ) != 0; 749 #endif /* _KERNEL */ 750 } 751 752 return (cached_result); 753 } 754 #endif /* __amd64 */ 755