1*4d703b5cSMark Powers /* 2*4d703b5cSMark Powers * CDDL HEADER START 3*4d703b5cSMark Powers * 4*4d703b5cSMark Powers * The contents of this file are subject to the terms of the 5*4d703b5cSMark Powers * Common Development and Distribution License (the "License"). 6*4d703b5cSMark Powers * You may not use this file except in compliance with the License. 7*4d703b5cSMark Powers * 8*4d703b5cSMark Powers * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*4d703b5cSMark Powers * or http://www.opensolaris.org/os/licensing. 10*4d703b5cSMark Powers * See the License for the specific language governing permissions 11*4d703b5cSMark Powers * and limitations under the License. 12*4d703b5cSMark Powers * 13*4d703b5cSMark Powers * When distributing Covered Code, include this CDDL HEADER in each 14*4d703b5cSMark Powers * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*4d703b5cSMark Powers * If applicable, add the following below this CDDL HEADER, with the 16*4d703b5cSMark Powers * fields enclosed by brackets "[]" replaced with your own identifying 17*4d703b5cSMark Powers * information: Portions Copyright [yyyy] [name of copyright owner] 18*4d703b5cSMark Powers * 19*4d703b5cSMark Powers * CDDL HEADER END 20*4d703b5cSMark Powers */ 21*4d703b5cSMark Powers /* 22*4d703b5cSMark Powers * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23*4d703b5cSMark Powers * Use is subject to license terms. 24*4d703b5cSMark Powers */ 25*4d703b5cSMark Powers 26*4d703b5cSMark Powers #ifndef _KERNEL 27*4d703b5cSMark Powers #include <strings.h> 28*4d703b5cSMark Powers #include <limits.h> 29*4d703b5cSMark Powers #include <assert.h> 30*4d703b5cSMark Powers #include <security/cryptoki.h> 31*4d703b5cSMark Powers #endif 32*4d703b5cSMark Powers 33*4d703b5cSMark Powers #include <sys/types.h> 34*4d703b5cSMark Powers #include <sys/kmem.h> 35*4d703b5cSMark Powers #include <modes/modes.h> 36*4d703b5cSMark Powers #include <sys/crypto/common.h> 37*4d703b5cSMark Powers #include <sys/crypto/impl.h> 38*4d703b5cSMark Powers #include <sys/byteorder.h> 39*4d703b5cSMark Powers 40*4d703b5cSMark Powers struct aes_block { 41*4d703b5cSMark Powers uint64_t a; 42*4d703b5cSMark Powers uint64_t b; 43*4d703b5cSMark Powers }; 44*4d703b5cSMark Powers 45*4d703b5cSMark Powers static void 46*4d703b5cSMark Powers gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res) 47*4d703b5cSMark Powers { 48*4d703b5cSMark Powers uint64_t R = { 0xe100000000000000ULL }; 49*4d703b5cSMark Powers struct aes_block z = { 0, 0 }; 50*4d703b5cSMark Powers struct aes_block v; 51*4d703b5cSMark Powers uint64_t x; 52*4d703b5cSMark Powers int i, j; 53*4d703b5cSMark Powers 54*4d703b5cSMark Powers v.a = ntohll(y[0]); 55*4d703b5cSMark Powers v.b = ntohll(y[1]); 56*4d703b5cSMark Powers 57*4d703b5cSMark Powers for (j = 0; j < 2; j++) { 58*4d703b5cSMark Powers x = ntohll(x_in[j]); 59*4d703b5cSMark Powers for (i = 0; i < 64; i++, x <<= 1) { 60*4d703b5cSMark Powers if (x & 0x8000000000000000ULL) { 61*4d703b5cSMark Powers z.a ^= v.a; 62*4d703b5cSMark Powers z.b ^= v.b; 63*4d703b5cSMark Powers } 64*4d703b5cSMark Powers if (v.b & 1ULL) { 65*4d703b5cSMark Powers v.b = (v.a << 63)|(v.b >> 1); 66*4d703b5cSMark Powers v.a = (v.a >> 1) ^ R; 67*4d703b5cSMark Powers } else { 68*4d703b5cSMark Powers v.b = (v.a << 63)|(v.b >> 1); 69*4d703b5cSMark Powers v.a = v.a >> 1; 70*4d703b5cSMark Powers } 71*4d703b5cSMark Powers } 72*4d703b5cSMark Powers } 73*4d703b5cSMark Powers res[0] = htonll(z.a); 74*4d703b5cSMark Powers res[1] = htonll(z.b); 75*4d703b5cSMark Powers } 76*4d703b5cSMark Powers 77*4d703b5cSMark Powers #define GHASH(c, d, t) \ 78*4d703b5cSMark Powers xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \ 79*4d703b5cSMark Powers gcm_mul((uint64_t *)(c)->gcm_ghash, (c)->gcm_H, (uint64_t *)(t)); 80*4d703b5cSMark Powers 81*4d703b5cSMark Powers /* 82*4d703b5cSMark Powers * Encrypt multiple blocks of data in GCM mode. Decrypt for GCM mode 83*4d703b5cSMark Powers * is done in another function. 84*4d703b5cSMark Powers */ 85*4d703b5cSMark Powers int 86*4d703b5cSMark Powers gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, 87*4d703b5cSMark Powers crypto_data_t *out, size_t block_size, 88*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 89*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 90*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 91*4d703b5cSMark Powers { 92*4d703b5cSMark Powers size_t remainder = length; 93*4d703b5cSMark Powers size_t need; 94*4d703b5cSMark Powers uint8_t *datap = (uint8_t *)data; 95*4d703b5cSMark Powers uint8_t *blockp; 96*4d703b5cSMark Powers uint8_t *lastp; 97*4d703b5cSMark Powers void *iov_or_mp; 98*4d703b5cSMark Powers offset_t offset; 99*4d703b5cSMark Powers uint8_t *out_data_1; 100*4d703b5cSMark Powers uint8_t *out_data_2; 101*4d703b5cSMark Powers size_t out_data_1_len; 102*4d703b5cSMark Powers uint64_t counter; 103*4d703b5cSMark Powers uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 104*4d703b5cSMark Powers 105*4d703b5cSMark Powers if (length + ctx->gcm_remainder_len < block_size) { 106*4d703b5cSMark Powers /* accumulate bytes here and return */ 107*4d703b5cSMark Powers bcopy(datap, 108*4d703b5cSMark Powers (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len, 109*4d703b5cSMark Powers length); 110*4d703b5cSMark Powers ctx->gcm_remainder_len += length; 111*4d703b5cSMark Powers ctx->gcm_copy_to = datap; 112*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 113*4d703b5cSMark Powers } 114*4d703b5cSMark Powers 115*4d703b5cSMark Powers lastp = (uint8_t *)ctx->gcm_cb; 116*4d703b5cSMark Powers if (out != NULL) 117*4d703b5cSMark Powers crypto_init_ptrs(out, &iov_or_mp, &offset); 118*4d703b5cSMark Powers 119*4d703b5cSMark Powers do { 120*4d703b5cSMark Powers /* Unprocessed data from last call. */ 121*4d703b5cSMark Powers if (ctx->gcm_remainder_len > 0) { 122*4d703b5cSMark Powers need = block_size - ctx->gcm_remainder_len; 123*4d703b5cSMark Powers 124*4d703b5cSMark Powers if (need > remainder) 125*4d703b5cSMark Powers return (CRYPTO_DATA_LEN_RANGE); 126*4d703b5cSMark Powers 127*4d703b5cSMark Powers bcopy(datap, &((uint8_t *)ctx->gcm_remainder) 128*4d703b5cSMark Powers [ctx->gcm_remainder_len], need); 129*4d703b5cSMark Powers 130*4d703b5cSMark Powers blockp = (uint8_t *)ctx->gcm_remainder; 131*4d703b5cSMark Powers } else { 132*4d703b5cSMark Powers blockp = datap; 133*4d703b5cSMark Powers } 134*4d703b5cSMark Powers 135*4d703b5cSMark Powers /* 136*4d703b5cSMark Powers * Increment counter. Counter bits are confined 137*4d703b5cSMark Powers * to the bottom 32 bits of the counter block. 138*4d703b5cSMark Powers */ 139*4d703b5cSMark Powers counter = ntohll(ctx->gcm_cb[1] & counter_mask); 140*4d703b5cSMark Powers counter = htonll(counter + 1); 141*4d703b5cSMark Powers counter &= counter_mask; 142*4d703b5cSMark Powers ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 143*4d703b5cSMark Powers 144*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, 145*4d703b5cSMark Powers (uint8_t *)ctx->gcm_tmp); 146*4d703b5cSMark Powers xor_block(blockp, (uint8_t *)ctx->gcm_tmp); 147*4d703b5cSMark Powers 148*4d703b5cSMark Powers lastp = (uint8_t *)ctx->gcm_tmp; 149*4d703b5cSMark Powers 150*4d703b5cSMark Powers ctx->gcm_processed_data_len += block_size; 151*4d703b5cSMark Powers 152*4d703b5cSMark Powers if (out == NULL) { 153*4d703b5cSMark Powers if (ctx->gcm_remainder_len > 0) { 154*4d703b5cSMark Powers bcopy(blockp, ctx->gcm_copy_to, 155*4d703b5cSMark Powers ctx->gcm_remainder_len); 156*4d703b5cSMark Powers bcopy(blockp + ctx->gcm_remainder_len, datap, 157*4d703b5cSMark Powers need); 158*4d703b5cSMark Powers } 159*4d703b5cSMark Powers } else { 160*4d703b5cSMark Powers crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, 161*4d703b5cSMark Powers &out_data_1_len, &out_data_2, block_size); 162*4d703b5cSMark Powers 163*4d703b5cSMark Powers /* copy block to where it belongs */ 164*4d703b5cSMark Powers if (out_data_1_len == block_size) { 165*4d703b5cSMark Powers copy_block(lastp, out_data_1); 166*4d703b5cSMark Powers } else { 167*4d703b5cSMark Powers bcopy(lastp, out_data_1, out_data_1_len); 168*4d703b5cSMark Powers if (out_data_2 != NULL) { 169*4d703b5cSMark Powers bcopy(lastp + out_data_1_len, 170*4d703b5cSMark Powers out_data_2, 171*4d703b5cSMark Powers block_size - out_data_1_len); 172*4d703b5cSMark Powers } 173*4d703b5cSMark Powers } 174*4d703b5cSMark Powers /* update offset */ 175*4d703b5cSMark Powers out->cd_offset += block_size; 176*4d703b5cSMark Powers } 177*4d703b5cSMark Powers 178*4d703b5cSMark Powers /* add ciphertext to the hash */ 179*4d703b5cSMark Powers GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); 180*4d703b5cSMark Powers 181*4d703b5cSMark Powers /* Update pointer to next block of data to be processed. */ 182*4d703b5cSMark Powers if (ctx->gcm_remainder_len != 0) { 183*4d703b5cSMark Powers datap += need; 184*4d703b5cSMark Powers ctx->gcm_remainder_len = 0; 185*4d703b5cSMark Powers } else { 186*4d703b5cSMark Powers datap += block_size; 187*4d703b5cSMark Powers } 188*4d703b5cSMark Powers 189*4d703b5cSMark Powers remainder = (size_t)&data[length] - (size_t)datap; 190*4d703b5cSMark Powers 191*4d703b5cSMark Powers /* Incomplete last block. */ 192*4d703b5cSMark Powers if (remainder > 0 && remainder < block_size) { 193*4d703b5cSMark Powers bcopy(datap, ctx->gcm_remainder, remainder); 194*4d703b5cSMark Powers ctx->gcm_remainder_len = remainder; 195*4d703b5cSMark Powers ctx->gcm_copy_to = datap; 196*4d703b5cSMark Powers goto out; 197*4d703b5cSMark Powers } 198*4d703b5cSMark Powers ctx->gcm_copy_to = NULL; 199*4d703b5cSMark Powers 200*4d703b5cSMark Powers } while (remainder > 0); 201*4d703b5cSMark Powers out: 202*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 203*4d703b5cSMark Powers } 204*4d703b5cSMark Powers 205*4d703b5cSMark Powers /* ARGSUSED */ 206*4d703b5cSMark Powers int 207*4d703b5cSMark Powers gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, 208*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 209*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 210*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 211*4d703b5cSMark Powers { 212*4d703b5cSMark Powers uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 213*4d703b5cSMark Powers uint8_t *ghash, *macp; 214*4d703b5cSMark Powers int i, rv; 215*4d703b5cSMark Powers 216*4d703b5cSMark Powers if (out->cd_length < 217*4d703b5cSMark Powers (ctx->gcm_remainder_len + ctx->gcm_tag_len)) { 218*4d703b5cSMark Powers return (CRYPTO_DATA_LEN_RANGE); 219*4d703b5cSMark Powers } 220*4d703b5cSMark Powers 221*4d703b5cSMark Powers ghash = (uint8_t *)ctx->gcm_ghash; 222*4d703b5cSMark Powers 223*4d703b5cSMark Powers if (ctx->gcm_remainder_len > 0) { 224*4d703b5cSMark Powers uint64_t counter; 225*4d703b5cSMark Powers uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp; 226*4d703b5cSMark Powers 227*4d703b5cSMark Powers /* 228*4d703b5cSMark Powers * Here is where we deal with data that is not a 229*4d703b5cSMark Powers * multiple of the block size. 230*4d703b5cSMark Powers */ 231*4d703b5cSMark Powers 232*4d703b5cSMark Powers /* 233*4d703b5cSMark Powers * Increment counter. 234*4d703b5cSMark Powers */ 235*4d703b5cSMark Powers counter = ntohll(ctx->gcm_cb[1] & counter_mask); 236*4d703b5cSMark Powers counter = htonll(counter + 1); 237*4d703b5cSMark Powers counter &= counter_mask; 238*4d703b5cSMark Powers ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 239*4d703b5cSMark Powers 240*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, 241*4d703b5cSMark Powers (uint8_t *)ctx->gcm_tmp); 242*4d703b5cSMark Powers 243*4d703b5cSMark Powers macp = (uint8_t *)ctx->gcm_remainder; 244*4d703b5cSMark Powers bzero(macp + ctx->gcm_remainder_len, 245*4d703b5cSMark Powers block_size - ctx->gcm_remainder_len); 246*4d703b5cSMark Powers 247*4d703b5cSMark Powers /* XOR with counter block */ 248*4d703b5cSMark Powers for (i = 0; i < ctx->gcm_remainder_len; i++) { 249*4d703b5cSMark Powers macp[i] ^= tmpp[i]; 250*4d703b5cSMark Powers } 251*4d703b5cSMark Powers 252*4d703b5cSMark Powers /* add ciphertext to the hash */ 253*4d703b5cSMark Powers GHASH(ctx, macp, ghash); 254*4d703b5cSMark Powers 255*4d703b5cSMark Powers ctx->gcm_processed_data_len += ctx->gcm_remainder_len; 256*4d703b5cSMark Powers } 257*4d703b5cSMark Powers 258*4d703b5cSMark Powers ctx->gcm_len_a_len_c[1] = htonll(ctx->gcm_processed_data_len << 3); 259*4d703b5cSMark Powers GHASH(ctx, ctx->gcm_len_a_len_c, ghash); 260*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, 261*4d703b5cSMark Powers (uint8_t *)ctx->gcm_J0); 262*4d703b5cSMark Powers xor_block((uint8_t *)ctx->gcm_J0, ghash); 263*4d703b5cSMark Powers 264*4d703b5cSMark Powers if (ctx->gcm_remainder_len > 0) { 265*4d703b5cSMark Powers rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len); 266*4d703b5cSMark Powers if (rv != CRYPTO_SUCCESS) 267*4d703b5cSMark Powers return (rv); 268*4d703b5cSMark Powers } 269*4d703b5cSMark Powers out->cd_offset += ctx->gcm_remainder_len; 270*4d703b5cSMark Powers ctx->gcm_remainder_len = 0; 271*4d703b5cSMark Powers rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len); 272*4d703b5cSMark Powers if (rv != CRYPTO_SUCCESS) 273*4d703b5cSMark Powers return (rv); 274*4d703b5cSMark Powers out->cd_offset += ctx->gcm_tag_len; 275*4d703b5cSMark Powers 276*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 277*4d703b5cSMark Powers } 278*4d703b5cSMark Powers 279*4d703b5cSMark Powers /* 280*4d703b5cSMark Powers * This will only deal with decrypting the last block of the input that 281*4d703b5cSMark Powers * might not be a multiple of block length. 282*4d703b5cSMark Powers */ 283*4d703b5cSMark Powers static void 284*4d703b5cSMark Powers gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index, 285*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 286*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 287*4d703b5cSMark Powers { 288*4d703b5cSMark Powers uint8_t *datap, *outp, *counterp; 289*4d703b5cSMark Powers uint64_t counter; 290*4d703b5cSMark Powers uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 291*4d703b5cSMark Powers int i; 292*4d703b5cSMark Powers 293*4d703b5cSMark Powers /* 294*4d703b5cSMark Powers * Increment counter. 295*4d703b5cSMark Powers * Counter bits are confined to the bottom 32 bits 296*4d703b5cSMark Powers */ 297*4d703b5cSMark Powers counter = ntohll(ctx->gcm_cb[1] & counter_mask); 298*4d703b5cSMark Powers counter = htonll(counter + 1); 299*4d703b5cSMark Powers counter &= counter_mask; 300*4d703b5cSMark Powers ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 301*4d703b5cSMark Powers 302*4d703b5cSMark Powers datap = (uint8_t *)ctx->gcm_remainder; 303*4d703b5cSMark Powers outp = &((ctx->gcm_pt_buf)[index]); 304*4d703b5cSMark Powers counterp = (uint8_t *)ctx->gcm_tmp; 305*4d703b5cSMark Powers 306*4d703b5cSMark Powers /* authentication tag */ 307*4d703b5cSMark Powers bzero((uint8_t *)ctx->gcm_tmp, block_size); 308*4d703b5cSMark Powers bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len); 309*4d703b5cSMark Powers 310*4d703b5cSMark Powers /* add ciphertext to the hash */ 311*4d703b5cSMark Powers GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); 312*4d703b5cSMark Powers 313*4d703b5cSMark Powers /* decrypt remaining ciphertext */ 314*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp); 315*4d703b5cSMark Powers 316*4d703b5cSMark Powers /* XOR with counter block */ 317*4d703b5cSMark Powers for (i = 0; i < ctx->gcm_remainder_len; i++) { 318*4d703b5cSMark Powers outp[i] = datap[i] ^ counterp[i]; 319*4d703b5cSMark Powers } 320*4d703b5cSMark Powers } 321*4d703b5cSMark Powers 322*4d703b5cSMark Powers /* ARGSUSED */ 323*4d703b5cSMark Powers int 324*4d703b5cSMark Powers gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, 325*4d703b5cSMark Powers crypto_data_t *out, size_t block_size, 326*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 327*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 328*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 329*4d703b5cSMark Powers { 330*4d703b5cSMark Powers size_t new_len; 331*4d703b5cSMark Powers uint8_t *new; 332*4d703b5cSMark Powers 333*4d703b5cSMark Powers /* 334*4d703b5cSMark Powers * Copy contiguous ciphertext input blocks to plaintext buffer. 335*4d703b5cSMark Powers * Ciphertext will be decrypted in the final. 336*4d703b5cSMark Powers */ 337*4d703b5cSMark Powers if (length > 0) { 338*4d703b5cSMark Powers new_len = ctx->gcm_pt_buf_len + length; 339*4d703b5cSMark Powers #ifdef _KERNEL 340*4d703b5cSMark Powers new = kmem_alloc(new_len, ctx->gcm_kmflag); 341*4d703b5cSMark Powers bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); 342*4d703b5cSMark Powers kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len); 343*4d703b5cSMark Powers #else 344*4d703b5cSMark Powers new = malloc(new_len); 345*4d703b5cSMark Powers bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); 346*4d703b5cSMark Powers free(ctx->gcm_pt_buf); 347*4d703b5cSMark Powers #endif 348*4d703b5cSMark Powers if (new == NULL) 349*4d703b5cSMark Powers return (CRYPTO_HOST_MEMORY); 350*4d703b5cSMark Powers 351*4d703b5cSMark Powers ctx->gcm_pt_buf = new; 352*4d703b5cSMark Powers ctx->gcm_pt_buf_len = new_len; 353*4d703b5cSMark Powers bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len], 354*4d703b5cSMark Powers length); 355*4d703b5cSMark Powers ctx->gcm_processed_data_len += length; 356*4d703b5cSMark Powers } 357*4d703b5cSMark Powers 358*4d703b5cSMark Powers ctx->gcm_remainder_len = 0; 359*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 360*4d703b5cSMark Powers } 361*4d703b5cSMark Powers 362*4d703b5cSMark Powers int 363*4d703b5cSMark Powers gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, 364*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 365*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 366*4d703b5cSMark Powers { 367*4d703b5cSMark Powers size_t pt_len; 368*4d703b5cSMark Powers size_t remainder; 369*4d703b5cSMark Powers uint8_t *ghash; 370*4d703b5cSMark Powers uint8_t *blockp; 371*4d703b5cSMark Powers uint8_t *cbp; 372*4d703b5cSMark Powers uint64_t counter; 373*4d703b5cSMark Powers uint64_t counter_mask = ntohll(0x00000000ffffffffULL); 374*4d703b5cSMark Powers int processed = 0, rv; 375*4d703b5cSMark Powers 376*4d703b5cSMark Powers ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len); 377*4d703b5cSMark Powers 378*4d703b5cSMark Powers pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; 379*4d703b5cSMark Powers ghash = (uint8_t *)ctx->gcm_ghash; 380*4d703b5cSMark Powers blockp = ctx->gcm_pt_buf; 381*4d703b5cSMark Powers remainder = pt_len; 382*4d703b5cSMark Powers while (remainder > 0) { 383*4d703b5cSMark Powers /* add ciphertext to the hash */ 384*4d703b5cSMark Powers GHASH(ctx, blockp, ghash); 385*4d703b5cSMark Powers 386*4d703b5cSMark Powers /* 387*4d703b5cSMark Powers * Increment counter. 388*4d703b5cSMark Powers * Counter bits are confined to the bottom 32 bits 389*4d703b5cSMark Powers */ 390*4d703b5cSMark Powers counter = ntohll(ctx->gcm_cb[1] & counter_mask); 391*4d703b5cSMark Powers counter = htonll(counter + 1); 392*4d703b5cSMark Powers counter &= counter_mask; 393*4d703b5cSMark Powers ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; 394*4d703b5cSMark Powers 395*4d703b5cSMark Powers cbp = (uint8_t *)ctx->gcm_tmp; 396*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp); 397*4d703b5cSMark Powers 398*4d703b5cSMark Powers /* XOR with ciphertext */ 399*4d703b5cSMark Powers xor_block(cbp, blockp); 400*4d703b5cSMark Powers 401*4d703b5cSMark Powers processed += block_size; 402*4d703b5cSMark Powers blockp += block_size; 403*4d703b5cSMark Powers remainder -= block_size; 404*4d703b5cSMark Powers 405*4d703b5cSMark Powers /* Incomplete last block */ 406*4d703b5cSMark Powers if (remainder > 0 && remainder < block_size) { 407*4d703b5cSMark Powers bcopy(blockp, ctx->gcm_remainder, remainder); 408*4d703b5cSMark Powers ctx->gcm_remainder_len = remainder; 409*4d703b5cSMark Powers /* 410*4d703b5cSMark Powers * not expecting anymore ciphertext, just 411*4d703b5cSMark Powers * compute plaintext for the remaining input 412*4d703b5cSMark Powers */ 413*4d703b5cSMark Powers gcm_decrypt_incomplete_block(ctx, block_size, 414*4d703b5cSMark Powers processed, encrypt_block, xor_block); 415*4d703b5cSMark Powers ctx->gcm_remainder_len = 0; 416*4d703b5cSMark Powers goto out; 417*4d703b5cSMark Powers } 418*4d703b5cSMark Powers } 419*4d703b5cSMark Powers out: 420*4d703b5cSMark Powers ctx->gcm_len_a_len_c[1] = htonll(pt_len << 3); 421*4d703b5cSMark Powers GHASH(ctx, ctx->gcm_len_a_len_c, ghash); 422*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, 423*4d703b5cSMark Powers (uint8_t *)ctx->gcm_J0); 424*4d703b5cSMark Powers xor_block((uint8_t *)ctx->gcm_J0, ghash); 425*4d703b5cSMark Powers 426*4d703b5cSMark Powers /* compare the input authentication tag with what we calculated */ 427*4d703b5cSMark Powers if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) { 428*4d703b5cSMark Powers /* They don't match */ 429*4d703b5cSMark Powers return (CRYPTO_INVALID_MAC); 430*4d703b5cSMark Powers } else { 431*4d703b5cSMark Powers rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len); 432*4d703b5cSMark Powers if (rv != CRYPTO_SUCCESS) 433*4d703b5cSMark Powers return (rv); 434*4d703b5cSMark Powers out->cd_offset += pt_len; 435*4d703b5cSMark Powers } 436*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 437*4d703b5cSMark Powers } 438*4d703b5cSMark Powers 439*4d703b5cSMark Powers static int 440*4d703b5cSMark Powers gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param) 441*4d703b5cSMark Powers { 442*4d703b5cSMark Powers size_t tag_len; 443*4d703b5cSMark Powers 444*4d703b5cSMark Powers /* 445*4d703b5cSMark Powers * Check the length of the authentication tag (in bits). 446*4d703b5cSMark Powers */ 447*4d703b5cSMark Powers tag_len = gcm_param->ulTagBits; 448*4d703b5cSMark Powers switch (tag_len) { 449*4d703b5cSMark Powers case 32: 450*4d703b5cSMark Powers case 64: 451*4d703b5cSMark Powers case 96: 452*4d703b5cSMark Powers case 104: 453*4d703b5cSMark Powers case 112: 454*4d703b5cSMark Powers case 120: 455*4d703b5cSMark Powers case 128: 456*4d703b5cSMark Powers break; 457*4d703b5cSMark Powers default: 458*4d703b5cSMark Powers return (CRYPTO_MECHANISM_PARAM_INVALID); 459*4d703b5cSMark Powers } 460*4d703b5cSMark Powers 461*4d703b5cSMark Powers if (gcm_param->ulIvLen == 0) 462*4d703b5cSMark Powers return (CRYPTO_MECHANISM_PARAM_INVALID); 463*4d703b5cSMark Powers 464*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 465*4d703b5cSMark Powers } 466*4d703b5cSMark Powers 467*4d703b5cSMark Powers static void 468*4d703b5cSMark Powers gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len, 469*4d703b5cSMark Powers gcm_ctx_t *ctx, size_t block_size, 470*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 471*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 472*4d703b5cSMark Powers { 473*4d703b5cSMark Powers uint8_t *cb; 474*4d703b5cSMark Powers ulong_t remainder = iv_len; 475*4d703b5cSMark Powers ulong_t processed = 0; 476*4d703b5cSMark Powers uint8_t *datap, *ghash; 477*4d703b5cSMark Powers uint64_t len_a_len_c[2]; 478*4d703b5cSMark Powers 479*4d703b5cSMark Powers ghash = (uint8_t *)ctx->gcm_ghash; 480*4d703b5cSMark Powers cb = (uint8_t *)ctx->gcm_cb; 481*4d703b5cSMark Powers if (iv_len == 12) { 482*4d703b5cSMark Powers bcopy(iv, cb, 12); 483*4d703b5cSMark Powers cb[12] = 0; 484*4d703b5cSMark Powers cb[13] = 0; 485*4d703b5cSMark Powers cb[14] = 0; 486*4d703b5cSMark Powers cb[15] = 1; 487*4d703b5cSMark Powers /* J0 will be used again in the final */ 488*4d703b5cSMark Powers copy_block(cb, (uint8_t *)ctx->gcm_J0); 489*4d703b5cSMark Powers } else { 490*4d703b5cSMark Powers /* GHASH the IV */ 491*4d703b5cSMark Powers do { 492*4d703b5cSMark Powers if (remainder < block_size) { 493*4d703b5cSMark Powers bzero(cb, block_size); 494*4d703b5cSMark Powers bcopy(&(iv[processed]), cb, remainder); 495*4d703b5cSMark Powers datap = (uint8_t *)cb; 496*4d703b5cSMark Powers remainder = 0; 497*4d703b5cSMark Powers } else { 498*4d703b5cSMark Powers datap = (uint8_t *)(&(iv[processed])); 499*4d703b5cSMark Powers processed += block_size; 500*4d703b5cSMark Powers remainder -= block_size; 501*4d703b5cSMark Powers } 502*4d703b5cSMark Powers GHASH(ctx, datap, ghash); 503*4d703b5cSMark Powers } while (remainder > 0); 504*4d703b5cSMark Powers 505*4d703b5cSMark Powers len_a_len_c[0] = 0; 506*4d703b5cSMark Powers len_a_len_c[1] = htonll(iv_len << 3); 507*4d703b5cSMark Powers GHASH(ctx, len_a_len_c, ctx->gcm_J0); 508*4d703b5cSMark Powers 509*4d703b5cSMark Powers /* J0 will be used again in the final */ 510*4d703b5cSMark Powers copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb); 511*4d703b5cSMark Powers } 512*4d703b5cSMark Powers } 513*4d703b5cSMark Powers 514*4d703b5cSMark Powers /* 515*4d703b5cSMark Powers * The following function is called at encrypt or decrypt init time 516*4d703b5cSMark Powers * for AES GCM mode. 517*4d703b5cSMark Powers */ 518*4d703b5cSMark Powers int 519*4d703b5cSMark Powers gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len, 520*4d703b5cSMark Powers unsigned char *auth_data, size_t auth_data_len, size_t block_size, 521*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 522*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 523*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 524*4d703b5cSMark Powers { 525*4d703b5cSMark Powers uint8_t *ghash, *datap, *authp; 526*4d703b5cSMark Powers size_t remainder, processed; 527*4d703b5cSMark Powers 528*4d703b5cSMark Powers /* encrypt zero block to get subkey H */ 529*4d703b5cSMark Powers bzero(ctx->gcm_H, sizeof (ctx->gcm_H)); 530*4d703b5cSMark Powers encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H, 531*4d703b5cSMark Powers (uint8_t *)ctx->gcm_H); 532*4d703b5cSMark Powers 533*4d703b5cSMark Powers gcm_format_initial_blocks(iv, iv_len, ctx, block_size, 534*4d703b5cSMark Powers copy_block, xor_block); 535*4d703b5cSMark Powers 536*4d703b5cSMark Powers authp = (uint8_t *)ctx->gcm_tmp; 537*4d703b5cSMark Powers ghash = (uint8_t *)ctx->gcm_ghash; 538*4d703b5cSMark Powers bzero(authp, block_size); 539*4d703b5cSMark Powers bzero(ghash, block_size); 540*4d703b5cSMark Powers 541*4d703b5cSMark Powers processed = 0; 542*4d703b5cSMark Powers remainder = auth_data_len; 543*4d703b5cSMark Powers do { 544*4d703b5cSMark Powers if (remainder < block_size) { 545*4d703b5cSMark Powers /* 546*4d703b5cSMark Powers * There's not a block full of data, pad rest of 547*4d703b5cSMark Powers * buffer with zero 548*4d703b5cSMark Powers */ 549*4d703b5cSMark Powers bzero(authp, block_size); 550*4d703b5cSMark Powers bcopy(&(auth_data[processed]), authp, remainder); 551*4d703b5cSMark Powers datap = (uint8_t *)authp; 552*4d703b5cSMark Powers remainder = 0; 553*4d703b5cSMark Powers } else { 554*4d703b5cSMark Powers datap = (uint8_t *)(&(auth_data[processed])); 555*4d703b5cSMark Powers processed += block_size; 556*4d703b5cSMark Powers remainder -= block_size; 557*4d703b5cSMark Powers } 558*4d703b5cSMark Powers 559*4d703b5cSMark Powers /* add auth data to the hash */ 560*4d703b5cSMark Powers GHASH(ctx, datap, ghash); 561*4d703b5cSMark Powers 562*4d703b5cSMark Powers } while (remainder > 0); 563*4d703b5cSMark Powers 564*4d703b5cSMark Powers return (CRYPTO_SUCCESS); 565*4d703b5cSMark Powers } 566*4d703b5cSMark Powers 567*4d703b5cSMark Powers int 568*4d703b5cSMark Powers gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, 569*4d703b5cSMark Powers int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), 570*4d703b5cSMark Powers void (*copy_block)(uint8_t *, uint8_t *), 571*4d703b5cSMark Powers void (*xor_block)(uint8_t *, uint8_t *)) 572*4d703b5cSMark Powers { 573*4d703b5cSMark Powers int rv; 574*4d703b5cSMark Powers CK_AES_GCM_PARAMS *gcm_param; 575*4d703b5cSMark Powers 576*4d703b5cSMark Powers if (param != NULL) { 577*4d703b5cSMark Powers gcm_param = (CK_AES_GCM_PARAMS *)param; 578*4d703b5cSMark Powers 579*4d703b5cSMark Powers if ((rv = gcm_validate_args(gcm_param)) != 0) { 580*4d703b5cSMark Powers return (rv); 581*4d703b5cSMark Powers } 582*4d703b5cSMark Powers 583*4d703b5cSMark Powers gcm_ctx->gcm_tag_len = gcm_param->ulTagBits; 584*4d703b5cSMark Powers gcm_ctx->gcm_tag_len >>= 3; 585*4d703b5cSMark Powers gcm_ctx->gcm_processed_data_len = 0; 586*4d703b5cSMark Powers 587*4d703b5cSMark Powers /* these values are in bits */ 588*4d703b5cSMark Powers gcm_ctx->gcm_len_a_len_c[0] = htonll(gcm_param->ulAADLen << 3); 589*4d703b5cSMark Powers 590*4d703b5cSMark Powers rv = CRYPTO_SUCCESS; 591*4d703b5cSMark Powers gcm_ctx->gcm_flags |= GCM_MODE; 592*4d703b5cSMark Powers } else { 593*4d703b5cSMark Powers rv = CRYPTO_MECHANISM_PARAM_INVALID; 594*4d703b5cSMark Powers goto out; 595*4d703b5cSMark Powers } 596*4d703b5cSMark Powers 597*4d703b5cSMark Powers if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen, 598*4d703b5cSMark Powers gcm_param->pAAD, gcm_param->ulAADLen, block_size, 599*4d703b5cSMark Powers encrypt_block, copy_block, xor_block) != 0) { 600*4d703b5cSMark Powers rv = CRYPTO_MECHANISM_PARAM_INVALID; 601*4d703b5cSMark Powers } 602*4d703b5cSMark Powers out: 603*4d703b5cSMark Powers return (rv); 604*4d703b5cSMark Powers } 605*4d703b5cSMark Powers 606*4d703b5cSMark Powers void * 607*4d703b5cSMark Powers gcm_alloc_ctx(int kmflag) 608*4d703b5cSMark Powers { 609*4d703b5cSMark Powers gcm_ctx_t *gcm_ctx; 610*4d703b5cSMark Powers 611*4d703b5cSMark Powers #ifdef _KERNEL 612*4d703b5cSMark Powers if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) 613*4d703b5cSMark Powers #else 614*4d703b5cSMark Powers if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL) 615*4d703b5cSMark Powers #endif 616*4d703b5cSMark Powers return (NULL); 617*4d703b5cSMark Powers 618*4d703b5cSMark Powers gcm_ctx->gcm_flags = GCM_MODE; 619*4d703b5cSMark Powers return (gcm_ctx); 620*4d703b5cSMark Powers } 621*4d703b5cSMark Powers 622*4d703b5cSMark Powers void 623*4d703b5cSMark Powers gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag) 624*4d703b5cSMark Powers { 625*4d703b5cSMark Powers ctx->gcm_kmflag = kmflag; 626*4d703b5cSMark Powers } 627