xref: /titanic_50/usr/src/common/crypto/modes/gcm.c (revision 4d703b5c9cb1bc29ace6c53cb50b2fe766e6370f)
1*4d703b5cSMark Powers /*
2*4d703b5cSMark Powers  * CDDL HEADER START
3*4d703b5cSMark Powers  *
4*4d703b5cSMark Powers  * The contents of this file are subject to the terms of the
5*4d703b5cSMark Powers  * Common Development and Distribution License (the "License").
6*4d703b5cSMark Powers  * You may not use this file except in compliance with the License.
7*4d703b5cSMark Powers  *
8*4d703b5cSMark Powers  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*4d703b5cSMark Powers  * or http://www.opensolaris.org/os/licensing.
10*4d703b5cSMark Powers  * See the License for the specific language governing permissions
11*4d703b5cSMark Powers  * and limitations under the License.
12*4d703b5cSMark Powers  *
13*4d703b5cSMark Powers  * When distributing Covered Code, include this CDDL HEADER in each
14*4d703b5cSMark Powers  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15*4d703b5cSMark Powers  * If applicable, add the following below this CDDL HEADER, with the
16*4d703b5cSMark Powers  * fields enclosed by brackets "[]" replaced with your own identifying
17*4d703b5cSMark Powers  * information: Portions Copyright [yyyy] [name of copyright owner]
18*4d703b5cSMark Powers  *
19*4d703b5cSMark Powers  * CDDL HEADER END
20*4d703b5cSMark Powers  */
21*4d703b5cSMark Powers /*
22*4d703b5cSMark Powers  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23*4d703b5cSMark Powers  * Use is subject to license terms.
24*4d703b5cSMark Powers  */
25*4d703b5cSMark Powers 
26*4d703b5cSMark Powers #ifndef _KERNEL
27*4d703b5cSMark Powers #include <strings.h>
28*4d703b5cSMark Powers #include <limits.h>
29*4d703b5cSMark Powers #include <assert.h>
30*4d703b5cSMark Powers #include <security/cryptoki.h>
31*4d703b5cSMark Powers #endif
32*4d703b5cSMark Powers 
33*4d703b5cSMark Powers #include <sys/types.h>
34*4d703b5cSMark Powers #include <sys/kmem.h>
35*4d703b5cSMark Powers #include <modes/modes.h>
36*4d703b5cSMark Powers #include <sys/crypto/common.h>
37*4d703b5cSMark Powers #include <sys/crypto/impl.h>
38*4d703b5cSMark Powers #include <sys/byteorder.h>
39*4d703b5cSMark Powers 
40*4d703b5cSMark Powers struct aes_block {
41*4d703b5cSMark Powers 	uint64_t a;
42*4d703b5cSMark Powers 	uint64_t b;
43*4d703b5cSMark Powers };
44*4d703b5cSMark Powers 
45*4d703b5cSMark Powers static void
46*4d703b5cSMark Powers gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
47*4d703b5cSMark Powers {
48*4d703b5cSMark Powers 	uint64_t R = { 0xe100000000000000ULL };
49*4d703b5cSMark Powers 	struct aes_block z = { 0, 0 };
50*4d703b5cSMark Powers 	struct aes_block v;
51*4d703b5cSMark Powers 	uint64_t x;
52*4d703b5cSMark Powers 	int i, j;
53*4d703b5cSMark Powers 
54*4d703b5cSMark Powers 	v.a = ntohll(y[0]);
55*4d703b5cSMark Powers 	v.b = ntohll(y[1]);
56*4d703b5cSMark Powers 
57*4d703b5cSMark Powers 	for (j = 0; j < 2; j++) {
58*4d703b5cSMark Powers 		x = ntohll(x_in[j]);
59*4d703b5cSMark Powers 		for (i = 0; i < 64; i++, x <<= 1) {
60*4d703b5cSMark Powers 			if (x & 0x8000000000000000ULL) {
61*4d703b5cSMark Powers 				z.a ^= v.a;
62*4d703b5cSMark Powers 				z.b ^= v.b;
63*4d703b5cSMark Powers 			}
64*4d703b5cSMark Powers 			if (v.b & 1ULL) {
65*4d703b5cSMark Powers 				v.b = (v.a << 63)|(v.b >> 1);
66*4d703b5cSMark Powers 				v.a = (v.a >> 1) ^ R;
67*4d703b5cSMark Powers 			} else {
68*4d703b5cSMark Powers 				v.b = (v.a << 63)|(v.b >> 1);
69*4d703b5cSMark Powers 				v.a = v.a >> 1;
70*4d703b5cSMark Powers 			}
71*4d703b5cSMark Powers 		}
72*4d703b5cSMark Powers 	}
73*4d703b5cSMark Powers 	res[0] = htonll(z.a);
74*4d703b5cSMark Powers 	res[1] = htonll(z.b);
75*4d703b5cSMark Powers }
76*4d703b5cSMark Powers 
77*4d703b5cSMark Powers #define	GHASH(c, d, t) \
78*4d703b5cSMark Powers 	xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
79*4d703b5cSMark Powers 	gcm_mul((uint64_t *)(c)->gcm_ghash, (c)->gcm_H, (uint64_t *)(t));
80*4d703b5cSMark Powers 
81*4d703b5cSMark Powers /*
82*4d703b5cSMark Powers  * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
83*4d703b5cSMark Powers  * is done in another function.
84*4d703b5cSMark Powers  */
85*4d703b5cSMark Powers int
86*4d703b5cSMark Powers gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
87*4d703b5cSMark Powers     crypto_data_t *out, size_t block_size,
88*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
89*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
90*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
91*4d703b5cSMark Powers {
92*4d703b5cSMark Powers 	size_t remainder = length;
93*4d703b5cSMark Powers 	size_t need;
94*4d703b5cSMark Powers 	uint8_t *datap = (uint8_t *)data;
95*4d703b5cSMark Powers 	uint8_t *blockp;
96*4d703b5cSMark Powers 	uint8_t *lastp;
97*4d703b5cSMark Powers 	void *iov_or_mp;
98*4d703b5cSMark Powers 	offset_t offset;
99*4d703b5cSMark Powers 	uint8_t *out_data_1;
100*4d703b5cSMark Powers 	uint8_t *out_data_2;
101*4d703b5cSMark Powers 	size_t out_data_1_len;
102*4d703b5cSMark Powers 	uint64_t counter;
103*4d703b5cSMark Powers 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
104*4d703b5cSMark Powers 
105*4d703b5cSMark Powers 	if (length + ctx->gcm_remainder_len < block_size) {
106*4d703b5cSMark Powers 		/* accumulate bytes here and return */
107*4d703b5cSMark Powers 		bcopy(datap,
108*4d703b5cSMark Powers 		    (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len,
109*4d703b5cSMark Powers 		    length);
110*4d703b5cSMark Powers 		ctx->gcm_remainder_len += length;
111*4d703b5cSMark Powers 		ctx->gcm_copy_to = datap;
112*4d703b5cSMark Powers 		return (CRYPTO_SUCCESS);
113*4d703b5cSMark Powers 	}
114*4d703b5cSMark Powers 
115*4d703b5cSMark Powers 	lastp = (uint8_t *)ctx->gcm_cb;
116*4d703b5cSMark Powers 	if (out != NULL)
117*4d703b5cSMark Powers 		crypto_init_ptrs(out, &iov_or_mp, &offset);
118*4d703b5cSMark Powers 
119*4d703b5cSMark Powers 	do {
120*4d703b5cSMark Powers 		/* Unprocessed data from last call. */
121*4d703b5cSMark Powers 		if (ctx->gcm_remainder_len > 0) {
122*4d703b5cSMark Powers 			need = block_size - ctx->gcm_remainder_len;
123*4d703b5cSMark Powers 
124*4d703b5cSMark Powers 			if (need > remainder)
125*4d703b5cSMark Powers 				return (CRYPTO_DATA_LEN_RANGE);
126*4d703b5cSMark Powers 
127*4d703b5cSMark Powers 			bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
128*4d703b5cSMark Powers 			    [ctx->gcm_remainder_len], need);
129*4d703b5cSMark Powers 
130*4d703b5cSMark Powers 			blockp = (uint8_t *)ctx->gcm_remainder;
131*4d703b5cSMark Powers 		} else {
132*4d703b5cSMark Powers 			blockp = datap;
133*4d703b5cSMark Powers 		}
134*4d703b5cSMark Powers 
135*4d703b5cSMark Powers 		/*
136*4d703b5cSMark Powers 		 * Increment counter. Counter bits are confined
137*4d703b5cSMark Powers 		 * to the bottom 32 bits of the counter block.
138*4d703b5cSMark Powers 		 */
139*4d703b5cSMark Powers 		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
140*4d703b5cSMark Powers 		counter = htonll(counter + 1);
141*4d703b5cSMark Powers 		counter &= counter_mask;
142*4d703b5cSMark Powers 		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
143*4d703b5cSMark Powers 
144*4d703b5cSMark Powers 		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
145*4d703b5cSMark Powers 		    (uint8_t *)ctx->gcm_tmp);
146*4d703b5cSMark Powers 		xor_block(blockp, (uint8_t *)ctx->gcm_tmp);
147*4d703b5cSMark Powers 
148*4d703b5cSMark Powers 		lastp = (uint8_t *)ctx->gcm_tmp;
149*4d703b5cSMark Powers 
150*4d703b5cSMark Powers 		ctx->gcm_processed_data_len += block_size;
151*4d703b5cSMark Powers 
152*4d703b5cSMark Powers 		if (out == NULL) {
153*4d703b5cSMark Powers 			if (ctx->gcm_remainder_len > 0) {
154*4d703b5cSMark Powers 				bcopy(blockp, ctx->gcm_copy_to,
155*4d703b5cSMark Powers 				    ctx->gcm_remainder_len);
156*4d703b5cSMark Powers 				bcopy(blockp + ctx->gcm_remainder_len, datap,
157*4d703b5cSMark Powers 				    need);
158*4d703b5cSMark Powers 			}
159*4d703b5cSMark Powers 		} else {
160*4d703b5cSMark Powers 			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
161*4d703b5cSMark Powers 			    &out_data_1_len, &out_data_2, block_size);
162*4d703b5cSMark Powers 
163*4d703b5cSMark Powers 			/* copy block to where it belongs */
164*4d703b5cSMark Powers 			if (out_data_1_len == block_size) {
165*4d703b5cSMark Powers 				copy_block(lastp, out_data_1);
166*4d703b5cSMark Powers 			} else {
167*4d703b5cSMark Powers 				bcopy(lastp, out_data_1, out_data_1_len);
168*4d703b5cSMark Powers 				if (out_data_2 != NULL) {
169*4d703b5cSMark Powers 					bcopy(lastp + out_data_1_len,
170*4d703b5cSMark Powers 					    out_data_2,
171*4d703b5cSMark Powers 					    block_size - out_data_1_len);
172*4d703b5cSMark Powers 				}
173*4d703b5cSMark Powers 			}
174*4d703b5cSMark Powers 			/* update offset */
175*4d703b5cSMark Powers 			out->cd_offset += block_size;
176*4d703b5cSMark Powers 		}
177*4d703b5cSMark Powers 
178*4d703b5cSMark Powers 		/* add ciphertext to the hash */
179*4d703b5cSMark Powers 		GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
180*4d703b5cSMark Powers 
181*4d703b5cSMark Powers 		/* Update pointer to next block of data to be processed. */
182*4d703b5cSMark Powers 		if (ctx->gcm_remainder_len != 0) {
183*4d703b5cSMark Powers 			datap += need;
184*4d703b5cSMark Powers 			ctx->gcm_remainder_len = 0;
185*4d703b5cSMark Powers 		} else {
186*4d703b5cSMark Powers 			datap += block_size;
187*4d703b5cSMark Powers 		}
188*4d703b5cSMark Powers 
189*4d703b5cSMark Powers 		remainder = (size_t)&data[length] - (size_t)datap;
190*4d703b5cSMark Powers 
191*4d703b5cSMark Powers 		/* Incomplete last block. */
192*4d703b5cSMark Powers 		if (remainder > 0 && remainder < block_size) {
193*4d703b5cSMark Powers 			bcopy(datap, ctx->gcm_remainder, remainder);
194*4d703b5cSMark Powers 			ctx->gcm_remainder_len = remainder;
195*4d703b5cSMark Powers 			ctx->gcm_copy_to = datap;
196*4d703b5cSMark Powers 			goto out;
197*4d703b5cSMark Powers 		}
198*4d703b5cSMark Powers 		ctx->gcm_copy_to = NULL;
199*4d703b5cSMark Powers 
200*4d703b5cSMark Powers 	} while (remainder > 0);
201*4d703b5cSMark Powers out:
202*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
203*4d703b5cSMark Powers }
204*4d703b5cSMark Powers 
205*4d703b5cSMark Powers /* ARGSUSED */
206*4d703b5cSMark Powers int
207*4d703b5cSMark Powers gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
208*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
209*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
210*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
211*4d703b5cSMark Powers {
212*4d703b5cSMark Powers 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
213*4d703b5cSMark Powers 	uint8_t *ghash, *macp;
214*4d703b5cSMark Powers 	int i, rv;
215*4d703b5cSMark Powers 
216*4d703b5cSMark Powers 	if (out->cd_length <
217*4d703b5cSMark Powers 	    (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
218*4d703b5cSMark Powers 		return (CRYPTO_DATA_LEN_RANGE);
219*4d703b5cSMark Powers 	}
220*4d703b5cSMark Powers 
221*4d703b5cSMark Powers 	ghash = (uint8_t *)ctx->gcm_ghash;
222*4d703b5cSMark Powers 
223*4d703b5cSMark Powers 	if (ctx->gcm_remainder_len > 0) {
224*4d703b5cSMark Powers 		uint64_t counter;
225*4d703b5cSMark Powers 		uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;
226*4d703b5cSMark Powers 
227*4d703b5cSMark Powers 		/*
228*4d703b5cSMark Powers 		 * Here is where we deal with data that is not a
229*4d703b5cSMark Powers 		 * multiple of the block size.
230*4d703b5cSMark Powers 		 */
231*4d703b5cSMark Powers 
232*4d703b5cSMark Powers 		/*
233*4d703b5cSMark Powers 		 * Increment counter.
234*4d703b5cSMark Powers 		 */
235*4d703b5cSMark Powers 		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
236*4d703b5cSMark Powers 		counter = htonll(counter + 1);
237*4d703b5cSMark Powers 		counter &= counter_mask;
238*4d703b5cSMark Powers 		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
239*4d703b5cSMark Powers 
240*4d703b5cSMark Powers 		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
241*4d703b5cSMark Powers 		    (uint8_t *)ctx->gcm_tmp);
242*4d703b5cSMark Powers 
243*4d703b5cSMark Powers 		macp = (uint8_t *)ctx->gcm_remainder;
244*4d703b5cSMark Powers 		bzero(macp + ctx->gcm_remainder_len,
245*4d703b5cSMark Powers 		    block_size - ctx->gcm_remainder_len);
246*4d703b5cSMark Powers 
247*4d703b5cSMark Powers 		/* XOR with counter block */
248*4d703b5cSMark Powers 		for (i = 0; i < ctx->gcm_remainder_len; i++) {
249*4d703b5cSMark Powers 			macp[i] ^= tmpp[i];
250*4d703b5cSMark Powers 		}
251*4d703b5cSMark Powers 
252*4d703b5cSMark Powers 		/* add ciphertext to the hash */
253*4d703b5cSMark Powers 		GHASH(ctx, macp, ghash);
254*4d703b5cSMark Powers 
255*4d703b5cSMark Powers 		ctx->gcm_processed_data_len += ctx->gcm_remainder_len;
256*4d703b5cSMark Powers 	}
257*4d703b5cSMark Powers 
258*4d703b5cSMark Powers 	ctx->gcm_len_a_len_c[1] = htonll(ctx->gcm_processed_data_len << 3);
259*4d703b5cSMark Powers 	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
260*4d703b5cSMark Powers 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
261*4d703b5cSMark Powers 	    (uint8_t *)ctx->gcm_J0);
262*4d703b5cSMark Powers 	xor_block((uint8_t *)ctx->gcm_J0, ghash);
263*4d703b5cSMark Powers 
264*4d703b5cSMark Powers 	if (ctx->gcm_remainder_len > 0) {
265*4d703b5cSMark Powers 		rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
266*4d703b5cSMark Powers 		if (rv != CRYPTO_SUCCESS)
267*4d703b5cSMark Powers 			return (rv);
268*4d703b5cSMark Powers 	}
269*4d703b5cSMark Powers 	out->cd_offset += ctx->gcm_remainder_len;
270*4d703b5cSMark Powers 	ctx->gcm_remainder_len = 0;
271*4d703b5cSMark Powers 	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
272*4d703b5cSMark Powers 	if (rv != CRYPTO_SUCCESS)
273*4d703b5cSMark Powers 		return (rv);
274*4d703b5cSMark Powers 	out->cd_offset += ctx->gcm_tag_len;
275*4d703b5cSMark Powers 
276*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
277*4d703b5cSMark Powers }
278*4d703b5cSMark Powers 
279*4d703b5cSMark Powers /*
280*4d703b5cSMark Powers  * This will only deal with decrypting the last block of the input that
281*4d703b5cSMark Powers  * might not be a multiple of block length.
282*4d703b5cSMark Powers  */
283*4d703b5cSMark Powers static void
284*4d703b5cSMark Powers gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
285*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
286*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
287*4d703b5cSMark Powers {
288*4d703b5cSMark Powers 	uint8_t *datap, *outp, *counterp;
289*4d703b5cSMark Powers 	uint64_t counter;
290*4d703b5cSMark Powers 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
291*4d703b5cSMark Powers 	int i;
292*4d703b5cSMark Powers 
293*4d703b5cSMark Powers 	/*
294*4d703b5cSMark Powers 	 * Increment counter.
295*4d703b5cSMark Powers 	 * Counter bits are confined to the bottom 32 bits
296*4d703b5cSMark Powers 	 */
297*4d703b5cSMark Powers 	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
298*4d703b5cSMark Powers 	counter = htonll(counter + 1);
299*4d703b5cSMark Powers 	counter &= counter_mask;
300*4d703b5cSMark Powers 	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
301*4d703b5cSMark Powers 
302*4d703b5cSMark Powers 	datap = (uint8_t *)ctx->gcm_remainder;
303*4d703b5cSMark Powers 	outp = &((ctx->gcm_pt_buf)[index]);
304*4d703b5cSMark Powers 	counterp = (uint8_t *)ctx->gcm_tmp;
305*4d703b5cSMark Powers 
306*4d703b5cSMark Powers 	/* authentication tag */
307*4d703b5cSMark Powers 	bzero((uint8_t *)ctx->gcm_tmp, block_size);
308*4d703b5cSMark Powers 	bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
309*4d703b5cSMark Powers 
310*4d703b5cSMark Powers 	/* add ciphertext to the hash */
311*4d703b5cSMark Powers 	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
312*4d703b5cSMark Powers 
313*4d703b5cSMark Powers 	/* decrypt remaining ciphertext */
314*4d703b5cSMark Powers 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
315*4d703b5cSMark Powers 
316*4d703b5cSMark Powers 	/* XOR with counter block */
317*4d703b5cSMark Powers 	for (i = 0; i < ctx->gcm_remainder_len; i++) {
318*4d703b5cSMark Powers 		outp[i] = datap[i] ^ counterp[i];
319*4d703b5cSMark Powers 	}
320*4d703b5cSMark Powers }
321*4d703b5cSMark Powers 
322*4d703b5cSMark Powers /* ARGSUSED */
323*4d703b5cSMark Powers int
324*4d703b5cSMark Powers gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
325*4d703b5cSMark Powers     crypto_data_t *out, size_t block_size,
326*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
327*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
328*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
329*4d703b5cSMark Powers {
330*4d703b5cSMark Powers 	size_t new_len;
331*4d703b5cSMark Powers 	uint8_t *new;
332*4d703b5cSMark Powers 
333*4d703b5cSMark Powers 	/*
334*4d703b5cSMark Powers 	 * Copy contiguous ciphertext input blocks to plaintext buffer.
335*4d703b5cSMark Powers 	 * Ciphertext will be decrypted in the final.
336*4d703b5cSMark Powers 	 */
337*4d703b5cSMark Powers 	if (length > 0) {
338*4d703b5cSMark Powers 		new_len = ctx->gcm_pt_buf_len + length;
339*4d703b5cSMark Powers #ifdef _KERNEL
340*4d703b5cSMark Powers 		new = kmem_alloc(new_len, ctx->gcm_kmflag);
341*4d703b5cSMark Powers 		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
342*4d703b5cSMark Powers 		kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
343*4d703b5cSMark Powers #else
344*4d703b5cSMark Powers 		new = malloc(new_len);
345*4d703b5cSMark Powers 		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
346*4d703b5cSMark Powers 		free(ctx->gcm_pt_buf);
347*4d703b5cSMark Powers #endif
348*4d703b5cSMark Powers 		if (new == NULL)
349*4d703b5cSMark Powers 			return (CRYPTO_HOST_MEMORY);
350*4d703b5cSMark Powers 
351*4d703b5cSMark Powers 		ctx->gcm_pt_buf = new;
352*4d703b5cSMark Powers 		ctx->gcm_pt_buf_len = new_len;
353*4d703b5cSMark Powers 		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
354*4d703b5cSMark Powers 		    length);
355*4d703b5cSMark Powers 		ctx->gcm_processed_data_len += length;
356*4d703b5cSMark Powers 	}
357*4d703b5cSMark Powers 
358*4d703b5cSMark Powers 	ctx->gcm_remainder_len = 0;
359*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
360*4d703b5cSMark Powers }
361*4d703b5cSMark Powers 
362*4d703b5cSMark Powers int
363*4d703b5cSMark Powers gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
364*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
365*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
366*4d703b5cSMark Powers {
367*4d703b5cSMark Powers 	size_t pt_len;
368*4d703b5cSMark Powers 	size_t remainder;
369*4d703b5cSMark Powers 	uint8_t *ghash;
370*4d703b5cSMark Powers 	uint8_t *blockp;
371*4d703b5cSMark Powers 	uint8_t *cbp;
372*4d703b5cSMark Powers 	uint64_t counter;
373*4d703b5cSMark Powers 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
374*4d703b5cSMark Powers 	int processed = 0, rv;
375*4d703b5cSMark Powers 
376*4d703b5cSMark Powers 	ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
377*4d703b5cSMark Powers 
378*4d703b5cSMark Powers 	pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
379*4d703b5cSMark Powers 	ghash = (uint8_t *)ctx->gcm_ghash;
380*4d703b5cSMark Powers 	blockp = ctx->gcm_pt_buf;
381*4d703b5cSMark Powers 	remainder = pt_len;
382*4d703b5cSMark Powers 	while (remainder > 0) {
383*4d703b5cSMark Powers 		/* add ciphertext to the hash */
384*4d703b5cSMark Powers 		GHASH(ctx, blockp, ghash);
385*4d703b5cSMark Powers 
386*4d703b5cSMark Powers 		/*
387*4d703b5cSMark Powers 		 * Increment counter.
388*4d703b5cSMark Powers 		 * Counter bits are confined to the bottom 32 bits
389*4d703b5cSMark Powers 		 */
390*4d703b5cSMark Powers 		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
391*4d703b5cSMark Powers 		counter = htonll(counter + 1);
392*4d703b5cSMark Powers 		counter &= counter_mask;
393*4d703b5cSMark Powers 		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
394*4d703b5cSMark Powers 
395*4d703b5cSMark Powers 		cbp = (uint8_t *)ctx->gcm_tmp;
396*4d703b5cSMark Powers 		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
397*4d703b5cSMark Powers 
398*4d703b5cSMark Powers 		/* XOR with ciphertext */
399*4d703b5cSMark Powers 		xor_block(cbp, blockp);
400*4d703b5cSMark Powers 
401*4d703b5cSMark Powers 		processed += block_size;
402*4d703b5cSMark Powers 		blockp += block_size;
403*4d703b5cSMark Powers 		remainder -= block_size;
404*4d703b5cSMark Powers 
405*4d703b5cSMark Powers 		/* Incomplete last block */
406*4d703b5cSMark Powers 		if (remainder > 0 && remainder < block_size) {
407*4d703b5cSMark Powers 			bcopy(blockp, ctx->gcm_remainder, remainder);
408*4d703b5cSMark Powers 			ctx->gcm_remainder_len = remainder;
409*4d703b5cSMark Powers 			/*
410*4d703b5cSMark Powers 			 * not expecting anymore ciphertext, just
411*4d703b5cSMark Powers 			 * compute plaintext for the remaining input
412*4d703b5cSMark Powers 			 */
413*4d703b5cSMark Powers 			gcm_decrypt_incomplete_block(ctx, block_size,
414*4d703b5cSMark Powers 			    processed, encrypt_block, xor_block);
415*4d703b5cSMark Powers 			ctx->gcm_remainder_len = 0;
416*4d703b5cSMark Powers 			goto out;
417*4d703b5cSMark Powers 		}
418*4d703b5cSMark Powers 	}
419*4d703b5cSMark Powers out:
420*4d703b5cSMark Powers 	ctx->gcm_len_a_len_c[1] = htonll(pt_len << 3);
421*4d703b5cSMark Powers 	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
422*4d703b5cSMark Powers 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
423*4d703b5cSMark Powers 	    (uint8_t *)ctx->gcm_J0);
424*4d703b5cSMark Powers 	xor_block((uint8_t *)ctx->gcm_J0, ghash);
425*4d703b5cSMark Powers 
426*4d703b5cSMark Powers 	/* compare the input authentication tag with what we calculated */
427*4d703b5cSMark Powers 	if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
428*4d703b5cSMark Powers 		/* They don't match */
429*4d703b5cSMark Powers 		return (CRYPTO_INVALID_MAC);
430*4d703b5cSMark Powers 	} else {
431*4d703b5cSMark Powers 		rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
432*4d703b5cSMark Powers 		if (rv != CRYPTO_SUCCESS)
433*4d703b5cSMark Powers 			return (rv);
434*4d703b5cSMark Powers 		out->cd_offset += pt_len;
435*4d703b5cSMark Powers 	}
436*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
437*4d703b5cSMark Powers }
438*4d703b5cSMark Powers 
439*4d703b5cSMark Powers static int
440*4d703b5cSMark Powers gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param)
441*4d703b5cSMark Powers {
442*4d703b5cSMark Powers 	size_t tag_len;
443*4d703b5cSMark Powers 
444*4d703b5cSMark Powers 	/*
445*4d703b5cSMark Powers 	 * Check the length of the authentication tag (in bits).
446*4d703b5cSMark Powers 	 */
447*4d703b5cSMark Powers 	tag_len = gcm_param->ulTagBits;
448*4d703b5cSMark Powers 	switch (tag_len) {
449*4d703b5cSMark Powers 	case 32:
450*4d703b5cSMark Powers 	case 64:
451*4d703b5cSMark Powers 	case 96:
452*4d703b5cSMark Powers 	case 104:
453*4d703b5cSMark Powers 	case 112:
454*4d703b5cSMark Powers 	case 120:
455*4d703b5cSMark Powers 	case 128:
456*4d703b5cSMark Powers 		break;
457*4d703b5cSMark Powers 	default:
458*4d703b5cSMark Powers 		return (CRYPTO_MECHANISM_PARAM_INVALID);
459*4d703b5cSMark Powers 	}
460*4d703b5cSMark Powers 
461*4d703b5cSMark Powers 	if (gcm_param->ulIvLen == 0)
462*4d703b5cSMark Powers 		return (CRYPTO_MECHANISM_PARAM_INVALID);
463*4d703b5cSMark Powers 
464*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
465*4d703b5cSMark Powers }
466*4d703b5cSMark Powers 
467*4d703b5cSMark Powers static void
468*4d703b5cSMark Powers gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
469*4d703b5cSMark Powers     gcm_ctx_t *ctx, size_t block_size,
470*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
471*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
472*4d703b5cSMark Powers {
473*4d703b5cSMark Powers 	uint8_t *cb;
474*4d703b5cSMark Powers 	ulong_t remainder = iv_len;
475*4d703b5cSMark Powers 	ulong_t processed = 0;
476*4d703b5cSMark Powers 	uint8_t *datap, *ghash;
477*4d703b5cSMark Powers 	uint64_t len_a_len_c[2];
478*4d703b5cSMark Powers 
479*4d703b5cSMark Powers 	ghash = (uint8_t *)ctx->gcm_ghash;
480*4d703b5cSMark Powers 	cb = (uint8_t *)ctx->gcm_cb;
481*4d703b5cSMark Powers 	if (iv_len == 12) {
482*4d703b5cSMark Powers 		bcopy(iv, cb, 12);
483*4d703b5cSMark Powers 		cb[12] = 0;
484*4d703b5cSMark Powers 		cb[13] = 0;
485*4d703b5cSMark Powers 		cb[14] = 0;
486*4d703b5cSMark Powers 		cb[15] = 1;
487*4d703b5cSMark Powers 		/* J0 will be used again in the final */
488*4d703b5cSMark Powers 		copy_block(cb, (uint8_t *)ctx->gcm_J0);
489*4d703b5cSMark Powers 	} else {
490*4d703b5cSMark Powers 		/* GHASH the IV */
491*4d703b5cSMark Powers 		do {
492*4d703b5cSMark Powers 			if (remainder < block_size) {
493*4d703b5cSMark Powers 				bzero(cb, block_size);
494*4d703b5cSMark Powers 				bcopy(&(iv[processed]), cb, remainder);
495*4d703b5cSMark Powers 				datap = (uint8_t *)cb;
496*4d703b5cSMark Powers 				remainder = 0;
497*4d703b5cSMark Powers 			} else {
498*4d703b5cSMark Powers 				datap = (uint8_t *)(&(iv[processed]));
499*4d703b5cSMark Powers 				processed += block_size;
500*4d703b5cSMark Powers 				remainder -= block_size;
501*4d703b5cSMark Powers 			}
502*4d703b5cSMark Powers 			GHASH(ctx, datap, ghash);
503*4d703b5cSMark Powers 		} while (remainder > 0);
504*4d703b5cSMark Powers 
505*4d703b5cSMark Powers 		len_a_len_c[0] = 0;
506*4d703b5cSMark Powers 		len_a_len_c[1] = htonll(iv_len << 3);
507*4d703b5cSMark Powers 		GHASH(ctx, len_a_len_c, ctx->gcm_J0);
508*4d703b5cSMark Powers 
509*4d703b5cSMark Powers 		/* J0 will be used again in the final */
510*4d703b5cSMark Powers 		copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb);
511*4d703b5cSMark Powers 	}
512*4d703b5cSMark Powers }
513*4d703b5cSMark Powers 
514*4d703b5cSMark Powers /*
515*4d703b5cSMark Powers  * The following function is called at encrypt or decrypt init time
516*4d703b5cSMark Powers  * for AES GCM mode.
517*4d703b5cSMark Powers  */
518*4d703b5cSMark Powers int
519*4d703b5cSMark Powers gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
520*4d703b5cSMark Powers     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
521*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
522*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
523*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
524*4d703b5cSMark Powers {
525*4d703b5cSMark Powers 	uint8_t *ghash, *datap, *authp;
526*4d703b5cSMark Powers 	size_t remainder, processed;
527*4d703b5cSMark Powers 
528*4d703b5cSMark Powers 	/* encrypt zero block to get subkey H */
529*4d703b5cSMark Powers 	bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
530*4d703b5cSMark Powers 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
531*4d703b5cSMark Powers 	    (uint8_t *)ctx->gcm_H);
532*4d703b5cSMark Powers 
533*4d703b5cSMark Powers 	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
534*4d703b5cSMark Powers 	    copy_block, xor_block);
535*4d703b5cSMark Powers 
536*4d703b5cSMark Powers 	authp = (uint8_t *)ctx->gcm_tmp;
537*4d703b5cSMark Powers 	ghash = (uint8_t *)ctx->gcm_ghash;
538*4d703b5cSMark Powers 	bzero(authp, block_size);
539*4d703b5cSMark Powers 	bzero(ghash, block_size);
540*4d703b5cSMark Powers 
541*4d703b5cSMark Powers 	processed = 0;
542*4d703b5cSMark Powers 	remainder = auth_data_len;
543*4d703b5cSMark Powers 	do {
544*4d703b5cSMark Powers 		if (remainder < block_size) {
545*4d703b5cSMark Powers 			/*
546*4d703b5cSMark Powers 			 * There's not a block full of data, pad rest of
547*4d703b5cSMark Powers 			 * buffer with zero
548*4d703b5cSMark Powers 			 */
549*4d703b5cSMark Powers 			bzero(authp, block_size);
550*4d703b5cSMark Powers 			bcopy(&(auth_data[processed]), authp, remainder);
551*4d703b5cSMark Powers 			datap = (uint8_t *)authp;
552*4d703b5cSMark Powers 			remainder = 0;
553*4d703b5cSMark Powers 		} else {
554*4d703b5cSMark Powers 			datap = (uint8_t *)(&(auth_data[processed]));
555*4d703b5cSMark Powers 			processed += block_size;
556*4d703b5cSMark Powers 			remainder -= block_size;
557*4d703b5cSMark Powers 		}
558*4d703b5cSMark Powers 
559*4d703b5cSMark Powers 		/* add auth data to the hash */
560*4d703b5cSMark Powers 		GHASH(ctx, datap, ghash);
561*4d703b5cSMark Powers 
562*4d703b5cSMark Powers 	} while (remainder > 0);
563*4d703b5cSMark Powers 
564*4d703b5cSMark Powers 	return (CRYPTO_SUCCESS);
565*4d703b5cSMark Powers }
566*4d703b5cSMark Powers 
567*4d703b5cSMark Powers int
568*4d703b5cSMark Powers gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
569*4d703b5cSMark Powers     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
570*4d703b5cSMark Powers     void (*copy_block)(uint8_t *, uint8_t *),
571*4d703b5cSMark Powers     void (*xor_block)(uint8_t *, uint8_t *))
572*4d703b5cSMark Powers {
573*4d703b5cSMark Powers 	int rv;
574*4d703b5cSMark Powers 	CK_AES_GCM_PARAMS *gcm_param;
575*4d703b5cSMark Powers 
576*4d703b5cSMark Powers 	if (param != NULL) {
577*4d703b5cSMark Powers 		gcm_param = (CK_AES_GCM_PARAMS *)param;
578*4d703b5cSMark Powers 
579*4d703b5cSMark Powers 		if ((rv = gcm_validate_args(gcm_param)) != 0) {
580*4d703b5cSMark Powers 			return (rv);
581*4d703b5cSMark Powers 		}
582*4d703b5cSMark Powers 
583*4d703b5cSMark Powers 		gcm_ctx->gcm_tag_len = gcm_param->ulTagBits;
584*4d703b5cSMark Powers 		gcm_ctx->gcm_tag_len >>= 3;
585*4d703b5cSMark Powers 		gcm_ctx->gcm_processed_data_len = 0;
586*4d703b5cSMark Powers 
587*4d703b5cSMark Powers 		/* these values are in bits */
588*4d703b5cSMark Powers 		gcm_ctx->gcm_len_a_len_c[0] = htonll(gcm_param->ulAADLen << 3);
589*4d703b5cSMark Powers 
590*4d703b5cSMark Powers 		rv = CRYPTO_SUCCESS;
591*4d703b5cSMark Powers 		gcm_ctx->gcm_flags |= GCM_MODE;
592*4d703b5cSMark Powers 	} else {
593*4d703b5cSMark Powers 		rv = CRYPTO_MECHANISM_PARAM_INVALID;
594*4d703b5cSMark Powers 		goto out;
595*4d703b5cSMark Powers 	}
596*4d703b5cSMark Powers 
597*4d703b5cSMark Powers 	if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen,
598*4d703b5cSMark Powers 	    gcm_param->pAAD, gcm_param->ulAADLen, block_size,
599*4d703b5cSMark Powers 	    encrypt_block, copy_block, xor_block) != 0) {
600*4d703b5cSMark Powers 		rv = CRYPTO_MECHANISM_PARAM_INVALID;
601*4d703b5cSMark Powers 	}
602*4d703b5cSMark Powers out:
603*4d703b5cSMark Powers 	return (rv);
604*4d703b5cSMark Powers }
605*4d703b5cSMark Powers 
606*4d703b5cSMark Powers void *
607*4d703b5cSMark Powers gcm_alloc_ctx(int kmflag)
608*4d703b5cSMark Powers {
609*4d703b5cSMark Powers 	gcm_ctx_t *gcm_ctx;
610*4d703b5cSMark Powers 
611*4d703b5cSMark Powers #ifdef _KERNEL
612*4d703b5cSMark Powers 	if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
613*4d703b5cSMark Powers #else
614*4d703b5cSMark Powers 	if ((gcm_ctx = calloc(1, sizeof (gcm_ctx_t))) == NULL)
615*4d703b5cSMark Powers #endif
616*4d703b5cSMark Powers 		return (NULL);
617*4d703b5cSMark Powers 
618*4d703b5cSMark Powers 	gcm_ctx->gcm_flags = GCM_MODE;
619*4d703b5cSMark Powers 	return (gcm_ctx);
620*4d703b5cSMark Powers }
621*4d703b5cSMark Powers 
622*4d703b5cSMark Powers void
623*4d703b5cSMark Powers gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag)
624*4d703b5cSMark Powers {
625*4d703b5cSMark Powers 	ctx->gcm_kmflag = kmflag;
626*4d703b5cSMark Powers }
627