xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softEncryptUtil.c (revision 14b24e2b79293068c8e016a69ef1d872fb5e2fd5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
25  */
26 
27 #include <pthread.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <strings.h>
31 #include <sys/types.h>
32 #include <security/cryptoki.h>
33 #include <modes/modes.h>
34 #include <arcfour.h>
35 #include "softSession.h"
36 #include "softObject.h"
37 #include "softOps.h"
38 #include "softCrypt.h"
39 #include "softRSA.h"
40 
41 /*
42  * Add padding bytes with the value of length of padding.
43  */
44 void
45 soft_add_pkcs7_padding(CK_BYTE *buf, int block_size, CK_ULONG data_len)
46 {
47 	(void) pkcs7_encode(NULL, data_len, buf, block_size, block_size);
48 }
49 
50 /*
51  * Perform encrypt init operation internally for the support of
52  * CKM_AES and CKM_DES MAC operations.
53  *
54  * This function is called with the session being held, and without
55  * its mutex taken.
56  */
57 CK_RV
58 soft_encrypt_init_internal(soft_session_t *session_p, CK_MECHANISM_PTR
59     pMechanism, soft_object_t *key_p)
60 {
61 	CK_RV rv;
62 
63 	(void) pthread_mutex_lock(&session_p->session_mutex);
64 
65 	/* Check to see if encrypt operation is already active */
66 	if (session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE) {
67 		(void) pthread_mutex_unlock(&session_p->session_mutex);
68 		return (CKR_OPERATION_ACTIVE);
69 	}
70 
71 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
72 
73 	(void) pthread_mutex_unlock(&session_p->session_mutex);
74 
75 	rv = soft_encrypt_init(session_p, pMechanism, key_p);
76 
77 	if (rv != CKR_OK) {
78 		(void) pthread_mutex_lock(&session_p->session_mutex);
79 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
80 		(void) pthread_mutex_unlock(&session_p->session_mutex);
81 	}
82 
83 	return (rv);
84 }
85 
86 /*
87  * soft_encrypt_init()
88  *
89  * Arguments:
90  *	session_p:	pointer to soft_session_t struct
91  *	pMechanism:	pointer to CK_MECHANISM struct provided by application
92  *	key_p:		pointer to key soft_object_t struct
93  *
94  * Description:
95  *	called by C_EncryptInit(). This function calls the corresponding
96  *	encrypt init routine based on the mechanism.
97  *
98  * Returns:
99  *	CKR_OK: success
100  *	CKR_HOST_MEMORY: run out of system memory
101  *	CKR_MECHANISM_PARAM_INVALID: invalid parameters in mechanism
102  *	CKR_MECHANISM_INVALID: invalid mechanism type
103  *	CKR_KEY_TYPE_INCONSISTENT: incorrect type of key to use
104  *		with the specified mechanism
105  */
106 CK_RV
107 soft_encrypt_init(soft_session_t *session_p, CK_MECHANISM_PTR pMechanism,
108     soft_object_t *key_p)
109 {
110 
111 	CK_RV rv;
112 
113 	switch (pMechanism->mechanism) {
114 
115 	case CKM_DES_ECB:
116 
117 		if (key_p->key_type != CKK_DES) {
118 			return (CKR_KEY_TYPE_INCONSISTENT);
119 		}
120 		goto ecb_common;
121 
122 	case CKM_DES3_ECB:
123 
124 		if ((key_p->key_type != CKK_DES2) &&
125 		    (key_p->key_type != CKK_DES3)) {
126 			return (CKR_KEY_TYPE_INCONSISTENT);
127 		}
128 
129 ecb_common:
130 		return (soft_des_crypt_init_common(session_p, pMechanism,
131 		    key_p, B_TRUE));
132 
133 	case CKM_DES_CBC:
134 	case CKM_DES_CBC_PAD:
135 
136 		if (key_p->key_type != CKK_DES) {
137 			return (CKR_KEY_TYPE_INCONSISTENT);
138 		}
139 
140 		goto cbc_common;
141 
142 	case CKM_DES3_CBC:
143 	case CKM_DES3_CBC_PAD:
144 	{
145 
146 		soft_des_ctx_t *soft_des_ctx;
147 
148 		if ((key_p->key_type != CKK_DES2) &&
149 		    (key_p->key_type != CKK_DES3)) {
150 			return (CKR_KEY_TYPE_INCONSISTENT);
151 		}
152 
153 cbc_common:
154 		if ((pMechanism->pParameter == NULL) ||
155 		    (pMechanism->ulParameterLen != DES_BLOCK_LEN)) {
156 			return (CKR_MECHANISM_PARAM_INVALID);
157 		}
158 
159 		rv = soft_des_crypt_init_common(session_p, pMechanism,
160 		    key_p, B_TRUE);
161 
162 		if (rv != CKR_OK)
163 			return (rv);
164 
165 		(void) pthread_mutex_lock(&session_p->session_mutex);
166 
167 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
168 		/* Copy Initialization Vector (IV) into the context. */
169 		(void) memcpy(soft_des_ctx->ivec, pMechanism->pParameter,
170 		    DES_BLOCK_LEN);
171 
172 		/* Allocate a context for DES cipher-block chaining. */
173 		soft_des_ctx->des_cbc = (void *)des_cbc_ctx_init(
174 		    soft_des_ctx->key_sched, soft_des_ctx->keysched_len,
175 		    soft_des_ctx->ivec, key_p->key_type);
176 
177 		if (soft_des_ctx->des_cbc == NULL) {
178 			bzero(soft_des_ctx->key_sched,
179 			    soft_des_ctx->keysched_len);
180 			free(soft_des_ctx->key_sched);
181 			free(session_p->encrypt.context);
182 			session_p->encrypt.context = NULL;
183 			rv = CKR_HOST_MEMORY;
184 		}
185 
186 		(void) pthread_mutex_unlock(&session_p->session_mutex);
187 
188 		return (rv);
189 	}
190 	case CKM_AES_ECB:
191 
192 		if (key_p->key_type != CKK_AES) {
193 			return (CKR_KEY_TYPE_INCONSISTENT);
194 		}
195 
196 		return (soft_aes_crypt_init_common(session_p, pMechanism,
197 		    key_p, B_TRUE));
198 
199 	case CKM_AES_CBC:
200 	case CKM_AES_CBC_PAD:
201 		if ((pMechanism->pParameter == NULL) ||
202 		    (pMechanism->ulParameterLen != AES_BLOCK_LEN)) {
203 			return (CKR_MECHANISM_PARAM_INVALID);
204 		}
205 	/* FALLTHRU */
206 	case CKM_AES_CMAC:
207 	{
208 		soft_aes_ctx_t *soft_aes_ctx;
209 
210 		if (key_p->key_type != CKK_AES) {
211 			return (CKR_KEY_TYPE_INCONSISTENT);
212 		}
213 
214 
215 		rv = soft_aes_crypt_init_common(session_p, pMechanism,
216 		    key_p, B_TRUE);
217 
218 		if (rv != CKR_OK)
219 			return (rv);
220 
221 		(void) pthread_mutex_lock(&session_p->session_mutex);
222 
223 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
224 		/* Copy Initialization Vector (IV) into the context. */
225 		if (pMechanism->mechanism == CKM_AES_CMAC) {
226 			(void) bzero(soft_aes_ctx->ivec, AES_BLOCK_LEN);
227 			/* Allocate a context for AES cipher-block chaining. */
228 			soft_aes_ctx->aes_cbc = (void *)aes_cmac_ctx_init(
229 			    soft_aes_ctx->key_sched,
230 			    soft_aes_ctx->keysched_len);
231 		} else {
232 			(void) memcpy(soft_aes_ctx->ivec,
233 			    pMechanism->pParameter,
234 			    AES_BLOCK_LEN);
235 			/* Allocate a context for AES cipher-block chaining. */
236 			soft_aes_ctx->aes_cbc = (void *)aes_cbc_ctx_init(
237 			    soft_aes_ctx->key_sched,
238 			    soft_aes_ctx->keysched_len,
239 			    soft_aes_ctx->ivec);
240 		}
241 		if (soft_aes_ctx->aes_cbc == NULL) {
242 			bzero(soft_aes_ctx->key_sched,
243 			    soft_aes_ctx->keysched_len);
244 			free(soft_aes_ctx->key_sched);
245 			free(session_p->encrypt.context);
246 			session_p->encrypt.context = NULL;
247 			rv = CKR_HOST_MEMORY;
248 		}
249 
250 		(void) pthread_mutex_unlock(&session_p->session_mutex);
251 
252 		return (rv);
253 	}
254 	case CKM_AES_CTR:
255 	{
256 		soft_aes_ctx_t *soft_aes_ctx;
257 
258 		if (key_p->key_type != CKK_AES) {
259 			return (CKR_KEY_TYPE_INCONSISTENT);
260 		}
261 
262 		if (pMechanism->pParameter == NULL ||
263 		    pMechanism->ulParameterLen != sizeof (CK_AES_CTR_PARAMS)) {
264 			return (CKR_MECHANISM_PARAM_INVALID);
265 		}
266 
267 		rv = soft_aes_crypt_init_common(session_p, pMechanism,
268 		    key_p, B_TRUE);
269 
270 		if (rv != CKR_OK)
271 			return (rv);
272 
273 		(void) pthread_mutex_lock(&session_p->session_mutex);
274 
275 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
276 		soft_aes_ctx->aes_cbc = aes_ctr_ctx_init(
277 		    soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len,
278 		    pMechanism->pParameter);
279 
280 		if (soft_aes_ctx->aes_cbc == NULL) {
281 			bzero(soft_aes_ctx->key_sched,
282 			    soft_aes_ctx->keysched_len);
283 			free(soft_aes_ctx->key_sched);
284 			free(session_p->encrypt.context);
285 			session_p->encrypt.context = NULL;
286 			rv = CKR_HOST_MEMORY;
287 		}
288 
289 		(void) pthread_mutex_unlock(&session_p->session_mutex);
290 
291 		return (rv);
292 	}
293 	case CKM_RC4:
294 
295 		if (key_p->key_type != CKK_RC4) {
296 			return (CKR_KEY_TYPE_INCONSISTENT);
297 		}
298 
299 		return (soft_arcfour_crypt_init(session_p, pMechanism, key_p,
300 		    B_TRUE));
301 
302 	case CKM_RSA_X_509:
303 	case CKM_RSA_PKCS:
304 
305 		if (key_p->key_type != CKK_RSA) {
306 			return (CKR_KEY_TYPE_INCONSISTENT);
307 		}
308 
309 		return (soft_rsa_crypt_init_common(session_p, pMechanism,
310 		    key_p, B_TRUE));
311 
312 	case CKM_BLOWFISH_CBC:
313 	{
314 		soft_blowfish_ctx_t *soft_blowfish_ctx;
315 
316 		if (key_p->key_type != CKK_BLOWFISH)
317 			return (CKR_KEY_TYPE_INCONSISTENT);
318 
319 		if ((pMechanism->pParameter == NULL) ||
320 		    (pMechanism->ulParameterLen != BLOWFISH_BLOCK_LEN))
321 			return (CKR_MECHANISM_PARAM_INVALID);
322 
323 		rv = soft_blowfish_crypt_init_common(session_p, pMechanism,
324 		    key_p, B_TRUE);
325 
326 		if (rv != CKR_OK)
327 			return (rv);
328 
329 		(void) pthread_mutex_lock(&session_p->session_mutex);
330 
331 		soft_blowfish_ctx =
332 		    (soft_blowfish_ctx_t *)session_p->encrypt.context;
333 		/* Copy Initialization Vector (IV) into the context. */
334 		(void) memcpy(soft_blowfish_ctx->ivec, pMechanism->pParameter,
335 		    BLOWFISH_BLOCK_LEN);
336 
337 		/* Allocate a context for Blowfish cipher-block chaining */
338 		soft_blowfish_ctx->blowfish_cbc =
339 		    (void *)blowfish_cbc_ctx_init(soft_blowfish_ctx->key_sched,
340 		    soft_blowfish_ctx->keysched_len,
341 		    soft_blowfish_ctx->ivec);
342 
343 		if (soft_blowfish_ctx->blowfish_cbc == NULL) {
344 			bzero(soft_blowfish_ctx->key_sched,
345 			    soft_blowfish_ctx->keysched_len);
346 			free(soft_blowfish_ctx->key_sched);
347 			free(session_p->encrypt.context);
348 			session_p->encrypt.context = NULL;
349 			rv = CKR_HOST_MEMORY;
350 		}
351 
352 		(void) pthread_mutex_unlock(&session_p->session_mutex);
353 
354 		return (rv);
355 	}
356 	default:
357 		return (CKR_MECHANISM_INVALID);
358 	}
359 }
360 
361 
362 /*
363  * soft_encrypt_common()
364  *
365  * Arguments:
366  *      session_p:	pointer to soft_session_t struct
367  *	pData:		pointer to the input data to be encrypted
368  *	ulDataLen:	length of the input data
369  *	pEncrypted:	pointer to the output data after encryption
370  *	pulEncryptedLen: pointer to the length of the output data
371  *	update:		boolean flag indicates caller is soft_encrypt
372  *			or soft_encrypt_update
373  *
374  * Description:
375  *      This function calls the corresponding encrypt routine based
376  *	on the mechanism.
377  *
378  * Returns:
379  *	see corresponding encrypt routine.
380  */
381 CK_RV
382 soft_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
383     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
384     CK_ULONG_PTR pulEncryptedLen, boolean_t update)
385 {
386 
387 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
388 
389 	switch (mechanism) {
390 
391 	case CKM_DES_ECB:
392 	case CKM_DES_CBC:
393 	case CKM_DES3_ECB:
394 	case CKM_DES3_CBC:
395 
396 		if (ulDataLen == 0) {
397 			*pulEncryptedLen = 0;
398 			return (CKR_OK);
399 		}
400 		/* FALLTHROUGH */
401 
402 	case CKM_DES_CBC_PAD:
403 	case CKM_DES3_CBC_PAD:
404 
405 		return (soft_des_encrypt_common(session_p, pData,
406 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
407 
408 	case CKM_AES_ECB:
409 	case CKM_AES_CBC:
410 	case CKM_AES_CTR:
411 
412 		if (ulDataLen == 0) {
413 			*pulEncryptedLen = 0;
414 			return (CKR_OK);
415 		}
416 		/* FALLTHROUGH */
417 
418 	case CKM_AES_CMAC:
419 	case CKM_AES_CBC_PAD:
420 
421 		return (soft_aes_encrypt_common(session_p, pData,
422 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
423 
424 	case CKM_BLOWFISH_CBC:
425 
426 		if (ulDataLen == 0) {
427 			*pulEncryptedLen = 0;
428 			return (CKR_OK);
429 		}
430 
431 		return (soft_blowfish_encrypt_common(session_p, pData,
432 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
433 
434 	case CKM_RC4:
435 
436 		if (ulDataLen == 0) {
437 			*pulEncryptedLen = 0;
438 			return (CKR_OK);
439 		}
440 
441 		return (soft_arcfour_crypt(&(session_p->encrypt), pData,
442 		    ulDataLen, pEncrypted, pulEncryptedLen));
443 
444 	case CKM_RSA_X_509:
445 	case CKM_RSA_PKCS:
446 
447 		return (soft_rsa_encrypt_common(session_p, pData,
448 		    ulDataLen, pEncrypted, pulEncryptedLen, mechanism));
449 
450 	default:
451 		return (CKR_MECHANISM_INVALID);
452 	}
453 }
454 
455 
456 /*
457  * soft_encrypt()
458  *
459  * Arguments:
460  *      session_p:	pointer to soft_session_t struct
461  *	pData:		pointer to the input data to be encrypted
462  *	ulDataLen:	length of the input data
463  *	pEncryptedData:	pointer to the output data after encryption
464  *	pulEncryptedDataLen: pointer to the length of the output data
465  *
466  * Description:
467  *      called by C_Encrypt(). This function calls the soft_encrypt_common
468  *	routine.
469  *
470  * Returns:
471  *	see soft_encrypt_common().
472  */
473 CK_RV
474 soft_encrypt(soft_session_t *session_p, CK_BYTE_PTR pData,
475     CK_ULONG ulDataLen, CK_BYTE_PTR pEncryptedData,
476     CK_ULONG_PTR pulEncryptedDataLen)
477 {
478 
479 	return (soft_encrypt_common(session_p, pData, ulDataLen,
480 	    pEncryptedData, pulEncryptedDataLen, B_FALSE));
481 }
482 
483 
484 /*
485  * soft_encrypt_update()
486  *
487  * Arguments:
488  *      session_p:	pointer to soft_session_t struct
489  *      pPart:		pointer to the input data to be digested
490  *      ulPartLen:	length of the input data
491  *	pEncryptedPart:	pointer to the ciphertext
492  *	pulEncryptedPartLen: pointer to the length of the ciphertext
493  *
494  * Description:
495  *      called by C_EncryptUpdate(). This function calls the
496  *	soft_encrypt_common routine (with update flag on).
497  *
498  * Returns:
499  *	see soft_encrypt_common().
500  */
501 CK_RV
502 soft_encrypt_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
503     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
504     CK_ULONG_PTR pulEncryptedPartLen)
505 {
506 
507 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
508 
509 	switch (mechanism) {
510 
511 	case CKM_DES_ECB:
512 	case CKM_DES_CBC:
513 	case CKM_DES_CBC_PAD:
514 	case CKM_DES3_ECB:
515 	case CKM_DES3_CBC:
516 	case CKM_DES3_CBC_PAD:
517 	case CKM_AES_ECB:
518 	case CKM_AES_CBC:
519 	case CKM_AES_CBC_PAD:
520 	case CKM_AES_CMAC:
521 	case CKM_AES_CTR:
522 	case CKM_BLOWFISH_CBC:
523 	case CKM_RC4:
524 
525 		return (soft_encrypt_common(session_p, pPart, ulPartLen,
526 		    pEncryptedPart, pulEncryptedPartLen, B_TRUE));
527 
528 	default:
529 		/* PKCS11: The mechanism only supports single-part operation. */
530 		return (CKR_MECHANISM_INVALID);
531 	}
532 }
533 
534 
535 /*
536  * soft_encrypt_final()
537  *
538  * Arguments:
539  *      session_p:		pointer to soft_session_t struct
540  *      pLastEncryptedPart:	pointer to the last encrypted data part
541  *      pulLastEncryptedPartLen: pointer to the length of the last
542  *				encrypted data part
543  *
544  * Description:
545  *      called by C_EncryptFinal().
546  *
547  * Returns:
548  *	CKR_OK: success
549  *	CKR_FUNCTION_FAILED: encrypt final function failed
550  *	CKR_DATA_LEN_RANGE: remaining buffer contains bad length
551  */
552 CK_RV
553 soft_encrypt_final(soft_session_t *session_p, CK_BYTE_PTR pLastEncryptedPart,
554     CK_ULONG_PTR pulLastEncryptedPartLen)
555 {
556 
557 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
558 	CK_ULONG out_len;
559 	CK_RV rv = CKR_OK;
560 	int rc;
561 
562 	(void) pthread_mutex_lock(&session_p->session_mutex);
563 
564 	if (session_p->encrypt.context == NULL) {
565 		rv = CKR_OPERATION_NOT_INITIALIZED;
566 		*pulLastEncryptedPartLen = 0;
567 		goto clean1;
568 	}
569 	switch (mechanism) {
570 
571 	case CKM_DES_CBC_PAD:
572 	case CKM_DES3_CBC_PAD:
573 	{
574 		soft_des_ctx_t *soft_des_ctx;
575 
576 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
577 		/*
578 		 * For CKM_DES_CBC_PAD, compute output length with
579 		 * padding. If the remaining buffer has one block
580 		 * of data, then output length will be two blocksize of
581 		 * ciphertext. If the remaining buffer has less than
582 		 * one block of data, then output length will be
583 		 * one blocksize.
584 		 */
585 		if (soft_des_ctx->remain_len == DES_BLOCK_LEN)
586 			out_len = 2 * DES_BLOCK_LEN;
587 		else
588 			out_len = DES_BLOCK_LEN;
589 
590 		if (pLastEncryptedPart == NULL) {
591 			/*
592 			 * Application asks for the length of the output
593 			 * buffer to hold the ciphertext.
594 			 */
595 			*pulLastEncryptedPartLen = out_len;
596 			goto clean1;
597 		} else {
598 			crypto_data_t out;
599 
600 			/* Copy remaining data to the output buffer. */
601 			(void) memcpy(pLastEncryptedPart, soft_des_ctx->data,
602 			    soft_des_ctx->remain_len);
603 
604 			/*
605 			 * Add padding bytes prior to encrypt final.
606 			 */
607 			soft_add_pkcs7_padding(pLastEncryptedPart +
608 			    soft_des_ctx->remain_len, DES_BLOCK_LEN,
609 			    soft_des_ctx->remain_len);
610 
611 			out.cd_format = CRYPTO_DATA_RAW;
612 			out.cd_offset = 0;
613 			out.cd_length = out_len;
614 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
615 			out.cd_raw.iov_len = out_len;
616 
617 			/* Encrypt multiple blocks of data. */
618 			rc = des_encrypt_contiguous_blocks(
619 			    (des_ctx_t *)soft_des_ctx->des_cbc,
620 			    (char *)pLastEncryptedPart, out_len, &out);
621 
622 			if (rc == 0) {
623 				*pulLastEncryptedPartLen = out_len;
624 			} else {
625 				*pulLastEncryptedPartLen = 0;
626 				rv = CKR_FUNCTION_FAILED;
627 			}
628 
629 			/* Cleanup memory space. */
630 			free(soft_des_ctx->des_cbc);
631 			bzero(soft_des_ctx->key_sched,
632 			    soft_des_ctx->keysched_len);
633 			free(soft_des_ctx->key_sched);
634 		}
635 
636 		break;
637 	}
638 	case CKM_DES_CBC:
639 	case CKM_DES_ECB:
640 	case CKM_DES3_CBC:
641 	case CKM_DES3_ECB:
642 	{
643 
644 		soft_des_ctx_t *soft_des_ctx;
645 
646 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
647 		/*
648 		 * CKM_DES_CBC and CKM_DES_ECB does not do any padding,
649 		 * so when the final is called, the remaining buffer
650 		 * should not contain any more data.
651 		 */
652 		*pulLastEncryptedPartLen = 0;
653 		if (soft_des_ctx->remain_len != 0) {
654 			rv = CKR_DATA_LEN_RANGE;
655 		} else {
656 			if (pLastEncryptedPart == NULL)
657 				goto clean1;
658 		}
659 
660 		/* Cleanup memory space. */
661 		free(soft_des_ctx->des_cbc);
662 		bzero(soft_des_ctx->key_sched, soft_des_ctx->keysched_len);
663 		free(soft_des_ctx->key_sched);
664 
665 		break;
666 	}
667 	case CKM_AES_CBC_PAD:
668 	{
669 		soft_aes_ctx_t *soft_aes_ctx;
670 
671 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
672 		/*
673 		 * For CKM_AES_CBC_PAD, compute output length with
674 		 * padding. If the remaining buffer has one block
675 		 * of data, then output length will be two blocksize of
676 		 * ciphertext. If the remaining buffer has less than
677 		 * one block of data, then output length will be
678 		 * one blocksize.
679 		 */
680 		if (soft_aes_ctx->remain_len == AES_BLOCK_LEN)
681 			out_len = 2 * AES_BLOCK_LEN;
682 		else
683 			out_len = AES_BLOCK_LEN;
684 
685 		if (pLastEncryptedPart == NULL) {
686 			/*
687 			 * Application asks for the length of the output
688 			 * buffer to hold the ciphertext.
689 			 */
690 			*pulLastEncryptedPartLen = out_len;
691 			goto clean1;
692 		} else {
693 			crypto_data_t out;
694 
695 			/* Copy remaining data to the output buffer. */
696 			(void) memcpy(pLastEncryptedPart, soft_aes_ctx->data,
697 			    soft_aes_ctx->remain_len);
698 
699 			/*
700 			 * Add padding bytes prior to encrypt final.
701 			 */
702 			soft_add_pkcs7_padding(pLastEncryptedPart +
703 			    soft_aes_ctx->remain_len, AES_BLOCK_LEN,
704 			    soft_aes_ctx->remain_len);
705 
706 			out.cd_format = CRYPTO_DATA_RAW;
707 			out.cd_offset = 0;
708 			out.cd_length = out_len;
709 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
710 			out.cd_raw.iov_len = out_len;
711 
712 			/* Encrypt multiple blocks of data. */
713 			rc = aes_encrypt_contiguous_blocks(
714 			    (aes_ctx_t *)soft_aes_ctx->aes_cbc,
715 			    (char *)pLastEncryptedPart, out_len, &out);
716 
717 			if (rc == 0) {
718 				*pulLastEncryptedPartLen = out_len;
719 			} else {
720 				*pulLastEncryptedPartLen = 0;
721 				rv = CKR_FUNCTION_FAILED;
722 			}
723 
724 			/* Cleanup memory space. */
725 			free(soft_aes_ctx->aes_cbc);
726 			bzero(soft_aes_ctx->key_sched,
727 			    soft_aes_ctx->keysched_len);
728 			free(soft_aes_ctx->key_sched);
729 		}
730 
731 		break;
732 	}
733 	case CKM_AES_CMAC:
734 	{
735 		soft_aes_ctx_t *soft_aes_ctx;
736 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
737 
738 		if (pLastEncryptedPart == NULL) {
739 			/*
740 			 * Application asks for the length of the output
741 			 * buffer to hold the ciphertext.
742 			 */
743 			*pulLastEncryptedPartLen = AES_BLOCK_LEN;
744 			goto clean1;
745 		} else {
746 			crypto_data_t out;
747 
748 			out.cd_format = CRYPTO_DATA_RAW;
749 			out.cd_offset = 0;
750 			out.cd_length = AES_BLOCK_LEN;
751 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
752 			out.cd_raw.iov_len = AES_BLOCK_LEN;
753 
754 			rc = cmac_mode_final(soft_aes_ctx->aes_cbc, &out,
755 			    aes_encrypt_block, aes_xor_block);
756 
757 			if (rc == 0) {
758 				*pulLastEncryptedPartLen = AES_BLOCK_LEN;
759 			} else {
760 				*pulLastEncryptedPartLen = 0;
761 				rv = CKR_FUNCTION_FAILED;
762 			}
763 
764 			/* Cleanup memory space. */
765 			free(soft_aes_ctx->aes_cbc);
766 			bzero(soft_aes_ctx->key_sched,
767 			    soft_aes_ctx->keysched_len);
768 			free(soft_aes_ctx->key_sched);
769 		}
770 
771 		break;
772 	}
773 	case CKM_AES_CBC:
774 	case CKM_AES_ECB:
775 	{
776 		soft_aes_ctx_t *soft_aes_ctx;
777 
778 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
779 		/*
780 		 * CKM_AES_CBC and CKM_AES_ECB does not do any padding,
781 		 * so when the final is called, the remaining buffer
782 		 * should not contain any more data.
783 		 */
784 		*pulLastEncryptedPartLen = 0;
785 		if (soft_aes_ctx->remain_len != 0) {
786 			rv = CKR_DATA_LEN_RANGE;
787 		} else {
788 			if (pLastEncryptedPart == NULL)
789 				goto clean1;
790 		}
791 
792 		/* Cleanup memory space. */
793 		free(soft_aes_ctx->aes_cbc);
794 		bzero(soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len);
795 		free(soft_aes_ctx->key_sched);
796 
797 		break;
798 	}
799 	case CKM_AES_CTR:
800 	{
801 		crypto_data_t out;
802 		soft_aes_ctx_t *soft_aes_ctx;
803 		ctr_ctx_t *ctr_ctx;
804 		size_t len;
805 
806 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
807 		ctr_ctx = soft_aes_ctx->aes_cbc;
808 		len = ctr_ctx->ctr_remainder_len;
809 
810 		if (pLastEncryptedPart == NULL) {
811 			*pulLastEncryptedPartLen = len;
812 			goto clean1;
813 		}
814 		if (len > 0) {
815 			out.cd_format = CRYPTO_DATA_RAW;
816 			out.cd_offset = 0;
817 			out.cd_length = len;
818 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
819 			out.cd_raw.iov_len = len;
820 
821 			rv = ctr_mode_final(ctr_ctx, &out, aes_encrypt_block);
822 		}
823 		if (rv == CRYPTO_BUFFER_TOO_SMALL) {
824 			*pulLastEncryptedPartLen = len;
825 			goto clean1;
826 		}
827 
828 		/* Cleanup memory space. */
829 		free(ctr_ctx);
830 		bzero(soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len);
831 		free(soft_aes_ctx->key_sched);
832 
833 		break;
834 	}
835 	case CKM_BLOWFISH_CBC:
836 	{
837 		soft_blowfish_ctx_t *soft_blowfish_ctx;
838 
839 		soft_blowfish_ctx =
840 		    (soft_blowfish_ctx_t *)session_p->encrypt.context;
841 		/*
842 		 * CKM_BLOWFISH_CBC does not do any padding, so when the
843 		 * final is called, the remaining buffer should not contain
844 		 * any more data
845 		 */
846 		*pulLastEncryptedPartLen = 0;
847 		if (soft_blowfish_ctx->remain_len != 0)
848 			rv = CKR_DATA_LEN_RANGE;
849 		else {
850 			if (pLastEncryptedPart == NULL)
851 				goto clean1;
852 		}
853 
854 		free(soft_blowfish_ctx->blowfish_cbc);
855 		bzero(soft_blowfish_ctx->key_sched,
856 		    soft_blowfish_ctx->keysched_len);
857 		free(soft_blowfish_ctx->key_sched);
858 		break;
859 	}
860 
861 	case CKM_RC4:
862 	{
863 		ARCFour_key *key = (ARCFour_key *)session_p->encrypt.context;
864 		/* Remaining data size is always zero for RC4. */
865 		*pulLastEncryptedPartLen = 0;
866 		if (pLastEncryptedPart == NULL)
867 			goto clean1;
868 		bzero(key, sizeof (*key));
869 		break;
870 	}
871 	default:
872 		/* PKCS11: The mechanism only supports single-part operation. */
873 		rv = CKR_MECHANISM_INVALID;
874 		break;
875 	}
876 
877 	free(session_p->encrypt.context);
878 	session_p->encrypt.context = NULL;
879 clean1:
880 	(void) pthread_mutex_unlock(&session_p->session_mutex);
881 
882 	return (rv);
883 }
884 
885 /*
886  * This function frees the allocated active crypto context and the
887  * lower level of allocated struct as needed.
888  * This function is called by the 1st tier of encrypt/decrypt routines
889  * or by the 2nd tier of session close routine. Since the 1st tier
890  * caller will always call this function without locking the session
891  * mutex and the 2nd tier caller will call with the lock, we add the
892  * third parameter "lock_held" to distinguish this case.
893  */
894 void
895 soft_crypt_cleanup(soft_session_t *session_p, boolean_t encrypt,
896     boolean_t lock_held)
897 {
898 
899 	crypto_active_op_t *active_op;
900 	boolean_t lock_true = B_TRUE;
901 
902 	if (!lock_held)
903 		(void) pthread_mutex_lock(&session_p->session_mutex);
904 
905 	active_op = (encrypt) ? &(session_p->encrypt) : &(session_p->decrypt);
906 
907 	switch (active_op->mech.mechanism) {
908 
909 	case CKM_DES_CBC_PAD:
910 	case CKM_DES3_CBC_PAD:
911 	case CKM_DES_CBC:
912 	case CKM_DES_ECB:
913 	case CKM_DES3_CBC:
914 	case CKM_DES3_ECB:
915 	{
916 
917 		soft_des_ctx_t *soft_des_ctx =
918 		    (soft_des_ctx_t *)active_op->context;
919 		des_ctx_t *des_ctx;
920 
921 		if (soft_des_ctx != NULL) {
922 			des_ctx = (des_ctx_t *)soft_des_ctx->des_cbc;
923 			if (des_ctx != NULL) {
924 				bzero(des_ctx->dc_keysched,
925 				    des_ctx->dc_keysched_len);
926 				free(soft_des_ctx->des_cbc);
927 			}
928 			bzero(soft_des_ctx->key_sched,
929 			    soft_des_ctx->keysched_len);
930 			free(soft_des_ctx->key_sched);
931 		}
932 		break;
933 	}
934 
935 	case CKM_AES_CBC_PAD:
936 	case CKM_AES_CBC:
937 	case CKM_AES_CMAC:
938 	case CKM_AES_ECB:
939 	{
940 		soft_aes_ctx_t *soft_aes_ctx =
941 		    (soft_aes_ctx_t *)active_op->context;
942 		aes_ctx_t *aes_ctx;
943 
944 		if (soft_aes_ctx != NULL) {
945 			aes_ctx = (aes_ctx_t *)soft_aes_ctx->aes_cbc;
946 			if (aes_ctx != NULL) {
947 				bzero(aes_ctx->ac_keysched,
948 				    aes_ctx->ac_keysched_len);
949 				free(soft_aes_ctx->aes_cbc);
950 			}
951 			bzero(soft_aes_ctx->key_sched,
952 			    soft_aes_ctx->keysched_len);
953 			free(soft_aes_ctx->key_sched);
954 		}
955 		break;
956 	}
957 
958 	case CKM_BLOWFISH_CBC:
959 	{
960 		soft_blowfish_ctx_t *soft_blowfish_ctx =
961 		    (soft_blowfish_ctx_t *)active_op->context;
962 		blowfish_ctx_t *blowfish_ctx;
963 
964 		if (soft_blowfish_ctx != NULL) {
965 			blowfish_ctx =
966 			    (blowfish_ctx_t *)soft_blowfish_ctx->blowfish_cbc;
967 			if (blowfish_ctx != NULL) {
968 				bzero(blowfish_ctx->bc_keysched,
969 				    blowfish_ctx->bc_keysched_len);
970 				free(soft_blowfish_ctx->blowfish_cbc);
971 			}
972 
973 			bzero(soft_blowfish_ctx->key_sched,
974 			    soft_blowfish_ctx->keysched_len);
975 			free(soft_blowfish_ctx->key_sched);
976 		}
977 		break;
978 	}
979 
980 	case CKM_RC4:
981 	{
982 		ARCFour_key *key = (ARCFour_key *)active_op->context;
983 
984 		if (key != NULL)
985 			bzero(key, sizeof (*key));
986 		break;
987 	}
988 
989 	case CKM_RSA_X_509:
990 	case CKM_RSA_PKCS:
991 	{
992 		soft_rsa_ctx_t *rsa_ctx =
993 		    (soft_rsa_ctx_t *)active_op->context;
994 
995 		if (rsa_ctx != NULL)
996 			if (rsa_ctx->key != NULL) {
997 				soft_cleanup_object(rsa_ctx->key);
998 				free(rsa_ctx->key);
999 			}
1000 
1001 		break;
1002 	}
1003 
1004 	} /* switch */
1005 
1006 	if (active_op->context != NULL) {
1007 		free(active_op->context);
1008 		active_op->context = NULL;
1009 	}
1010 
1011 	active_op->flags = 0;
1012 
1013 	if (!lock_held)
1014 		SES_REFRELE(session_p, lock_true);
1015 }
1016