xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softRSA.c (revision a38ddfee9c8c6b6c5a2947ff52fd2338362a4444)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <pthread.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <strings.h>
32 #include <sys/types.h>
33 #include <security/cryptoki.h>
34 #include <bignum.h>
35 #include "softGlobal.h"
36 #include "softSession.h"
37 #include "softObject.h"
38 #include "softOps.h"
39 #include "softRSA.h"
40 #include "softMAC.h"
41 #include "softRandom.h"
42 #include "softCrypt.h"
43 
44 CK_RV
45 soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
46     CK_BYTE_PTR out, int realpublic)
47 {
48 
49 	CK_RV rv = CKR_OK;
50 
51 /* EXPORT DELETE START */
52 
53 	uchar_t expo[MAX_KEY_ATTR_BUFLEN];
54 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
55 	uint32_t expo_len = sizeof (expo);
56 	uint32_t modulus_len = sizeof (modulus);
57 	BIGNUM msg;
58 	RSAkey *rsakey;
59 
60 	if (realpublic) {
61 		rv = soft_get_public_value(key, CKA_PUBLIC_EXPONENT, expo,
62 		    &expo_len);
63 		if (rv != CKR_OK) {
64 			goto clean1;
65 		}
66 	} else {
67 		rv = soft_get_private_value(key, CKA_PRIVATE_EXPONENT, expo,
68 		    &expo_len);
69 		if (rv != CKR_OK) {
70 			goto clean1;
71 		}
72 	}
73 
74 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
75 	if (rv != CKR_OK) {
76 		goto clean1;
77 	}
78 
79 	if (expo_len > modulus_len) {
80 		rv = CKR_KEY_SIZE_RANGE;
81 		goto clean1;
82 	}
83 
84 	rsakey = calloc(1, sizeof (RSAkey));
85 	if (rsakey == NULL) {
86 		rv = CKR_HOST_MEMORY;
87 		goto clean1;
88 	}
89 
90 	if (RSA_key_init(rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
91 		rv = CKR_HOST_MEMORY;
92 		goto clean4;
93 	}
94 
95 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
96 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
97 		rv = CKR_HOST_MEMORY;
98 		goto clean5;
99 	}
100 
101 	/* Convert octet string exponent to big integer format. */
102 	bytestring2bignum(&(rsakey->e), expo, expo_len);
103 
104 	/* Convert octet string modulus to big integer format. */
105 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
106 
107 	/* Convert octet string input data to big integer format. */
108 	bytestring2bignum(&msg, (uchar_t *)in, in_len);
109 
110 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
111 		rv = CKR_DATA_LEN_RANGE;
112 		goto clean6;
113 	}
114 
115 	/* Perform RSA computation on big integer input data. */
116 	if (big_modexp(&msg, &msg, &(rsakey->e), &(rsakey->n), NULL) !=
117 	    BIG_OK) {
118 		rv = CKR_HOST_MEMORY;
119 		goto clean6;
120 	}
121 
122 	/* Convert the big integer output data to octet string. */
123 	bignum2bytestring((uchar_t *)out, &msg, modulus_len);
124 
125 clean6:
126 	big_finish(&msg);
127 clean5:
128 	RSA_key_finish(rsakey);
129 clean4:
130 	free(rsakey);
131 clean1:
132 
133 /* EXPORT DELETE END */
134 
135 	return (rv);
136 }
137 
138 
139 CK_RV
140 soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
141     CK_BYTE_PTR out)
142 {
143 
144 	CK_RV rv = CKR_OK;
145 
146 /* EXPORT DELETE START */
147 
148 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
149 	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
150 	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
151 	uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
152 	uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
153 	uchar_t coef[MAX_KEY_ATTR_BUFLEN];
154 	uint32_t modulus_len = sizeof (modulus);
155 	uint32_t prime1_len = sizeof (prime1);
156 	uint32_t prime2_len = sizeof (prime2);
157 	uint32_t expo1_len = sizeof (expo1);
158 	uint32_t expo2_len = sizeof (expo2);
159 	uint32_t coef_len = sizeof (coef);
160 	BIGNUM msg;
161 	RSAkey *rsakey;
162 
163 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
164 	if (rv != CKR_OK) {
165 		goto clean1;
166 	}
167 
168 	rv = soft_get_private_value(key, CKA_PRIME_1, prime1, &prime1_len);
169 
170 	if ((prime1_len == 0) && (rv == CKR_OK)) {
171 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
172 		goto clean1;
173 	} else {
174 		if (rv != CKR_OK)
175 			goto clean1;
176 	}
177 
178 	rv = soft_get_private_value(key, CKA_PRIME_2, prime2, &prime2_len);
179 
180 	if ((prime2_len == 0) && (rv == CKR_OK)) {
181 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
182 		goto clean1;
183 	} else {
184 		if (rv != CKR_OK)
185 			goto clean1;
186 	}
187 
188 	rv = soft_get_private_value(key, CKA_EXPONENT_1, expo1, &expo1_len);
189 
190 	if ((expo1_len == 0) && (rv == CKR_OK)) {
191 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
192 		goto clean1;
193 	} else {
194 		if (rv != CKR_OK)
195 			goto clean1;
196 	}
197 
198 	rv = soft_get_private_value(key, CKA_EXPONENT_2, expo2, &expo2_len);
199 
200 	if ((expo2_len == 0) && (rv == CKR_OK)) {
201 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
202 		goto clean1;
203 	} else {
204 		if (rv != CKR_OK)
205 			goto clean1;
206 	}
207 
208 	rv = soft_get_private_value(key, CKA_COEFFICIENT, coef, &coef_len);
209 
210 	if ((coef_len == 0) && (rv == CKR_OK)) {
211 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
212 		goto clean1;
213 	} else {
214 		if (rv != CKR_OK)
215 			goto clean1;
216 	}
217 
218 	rsakey = calloc(1, sizeof (RSAkey));
219 	if (rsakey == NULL) {
220 		rv = CKR_HOST_MEMORY;
221 		goto clean1;
222 	}
223 
224 	/* psize and qsize for RSA_key_init is in bits. */
225 	if (RSA_key_init(rsakey, prime2_len * 8, prime1_len * 8) != BIG_OK) {
226 		rv = CKR_HOST_MEMORY;
227 		goto clean8;
228 	}
229 
230 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
231 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
232 		rv = CKR_HOST_MEMORY;
233 		goto clean9;
234 	}
235 
236 	/* Convert octet string input data to big integer format. */
237 	bytestring2bignum(&msg, (uchar_t *)in, in_len);
238 
239 	/* Convert octet string modulus to big integer format. */
240 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
241 
242 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
243 		rv = CKR_DATA_LEN_RANGE;
244 		goto clean10;
245 	}
246 
247 	/* Convert the rest of private key attributes to big integer format. */
248 	bytestring2bignum(&(rsakey->dmodpminus1), expo2, expo2_len);
249 	bytestring2bignum(&(rsakey->dmodqminus1), expo1, expo1_len);
250 	bytestring2bignum(&(rsakey->p), prime2, prime2_len);
251 	bytestring2bignum(&(rsakey->q), prime1, prime1_len);
252 	bytestring2bignum(&(rsakey->pinvmodq), coef, coef_len);
253 
254 	if ((big_cmp_abs(&(rsakey->dmodpminus1), &(rsakey->p)) > 0) ||
255 	    (big_cmp_abs(&(rsakey->dmodqminus1), &(rsakey->q)) > 0) ||
256 	    (big_cmp_abs(&(rsakey->pinvmodq), &(rsakey->q)) > 0)) {
257 		rv = CKR_KEY_SIZE_RANGE;
258 		goto clean10;
259 	}
260 
261 	/* Perform RSA computation on big integer input data. */
262 	if (big_modexp_crt(&msg, &msg, &(rsakey->dmodpminus1),
263 	    &(rsakey->dmodqminus1), &(rsakey->p), &(rsakey->q),
264 	    &(rsakey->pinvmodq), NULL, NULL) != BIG_OK) {
265 		rv = CKR_HOST_MEMORY;
266 		goto clean10;
267 	}
268 
269 	/* Convert the big integer output data to octet string. */
270 	bignum2bytestring((uchar_t *)out, &msg, modulus_len);
271 
272 clean10:
273 	big_finish(&msg);
274 clean9:
275 	RSA_key_finish(rsakey);
276 clean8:
277 	free(rsakey);
278 clean1:
279 
280 /* EXPORT DELETE END */
281 
282 	return (rv);
283 }
284 
285 /*
286  * Allocate a RSA context for the active encryption or decryption operation.
287  * This function is called without the session lock held.
288  */
289 CK_RV
290 soft_rsa_crypt_init_common(soft_session_t *session_p,
291     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
292     boolean_t encrypt)
293 {
294 
295 	soft_rsa_ctx_t *rsa_ctx;
296 	soft_object_t *tmp_key = NULL;
297 	CK_RV rv;
298 
299 	rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
300 	if (rsa_ctx == NULL) {
301 		return (CKR_HOST_MEMORY);
302 	}
303 
304 	/*
305 	 * Make a copy of the encryption or decryption key, and save it
306 	 * in the RSA crypto context since it will be used later for
307 	 * encryption/decryption. We don't want to hold any object reference
308 	 * on this original key while doing encryption/decryption.
309 	 */
310 	(void) pthread_mutex_lock(&key_p->object_mutex);
311 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
312 	    NULL);
313 
314 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
315 		/* Most likely we ran out of space. */
316 		(void) pthread_mutex_unlock(&key_p->object_mutex);
317 		free(rsa_ctx);
318 		return (rv);
319 	}
320 
321 	/* No need to hold the lock on the old object. */
322 	(void) pthread_mutex_unlock(&key_p->object_mutex);
323 	rsa_ctx->key = tmp_key;
324 
325 	(void) pthread_mutex_lock(&session_p->session_mutex);
326 	if (encrypt) {
327 		/* Called by C_EncryptInit. */
328 		session_p->encrypt.context = rsa_ctx;
329 		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
330 	} else {
331 		/* Called by C_DecryptInit. */
332 		session_p->decrypt.context = rsa_ctx;
333 		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
334 	}
335 	(void) pthread_mutex_unlock(&session_p->session_mutex);
336 
337 	return (CKR_OK);
338 }
339 
340 CK_RV
341 soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
342     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
343     CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
344 {
345 
346 	soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
347 	soft_object_t *key = rsa_ctx->key;
348 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
349 	uint32_t modulus_len = sizeof (modulus);
350 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
351 	CK_BYTE	cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
352 	CK_RV rv = CKR_OK;
353 
354 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
355 	if (rv != CKR_OK) {
356 		goto clean_exit;
357 	}
358 
359 	if (pEncrypted == NULL) {
360 		/*
361 		 * Application asks for the length of the output buffer
362 		 * to hold the ciphertext.
363 		 */
364 		*pulEncryptedLen = modulus_len;
365 		rv = CKR_OK;
366 		goto clean1;
367 	}
368 
369 	if (mechanism == CKM_RSA_PKCS) {
370 		/*
371 		 * Input data length needs to be <=
372 		 * modulus length-MIN_PKCS1_PADLEN.
373 		 */
374 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
375 			*pulEncryptedLen = modulus_len;
376 			rv = CKR_DATA_LEN_RANGE;
377 			goto clean_exit;
378 		}
379 	} else {
380 		/* Input data length needs to be <= modulus length. */
381 		if (ulDataLen > (CK_ULONG)modulus_len) {
382 			*pulEncryptedLen = modulus_len;
383 			rv = CKR_DATA_LEN_RANGE;
384 			goto clean_exit;
385 		}
386 	}
387 
388 	/* Is the application-supplied buffer large enough? */
389 	if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
390 		*pulEncryptedLen = modulus_len;
391 		rv = CKR_BUFFER_TOO_SMALL;
392 		goto clean1;
393 	}
394 
395 	if (mechanism == CKM_RSA_PKCS) {
396 		/*
397 		 * Add PKCS padding to the input data to format a block
398 		 * type "02" encryption block.
399 		 */
400 		rv = soft_encrypt_rsa_pkcs_encode(pData, ulDataLen, plain_data,
401 		    modulus_len);
402 
403 		if (rv != CKR_OK)
404 			goto clean_exit;
405 	} else {
406 		/* Pad zeros for the leading bytes of the input data. */
407 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
408 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
409 		    ulDataLen);
410 	}
411 
412 	rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
413 	if (rv == CKR_OK) {
414 		(void) memcpy(pEncrypted, cipher_data, modulus_len);
415 		*pulEncryptedLen = modulus_len;
416 	}
417 
418 clean_exit:
419 	(void) pthread_mutex_lock(&session_p->session_mutex);
420 	free(session_p->encrypt.context);
421 	session_p->encrypt.context = NULL;
422 	(void) pthread_mutex_unlock(&session_p->session_mutex);
423 	soft_cleanup_object(key);
424 	free(key);
425 clean1:
426 	return (rv);
427 }
428 
429 
430 CK_RV
431 soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
432     CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
433     CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
434 {
435 
436 	soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
437 	soft_object_t *key = rsa_ctx->key;
438 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
439 	uint32_t modulus_len = sizeof (modulus);
440 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
441 	CK_RV rv = CKR_OK;
442 
443 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
444 	if (rv != CKR_OK) {
445 		goto clean_exit;
446 	}
447 
448 	if (ulEncryptedLen != (CK_ULONG)modulus_len) {
449 		rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
450 		goto clean_exit;
451 	}
452 
453 	if (pData == NULL) {
454 		/*
455 		 * Application asks for the length of the output buffer
456 		 * to hold the recovered data.
457 		 */
458 		*pulDataLen = modulus_len;
459 		rv = CKR_OK;
460 		goto clean1;
461 	}
462 
463 	if (mechanism == CKM_RSA_X_509) {
464 		if (*pulDataLen < (CK_ULONG)modulus_len) {
465 			*pulDataLen = modulus_len;
466 			rv = CKR_BUFFER_TOO_SMALL;
467 			goto clean1;
468 		}
469 	}
470 
471 	rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
472 	if (rv != CKR_OK) {
473 		goto clean_exit;
474 	}
475 
476 	if (mechanism == CKM_RSA_PKCS) {
477 		int plain_len = modulus_len;
478 		uint32_t num_padding;
479 
480 		/* Strip off the PKCS block formatting data. */
481 		rv = soft_decrypt_rsa_pkcs_decode(plain_data, &plain_len);
482 		if (rv != CKR_OK)
483 			goto clean_exit;
484 
485 		num_padding = modulus_len - plain_len;
486 		if (ulEncryptedLen - num_padding > *pulDataLen) {
487 			*pulDataLen = plain_len;
488 			rv = CKR_BUFFER_TOO_SMALL;
489 			goto clean1;
490 		}
491 
492 		(void) memcpy(pData, &plain_data[num_padding], plain_len);
493 		*pulDataLen = plain_len;
494 	} else {
495 		(void) memcpy(pData, plain_data, modulus_len);
496 		*pulDataLen = modulus_len;
497 	}
498 
499 clean_exit:
500 	(void) pthread_mutex_lock(&session_p->session_mutex);
501 	free(session_p->decrypt.context);
502 	session_p->decrypt.context = NULL;
503 	(void) pthread_mutex_unlock(&session_p->session_mutex);
504 	soft_cleanup_object(key);
505 	free(key);
506 
507 clean1:
508 	return (rv);
509 }
510 
511 /*
512  * Allocate a RSA context for the active sign or verify operation.
513  * This function is called without the session lock held.
514  */
515 CK_RV
516 soft_rsa_sign_verify_init_common(soft_session_t *session_p,
517     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
518     boolean_t sign)
519 {
520 	CK_RV rv = CKR_OK;
521 	soft_rsa_ctx_t *rsa_ctx;
522 	CK_MECHANISM digest_mech;
523 	soft_object_t *tmp_key = NULL;
524 
525 	if (sign) {
526 		if ((key_p->class != CKO_PRIVATE_KEY) ||
527 		    (key_p->key_type != CKK_RSA))
528 			return (CKR_KEY_TYPE_INCONSISTENT);
529 	} else {
530 		if ((key_p->class != CKO_PUBLIC_KEY) ||
531 		    (key_p->key_type != CKK_RSA))
532 			return (CKR_KEY_TYPE_INCONSISTENT);
533 	}
534 
535 	switch (pMechanism->mechanism) {
536 	case CKM_MD5_RSA_PKCS:
537 		digest_mech.mechanism = CKM_MD5;
538 		rv = soft_digest_init_internal(session_p, &digest_mech);
539 		if (rv != CKR_OK)
540 			return (rv);
541 		break;
542 
543 	case CKM_SHA1_RSA_PKCS:
544 		digest_mech.mechanism = CKM_SHA_1;
545 		digest_mech.pParameter = pMechanism->pParameter;
546 		digest_mech.ulParameterLen = pMechanism->ulParameterLen;
547 		rv = soft_digest_init_internal(session_p, &digest_mech);
548 		if (rv != CKR_OK)
549 			return (rv);
550 		break;
551 
552 	case CKM_SHA256_RSA_PKCS:
553 		digest_mech.mechanism = CKM_SHA256;
554 		rv = soft_digest_init_internal(session_p, &digest_mech);
555 		if (rv != CKR_OK)
556 			return (rv);
557 		break;
558 
559 	case CKM_SHA384_RSA_PKCS:
560 		digest_mech.mechanism = CKM_SHA384;
561 		rv = soft_digest_init_internal(session_p, &digest_mech);
562 		if (rv != CKR_OK)
563 			return (rv);
564 		break;
565 
566 	case CKM_SHA512_RSA_PKCS:
567 		digest_mech.mechanism = CKM_SHA512;
568 		rv = soft_digest_init_internal(session_p, &digest_mech);
569 		if (rv != CKR_OK)
570 			return (rv);
571 		break;
572 	}
573 
574 	rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));
575 
576 	if (rsa_ctx == NULL) {
577 		rv = CKR_HOST_MEMORY;
578 		goto clean_exit;
579 	}
580 
581 	(void) pthread_mutex_lock(&key_p->object_mutex);
582 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
583 	    NULL);
584 
585 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
586 		/* Most likely we ran out of space. */
587 		(void) pthread_mutex_unlock(&key_p->object_mutex);
588 		free(rsa_ctx);
589 		goto clean_exit;
590 	}
591 
592 	/* No need to hold the lock on the old object. */
593 	(void) pthread_mutex_unlock(&key_p->object_mutex);
594 	rsa_ctx->key = tmp_key;
595 
596 	(void) pthread_mutex_lock(&session_p->session_mutex);
597 
598 	if (sign) {
599 		session_p->sign.context = rsa_ctx;
600 		session_p->sign.mech.mechanism = pMechanism->mechanism;
601 	} else {
602 		session_p->verify.context = rsa_ctx;
603 		session_p->verify.mech.mechanism = pMechanism->mechanism;
604 	}
605 
606 	(void) pthread_mutex_unlock(&session_p->session_mutex);
607 
608 	return (CKR_OK);
609 
610 clean_exit:
611 	(void) pthread_mutex_lock(&session_p->session_mutex);
612 	if (session_p->digest.context != NULL) {
613 		free(session_p->digest.context);
614 		session_p->digest.context = NULL;
615 		session_p->digest.flags = 0;
616 	}
617 	(void) pthread_mutex_unlock(&session_p->session_mutex);
618 	return (rv);
619 
620 }
621 
622 
623 CK_RV
624 soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
625     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
626     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
627 {
628 
629 	CK_RV rv = CKR_OK;
630 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
631 	soft_object_t *key = rsa_ctx->key;
632 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
633 	uint32_t modulus_len = sizeof (modulus);
634 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
635 	CK_BYTE	signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
636 
637 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
638 	if (rv != CKR_OK) {
639 		goto clean_exit;
640 	}
641 
642 	if (pSigned == NULL) {
643 		/* Application asks for the length of the output buffer. */
644 		*pulSignedLen = modulus_len;
645 		rv = CKR_OK;
646 		goto clean1;
647 	}
648 
649 	switch (mechanism) {
650 
651 	case CKM_RSA_PKCS:
652 
653 		/*
654 		 * Input data length needs to be <=
655 		 * modulus length-MIN_PKCS1_PADLEN.
656 		 */
657 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
658 			*pulSignedLen = modulus_len;
659 			rv = CKR_DATA_LEN_RANGE;
660 			goto clean_exit;
661 		}
662 		break;
663 
664 	case CKM_RSA_X_509:
665 
666 		/* Input data length needs to be <= modulus length. */
667 		if (ulDataLen > (CK_ULONG)modulus_len) {
668 			*pulSignedLen = modulus_len;
669 			rv = CKR_DATA_LEN_RANGE;
670 			goto clean_exit;
671 		}
672 		break;
673 	}
674 
675 	/* Is the application-supplied buffer large enough? */
676 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
677 		*pulSignedLen = modulus_len;
678 		rv = CKR_BUFFER_TOO_SMALL;
679 		goto clean1;
680 	}
681 
682 	switch (mechanism) {
683 
684 	case CKM_RSA_PKCS:
685 	case CKM_MD5_RSA_PKCS:
686 	case CKM_SHA1_RSA_PKCS:
687 	case CKM_SHA256_RSA_PKCS:
688 	case CKM_SHA384_RSA_PKCS:
689 	case CKM_SHA512_RSA_PKCS:
690 		/*
691 		 * Add PKCS padding to the input data to format a block
692 		 * type "01" encryption block.
693 		 */
694 		rv = soft_sign_rsa_pkcs_encode(pData, ulDataLen, plain_data,
695 		    modulus_len);
696 
697 		if (rv != CKR_OK) {
698 			goto clean_exit;
699 		}
700 		break;
701 
702 	case CKM_RSA_X_509:
703 
704 		/* Pad zeros for the leading bytes of the input data. */
705 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
706 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
707 		    ulDataLen);
708 		break;
709 	}
710 
711 	/*
712 	 * Perform RSA encryption with the signer's RSA private key
713 	 * for signature process.
714 	 */
715 	rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);
716 
717 	if (rv == CKR_OK) {
718 		(void) memcpy(pSigned, signed_data, modulus_len);
719 		*pulSignedLen = modulus_len;
720 	}
721 
722 clean_exit:
723 	(void) pthread_mutex_lock(&session_p->session_mutex);
724 	free(session_p->sign.context);
725 	session_p->sign.context = NULL;
726 	if (session_p->digest.context != NULL) {
727 		free(session_p->digest.context);
728 		session_p->digest.context = NULL;
729 		session_p->digest.flags = 0;
730 	}
731 	(void) pthread_mutex_unlock(&session_p->session_mutex);
732 	soft_cleanup_object(key);
733 	free(key);
734 
735 clean1:
736 	return (rv);
737 }
738 
739 
740 CK_RV
741 soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
742     CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
743     CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
744 {
745 
746 	CK_RV rv = CKR_OK;
747 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
748 	soft_object_t *key = rsa_ctx->key;
749 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
750 	uint32_t modulus_len = sizeof (modulus);
751 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
752 
753 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
754 	if (rv != CKR_OK) {
755 		goto clean_exit;
756 	}
757 
758 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
759 		rv = CKR_SIGNATURE_LEN_RANGE;
760 		goto clean_exit;
761 	}
762 
763 	/*
764 	 * Perform RSA decryption with the signer's RSA public key
765 	 * for verification process.
766 	 */
767 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
768 	if (rv == CKR_OK) {
769 		switch (mechanism) {
770 
771 		case CKM_RSA_PKCS:
772 		case CKM_MD5_RSA_PKCS:
773 		case CKM_SHA1_RSA_PKCS:
774 		case CKM_SHA256_RSA_PKCS:
775 		case CKM_SHA384_RSA_PKCS:
776 		case CKM_SHA512_RSA_PKCS:
777 		{
778 			/*
779 			 * Strip off the encoded padding bytes in front of the
780 			 * recovered data, then compare the recovered data with
781 			 * the original data.
782 			 */
783 			int data_len = modulus_len;
784 
785 			rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
786 			if (rv != CKR_OK) {
787 				goto clean_exit;
788 			}
789 
790 			if ((CK_ULONG)data_len != ulDataLen) {
791 				rv = CKR_SIGNATURE_LEN_RANGE;
792 				goto clean_exit;
793 			} else if (memcmp(pData,
794 			    &plain_data[modulus_len - data_len],
795 			    ulDataLen) != 0) {
796 				rv = CKR_SIGNATURE_INVALID;
797 				goto clean_exit;
798 			}
799 			break;
800 		}
801 
802 		case CKM_RSA_X_509:
803 			/*
804 			 * Strip off the encoded padding bytes in front of the
805 			 * recovered plain_data, then compare the input data
806 			 * with the recovered data.
807 			 */
808 			if (memcmp(pData,
809 			    plain_data + ulSignatureLen - ulDataLen,
810 			    ulDataLen) != 0) {
811 				rv = CKR_SIGNATURE_INVALID;
812 				goto clean_exit;
813 			}
814 			break;
815 		}
816 	}
817 
818 	if (rv == CKR_DATA_LEN_RANGE) {
819 		if ((mechanism == CKM_MD5_RSA_PKCS) ||
820 		    (mechanism == CKM_SHA1_RSA_PKCS) ||
821 		    (mechanism == CKM_SHA256_RSA_PKCS) ||
822 		    (mechanism == CKM_SHA384_RSA_PKCS) ||
823 		    (mechanism == CKM_SHA512_RSA_PKCS))
824 			rv = CKR_SIGNATURE_INVALID;
825 	}
826 
827 clean_exit:
828 	(void) pthread_mutex_lock(&session_p->session_mutex);
829 	free(session_p->verify.context);
830 	session_p->verify.context = NULL;
831 	if (session_p->digest.context != NULL) {
832 		free(session_p->digest.context);
833 		session_p->digest.context = NULL;
834 		session_p->digest.flags = 0;
835 	}
836 	(void) pthread_mutex_unlock(&session_p->session_mutex);
837 	soft_cleanup_object(key);
838 	free(key);
839 	return (rv);
840 }
841 
842 CK_RV
843 soft_genRSAkey_set_attribute(soft_object_t *key, RSAkey *rsakey,
844     CK_ATTRIBUTE_TYPE type, uint32_t modulus_len, boolean_t public)
845 {
846 
847 	uchar_t	*buf, *buf1;
848 	uint32_t buflen;
849 	CK_RV rv = CKR_OK;
850 	biginteger_t *dst = NULL;
851 	biginteger_t src;
852 
853 	/*
854 	 * Allocate the buffer used to store the value of key fields
855 	 * for bignum2bytestring. Since bignum only deals with a buffer
856 	 * whose size is multiple of sizeof (BIG_CHUNK_TYPE),
857 	 * modulus_len is rounded up to be multiple of that.
858 	 */
859 	if ((buf1 = malloc((modulus_len + sizeof (BIG_CHUNK_TYPE) - 1) &
860 	    ~(sizeof (BIG_CHUNK_TYPE) - 1))) == NULL) {
861 		rv = CKR_HOST_MEMORY;
862 		goto cleanexit;
863 	}
864 
865 	buf = buf1;
866 
867 	switch (type) {
868 
869 	case CKA_MODULUS:
870 
871 		buflen = rsakey->n.len * (int)sizeof (BIG_CHUNK_TYPE);
872 		bignum2bytestring(buf, &rsakey->n, buflen);
873 		if (public)
874 			dst = OBJ_PUB_RSA_MOD(key);
875 		else
876 			dst = OBJ_PRI_RSA_MOD(key);
877 		break;
878 
879 	case CKA_PUBLIC_EXPONENT:
880 
881 		buflen = rsakey->e.len * (int)sizeof (BIG_CHUNK_TYPE);
882 		bignum2bytestring(buf, &rsakey->e, buflen);
883 		if (public)
884 			dst = OBJ_PUB_RSA_PUBEXPO(key);
885 		else
886 			dst = OBJ_PRI_RSA_PUBEXPO(key);
887 		break;
888 
889 	case CKA_PRIVATE_EXPONENT:
890 
891 		buflen = rsakey->d.len * (int)sizeof (BIG_CHUNK_TYPE);
892 		bignum2bytestring(buf, &rsakey->d, buflen);
893 		dst = OBJ_PRI_RSA_PRIEXPO(key);
894 		break;
895 
896 	case CKA_PRIME_1:
897 
898 		buflen = rsakey->q.len * (int)sizeof (BIG_CHUNK_TYPE);
899 		bignum2bytestring(buf, &rsakey->q, buflen);
900 		dst = OBJ_PRI_RSA_PRIME1(key);
901 		break;
902 
903 	case CKA_PRIME_2:
904 
905 		buflen = rsakey->p.len * (int)sizeof (BIG_CHUNK_TYPE);
906 		bignum2bytestring(buf, &rsakey->p, buflen);
907 		dst = OBJ_PRI_RSA_PRIME2(key);
908 		break;
909 
910 	case CKA_EXPONENT_1:
911 
912 		buflen = rsakey->dmodqminus1.len *
913 		    (int)sizeof (BIG_CHUNK_TYPE);
914 		bignum2bytestring(buf, &rsakey->dmodqminus1, buflen);
915 		dst = OBJ_PRI_RSA_EXPO1(key);
916 		break;
917 
918 	case CKA_EXPONENT_2:
919 
920 		buflen = rsakey->dmodpminus1.len *
921 		    (int)sizeof (BIG_CHUNK_TYPE);
922 		bignum2bytestring(buf, &rsakey->dmodpminus1, buflen);
923 		dst = OBJ_PRI_RSA_EXPO2(key);
924 		break;
925 
926 	case CKA_COEFFICIENT:
927 
928 		buflen = rsakey->pinvmodq.len * (int)sizeof (BIG_CHUNK_TYPE);
929 		bignum2bytestring(buf, &rsakey->pinvmodq, buflen);
930 		dst = OBJ_PRI_RSA_COEF(key);
931 		break;
932 	}
933 
934 	while (buf[0] == 0) {	/* remove proceeding 0x00 */
935 		buf++;
936 		buflen--;
937 	}
938 
939 	src.big_value_len = buflen;
940 
941 	if ((src.big_value = malloc(buflen)) == NULL) {
942 		rv = CKR_HOST_MEMORY;
943 		goto cleanexit;
944 	}
945 	(void) memcpy(src.big_value, buf, buflen);
946 
947 	/* Copy the attribute in the key object. */
948 	copy_bigint_attr(&src, dst);
949 
950 cleanexit:
951 	free(buf1);
952 	return (rv);
953 
954 }
955 
956 
957 CK_RV
958 generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM * pubexp,
959     boolean_t token_obj)
960 {
961 	CK_RV		rv = CKR_OK;
962 
963 /* EXPORT DELETE START */
964 
965 	BIGNUM		a, b, c, d, e, f, g, h;
966 	int		len, keylen, size;
967 	BIG_ERR_CODE	brv = BIG_OK;
968 
969 	size = psize + qsize;
970 	keylen = BITLEN2BIGNUMLEN(size);
971 	len = keylen * 2 + 1;
972 	key->size = size;
973 
974 	a.malloced = 0;
975 	b.malloced = 0;
976 	c.malloced = 0;
977 	d.malloced = 0;
978 	e.malloced = 0;
979 	f.malloced = 0;
980 	g.malloced = 0;
981 	h.malloced = 0;
982 
983 	if ((big_init(&a, len) != BIG_OK) ||
984 	    (big_init(&b, len) != BIG_OK) ||
985 	    (big_init(&c, len) != BIG_OK) ||
986 	    (big_init(&d, len) != BIG_OK) ||
987 	    (big_init(&e, len) != BIG_OK) ||
988 	    (big_init(&f, len) != BIG_OK) ||
989 	    (big_init(&g, len) != BIG_OK) ||
990 	    (big_init(&h, len) != BIG_OK)) {
991 		big_finish(&h);
992 		big_finish(&g);
993 		big_finish(&f);
994 		big_finish(&e);
995 		big_finish(&d);
996 		big_finish(&c);
997 		big_finish(&b);
998 		big_finish(&a);
999 
1000 		return (CKR_HOST_MEMORY);
1001 	}
1002 nextp:
1003 	if ((brv = random_bignum(&a, psize, token_obj)) != BIG_OK) {
1004 		goto ret;
1005 	}
1006 
1007 	if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
1008 		goto ret;
1009 	}
1010 	(void) big_sub_pos(&a, &b, &big_One);
1011 	if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
1012 		goto ret;
1013 	}
1014 	if (big_cmp_abs(&f, &big_One) != 0) {
1015 		goto nextp;
1016 	}
1017 
1018 	if ((brv = random_bignum(&c, qsize, token_obj)) != BIG_OK) {
1019 		goto ret;
1020 	}
1021 
1022 nextq:
1023 	(void) big_add(&a, &c, &big_Two);
1024 
1025 	if (big_bitlength(&a) != qsize) {
1026 		goto nextp;
1027 	}
1028 	if (big_cmp_abs(&a, &b) == 0) {
1029 		goto nextp;
1030 	}
1031 	if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
1032 		goto ret;
1033 	}
1034 	if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
1035 		goto ret;
1036 	}
1037 	if (big_bitlength(&g) != size) {
1038 		goto nextp;
1039 	}
1040 
1041 	(void) big_sub_pos(&a, &b, &big_One);
1042 	(void) big_sub_pos(&d, &c, &big_One);
1043 
1044 	if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
1045 		goto ret;
1046 	}
1047 	if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
1048 		goto ret;
1049 	}
1050 	if (big_cmp_abs(&f, &big_One) != 0) {
1051 		goto nextq;
1052 	} else {
1053 		(void) big_copy(&e, pubexp);
1054 	}
1055 	if (d.sign == -1) {
1056 		if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
1057 			goto ret;
1058 		}
1059 	}
1060 	(void) big_copy(&(key->p), &b);
1061 	(void) big_copy(&(key->q), &c);
1062 	(void) big_copy(&(key->n), &g);
1063 	(void) big_copy(&(key->d), &d);
1064 	(void) big_copy(&(key->e), &e);
1065 
1066 	if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
1067 		goto ret;
1068 	}
1069 	if (f.sign == -1) {
1070 		if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
1071 			goto ret;
1072 		}
1073 	}
1074 	(void) big_copy(&(key->pinvmodq), &f);
1075 
1076 	(void) big_sub(&a, &b, &big_One);
1077 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
1078 		goto ret;
1079 	}
1080 	(void) big_copy(&(key->dmodpminus1), &f);
1081 	(void) big_sub(&a, &c, &big_One);
1082 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
1083 		goto ret;
1084 	}
1085 	(void) big_copy(&(key->dmodqminus1), &f);
1086 
1087 	if ((brv = random_bignum(&h, size, token_obj)) != BIG_OK) {
1088 		goto ret;
1089 	}
1090 	if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
1091 		goto ret;
1092 	}
1093 	if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
1094 		goto ret;
1095 	}
1096 
1097 	if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
1098 		goto ret;
1099 	}
1100 
1101 	if (big_cmp_abs(&b, &h) != 0) {
1102 		rv = generate_rsa_key(key, psize, qsize, pubexp, token_obj);
1103 		goto ret1;
1104 	} else {
1105 		brv = BIG_OK;
1106 	}
1107 
1108 ret:
1109 	rv = convert_rv(brv);
1110 ret1:
1111 	big_finish(&h);
1112 	big_finish(&g);
1113 	big_finish(&f);
1114 	big_finish(&e);
1115 	big_finish(&d);
1116 	big_finish(&c);
1117 	big_finish(&b);
1118 	big_finish(&a);
1119 
1120 /* EXPORT DELETE END */
1121 
1122 	return (rv);
1123 }
1124 
1125 
1126 CK_RV
1127 soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
1128 {
1129 
1130 	CK_RV rv = CKR_OK;
1131 	uint32_t modulus_len;
1132 	uchar_t	pub_expo[MAX_KEY_ATTR_BUFLEN];
1133 	uint32_t pub_expo_len = sizeof (pub_expo);
1134 	BIGNUM	public_exponent = {0};
1135 	RSAkey	rsakey = {0};
1136 	CK_ATTRIBUTE template;
1137 
1138 	if ((pubkey == NULL) || (prikey == NULL)) {
1139 		return (CKR_ARGUMENTS_BAD);
1140 	}
1141 
1142 	template.pValue = malloc(sizeof (CK_ULONG));
1143 
1144 	if (template.pValue == NULL) {
1145 		return (CKR_HOST_MEMORY);
1146 	}
1147 
1148 	template.ulValueLen = sizeof (CK_ULONG);
1149 
1150 	rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
1151 	    &template);
1152 
1153 	if (rv != CKR_OK) {
1154 		goto clean0;
1155 	}
1156 
1157 #ifdef	__sparcv9
1158 	/* LINTED */
1159 	modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
1160 #else	/* !__sparcv9 */
1161 	modulus_len = *((CK_ULONG *)(template.pValue));
1162 #endif	/* __sparcv9 */
1163 
1164 	/* Convert modulus length from bit length to byte length. */
1165 	modulus_len = (modulus_len + 7) / 8;
1166 
1167 	/* Modulus length needs to be between min key size and max key size. */
1168 	if ((modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES) ||
1169 	    (modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)) {
1170 		rv = CKR_ATTRIBUTE_VALUE_INVALID;
1171 		goto clean0;
1172 	}
1173 
1174 	rv = soft_get_public_value(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
1175 	    &pub_expo_len);
1176 	if (rv != CKR_OK) {
1177 		goto clean0;
1178 	}
1179 
1180 	/* Create a public exponent in bignum format. */
1181 	if (big_init(&public_exponent, CHARLEN2BIGNUMLEN(modulus_len)) !=
1182 	    BIG_OK) {
1183 		rv = CKR_HOST_MEMORY;
1184 		goto clean0;
1185 	}
1186 	bytestring2bignum(&public_exponent, pub_expo, pub_expo_len);
1187 
1188 	if (RSA_key_init(&rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
1189 		rv = CKR_HOST_MEMORY;
1190 		goto clean2;
1191 	}
1192 
1193 	/* Generate RSA key pair. */
1194 	if ((rv = generate_rsa_key(&rsakey, modulus_len * 4, modulus_len * 4,
1195 	    &public_exponent, (IS_TOKEN_OBJECT(pubkey) ||
1196 	    IS_TOKEN_OBJECT(prikey)))) != CKR_OK) {
1197 		goto clean3;
1198 	}
1199 
1200 	/*
1201 	 * Add modulus in public template, and add all eight key fields
1202 	 * in private template.
1203 	 */
1204 	if ((rv = soft_genRSAkey_set_attribute(pubkey, &rsakey,
1205 	    CKA_MODULUS, modulus_len, B_TRUE)) != CKR_OK) {
1206 		goto clean3;
1207 	}
1208 
1209 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1210 	    CKA_MODULUS, modulus_len, B_FALSE)) != CKR_OK) {
1211 		goto clean3;
1212 	}
1213 
1214 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1215 	    CKA_PRIVATE_EXPONENT, modulus_len, B_FALSE)) != CKR_OK) {
1216 		goto clean3;
1217 	}
1218 
1219 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1220 	    CKA_PUBLIC_EXPONENT, modulus_len, B_FALSE)) != CKR_OK) {
1221 		goto clean3;
1222 	}
1223 
1224 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1225 	    CKA_PRIME_1, modulus_len, B_FALSE)) != CKR_OK) {
1226 		goto clean3;
1227 	}
1228 
1229 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1230 	    CKA_PRIME_2, modulus_len, B_FALSE)) != CKR_OK) {
1231 		goto clean3;
1232 	}
1233 
1234 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1235 	    CKA_EXPONENT_1, modulus_len, B_FALSE)) != CKR_OK) {
1236 		goto clean3;
1237 	}
1238 
1239 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1240 	    CKA_EXPONENT_2, modulus_len, B_FALSE)) != CKR_OK) {
1241 		goto clean3;
1242 	}
1243 
1244 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1245 	    CKA_COEFFICIENT, modulus_len, B_FALSE)) != CKR_OK) {
1246 		goto clean3;
1247 	}
1248 
1249 clean3:
1250 	RSA_key_finish(&rsakey);
1251 clean2:
1252 	big_finish(&public_exponent);
1253 clean0:
1254 	free(template.pValue);
1255 
1256 	return (rv);
1257 }
1258 
1259 
1260 CK_ULONG
1261 get_rsa_sha1_prefix(CK_MECHANISM_PTR mech, CK_BYTE_PTR *prefix) {
1262 	if (mech->pParameter == NULL) {
1263 		*prefix = (CK_BYTE *)SHA1_DER_PREFIX;
1264 		return (SHA1_DER_PREFIX_Len);
1265 	}
1266 
1267 	*prefix = (CK_BYTE *)SHA1_DER_PREFIX_OID;
1268 	return (SHA1_DER_PREFIX_OID_Len);
1269 }
1270 
1271 CK_RV
1272 soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1273     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1274     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1275 {
1276 
1277 	CK_RV rv = CKR_OK;
1278 	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space enough for all mechs */
1279 	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
1280 	/* space enough for all mechs */
1281 	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1282 	CK_ULONG der_data_len;
1283 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
1284 	soft_object_t *key = rsa_ctx->key;
1285 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1286 	uint32_t modulus_len = sizeof (modulus);
1287 	CK_ULONG der_len;
1288 	CK_BYTE_PTR der_prefix;
1289 
1290 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
1291 	if (rv != CKR_OK) {
1292 		(void) pthread_mutex_lock(&session_p->session_mutex);
1293 		free(session_p->digest.context);
1294 		session_p->digest.context = NULL;
1295 		session_p->digest.flags = 0;
1296 		(void) pthread_mutex_unlock(&session_p->session_mutex);
1297 		soft_cleanup_object(key);
1298 		free(key);
1299 		goto clean1;
1300 	}
1301 
1302 	/* Check arguments before performing message digest. */
1303 	if (pSigned == NULL) {
1304 		/* Application asks for the length of the output buffer. */
1305 		*pulSignedLen = modulus_len;
1306 		rv = CKR_OK;
1307 		goto clean1;
1308 	}
1309 
1310 	/* Is the application-supplied buffer large enough? */
1311 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
1312 		*pulSignedLen = modulus_len;
1313 		rv = CKR_BUFFER_TOO_SMALL;
1314 		goto clean1;
1315 	}
1316 
1317 	if (Final) {
1318 		rv = soft_digest_final(session_p, hash, &hash_len);
1319 	} else {
1320 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1321 	}
1322 
1323 	if (rv != CKR_OK) {
1324 		/* free the signature key */
1325 		soft_cleanup_object(key);
1326 		free(key);
1327 		goto clean_exit;
1328 	}
1329 
1330 	/*
1331 	 * Prepare the DER encoding of the DigestInfo value by setting it to:
1332 	 *	<MECH>_DER_PREFIX || H
1333 	 *
1334 	 * See rsa_impl.c for more details.
1335 	 */
1336 	switch (session_p->digest.mech.mechanism) {
1337 	case CKM_MD5:
1338 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1339 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1340 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1341 		break;
1342 	case CKM_SHA_1:
1343 		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1344 		    &der_prefix);
1345 		(void) memcpy(der_data, der_prefix, der_len);
1346 		(void) memcpy(der_data + der_len, hash, hash_len);
1347 		der_data_len = der_len + hash_len;
1348 		break;
1349 	case CKM_SHA256:
1350 		(void) memcpy(der_data, SHA256_DER_PREFIX,
1351 		    SHA2_DER_PREFIX_Len);
1352 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1353 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1354 		break;
1355 	case CKM_SHA384:
1356 		(void) memcpy(der_data, SHA384_DER_PREFIX,
1357 		    SHA2_DER_PREFIX_Len);
1358 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1359 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1360 		break;
1361 	case CKM_SHA512:
1362 		(void) memcpy(der_data, SHA512_DER_PREFIX,
1363 		    SHA2_DER_PREFIX_Len);
1364 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1365 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1366 		break;
1367 	}
1368 
1369 	/*
1370 	 * Now, we are ready to sign the DER_ENCODED data
1371 	 * soft_rsa_sign_common() will free the signature key.
1372 	 */
1373 	rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
1374 	    pSigned, pulSignedLen, mechanism);
1375 
1376 clean_exit:
1377 	(void) pthread_mutex_lock(&session_p->session_mutex);
1378 	/* soft_digest_common() has freed the digest context */
1379 	session_p->digest.flags = 0;
1380 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1381 
1382 clean1:
1383 	return (rv);
1384 }
1385 
1386 
1387 CK_RV
1388 soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1389     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1390     CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1391 {
1392 
1393 	CK_RV rv = CKR_OK;
1394 	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space for all mechs */
1395 	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
1396 	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1397 	CK_ULONG der_data_len;
1398 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1399 	soft_object_t *key = rsa_ctx->key;
1400 	CK_ULONG der_len;
1401 	CK_BYTE_PTR der_prefix;
1402 
1403 	if (Final) {
1404 		rv = soft_digest_final(session_p, hash, &hash_len);
1405 	} else {
1406 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1407 	}
1408 
1409 	if (rv != CKR_OK) {
1410 		/* free the verification key */
1411 		soft_cleanup_object(key);
1412 		free(key);
1413 		goto clean_exit;
1414 	}
1415 
1416 	/*
1417 	 * Prepare the DER encoding of the DigestInfo value as follows:
1418 	 * MD5:		MD5_DER_PREFIX || H
1419 	 * SHA-1:	SHA1_DER_PREFIX || H
1420 	 * SHA2:	SHA2_DER_PREFIX || H
1421 	 *
1422 	 * See rsa_impl.c for more details.
1423 	 */
1424 	switch (session_p->digest.mech.mechanism) {
1425 	case CKM_MD5:
1426 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1427 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1428 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1429 		break;
1430 	case CKM_SHA_1:
1431 		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1432 		    &der_prefix);
1433 		(void) memcpy(der_data, der_prefix, der_len);
1434 		(void) memcpy(der_data + der_len, hash, hash_len);
1435 		der_data_len = der_len + hash_len;
1436 		break;
1437 	case CKM_SHA256:
1438 		(void) memcpy(der_data, SHA256_DER_PREFIX,
1439 		    SHA2_DER_PREFIX_Len);
1440 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1441 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1442 		break;
1443 	case CKM_SHA384:
1444 		(void) memcpy(der_data, SHA384_DER_PREFIX,
1445 		    SHA2_DER_PREFIX_Len);
1446 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1447 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1448 		break;
1449 	case CKM_SHA512:
1450 		(void) memcpy(der_data, SHA512_DER_PREFIX,
1451 		    SHA2_DER_PREFIX_Len);
1452 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1453 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1454 		break;
1455 	}
1456 
1457 	/*
1458 	 * Now, we are ready to verify the DER_ENCODED data using signature.
1459 	 * soft_rsa_verify_common() will free the verification key.
1460 	 */
1461 	rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
1462 	    pSigned, ulSignedLen, mechanism);
1463 
1464 clean_exit:
1465 	(void) pthread_mutex_lock(&session_p->session_mutex);
1466 	/* soft_digest_common() has freed the digest context */
1467 	session_p->digest.flags = 0;
1468 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1469 
1470 	return (rv);
1471 
1472 }
1473 
1474 
1475 CK_RV
1476 soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
1477     CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
1478 {
1479 
1480 	CK_RV rv = CKR_OK;
1481 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1482 	CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
1483 	soft_object_t *key = rsa_ctx->key;
1484 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1485 	uint32_t modulus_len = sizeof (modulus);
1486 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1487 
1488 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
1489 	if (rv != CKR_OK) {
1490 		goto clean_exit;
1491 	}
1492 
1493 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
1494 		rv = CKR_SIGNATURE_LEN_RANGE;
1495 		goto clean_exit;
1496 	}
1497 
1498 	/*
1499 	 * Perform RSA decryption with the signer's RSA public key
1500 	 * for verification process.
1501 	 */
1502 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
1503 	if (rv == CKR_OK) {
1504 		switch (mechanism) {
1505 
1506 		case CKM_RSA_PKCS:
1507 		{
1508 			/*
1509 			 * Strip off the encoded padding bytes in front of the
1510 			 * recovered data.
1511 			 */
1512 			int data_len = modulus_len;
1513 
1514 			rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1515 			if (rv != CKR_OK) {
1516 				goto clean_exit;
1517 			}
1518 
1519 			/*
1520 			 * If application asks for the length of the output
1521 			 * buffer?
1522 			 */
1523 			if (pData == NULL) {
1524 				*pulDataLen = data_len;
1525 				rv = CKR_OK;
1526 				goto clean1;
1527 			}
1528 
1529 			/* Is the application-supplied buffer large enough? */
1530 			if (*pulDataLen < (CK_ULONG)data_len) {
1531 				*pulDataLen = data_len;
1532 				rv = CKR_BUFFER_TOO_SMALL;
1533 				goto clean1;
1534 			}
1535 
1536 			(void) memcpy(pData,
1537 			    &plain_data[modulus_len - data_len], data_len);
1538 			*pulDataLen = data_len;
1539 
1540 			break;
1541 		}
1542 
1543 		case CKM_RSA_X_509:
1544 			/*
1545 			 * If application asks for the length of the output
1546 			 * buffer?
1547 			 */
1548 			if (pData == NULL) {
1549 				*pulDataLen = modulus_len;
1550 				rv = CKR_OK;
1551 				goto clean1;
1552 			}
1553 
1554 			/* Is the application-supplied buffer large enough? */
1555 			if (*pulDataLen < (CK_ULONG)modulus_len) {
1556 				*pulDataLen = modulus_len;
1557 				rv = CKR_BUFFER_TOO_SMALL;
1558 				goto clean1;
1559 			}
1560 
1561 			(void) memcpy(pData, plain_data, modulus_len);
1562 			*pulDataLen = modulus_len;
1563 
1564 			break;
1565 		}
1566 	}
1567 
1568 clean_exit:
1569 	(void) pthread_mutex_lock(&session_p->session_mutex);
1570 	free(session_p->verify.context);
1571 	session_p->verify.context = NULL;
1572 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1573 	soft_cleanup_object(key);
1574 	free(key);
1575 
1576 clean1:
1577 	return (rv);
1578 }
1579