xref: /titanic_52/usr/src/lib/pkcs11/pkcs11_softtoken/common/softRSA.c (revision bdfc6d18da790deeec2e0eb09c625902defe2498)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <pthread.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <strings.h>
33 #include <sys/types.h>
34 #include <security/cryptoki.h>
35 #include <bignum.h>
36 #include "softGlobal.h"
37 #include "softSession.h"
38 #include "softObject.h"
39 #include "softOps.h"
40 #include "softRSA.h"
41 #include "softMAC.h"
42 #include "softRandom.h"
43 #include "softCrypt.h"
44 
45 CK_RV
46 soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
47     CK_BYTE_PTR out, int realpublic)
48 {
49 
50 	CK_RV rv = CKR_OK;
51 
52 /* EXPORT DELETE START */
53 
54 	uchar_t expo[MAX_KEY_ATTR_BUFLEN];
55 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
56 	uint32_t expo_len = sizeof (expo);
57 	uint32_t modulus_len = sizeof (modulus);
58 	BIGNUM msg;
59 	RSAkey *rsakey;
60 
61 	if (realpublic) {
62 		rv = soft_get_public_attr(key, CKA_PUBLIC_EXPONENT, expo,
63 		    &expo_len);
64 		if (rv != CKR_OK) {
65 			goto clean1;
66 		}
67 	} else {
68 		rv = soft_get_private_attr(key, CKA_PRIVATE_EXPONENT, expo,
69 		    &expo_len);
70 		if (rv != CKR_OK) {
71 			goto clean1;
72 		}
73 	}
74 
75 	rv = soft_get_public_attr(key, CKA_MODULUS, modulus, &modulus_len);
76 	if (rv != CKR_OK) {
77 		goto clean1;
78 	}
79 
80 	if (expo_len > modulus_len) {
81 		rv = CKR_KEY_SIZE_RANGE;
82 		goto clean1;
83 	}
84 
85 	rsakey = calloc(1, sizeof (RSAkey));
86 	if (rsakey == NULL) {
87 		rv = CKR_HOST_MEMORY;
88 		goto clean1;
89 	}
90 
91 	if (RSA_key_init(rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
92 		rv = CKR_HOST_MEMORY;
93 		goto clean4;
94 	}
95 
96 	/* Size for big_init is in (32-bit) words. */
97 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
98 	    (int)sizeof (uint32_t)) != BIG_OK) {
99 		rv = CKR_HOST_MEMORY;
100 		goto clean5;
101 	}
102 
103 	/* Convert octet string exponent to big integer format. */
104 	bytestring2bignum(&(rsakey->e), expo, expo_len);
105 
106 	/* Convert octet string modulus to big integer format. */
107 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
108 
109 	/* Convert octet string input data to big integer format. */
110 	bytestring2bignum(&msg, (uchar_t *)in, in_len);
111 
112 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
113 		rv = CKR_DATA_LEN_RANGE;
114 		goto clean6;
115 	}
116 
117 	/* Perform RSA computation on big integer input data. */
118 	if (big_modexp(&msg, &msg, &(rsakey->e), &(rsakey->n), NULL) !=
119 	    BIG_OK) {
120 		rv = CKR_HOST_MEMORY;
121 		goto clean6;
122 	}
123 
124 	/* Convert the big integer output data to octet string. */
125 	bignum2bytestring((uchar_t *)out, &msg, modulus_len);
126 
127 clean6:
128 	big_finish(&msg);
129 clean5:
130 	RSA_key_finish(rsakey);
131 clean4:
132 	free(rsakey);
133 clean1:
134 
135 /* EXPORT DELETE END */
136 
137 	return (rv);
138 }
139 
140 
141 CK_RV
142 soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
143     CK_BYTE_PTR out)
144 {
145 
146 	CK_RV rv = CKR_OK;
147 
148 /* EXPORT DELETE START */
149 
150 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
151 	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
152 	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
153 	uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
154 	uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
155 	uchar_t coef[MAX_KEY_ATTR_BUFLEN];
156 	uint32_t modulus_len = sizeof (modulus);
157 	uint32_t prime1_len = sizeof (prime1);
158 	uint32_t prime2_len = sizeof (prime2);
159 	uint32_t expo1_len = sizeof (expo1);
160 	uint32_t expo2_len = sizeof (expo2);
161 	uint32_t coef_len = sizeof (coef);
162 	BIGNUM msg;
163 	RSAkey *rsakey;
164 
165 	rv = soft_get_private_attr(key, CKA_MODULUS, modulus, &modulus_len);
166 	if (rv != CKR_OK) {
167 		goto clean1;
168 	}
169 
170 	rv = soft_get_private_attr(key, CKA_PRIME_1, prime1, &prime1_len);
171 
172 	if ((prime1_len == 0) && (rv == CKR_OK)) {
173 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
174 		goto clean1;
175 	} else {
176 		if (rv != CKR_OK)
177 			goto clean1;
178 	}
179 
180 	rv = soft_get_private_attr(key, CKA_PRIME_2, prime2, &prime2_len);
181 
182 	if ((prime2_len == 0) && (rv == CKR_OK)) {
183 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
184 		goto clean1;
185 	} else {
186 		if (rv != CKR_OK)
187 			goto clean1;
188 	}
189 
190 	rv = soft_get_private_attr(key, CKA_EXPONENT_1, expo1, &expo1_len);
191 
192 	if ((expo1_len == 0) && (rv == CKR_OK)) {
193 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
194 		goto clean1;
195 	} else {
196 		if (rv != CKR_OK)
197 			goto clean1;
198 	}
199 
200 	rv = soft_get_private_attr(key, CKA_EXPONENT_2, expo2, &expo2_len);
201 
202 	if ((expo2_len == 0) && (rv == CKR_OK)) {
203 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
204 		goto clean1;
205 	} else {
206 		if (rv != CKR_OK)
207 			goto clean1;
208 	}
209 
210 	rv = soft_get_private_attr(key, CKA_COEFFICIENT, coef, &coef_len);
211 
212 	if ((coef_len == 0) && (rv == CKR_OK)) {
213 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
214 		goto clean1;
215 	} else {
216 		if (rv != CKR_OK)
217 			goto clean1;
218 	}
219 
220 	rsakey = calloc(1, sizeof (RSAkey));
221 	if (rsakey == NULL) {
222 		rv = CKR_HOST_MEMORY;
223 		goto clean1;
224 	}
225 
226 	/* psize and qsize for RSA_key_init is in bits. */
227 	if (RSA_key_init(rsakey, prime2_len * 8, prime1_len * 8) != BIG_OK) {
228 		rv = CKR_HOST_MEMORY;
229 		goto clean8;
230 	}
231 
232 	/* Size for big_init is in (32-bit) words. */
233 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
234 	    (int)sizeof (uint32_t)) != BIG_OK) {
235 		rv = CKR_HOST_MEMORY;
236 		goto clean9;
237 	}
238 
239 	/* Convert octet string input data to big integer format. */
240 	bytestring2bignum(&msg, (uchar_t *)in, in_len);
241 
242 	/* Convert octet string modulus to big integer format. */
243 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
244 
245 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
246 		rv = CKR_DATA_LEN_RANGE;
247 		goto clean10;
248 	}
249 
250 	/* Convert the rest of private key attributes to big integer format. */
251 	bytestring2bignum(&(rsakey->dmodpminus1), expo2, expo2_len);
252 	bytestring2bignum(&(rsakey->dmodqminus1), expo1, expo1_len);
253 	bytestring2bignum(&(rsakey->p), prime2, prime2_len);
254 	bytestring2bignum(&(rsakey->q), prime1, prime1_len);
255 	bytestring2bignum(&(rsakey->pinvmodq), coef, coef_len);
256 
257 	if ((big_cmp_abs(&(rsakey->dmodpminus1), &(rsakey->p)) > 0) ||
258 	    (big_cmp_abs(&(rsakey->dmodqminus1), &(rsakey->q)) > 0) ||
259 	    (big_cmp_abs(&(rsakey->pinvmodq), &(rsakey->q)) > 0)) {
260 		rv = CKR_KEY_SIZE_RANGE;
261 		goto clean10;
262 	}
263 
264 	/* Perform RSA computation on big integer input data. */
265 	if (big_modexp_crt(&msg, &msg, &(rsakey->dmodpminus1),
266 	    &(rsakey->dmodqminus1), &(rsakey->p), &(rsakey->q),
267 	    &(rsakey->pinvmodq), NULL, NULL) != BIG_OK) {
268 		rv = CKR_HOST_MEMORY;
269 		goto clean10;
270 	}
271 
272 	/* Convert the big integer output data to octet string. */
273 	bignum2bytestring((uchar_t *)out, &msg, modulus_len);
274 
275 clean10:
276 	big_finish(&msg);
277 clean9:
278 	RSA_key_finish(rsakey);
279 clean8:
280 	free(rsakey);
281 clean1:
282 
283 /* EXPORT DELETE END */
284 
285 	return (rv);
286 }
287 
288 /*
289  * Allocate a RSA context for the active encryption or decryption operation.
290  * This function is called without the session lock held.
291  */
292 CK_RV
293 soft_rsa_crypt_init_common(soft_session_t *session_p,
294     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
295     boolean_t encrypt)
296 {
297 
298 	soft_rsa_ctx_t *rsa_ctx;
299 	soft_object_t *tmp_key = NULL;
300 	CK_RV rv;
301 
302 	rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
303 	if (rsa_ctx == NULL) {
304 		return (CKR_HOST_MEMORY);
305 	}
306 
307 	/*
308 	 * Make a copy of the encryption or decryption key, and save it
309 	 * in the RSA crypto context since it will be used later for
310 	 * encryption/decryption. We don't want to hold any object reference
311 	 * on this original key while doing encryption/decryption.
312 	 */
313 	(void) pthread_mutex_lock(&key_p->object_mutex);
314 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
315 	    NULL);
316 
317 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
318 		/* Most likely we ran out of space. */
319 		(void) pthread_mutex_unlock(&key_p->object_mutex);
320 		free(rsa_ctx);
321 		return (rv);
322 	}
323 
324 	/* No need to hold the lock on the old object. */
325 	(void) pthread_mutex_unlock(&key_p->object_mutex);
326 	rsa_ctx->key = tmp_key;
327 
328 	(void) pthread_mutex_lock(&session_p->session_mutex);
329 	if (encrypt) {
330 		/* Called by C_EncryptInit. */
331 		session_p->encrypt.context = rsa_ctx;
332 		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
333 	} else {
334 		/* Called by C_DecryptInit. */
335 		session_p->decrypt.context = rsa_ctx;
336 		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
337 	}
338 	(void) pthread_mutex_unlock(&session_p->session_mutex);
339 
340 	return (CKR_OK);
341 }
342 
343 CK_RV
344 soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
345     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
346     CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
347 {
348 
349 	soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
350 	soft_object_t *key = rsa_ctx->key;
351 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
352 	uint32_t modulus_len = sizeof (modulus);
353 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
354 	CK_BYTE	cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
355 	CK_RV rv = CKR_OK;
356 
357 	rv = soft_get_public_attr(key, CKA_MODULUS, modulus, &modulus_len);
358 	if (rv != CKR_OK) {
359 		goto clean_exit;
360 	}
361 
362 	if (pEncrypted == NULL) {
363 		/*
364 		 * Application asks for the length of the output buffer
365 		 * to hold the ciphertext.
366 		 */
367 		*pulEncryptedLen = modulus_len;
368 		rv = CKR_OK;
369 		goto clean1;
370 	}
371 
372 	if (mechanism == CKM_RSA_PKCS) {
373 		/*
374 		 * Input data length needs to be <=
375 		 * modulus length-MIN_PKCS1_PADLEN.
376 		 */
377 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
378 			*pulEncryptedLen = modulus_len;
379 			rv = CKR_DATA_LEN_RANGE;
380 			goto clean_exit;
381 		}
382 	} else {
383 		/* Input data length needs to be <= modulus length. */
384 		if (ulDataLen > (CK_ULONG)modulus_len) {
385 			*pulEncryptedLen = modulus_len;
386 			rv = CKR_DATA_LEN_RANGE;
387 			goto clean_exit;
388 		}
389 	}
390 
391 	/* Is the application-supplied buffer large enough? */
392 	if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
393 		*pulEncryptedLen = modulus_len;
394 		rv = CKR_BUFFER_TOO_SMALL;
395 		goto clean1;
396 	}
397 
398 	if (mechanism == CKM_RSA_PKCS) {
399 		/*
400 		 * Add PKCS padding to the input data to format a block
401 		 * type "02" encryption block.
402 		 */
403 		rv = soft_encrypt_rsa_pkcs_encode(pData, ulDataLen, plain_data,
404 		    modulus_len);
405 
406 		if (rv != CKR_OK)
407 			goto clean_exit;
408 	} else {
409 		/* Pad zeros for the leading bytes of the input data. */
410 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
411 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
412 		    ulDataLen);
413 	}
414 
415 	rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
416 	if (rv == CKR_OK) {
417 		(void) memcpy(pEncrypted, cipher_data, modulus_len);
418 		*pulEncryptedLen = modulus_len;
419 	}
420 
421 clean_exit:
422 	(void) pthread_mutex_lock(&session_p->session_mutex);
423 	free(session_p->encrypt.context);
424 	session_p->encrypt.context = NULL;
425 	(void) pthread_mutex_unlock(&session_p->session_mutex);
426 	soft_cleanup_object(key);
427 	free(key);
428 clean1:
429 	return (rv);
430 }
431 
432 
433 CK_RV
434 soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
435     CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
436     CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
437 {
438 
439 	soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
440 	soft_object_t *key = rsa_ctx->key;
441 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
442 	uint32_t modulus_len = sizeof (modulus);
443 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
444 	CK_RV rv = CKR_OK;
445 
446 	rv = soft_get_private_attr(key, CKA_MODULUS, modulus, &modulus_len);
447 	if (rv != CKR_OK) {
448 		goto clean_exit;
449 	}
450 
451 	if (ulEncryptedLen != (CK_ULONG)modulus_len) {
452 		rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
453 		goto clean_exit;
454 	}
455 
456 	if (pData == NULL) {
457 		/*
458 		 * Application asks for the length of the output buffer
459 		 * to hold the recovered data.
460 		 */
461 		*pulDataLen = modulus_len;
462 		rv = CKR_OK;
463 		goto clean1;
464 	}
465 
466 	if (mechanism == CKM_RSA_X_509) {
467 		if (*pulDataLen < (CK_ULONG)modulus_len) {
468 			*pulDataLen = modulus_len;
469 			rv = CKR_BUFFER_TOO_SMALL;
470 			goto clean1;
471 		}
472 	}
473 
474 	rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
475 	if (rv != CKR_OK) {
476 		goto clean_exit;
477 	}
478 
479 	if (mechanism == CKM_RSA_PKCS) {
480 		int plain_len = modulus_len;
481 		uint32_t num_padding;
482 
483 		/* Strip off the PKCS block formatting data. */
484 		rv = soft_decrypt_rsa_pkcs_decode(plain_data, &plain_len);
485 		if (rv != CKR_OK)
486 			goto clean_exit;
487 
488 		num_padding = modulus_len - plain_len;
489 		if (ulEncryptedLen - num_padding > *pulDataLen) {
490 			*pulDataLen = plain_len;
491 			rv = CKR_BUFFER_TOO_SMALL;
492 			goto clean1;
493 		}
494 
495 		(void) memcpy(pData, &plain_data[num_padding], plain_len);
496 		*pulDataLen = plain_len;
497 	} else {
498 		(void) memcpy(pData, plain_data, modulus_len);
499 		*pulDataLen = modulus_len;
500 	}
501 
502 clean_exit:
503 	(void) pthread_mutex_lock(&session_p->session_mutex);
504 	free(session_p->decrypt.context);
505 	session_p->decrypt.context = NULL;
506 	(void) pthread_mutex_unlock(&session_p->session_mutex);
507 	soft_cleanup_object(key);
508 	free(key);
509 
510 clean1:
511 	return (rv);
512 }
513 
514 /*
515  * Allocate a RSA context for the active sign or verify operation.
516  * This function is called without the session lock held.
517  */
518 CK_RV
519 soft_rsa_sign_verify_init_common(soft_session_t *session_p,
520     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
521     boolean_t sign)
522 {
523 	CK_RV rv = CKR_OK;
524 	soft_rsa_ctx_t *rsa_ctx;
525 	CK_MECHANISM digest_mech;
526 	soft_object_t *tmp_key = NULL;
527 
528 	if (sign) {
529 		if ((key_p->class != CKO_PRIVATE_KEY) ||
530 		    (key_p->key_type != CKK_RSA))
531 			return (CKR_KEY_TYPE_INCONSISTENT);
532 	} else {
533 		if ((key_p->class != CKO_PUBLIC_KEY) ||
534 		    (key_p->key_type != CKK_RSA))
535 			return (CKR_KEY_TYPE_INCONSISTENT);
536 	}
537 
538 	switch (pMechanism->mechanism) {
539 	case CKM_MD5_RSA_PKCS:
540 		digest_mech.mechanism = CKM_MD5;
541 		rv = soft_digest_init_internal(session_p, &digest_mech);
542 		if (rv != CKR_OK)
543 			return (rv);
544 		break;
545 
546 	case CKM_SHA1_RSA_PKCS:
547 		digest_mech.mechanism = CKM_SHA_1;
548 		rv = soft_digest_init_internal(session_p, &digest_mech);
549 		if (rv != CKR_OK)
550 			return (rv);
551 		break;
552 	}
553 
554 	rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));
555 
556 	if (rsa_ctx == NULL) {
557 		rv = CKR_HOST_MEMORY;
558 		goto clean_exit;
559 	}
560 
561 	(void) pthread_mutex_lock(&key_p->object_mutex);
562 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
563 	    NULL);
564 
565 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
566 		/* Most likely we ran out of space. */
567 		(void) pthread_mutex_unlock(&key_p->object_mutex);
568 		free(rsa_ctx);
569 		goto clean_exit;
570 	}
571 
572 	/* No need to hold the lock on the old object. */
573 	(void) pthread_mutex_unlock(&key_p->object_mutex);
574 	rsa_ctx->key = tmp_key;
575 
576 	(void) pthread_mutex_lock(&session_p->session_mutex);
577 
578 	if (sign) {
579 		session_p->sign.context = rsa_ctx;
580 		session_p->sign.mech.mechanism = pMechanism->mechanism;
581 	} else {
582 		session_p->verify.context = rsa_ctx;
583 		session_p->verify.mech.mechanism = pMechanism->mechanism;
584 	}
585 
586 	(void) pthread_mutex_unlock(&session_p->session_mutex);
587 
588 	return (CKR_OK);
589 
590 clean_exit:
591 	(void) pthread_mutex_lock(&session_p->session_mutex);
592 	if (session_p->digest.context != NULL) {
593 		free(session_p->digest.context);
594 		session_p->digest.context = NULL;
595 		session_p->digest.flags = 0;
596 	}
597 	(void) pthread_mutex_unlock(&session_p->session_mutex);
598 	return (rv);
599 
600 }
601 
602 
603 CK_RV
604 soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
605     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
606     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
607 {
608 
609 	CK_RV rv = CKR_OK;
610 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
611 	soft_object_t *key = rsa_ctx->key;
612 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
613 	uint32_t modulus_len = sizeof (modulus);
614 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
615 	CK_BYTE	signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
616 
617 	rv = soft_get_private_attr(key, CKA_MODULUS, modulus, &modulus_len);
618 	if (rv != CKR_OK) {
619 		goto clean_exit;
620 	}
621 
622 	if (pSigned == NULL) {
623 		/* Application asks for the length of the output buffer. */
624 		*pulSignedLen = modulus_len;
625 		rv = CKR_OK;
626 		goto clean1;
627 	}
628 
629 	switch (mechanism) {
630 
631 	case CKM_RSA_PKCS:
632 
633 		/*
634 		 * Input data length needs to be <=
635 		 * modulus length-MIN_PKCS1_PADLEN.
636 		 */
637 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
638 			*pulSignedLen = modulus_len;
639 			rv = CKR_DATA_LEN_RANGE;
640 			goto clean_exit;
641 		}
642 		break;
643 
644 	case CKM_RSA_X_509:
645 
646 		/* Input data length needs to be <= modulus length. */
647 		if (ulDataLen > (CK_ULONG)modulus_len) {
648 			*pulSignedLen = modulus_len;
649 			rv = CKR_DATA_LEN_RANGE;
650 			goto clean_exit;
651 		}
652 		break;
653 	}
654 
655 	/* Is the application-supplied buffer large enough? */
656 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
657 		*pulSignedLen = modulus_len;
658 		rv = CKR_BUFFER_TOO_SMALL;
659 		goto clean1;
660 	}
661 
662 	switch (mechanism) {
663 
664 	case CKM_RSA_PKCS:
665 	case CKM_MD5_RSA_PKCS:
666 	case CKM_SHA1_RSA_PKCS:
667 		/*
668 		 * Add PKCS padding to the input data to format a block
669 		 * type "01" encryption block.
670 		 */
671 		rv = soft_sign_rsa_pkcs_encode(pData, ulDataLen, plain_data,
672 		    modulus_len);
673 
674 		if (rv != CKR_OK) {
675 			goto clean_exit;
676 		}
677 		break;
678 
679 	case CKM_RSA_X_509:
680 
681 		/* Pad zeros for the leading bytes of the input data. */
682 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
683 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
684 		    ulDataLen);
685 		break;
686 	}
687 
688 	/*
689 	 * Perform RSA encryption with the signer's RSA private key
690 	 * for signature process.
691 	 */
692 	rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);
693 
694 	if (rv == CKR_OK) {
695 		(void) memcpy(pSigned, signed_data, modulus_len);
696 		*pulSignedLen = modulus_len;
697 	}
698 
699 clean_exit:
700 	(void) pthread_mutex_lock(&session_p->session_mutex);
701 	free(session_p->sign.context);
702 	session_p->sign.context = NULL;
703 	if (session_p->digest.context != NULL) {
704 		free(session_p->digest.context);
705 		session_p->digest.context = NULL;
706 		session_p->digest.flags = 0;
707 	}
708 	(void) pthread_mutex_unlock(&session_p->session_mutex);
709 	soft_cleanup_object(key);
710 	free(key);
711 
712 clean1:
713 	return (rv);
714 }
715 
716 
717 CK_RV
718 soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
719     CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
720     CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
721 {
722 
723 	CK_RV rv = CKR_OK;
724 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
725 	soft_object_t *key = rsa_ctx->key;
726 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
727 	uint32_t modulus_len = sizeof (modulus);
728 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
729 
730 	rv = soft_get_public_attr(key, CKA_MODULUS, modulus, &modulus_len);
731 	if (rv != CKR_OK) {
732 		goto clean_exit;
733 	}
734 
735 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
736 		rv = CKR_SIGNATURE_LEN_RANGE;
737 		goto clean_exit;
738 	}
739 
740 	/*
741 	 * Perform RSA decryption with the signer's RSA public key
742 	 * for verification process.
743 	 */
744 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
745 	if (rv == CKR_OK) {
746 		switch (mechanism) {
747 
748 		case CKM_RSA_PKCS:
749 		case CKM_MD5_RSA_PKCS:
750 		case CKM_SHA1_RSA_PKCS:
751 		{
752 			/*
753 			 * Strip off the encoded padding bytes in front of the
754 			 * recovered data, then compare the recovered data with
755 			 * the original data.
756 			 */
757 			int data_len = modulus_len;
758 
759 			rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
760 			if (rv != CKR_OK) {
761 				goto clean_exit;
762 			}
763 
764 			if ((CK_ULONG)data_len != ulDataLen) {
765 				rv = CKR_SIGNATURE_LEN_RANGE;
766 				goto clean_exit;
767 			} else if (memcmp(pData,
768 			    &plain_data[modulus_len - data_len],
769 			    ulDataLen) != 0) {
770 				rv = CKR_SIGNATURE_INVALID;
771 				goto clean_exit;
772 			}
773 			break;
774 		}
775 
776 		case CKM_RSA_X_509:
777 			/*
778 			 * Strip off the encoded padding bytes in front of the
779 			 * recovered plain_data, then compare the input data
780 			 * with the recovered data.
781 			 */
782 			if (memcmp(pData,
783 			    plain_data + ulSignatureLen - ulDataLen,
784 			    ulDataLen) != 0) {
785 				rv = CKR_SIGNATURE_INVALID;
786 				goto clean_exit;
787 			}
788 			break;
789 		}
790 	}
791 
792 	if (rv == CKR_DATA_LEN_RANGE) {
793 		if ((mechanism == CKM_MD5_RSA_PKCS) ||
794 		    (mechanism == CKM_SHA1_RSA_PKCS))
795 			rv = CKR_SIGNATURE_INVALID;
796 	}
797 
798 clean_exit:
799 	(void) pthread_mutex_lock(&session_p->session_mutex);
800 	free(session_p->verify.context);
801 	session_p->verify.context = NULL;
802 	if (session_p->digest.context != NULL) {
803 		free(session_p->digest.context);
804 		session_p->digest.context = NULL;
805 		session_p->digest.flags = 0;
806 	}
807 	(void) pthread_mutex_unlock(&session_p->session_mutex);
808 	soft_cleanup_object(key);
809 	free(key);
810 	return (rv);
811 }
812 
813 CK_RV
814 soft_genRSAkey_set_attribute(soft_object_t *key, RSAkey *rsakey,
815     CK_ATTRIBUTE_TYPE type, uint32_t modulus_len, boolean_t public)
816 {
817 
818 	uchar_t	*buf, *buf1;
819 	uint32_t buflen;
820 	CK_RV rv = CKR_OK;
821 	biginteger_t *dst = NULL;
822 	biginteger_t src;
823 
824 	/*
825 	 * Allocate the buffer used to store the value of key fields
826 	 * for bignum2bytestring. Since bignum only deals with a buffer
827 	 * whose size is multiple of 4, modulus_len is rounded up to be
828 	 * multiple of 4.
829 	 */
830 	if ((buf1 = malloc((modulus_len + 3) & ~3)) == NULL) {
831 		rv = CKR_HOST_MEMORY;
832 		goto cleanexit;
833 	}
834 
835 	buf = buf1;
836 
837 	switch (type) {
838 
839 	case CKA_MODULUS:
840 
841 		buflen = rsakey->n.len * (int)sizeof (uint32_t);
842 		bignum2bytestring(buf, &rsakey->n, buflen);
843 		if (public)
844 			dst = OBJ_PUB_RSA_MOD(key);
845 		else
846 			dst = OBJ_PRI_RSA_MOD(key);
847 		break;
848 
849 	case CKA_PUBLIC_EXPONENT:
850 
851 		buflen = rsakey->e.len * (int)sizeof (uint32_t);
852 		bignum2bytestring(buf, &rsakey->e, buflen);
853 		if (public)
854 			dst = OBJ_PUB_RSA_PUBEXPO(key);
855 		else
856 			dst = OBJ_PRI_RSA_PUBEXPO(key);
857 		break;
858 
859 	case CKA_PRIVATE_EXPONENT:
860 
861 		buflen = rsakey->d.len * (int)sizeof (uint32_t);
862 		bignum2bytestring(buf, &rsakey->d, buflen);
863 		dst = OBJ_PRI_RSA_PRIEXPO(key);
864 		break;
865 
866 	case CKA_PRIME_1:
867 
868 		buflen = rsakey->q.len * (int)sizeof (uint32_t);
869 		bignum2bytestring(buf, &rsakey->q, buflen);
870 		dst = OBJ_PRI_RSA_PRIME1(key);
871 		break;
872 
873 	case CKA_PRIME_2:
874 
875 		buflen = rsakey->p.len * (int)sizeof (uint32_t);
876 		bignum2bytestring(buf, &rsakey->p, buflen);
877 		dst = OBJ_PRI_RSA_PRIME2(key);
878 		break;
879 
880 	case CKA_EXPONENT_1:
881 
882 		buflen = rsakey->dmodqminus1.len * (int)sizeof (uint32_t);
883 		bignum2bytestring(buf, &rsakey->dmodqminus1, buflen);
884 		dst = OBJ_PRI_RSA_EXPO1(key);
885 		break;
886 
887 	case CKA_EXPONENT_2:
888 
889 		buflen = rsakey->dmodpminus1.len * (int)sizeof (uint32_t);
890 		bignum2bytestring(buf, &rsakey->dmodpminus1, buflen);
891 		dst = OBJ_PRI_RSA_EXPO2(key);
892 		break;
893 
894 	case CKA_COEFFICIENT:
895 
896 		buflen = rsakey->pinvmodq.len * (int)sizeof (uint32_t);
897 		bignum2bytestring(buf, &rsakey->pinvmodq, buflen);
898 		dst = OBJ_PRI_RSA_COEF(key);
899 		break;
900 	}
901 
902 	while (buf[0] == 0) {	/* remove proceeding 0x00 */
903 		buf++;
904 		buflen--;
905 	}
906 
907 	src.big_value_len = buflen;
908 
909 	if ((src.big_value = malloc(buflen)) == NULL) {
910 		rv = CKR_HOST_MEMORY;
911 		goto cleanexit;
912 	}
913 	(void) memcpy(src.big_value, buf, buflen);
914 
915 	/* Copy the attribute in the key object. */
916 	copy_bigint_attr(&src, dst);
917 
918 cleanexit:
919 	free(buf1);
920 	return (rv);
921 
922 }
923 
924 
925 CK_RV
926 generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM * pubexp,
927     boolean_t token_obj)
928 {
929 	CK_RV		rv = CKR_OK;
930 
931 /* EXPORT DELETE START */
932 
933 	BIGNUM		a, b, c, d, e, f, g, h;
934 	int		len, keylen, size;
935 	BIG_ERR_CODE	brv = BIG_OK;
936 
937 	size = psize + qsize;
938 	keylen = (size + 31) / 32;
939 	len = keylen * 2 + 1;
940 	key->size = size;
941 
942 	a.malloced = 0;
943 	b.malloced = 0;
944 	c.malloced = 0;
945 	d.malloced = 0;
946 	e.malloced = 0;
947 	f.malloced = 0;
948 	g.malloced = 0;
949 	h.malloced = 0;
950 
951 	if ((big_init(&a, len) != BIG_OK) ||
952 	    (big_init(&b, len) != BIG_OK) ||
953 	    (big_init(&c, len) != BIG_OK) ||
954 	    (big_init(&d, len) != BIG_OK) ||
955 	    (big_init(&e, len) != BIG_OK) ||
956 	    (big_init(&f, len) != BIG_OK) ||
957 	    (big_init(&g, len) != BIG_OK) ||
958 	    (big_init(&h, len) != BIG_OK)) {
959 		big_finish(&h);
960 		big_finish(&g);
961 		big_finish(&f);
962 		big_finish(&e);
963 		big_finish(&d);
964 		big_finish(&c);
965 		big_finish(&b);
966 		big_finish(&a);
967 
968 		return (CKR_HOST_MEMORY);
969 	}
970 nextp:
971 	if ((brv = random_bignum(&a, psize, token_obj)) != BIG_OK) {
972 		goto ret;
973 	}
974 
975 	if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
976 		goto ret;
977 	}
978 	(void) big_sub_pos(&a, &b, &One);
979 	if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
980 		goto ret;
981 	}
982 	if (big_cmp_abs(&f, &One) != 0) {
983 		goto nextp;
984 	}
985 
986 	if ((brv = random_bignum(&c, qsize, token_obj)) != BIG_OK) {
987 		goto ret;
988 	}
989 
990 nextq:
991 	(void) big_add(&a, &c, &Two);
992 
993 	if (big_bitlength(&a) != qsize) {
994 		goto nextp;
995 	}
996 	if (big_cmp_abs(&a, &b) == 0) {
997 		goto nextp;
998 	}
999 	if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
1000 		goto ret;
1001 	}
1002 	if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
1003 		goto ret;
1004 	}
1005 	if (big_bitlength(&g) != size) {
1006 		goto nextp;
1007 	}
1008 
1009 	(void) big_sub_pos(&a, &b, &One);
1010 	(void) big_sub_pos(&d, &c, &One);
1011 
1012 	if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
1013 		goto ret;
1014 	}
1015 	if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
1016 		goto ret;
1017 	}
1018 	if (big_cmp_abs(&f, &One) != 0) {
1019 		goto nextq;
1020 	} else {
1021 		(void) big_copy(&e, pubexp);
1022 	}
1023 	if (d.sign == -1) {
1024 		if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
1025 			goto ret;
1026 		}
1027 	}
1028 	(void) big_copy(&(key->p), &b);
1029 	(void) big_copy(&(key->q), &c);
1030 	(void) big_copy(&(key->n), &g);
1031 	(void) big_copy(&(key->d), &d);
1032 	(void) big_copy(&(key->e), &e);
1033 
1034 	if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
1035 		goto ret;
1036 	}
1037 	if (f.sign == -1) {
1038 		if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
1039 			goto ret;
1040 		}
1041 	}
1042 	(void) big_copy(&(key->pinvmodq), &f);
1043 
1044 	(void) big_sub(&a, &b, &One);
1045 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
1046 		goto ret;
1047 	}
1048 	(void) big_copy(&(key->dmodpminus1), &f);
1049 	(void) big_sub(&a, &c, &One);
1050 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
1051 		goto ret;
1052 	}
1053 	(void) big_copy(&(key->dmodqminus1), &f);
1054 
1055 	if ((brv = random_bignum(&h, size, token_obj)) != BIG_OK) {
1056 		goto ret;
1057 	}
1058 	if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
1059 		goto ret;
1060 	}
1061 	if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
1062 		goto ret;
1063 	}
1064 
1065 	if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
1066 		goto ret;
1067 	}
1068 
1069 	if (big_cmp_abs(&b, &h) != 0) {
1070 		rv = generate_rsa_key(key, psize, qsize, pubexp, token_obj);
1071 		goto ret1;
1072 	} else {
1073 		brv = BIG_OK;
1074 	}
1075 
1076 ret:
1077 	rv = convert_rv(brv);
1078 ret1:
1079 	big_finish(&h);
1080 	big_finish(&g);
1081 	big_finish(&f);
1082 	big_finish(&e);
1083 	big_finish(&d);
1084 	big_finish(&c);
1085 	big_finish(&b);
1086 	big_finish(&a);
1087 
1088 /* EXPORT DELETE END */
1089 
1090 	return (rv);
1091 }
1092 
1093 
1094 CK_RV
1095 soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
1096 {
1097 
1098 	CK_RV rv = CKR_OK;
1099 	uint32_t modulus_len;
1100 	uchar_t	pub_expo[MAX_KEY_ATTR_BUFLEN];
1101 	uint32_t pub_expo_len = sizeof (pub_expo);
1102 	BIGNUM	public_exponent = {0};
1103 	RSAkey	rsakey = {0};
1104 	CK_ATTRIBUTE template;
1105 
1106 	if ((pubkey == NULL) || (prikey == NULL)) {
1107 		return (CKR_ARGUMENTS_BAD);
1108 	}
1109 
1110 	template.pValue = malloc(sizeof (CK_ULONG));
1111 
1112 	if (template.pValue == NULL) {
1113 		return (CKR_HOST_MEMORY);
1114 	}
1115 
1116 	template.ulValueLen = sizeof (CK_ULONG);
1117 
1118 	rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
1119 	    &template);
1120 
1121 	if (rv != CKR_OK) {
1122 		goto clean0;
1123 	}
1124 
1125 #ifdef	__sparcv9
1126 	/* LINTED */
1127 	modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
1128 #else	/* !__sparcv9 */
1129 	modulus_len = *((CK_ULONG *)(template.pValue));
1130 #endif	/* __sparcv9 */
1131 
1132 	/* Convert modulus length from bit length to byte length. */
1133 	modulus_len = (modulus_len + 7) / 8;
1134 
1135 	/* Modulus length needs to be between min key size and max key size. */
1136 	if ((modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES) ||
1137 	    (modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)) {
1138 		rv = CKR_ATTRIBUTE_VALUE_INVALID;
1139 		goto clean0;
1140 	}
1141 
1142 	rv = soft_get_public_attr(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
1143 	    &pub_expo_len);
1144 	if (rv != CKR_OK) {
1145 		goto clean0;
1146 	}
1147 
1148 	/* Create a public exponent in bignum format. */
1149 	if (big_init(&public_exponent, (modulus_len + 3)/4) != BIG_OK) {
1150 		rv = CKR_HOST_MEMORY;
1151 		goto clean0;
1152 	}
1153 	bytestring2bignum(&public_exponent, pub_expo, pub_expo_len);
1154 
1155 	if (RSA_key_init(&rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
1156 		rv = CKR_HOST_MEMORY;
1157 		goto clean2;
1158 	}
1159 
1160 	/* Generate RSA key pair. */
1161 	if ((rv = generate_rsa_key(&rsakey, modulus_len * 4, modulus_len * 4,
1162 	    &public_exponent, (IS_TOKEN_OBJECT(pubkey) ||
1163 	    IS_TOKEN_OBJECT(prikey)))) != CKR_OK) {
1164 		goto clean3;
1165 	}
1166 
1167 	/*
1168 	 * Add modulus in public template, and add all eight key fields
1169 	 * in private template.
1170 	 */
1171 	if ((rv = soft_genRSAkey_set_attribute(pubkey, &rsakey,
1172 	    CKA_MODULUS, modulus_len, B_TRUE)) != CKR_OK) {
1173 		goto clean3;
1174 	}
1175 
1176 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1177 	    CKA_MODULUS, modulus_len, B_FALSE)) != CKR_OK) {
1178 		goto clean3;
1179 	}
1180 
1181 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1182 	    CKA_PRIVATE_EXPONENT, modulus_len, B_FALSE)) != CKR_OK) {
1183 		goto clean3;
1184 	}
1185 
1186 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1187 	    CKA_PUBLIC_EXPONENT, modulus_len, B_FALSE)) != CKR_OK) {
1188 		goto clean3;
1189 	}
1190 
1191 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1192 	    CKA_PRIME_1, modulus_len, B_FALSE)) != CKR_OK) {
1193 		goto clean3;
1194 	}
1195 
1196 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1197 	    CKA_PRIME_2, modulus_len, B_FALSE)) != CKR_OK) {
1198 		goto clean3;
1199 	}
1200 
1201 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1202 	    CKA_EXPONENT_1, modulus_len, B_FALSE)) != CKR_OK) {
1203 		goto clean3;
1204 	}
1205 
1206 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1207 	    CKA_EXPONENT_2, modulus_len, B_FALSE)) != CKR_OK) {
1208 		goto clean3;
1209 	}
1210 
1211 	if ((rv = soft_genRSAkey_set_attribute(prikey, &rsakey,
1212 	    CKA_COEFFICIENT, modulus_len, B_FALSE)) != CKR_OK) {
1213 		goto clean3;
1214 	}
1215 
1216 clean3:
1217 	RSA_key_finish(&rsakey);
1218 clean2:
1219 	big_finish(&public_exponent);
1220 clean0:
1221 	free(template.pValue);
1222 
1223 	return (rv);
1224 }
1225 
1226 CK_RV
1227 soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1228     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1229     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1230 {
1231 
1232 	CK_RV rv = CKR_OK;
1233 	CK_BYTE hash[SHA1_HASH_SIZE];  /* space enough for SHA1 and MD5 */
1234 	CK_ULONG hash_len = SHA1_HASH_SIZE;
1235 	/* space enough for SHA1 and MD5 */
1236 	CK_BYTE der_data[SHA1_HASH_SIZE + SHA1_DER_PREFIX_Len];
1237 	CK_ULONG der_data_len;
1238 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
1239 	soft_object_t *key = rsa_ctx->key;
1240 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1241 	uint32_t modulus_len = sizeof (modulus);
1242 
1243 	rv = soft_get_private_attr(key, CKA_MODULUS, modulus, &modulus_len);
1244 	if (rv != CKR_OK) {
1245 		(void) pthread_mutex_lock(&session_p->session_mutex);
1246 		free(session_p->digest.context);
1247 		session_p->digest.context = NULL;
1248 		session_p->digest.flags = 0;
1249 		(void) pthread_mutex_unlock(&session_p->session_mutex);
1250 		soft_cleanup_object(key);
1251 		free(key);
1252 		goto clean1;
1253 	}
1254 
1255 	/* Check arguments before performing message digest. */
1256 	if (pSigned == NULL) {
1257 		/* Application asks for the length of the output buffer. */
1258 		*pulSignedLen = modulus_len;
1259 		rv = CKR_OK;
1260 		goto clean1;
1261 	}
1262 
1263 	/* Is the application-supplied buffer large enough? */
1264 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
1265 		*pulSignedLen = modulus_len;
1266 		rv = CKR_BUFFER_TOO_SMALL;
1267 		goto clean1;
1268 	}
1269 
1270 	if (Final) {
1271 		rv = soft_digest_final(session_p, hash, &hash_len);
1272 	} else {
1273 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1274 	}
1275 
1276 	if (rv != CKR_OK) {
1277 		/* free the signature key */
1278 		soft_cleanup_object(key);
1279 		free(key);
1280 		goto clean_exit;
1281 	}
1282 
1283 	/*
1284 	 * Prepare the DER encoding of the DigestInfo value as follows:
1285 	 * MD5:		MD5_DER_PREFIX || H
1286 	 * SHA-1:	SHA1_DER_PREFIX || H
1287 	 *
1288 	 * See rsa_impl.c for more details.
1289 	 */
1290 	if (session_p->digest.mech.mechanism == CKM_MD5) {
1291 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1292 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1293 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1294 	} else { /* SHA_1 */
1295 		(void) memcpy(der_data, SHA1_DER_PREFIX, SHA1_DER_PREFIX_Len);
1296 		(void) memcpy(der_data + SHA1_DER_PREFIX_Len, hash, hash_len);
1297 		der_data_len = SHA1_DER_PREFIX_Len + hash_len;
1298 	}
1299 
1300 	/*
1301 	 * Now, we are ready to sign the DER_ENCODED data
1302 	 * soft_rsa_sign_common() will free the signature key.
1303 	 */
1304 	rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
1305 	    pSigned, pulSignedLen, mechanism);
1306 
1307 clean_exit:
1308 	(void) pthread_mutex_lock(&session_p->session_mutex);
1309 	/* soft_digest_common() has freed the digest context */
1310 	session_p->digest.flags = 0;
1311 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1312 
1313 clean1:
1314 	return (rv);
1315 }
1316 
1317 
1318 CK_RV
1319 soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1320     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1321     CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1322 {
1323 
1324 	CK_RV rv = CKR_OK;
1325 	CK_BYTE hash[SHA1_HASH_SIZE];  /* space enough for SHA1 and MD5 */
1326 	CK_ULONG hash_len = SHA1_HASH_SIZE;
1327 	/* space enough for SHA1 and MD5 */
1328 	CK_BYTE der_data[SHA1_HASH_SIZE + SHA1_DER_PREFIX_Len];
1329 	CK_ULONG der_data_len;
1330 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1331 	soft_object_t *key = rsa_ctx->key;
1332 
1333 	if (Final) {
1334 		rv = soft_digest_final(session_p, hash, &hash_len);
1335 	} else {
1336 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1337 	}
1338 
1339 	if (rv != CKR_OK) {
1340 		/* free the verification key */
1341 		soft_cleanup_object(key);
1342 		free(key);
1343 		goto clean_exit;
1344 	}
1345 
1346 	/*
1347 	 * Prepare the DER encoding of the DigestInfo value as follows:
1348 	 * MD5:		MD5_DER_PREFIX || H
1349 	 * SHA-1:	SHA1_DER_PREFIX || H
1350 	 *
1351 	 * See rsa_impl.c for more details.
1352 	 */
1353 	if (session_p->digest.mech.mechanism == CKM_MD5) {
1354 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1355 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1356 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1357 	} else { /* SHA_1 */
1358 		(void) memcpy(der_data, SHA1_DER_PREFIX, SHA1_DER_PREFIX_Len);
1359 		(void) memcpy(der_data + SHA1_DER_PREFIX_Len, hash, hash_len);
1360 		der_data_len = SHA1_DER_PREFIX_Len + hash_len;
1361 	}
1362 
1363 	/*
1364 	 * Now, we are ready to verify the DER_ENCODED data using signature.
1365 	 * soft_rsa_verify_common() will free the verification key.
1366 	 */
1367 	rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
1368 	    pSigned, ulSignedLen, mechanism);
1369 
1370 clean_exit:
1371 	(void) pthread_mutex_lock(&session_p->session_mutex);
1372 	/* soft_digest_common() has freed the digest context */
1373 	session_p->digest.flags = 0;
1374 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1375 
1376 	return (rv);
1377 
1378 }
1379 
1380 
1381 CK_RV
1382 soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
1383     CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
1384 {
1385 
1386 	CK_RV rv = CKR_OK;
1387 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1388 	CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
1389 	soft_object_t *key = rsa_ctx->key;
1390 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1391 	uint32_t modulus_len = sizeof (modulus);
1392 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1393 
1394 	rv = soft_get_public_attr(key, CKA_MODULUS, modulus, &modulus_len);
1395 	if (rv != CKR_OK) {
1396 		goto clean_exit;
1397 	}
1398 
1399 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
1400 		rv = CKR_SIGNATURE_LEN_RANGE;
1401 		goto clean_exit;
1402 	}
1403 
1404 	/*
1405 	 * Perform RSA decryption with the signer's RSA public key
1406 	 * for verification process.
1407 	 */
1408 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
1409 	if (rv == CKR_OK) {
1410 		switch (mechanism) {
1411 
1412 		case CKM_RSA_PKCS:
1413 		{
1414 			/*
1415 			 * Strip off the encoded padding bytes in front of the
1416 			 * recovered data.
1417 			 */
1418 			int data_len = modulus_len;
1419 
1420 			rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1421 			if (rv != CKR_OK) {
1422 				goto clean_exit;
1423 			}
1424 
1425 			/*
1426 			 * If application asks for the length of the output
1427 			 * buffer?
1428 			 */
1429 			if (pData == NULL) {
1430 				*pulDataLen = data_len;
1431 				rv = CKR_OK;
1432 				goto clean1;
1433 			}
1434 
1435 			/* Is the application-supplied buffer large enough? */
1436 			if (*pulDataLen < (CK_ULONG)data_len) {
1437 				*pulDataLen = data_len;
1438 				rv = CKR_BUFFER_TOO_SMALL;
1439 				goto clean1;
1440 			}
1441 
1442 			(void) memcpy(pData,
1443 			    &plain_data[modulus_len - data_len], data_len);
1444 			*pulDataLen = data_len;
1445 
1446 			break;
1447 		}
1448 
1449 		case CKM_RSA_X_509:
1450 			/*
1451 			 * If application asks for the length of the output
1452 			 * buffer?
1453 			 */
1454 			if (pData == NULL) {
1455 				*pulDataLen = modulus_len;
1456 				rv = CKR_OK;
1457 				goto clean1;
1458 			}
1459 
1460 			/* Is the application-supplied buffer large enough? */
1461 			if (*pulDataLen < (CK_ULONG)modulus_len) {
1462 				*pulDataLen = modulus_len;
1463 				rv = CKR_BUFFER_TOO_SMALL;
1464 				goto clean1;
1465 			}
1466 
1467 			(void) memcpy(pData, plain_data, modulus_len);
1468 			*pulDataLen = modulus_len;
1469 
1470 			break;
1471 		}
1472 	}
1473 
1474 clean_exit:
1475 	(void) pthread_mutex_lock(&session_p->session_mutex);
1476 	free(session_p->verify.context);
1477 	session_p->verify.context = NULL;
1478 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1479 	soft_cleanup_object(key);
1480 	free(key);
1481 
1482 clean1:
1483 	return (rv);
1484 }
1485