xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softRSA.c (revision 694c35faa87b858ecdadfe4fc592615f4eefbb07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <pthread.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <strings.h>
30 #include <sys/types.h>
31 #include <security/cryptoki.h>
32 #include <cryptoutil.h>
33 #include "softGlobal.h"
34 #include "softSession.h"
35 #include "softObject.h"
36 #include "softOps.h"
37 #include "softRSA.h"
38 #include "softMAC.h"
39 #include "softCrypt.h"
40 
41 CK_RV
soft_rsa_encrypt(soft_object_t * key,CK_BYTE_PTR in,uint32_t in_len,CK_BYTE_PTR out,int realpublic)42 soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
43     CK_BYTE_PTR out, int realpublic)
44 {
45 
46 	CK_RV rv = CKR_OK;
47 
48 	uchar_t expo[MAX_KEY_ATTR_BUFLEN];
49 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
50 	uint32_t expo_len = sizeof (expo);
51 	uint32_t modulus_len = sizeof (modulus);
52 	RSAbytekey k;
53 
54 	if (realpublic) {
55 		rv = soft_get_public_value(key, CKA_PUBLIC_EXPONENT, expo,
56 		    &expo_len);
57 		if (rv != CKR_OK) {
58 			goto clean1;
59 		}
60 	} else {
61 		rv = soft_get_private_value(key, CKA_PRIVATE_EXPONENT, expo,
62 		    &expo_len);
63 		if (rv != CKR_OK) {
64 			goto clean1;
65 		}
66 	}
67 
68 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
69 	if (rv != CKR_OK) {
70 		goto clean1;
71 	}
72 
73 	k.modulus = modulus;
74 	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
75 	k.pubexpo = expo;
76 	k.pubexpo_bytes = expo_len;
77 	k.rfunc = NULL;
78 
79 	rv = rsa_encrypt(&k, in, in_len, out);
80 
81 clean1:
82 
83 	return (rv);
84 }
85 
86 
87 CK_RV
soft_rsa_decrypt(soft_object_t * key,CK_BYTE_PTR in,uint32_t in_len,CK_BYTE_PTR out)88 soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
89     CK_BYTE_PTR out)
90 {
91 
92 	CK_RV rv = CKR_OK;
93 
94 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
95 	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
96 	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
97 	uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
98 	uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
99 	uchar_t coef[MAX_KEY_ATTR_BUFLEN];
100 	uint32_t modulus_len = sizeof (modulus);
101 	uint32_t prime1_len = sizeof (prime1);
102 	uint32_t prime2_len = sizeof (prime2);
103 	uint32_t expo1_len = sizeof (expo1);
104 	uint32_t expo2_len = sizeof (expo2);
105 	uint32_t coef_len = sizeof (coef);
106 	RSAbytekey k;
107 
108 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
109 	if (rv != CKR_OK) {
110 		goto clean1;
111 	}
112 
113 	rv = soft_get_private_value(key, CKA_PRIME_1, prime1, &prime1_len);
114 
115 	if ((prime1_len == 0) && (rv == CKR_OK)) {
116 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
117 		goto clean1;
118 	} else {
119 		if (rv != CKR_OK)
120 			goto clean1;
121 	}
122 
123 	rv = soft_get_private_value(key, CKA_PRIME_2, prime2, &prime2_len);
124 
125 	if ((prime2_len == 0) && (rv == CKR_OK)) {
126 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
127 		goto clean1;
128 	} else {
129 		if (rv != CKR_OK)
130 			goto clean1;
131 	}
132 
133 	rv = soft_get_private_value(key, CKA_EXPONENT_1, expo1, &expo1_len);
134 
135 	if ((expo1_len == 0) && (rv == CKR_OK)) {
136 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
137 		goto clean1;
138 	} else {
139 		if (rv != CKR_OK)
140 			goto clean1;
141 	}
142 
143 	rv = soft_get_private_value(key, CKA_EXPONENT_2, expo2, &expo2_len);
144 
145 	if ((expo2_len == 0) && (rv == CKR_OK)) {
146 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
147 		goto clean1;
148 	} else {
149 		if (rv != CKR_OK)
150 			goto clean1;
151 	}
152 
153 	rv = soft_get_private_value(key, CKA_COEFFICIENT, coef, &coef_len);
154 
155 	if ((coef_len == 0) && (rv == CKR_OK)) {
156 		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
157 		goto clean1;
158 	} else {
159 		if (rv != CKR_OK)
160 			goto clean1;
161 	}
162 
163 	k.modulus = modulus;
164 	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
165 	k.prime1 = prime1;
166 	k.prime1_bytes = prime1_len;
167 	k.prime2 = prime2;
168 	k.prime2_bytes = prime2_len;
169 	k.expo1 = expo1;
170 	k.expo1_bytes = expo1_len;
171 	k.expo2 = expo2;
172 	k.expo2_bytes = expo2_len;
173 	k.coeff = coef;
174 	k.coeff_bytes = coef_len;
175 	k.rfunc = NULL;
176 
177 	rv = rsa_decrypt(&k, in, in_len, out);
178 
179 clean1:
180 
181 	return (rv);
182 }
183 
184 /*
185  * Allocate a RSA context for the active encryption or decryption operation.
186  * This function is called without the session lock held.
187  */
188 CK_RV
soft_rsa_crypt_init_common(soft_session_t * session_p,CK_MECHANISM_PTR pMechanism,soft_object_t * key_p,boolean_t encrypt)189 soft_rsa_crypt_init_common(soft_session_t *session_p,
190     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
191     boolean_t encrypt)
192 {
193 
194 	soft_rsa_ctx_t *rsa_ctx;
195 	soft_object_t *tmp_key = NULL;
196 	CK_RV rv;
197 
198 	rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
199 	if (rsa_ctx == NULL) {
200 		return (CKR_HOST_MEMORY);
201 	}
202 
203 	/*
204 	 * Make a copy of the encryption or decryption key, and save it
205 	 * in the RSA crypto context since it will be used later for
206 	 * encryption/decryption. We don't want to hold any object reference
207 	 * on this original key while doing encryption/decryption.
208 	 */
209 	(void) pthread_mutex_lock(&key_p->object_mutex);
210 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
211 	    NULL);
212 
213 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
214 		/* Most likely we ran out of space. */
215 		(void) pthread_mutex_unlock(&key_p->object_mutex);
216 		free(rsa_ctx);
217 		return (rv);
218 	}
219 
220 	/* No need to hold the lock on the old object. */
221 	(void) pthread_mutex_unlock(&key_p->object_mutex);
222 	rsa_ctx->key = tmp_key;
223 
224 	(void) pthread_mutex_lock(&session_p->session_mutex);
225 	if (encrypt) {
226 		/* Called by C_EncryptInit. */
227 		session_p->encrypt.context = rsa_ctx;
228 		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
229 	} else {
230 		/* Called by C_DecryptInit. */
231 		session_p->decrypt.context = rsa_ctx;
232 		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
233 	}
234 	(void) pthread_mutex_unlock(&session_p->session_mutex);
235 
236 	return (CKR_OK);
237 }
238 
239 CK_RV
soft_rsa_encrypt_common(soft_session_t * session_p,CK_BYTE_PTR pData,CK_ULONG ulDataLen,CK_BYTE_PTR pEncrypted,CK_ULONG_PTR pulEncryptedLen,CK_MECHANISM_TYPE mechanism)240 soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
241     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
242     CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
243 {
244 
245 	soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
246 	soft_object_t *key = rsa_ctx->key;
247 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
248 	uint32_t modulus_len = sizeof (modulus);
249 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
250 	CK_BYTE	cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
251 	CK_RV rv = CKR_OK;
252 
253 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
254 	if (rv != CKR_OK) {
255 		goto clean_exit;
256 	}
257 
258 	if (pEncrypted == NULL) {
259 		/*
260 		 * Application asks for the length of the output buffer
261 		 * to hold the ciphertext.
262 		 */
263 		*pulEncryptedLen = modulus_len;
264 		rv = CKR_OK;
265 		goto clean1;
266 	}
267 
268 	if (mechanism == CKM_RSA_PKCS) {
269 		/*
270 		 * Input data length needs to be <=
271 		 * modulus length-MIN_PKCS1_PADLEN.
272 		 */
273 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
274 			*pulEncryptedLen = modulus_len;
275 			rv = CKR_DATA_LEN_RANGE;
276 			goto clean_exit;
277 		}
278 	} else {
279 		/* Input data length needs to be <= modulus length. */
280 		if (ulDataLen > (CK_ULONG)modulus_len) {
281 			*pulEncryptedLen = modulus_len;
282 			rv = CKR_DATA_LEN_RANGE;
283 			goto clean_exit;
284 		}
285 	}
286 
287 	/* Is the application-supplied buffer large enough? */
288 	if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
289 		*pulEncryptedLen = modulus_len;
290 		rv = CKR_BUFFER_TOO_SMALL;
291 		goto clean1;
292 	}
293 
294 	if (mechanism == CKM_RSA_PKCS) {
295 		/*
296 		 * Add PKCS padding to the input data to format a block
297 		 * type "02" encryption block.
298 		 */
299 		rv = pkcs1_encode(PKCS1_ENCRYPT, pData, ulDataLen, plain_data,
300 		    modulus_len);
301 
302 		if (rv != CKR_OK)
303 			goto clean_exit;
304 	} else {
305 		/* Pad zeros for the leading bytes of the input data. */
306 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
307 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
308 		    ulDataLen);
309 	}
310 
311 	rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
312 	if (rv == CKR_OK) {
313 		(void) memcpy(pEncrypted, cipher_data, modulus_len);
314 		*pulEncryptedLen = modulus_len;
315 	}
316 
317 clean_exit:
318 	(void) pthread_mutex_lock(&session_p->session_mutex);
319 	free(session_p->encrypt.context);
320 	session_p->encrypt.context = NULL;
321 	(void) pthread_mutex_unlock(&session_p->session_mutex);
322 	soft_cleanup_object(key);
323 	free(key);
324 clean1:
325 	return (rv);
326 }
327 
328 
329 CK_RV
soft_rsa_decrypt_common(soft_session_t * session_p,CK_BYTE_PTR pEncrypted,CK_ULONG ulEncryptedLen,CK_BYTE_PTR pData,CK_ULONG_PTR pulDataLen,CK_MECHANISM_TYPE mechanism)330 soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
331     CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
332     CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
333 {
334 
335 	soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
336 	soft_object_t *key = rsa_ctx->key;
337 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
338 	uint32_t modulus_len = sizeof (modulus);
339 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
340 	CK_RV rv = CKR_OK;
341 
342 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
343 	if (rv != CKR_OK) {
344 		goto clean_exit;
345 	}
346 
347 	if (ulEncryptedLen != (CK_ULONG)modulus_len) {
348 		rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
349 		goto clean_exit;
350 	}
351 
352 	if (pData == NULL) {
353 		/*
354 		 * Application asks for the length of the output buffer
355 		 * to hold the recovered data.
356 		 */
357 		*pulDataLen = modulus_len;
358 		rv = CKR_OK;
359 		goto clean1;
360 	}
361 
362 	if (mechanism == CKM_RSA_X_509) {
363 		if (*pulDataLen < (CK_ULONG)modulus_len) {
364 			*pulDataLen = modulus_len;
365 			rv = CKR_BUFFER_TOO_SMALL;
366 			goto clean1;
367 		}
368 	}
369 
370 	rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
371 	if (rv != CKR_OK) {
372 		goto clean_exit;
373 	}
374 
375 	if (mechanism == CKM_RSA_PKCS) {
376 		size_t plain_len = modulus_len;
377 		size_t num_padding;
378 
379 		/* Strip off the PKCS block formatting data. */
380 		rv = pkcs1_decode(PKCS1_DECRYPT, plain_data, &plain_len);
381 		if (rv != CKR_OK)
382 			goto clean_exit;
383 
384 		num_padding = modulus_len - plain_len;
385 		if (ulEncryptedLen - num_padding > *pulDataLen) {
386 			*pulDataLen = plain_len;
387 			rv = CKR_BUFFER_TOO_SMALL;
388 			goto clean1;
389 		}
390 
391 		(void) memcpy(pData, &plain_data[num_padding], plain_len);
392 		*pulDataLen = plain_len;
393 	} else {
394 		(void) memcpy(pData, plain_data, modulus_len);
395 		*pulDataLen = modulus_len;
396 	}
397 
398 clean_exit:
399 	(void) pthread_mutex_lock(&session_p->session_mutex);
400 	free(session_p->decrypt.context);
401 	session_p->decrypt.context = NULL;
402 	(void) pthread_mutex_unlock(&session_p->session_mutex);
403 	soft_cleanup_object(key);
404 	free(key);
405 
406 clean1:
407 	return (rv);
408 }
409 
410 /*
411  * Allocate a RSA context for the active sign or verify operation.
412  * This function is called without the session lock held.
413  */
414 CK_RV
soft_rsa_sign_verify_init_common(soft_session_t * session_p,CK_MECHANISM_PTR pMechanism,soft_object_t * key_p,boolean_t sign)415 soft_rsa_sign_verify_init_common(soft_session_t *session_p,
416     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
417     boolean_t sign)
418 {
419 	CK_RV rv = CKR_OK;
420 	soft_rsa_ctx_t *rsa_ctx;
421 	CK_MECHANISM digest_mech;
422 	soft_object_t *tmp_key = NULL;
423 
424 	if (sign) {
425 		if ((key_p->class != CKO_PRIVATE_KEY) ||
426 		    (key_p->key_type != CKK_RSA))
427 			return (CKR_KEY_TYPE_INCONSISTENT);
428 	} else {
429 		if ((key_p->class != CKO_PUBLIC_KEY) ||
430 		    (key_p->key_type != CKK_RSA))
431 			return (CKR_KEY_TYPE_INCONSISTENT);
432 	}
433 
434 	switch (pMechanism->mechanism) {
435 	case CKM_MD5_RSA_PKCS:
436 		digest_mech.mechanism = CKM_MD5;
437 		rv = soft_digest_init_internal(session_p, &digest_mech);
438 		if (rv != CKR_OK)
439 			return (rv);
440 		break;
441 
442 	case CKM_SHA1_RSA_PKCS:
443 		digest_mech.mechanism = CKM_SHA_1;
444 		digest_mech.pParameter = pMechanism->pParameter;
445 		digest_mech.ulParameterLen = pMechanism->ulParameterLen;
446 		rv = soft_digest_init_internal(session_p, &digest_mech);
447 		if (rv != CKR_OK)
448 			return (rv);
449 		break;
450 
451 	case CKM_SHA256_RSA_PKCS:
452 		digest_mech.mechanism = CKM_SHA256;
453 		rv = soft_digest_init_internal(session_p, &digest_mech);
454 		if (rv != CKR_OK)
455 			return (rv);
456 		break;
457 
458 	case CKM_SHA384_RSA_PKCS:
459 		digest_mech.mechanism = CKM_SHA384;
460 		rv = soft_digest_init_internal(session_p, &digest_mech);
461 		if (rv != CKR_OK)
462 			return (rv);
463 		break;
464 
465 	case CKM_SHA512_RSA_PKCS:
466 		digest_mech.mechanism = CKM_SHA512;
467 		rv = soft_digest_init_internal(session_p, &digest_mech);
468 		if (rv != CKR_OK)
469 			return (rv);
470 		break;
471 	}
472 
473 	rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));
474 
475 	if (rsa_ctx == NULL) {
476 		rv = CKR_HOST_MEMORY;
477 		goto clean_exit;
478 	}
479 
480 	(void) pthread_mutex_lock(&key_p->object_mutex);
481 	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
482 	    NULL);
483 
484 	if ((rv != CKR_OK) || (tmp_key == NULL)) {
485 		/* Most likely we ran out of space. */
486 		(void) pthread_mutex_unlock(&key_p->object_mutex);
487 		free(rsa_ctx);
488 		goto clean_exit;
489 	}
490 
491 	/* No need to hold the lock on the old object. */
492 	(void) pthread_mutex_unlock(&key_p->object_mutex);
493 	rsa_ctx->key = tmp_key;
494 
495 	(void) pthread_mutex_lock(&session_p->session_mutex);
496 
497 	if (sign) {
498 		session_p->sign.context = rsa_ctx;
499 		session_p->sign.mech.mechanism = pMechanism->mechanism;
500 	} else {
501 		session_p->verify.context = rsa_ctx;
502 		session_p->verify.mech.mechanism = pMechanism->mechanism;
503 	}
504 
505 	(void) pthread_mutex_unlock(&session_p->session_mutex);
506 
507 	return (CKR_OK);
508 
509 clean_exit:
510 	(void) pthread_mutex_lock(&session_p->session_mutex);
511 	if (session_p->digest.context != NULL) {
512 		free(session_p->digest.context);
513 		session_p->digest.context = NULL;
514 		session_p->digest.flags = 0;
515 	}
516 	(void) pthread_mutex_unlock(&session_p->session_mutex);
517 	return (rv);
518 
519 }
520 
521 
522 CK_RV
soft_rsa_sign_common(soft_session_t * session_p,CK_BYTE_PTR pData,CK_ULONG ulDataLen,CK_BYTE_PTR pSigned,CK_ULONG_PTR pulSignedLen,CK_MECHANISM_TYPE mechanism)523 soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
524     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
525     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
526 {
527 
528 	CK_RV rv = CKR_OK;
529 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
530 	soft_object_t *key = rsa_ctx->key;
531 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
532 	uint32_t modulus_len = sizeof (modulus);
533 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
534 	CK_BYTE	signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
535 
536 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
537 	if (rv != CKR_OK) {
538 		goto clean_exit;
539 	}
540 
541 	if (pSigned == NULL) {
542 		/* Application asks for the length of the output buffer. */
543 		*pulSignedLen = modulus_len;
544 		rv = CKR_OK;
545 		goto clean1;
546 	}
547 
548 	switch (mechanism) {
549 
550 	case CKM_RSA_PKCS:
551 
552 		/*
553 		 * Input data length needs to be <=
554 		 * modulus length-MIN_PKCS1_PADLEN.
555 		 */
556 		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
557 			*pulSignedLen = modulus_len;
558 			rv = CKR_DATA_LEN_RANGE;
559 			goto clean_exit;
560 		}
561 		break;
562 
563 	case CKM_RSA_X_509:
564 
565 		/* Input data length needs to be <= modulus length. */
566 		if (ulDataLen > (CK_ULONG)modulus_len) {
567 			*pulSignedLen = modulus_len;
568 			rv = CKR_DATA_LEN_RANGE;
569 			goto clean_exit;
570 		}
571 		break;
572 	}
573 
574 	/* Is the application-supplied buffer large enough? */
575 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
576 		*pulSignedLen = modulus_len;
577 		rv = CKR_BUFFER_TOO_SMALL;
578 		goto clean1;
579 	}
580 
581 	switch (mechanism) {
582 
583 	case CKM_RSA_PKCS:
584 	case CKM_MD5_RSA_PKCS:
585 	case CKM_SHA1_RSA_PKCS:
586 	case CKM_SHA256_RSA_PKCS:
587 	case CKM_SHA384_RSA_PKCS:
588 	case CKM_SHA512_RSA_PKCS:
589 		/*
590 		 * Add PKCS padding to the input data to format a block
591 		 * type "01" encryption block.
592 		 */
593 		rv = pkcs1_encode(PKCS1_SIGN, pData, ulDataLen, plain_data,
594 		    modulus_len);
595 
596 		if (rv != CKR_OK) {
597 			goto clean_exit;
598 		}
599 		break;
600 
601 	case CKM_RSA_X_509:
602 
603 		/* Pad zeros for the leading bytes of the input data. */
604 		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
605 		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
606 		    ulDataLen);
607 		break;
608 	}
609 
610 	/*
611 	 * Perform RSA encryption with the signer's RSA private key
612 	 * for signature process.
613 	 */
614 	rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);
615 
616 	if (rv == CKR_OK) {
617 		(void) memcpy(pSigned, signed_data, modulus_len);
618 		*pulSignedLen = modulus_len;
619 	}
620 
621 clean_exit:
622 	(void) pthread_mutex_lock(&session_p->session_mutex);
623 	free(session_p->sign.context);
624 	session_p->sign.context = NULL;
625 	if (session_p->digest.context != NULL) {
626 		free(session_p->digest.context);
627 		session_p->digest.context = NULL;
628 		session_p->digest.flags = 0;
629 	}
630 	(void) pthread_mutex_unlock(&session_p->session_mutex);
631 	soft_cleanup_object(key);
632 	free(key);
633 
634 clean1:
635 	return (rv);
636 }
637 
638 
639 CK_RV
soft_rsa_verify_common(soft_session_t * session_p,CK_BYTE_PTR pData,CK_ULONG ulDataLen,CK_BYTE_PTR pSignature,CK_ULONG ulSignatureLen,CK_MECHANISM_TYPE mechanism)640 soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
641     CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
642     CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
643 {
644 
645 	CK_RV rv = CKR_OK;
646 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
647 	soft_object_t *key = rsa_ctx->key;
648 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
649 	uint32_t modulus_len = sizeof (modulus);
650 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
651 
652 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
653 	if (rv != CKR_OK) {
654 		goto clean_exit;
655 	}
656 
657 	if (ulDataLen == 0) {
658 		rv = CKR_DATA_LEN_RANGE;
659 		goto clean_exit;
660 	}
661 
662 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
663 		rv = CKR_SIGNATURE_LEN_RANGE;
664 		goto clean_exit;
665 	}
666 
667 	/*
668 	 * Perform RSA decryption with the signer's RSA public key
669 	 * for verification process.
670 	 */
671 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
672 	if (rv == CKR_OK) {
673 		switch (mechanism) {
674 
675 		case CKM_RSA_PKCS:
676 		case CKM_MD5_RSA_PKCS:
677 		case CKM_SHA1_RSA_PKCS:
678 		case CKM_SHA256_RSA_PKCS:
679 		case CKM_SHA384_RSA_PKCS:
680 		case CKM_SHA512_RSA_PKCS:
681 		{
682 			/*
683 			 * Strip off the encoded padding bytes in front of the
684 			 * recovered data, then compare the recovered data with
685 			 * the original data.
686 			 */
687 			size_t data_len = modulus_len;
688 
689 			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
690 			if (rv != CKR_OK) {
691 				goto clean_exit;
692 			}
693 
694 			if ((CK_ULONG)data_len != ulDataLen) {
695 				rv = CKR_DATA_LEN_RANGE;
696 				goto clean_exit;
697 			} else if (memcmp(pData,
698 			    &plain_data[modulus_len - data_len],
699 			    ulDataLen) != 0) {
700 				rv = CKR_SIGNATURE_INVALID;
701 				goto clean_exit;
702 			}
703 			break;
704 		}
705 
706 		case CKM_RSA_X_509:
707 			/*
708 			 * Strip off the encoded padding bytes in front of the
709 			 * recovered plain_data, then compare the input data
710 			 * with the recovered data.
711 			 */
712 			if (memcmp(pData,
713 			    plain_data + ulSignatureLen - ulDataLen,
714 			    ulDataLen) != 0) {
715 				rv = CKR_SIGNATURE_INVALID;
716 				goto clean_exit;
717 			}
718 			break;
719 		}
720 	}
721 
722 	if (rv == CKR_DATA_LEN_RANGE) {
723 		if ((mechanism == CKM_MD5_RSA_PKCS) ||
724 		    (mechanism == CKM_SHA1_RSA_PKCS) ||
725 		    (mechanism == CKM_SHA256_RSA_PKCS) ||
726 		    (mechanism == CKM_SHA384_RSA_PKCS) ||
727 		    (mechanism == CKM_SHA512_RSA_PKCS))
728 			rv = CKR_SIGNATURE_INVALID;
729 	}
730 
731 clean_exit:
732 	(void) pthread_mutex_lock(&session_p->session_mutex);
733 	free(session_p->verify.context);
734 	session_p->verify.context = NULL;
735 	if (session_p->digest.context != NULL) {
736 		free(session_p->digest.context);
737 		session_p->digest.context = NULL;
738 		session_p->digest.flags = 0;
739 	}
740 	(void) pthread_mutex_unlock(&session_p->session_mutex);
741 	soft_cleanup_object(key);
742 	free(key);
743 	return (rv);
744 }
745 
746 CK_RV
soft_genRSAkey_set_attribute(soft_object_t * key,CK_ATTRIBUTE_TYPE type,uchar_t * buf,uint32_t buflen,boolean_t public)747 soft_genRSAkey_set_attribute(soft_object_t *key, CK_ATTRIBUTE_TYPE type,
748     uchar_t *buf, uint32_t buflen, boolean_t public)
749 {
750 	CK_RV rv = CKR_OK;
751 	biginteger_t *dst = NULL;
752 	biginteger_t src;
753 
754 	switch (type) {
755 
756 	case CKA_MODULUS:
757 
758 		if (public)
759 			dst = OBJ_PUB_RSA_MOD(key);
760 		else
761 			dst = OBJ_PRI_RSA_MOD(key);
762 		break;
763 
764 	case CKA_PUBLIC_EXPONENT:
765 
766 		if (public)
767 			dst = OBJ_PUB_RSA_PUBEXPO(key);
768 		else
769 			dst = OBJ_PRI_RSA_PUBEXPO(key);
770 		break;
771 
772 	case CKA_PRIVATE_EXPONENT:
773 
774 		dst = OBJ_PRI_RSA_PRIEXPO(key);
775 		break;
776 
777 	case CKA_PRIME_1:
778 
779 		dst = OBJ_PRI_RSA_PRIME1(key);
780 		break;
781 
782 	case CKA_PRIME_2:
783 
784 		dst = OBJ_PRI_RSA_PRIME2(key);
785 		break;
786 
787 	case CKA_EXPONENT_1:
788 
789 		dst = OBJ_PRI_RSA_EXPO1(key);
790 		break;
791 
792 	case CKA_EXPONENT_2:
793 
794 		dst = OBJ_PRI_RSA_EXPO2(key);
795 		break;
796 
797 	case CKA_COEFFICIENT:
798 
799 		dst = OBJ_PRI_RSA_COEF(key);
800 		break;
801 	}
802 
803 	/* Note: no explanation found for why this is needed */
804 	while (buf[0] == 0) {	/* remove proceeding 0x00 */
805 		buf++;
806 		buflen--;
807 	}
808 
809 	if ((rv = dup_bigint_attr(&src, buf, buflen)) != CKR_OK)
810 		goto cleanexit;
811 
812 	/* Copy the attribute in the key object. */
813 	copy_bigint_attr(&src, dst);
814 
815 cleanexit:
816 	return (rv);
817 
818 }
819 
820 
821 CK_RV
soft_rsa_genkey_pair(soft_object_t * pubkey,soft_object_t * prikey)822 soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
823 {
824 	CK_RV rv = CKR_OK;
825 	CK_ATTRIBUTE template;
826 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
827 	uint32_t modulus_len;
828 	uchar_t pub_expo[MAX_KEY_ATTR_BUFLEN];
829 	uint32_t pub_expo_len = sizeof (pub_expo);
830 	uchar_t private_exponent[MAX_KEY_ATTR_BUFLEN];
831 	uint32_t private_exponent_len = sizeof (private_exponent);
832 	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
833 	uint32_t prime1_len = sizeof (prime1);
834 	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
835 	uint32_t prime2_len = sizeof (prime2);
836 	uchar_t exponent1[MAX_KEY_ATTR_BUFLEN];
837 	uint32_t exponent1_len = sizeof (exponent1);
838 	uchar_t exponent2[MAX_KEY_ATTR_BUFLEN];
839 	uint32_t exponent2_len = sizeof (exponent2);
840 	uchar_t coefficient[MAX_KEY_ATTR_BUFLEN];
841 	uint32_t coefficient_len = sizeof (coefficient);
842 	RSAbytekey k;
843 
844 	if ((pubkey == NULL) || (prikey == NULL)) {
845 		return (CKR_ARGUMENTS_BAD);
846 	}
847 
848 	template.pValue = malloc(sizeof (CK_ULONG));
849 	if (template.pValue == NULL) {
850 		return (CKR_HOST_MEMORY);
851 	}
852 	template.ulValueLen = sizeof (CK_ULONG);
853 
854 	rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
855 	    &template);
856 	if (rv != CKR_OK) {
857 		free(template.pValue);
858 		goto clean0;
859 	}
860 
861 #ifdef	__sparcv9
862 	/* LINTED */
863 	modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
864 #else	/* !__sparcv9 */
865 	modulus_len = *((CK_ULONG *)(template.pValue));
866 #endif	/* __sparcv9 */
867 
868 	free(template.pValue);
869 
870 	rv = soft_get_public_value(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
871 	    &pub_expo_len);
872 	if (rv != CKR_OK) {
873 		goto clean0;
874 	}
875 
876 	/* Inputs to RSA key pair generation */
877 	k.modulus_bits = modulus_len;		/* save modulus len in bits  */
878 	modulus_len = CRYPTO_BITS2BYTES(modulus_len);	/* convert to bytes */
879 	k.modulus = modulus;
880 	k.pubexpo = pub_expo;
881 	k.pubexpo_bytes = pub_expo_len;
882 	k.rfunc = (IS_TOKEN_OBJECT(pubkey) || IS_TOKEN_OBJECT(prikey)) ?
883 	    pkcs11_get_random : pkcs11_get_urandom;
884 
885 	/* Outputs from RSA key pair generation */
886 	k.privexpo = private_exponent;
887 	k.privexpo_bytes = private_exponent_len;
888 	k.prime1 = prime1;
889 	k.prime1_bytes = prime1_len;
890 	k.prime2 = prime2;
891 	k.prime2_bytes = prime2_len;
892 	k.expo1 = exponent1;
893 	k.expo1_bytes = exponent1_len;
894 	k.expo2 = exponent2;
895 	k.expo2_bytes = exponent2_len;
896 	k.coeff = coefficient;
897 	k.coeff_bytes = coefficient_len;
898 
899 	rv = rsa_genkey_pair(&k);
900 
901 	if (rv != CKR_OK) {
902 		goto clean0;
903 	}
904 
905 	/*
906 	 * Add modulus in public template, and add all eight key fields
907 	 * in private template.
908 	 */
909 	if ((rv = soft_genRSAkey_set_attribute(pubkey, CKA_MODULUS,
910 	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_TRUE)) != CKR_OK) {
911 		goto clean0;
912 	}
913 
914 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_MODULUS,
915 	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_FALSE)) != CKR_OK) {
916 		goto clean0;
917 	}
918 
919 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIVATE_EXPONENT,
920 	    private_exponent, k.privexpo_bytes, B_FALSE)) != CKR_OK) {
921 		goto clean0;
922 	}
923 
924 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PUBLIC_EXPONENT,
925 	    pub_expo, k.pubexpo_bytes, B_FALSE)) != CKR_OK) {
926 		goto clean0;
927 	}
928 
929 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_1,
930 	    prime1, k.prime1_bytes, B_FALSE)) != CKR_OK) {
931 		goto clean0;
932 	}
933 
934 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_2,
935 	    prime2, k.prime2_bytes, B_FALSE)) != CKR_OK) {
936 		goto clean0;
937 	}
938 
939 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_1,
940 	    exponent1, k.expo1_bytes, B_FALSE)) != CKR_OK) {
941 		goto clean0;
942 	}
943 
944 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_2,
945 	    exponent2, k.expo2_bytes, B_FALSE)) != CKR_OK) {
946 		goto clean0;
947 	}
948 
949 	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_COEFFICIENT,
950 	    coefficient, k.coeff_bytes, B_FALSE)) != CKR_OK) {
951 		goto clean0;
952 	}
953 
954 clean0:
955 	return (rv);
956 }
957 
958 
959 CK_ULONG
get_rsa_sha1_prefix(CK_MECHANISM_PTR mech,CK_BYTE_PTR * prefix)960 get_rsa_sha1_prefix(CK_MECHANISM_PTR mech, CK_BYTE_PTR *prefix) {
961 	if (mech->pParameter == NULL) {
962 		*prefix = (CK_BYTE *)SHA1_DER_PREFIX;
963 		return (SHA1_DER_PREFIX_Len);
964 	}
965 
966 	*prefix = (CK_BYTE *)SHA1_DER_PREFIX_OID;
967 	return (SHA1_DER_PREFIX_OID_Len);
968 }
969 
970 CK_RV
soft_rsa_digest_sign_common(soft_session_t * session_p,CK_BYTE_PTR pData,CK_ULONG ulDataLen,CK_BYTE_PTR pSigned,CK_ULONG_PTR pulSignedLen,CK_MECHANISM_TYPE mechanism,boolean_t Final)971 soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
972     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
973     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
974 {
975 
976 	CK_RV rv = CKR_OK;
977 	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space enough for all mechs */
978 	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
979 	/* space enough for all mechs */
980 	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
981 	CK_ULONG der_data_len;
982 	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
983 	soft_object_t *key = rsa_ctx->key;
984 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
985 	uint32_t modulus_len = sizeof (modulus);
986 	CK_ULONG der_len;
987 	CK_BYTE_PTR der_prefix;
988 
989 	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
990 	if (rv != CKR_OK) {
991 		(void) pthread_mutex_lock(&session_p->session_mutex);
992 		free(session_p->digest.context);
993 		session_p->digest.context = NULL;
994 		session_p->digest.flags = 0;
995 		(void) pthread_mutex_unlock(&session_p->session_mutex);
996 		soft_cleanup_object(key);
997 		free(key);
998 		goto clean1;
999 	}
1000 
1001 	/* Check arguments before performing message digest. */
1002 	if (pSigned == NULL) {
1003 		/* Application asks for the length of the output buffer. */
1004 		*pulSignedLen = modulus_len;
1005 		rv = CKR_OK;
1006 		goto clean1;
1007 	}
1008 
1009 	/* Is the application-supplied buffer large enough? */
1010 	if (*pulSignedLen < (CK_ULONG)modulus_len) {
1011 		*pulSignedLen = modulus_len;
1012 		rv = CKR_BUFFER_TOO_SMALL;
1013 		goto clean1;
1014 	}
1015 
1016 	if (Final) {
1017 		rv = soft_digest_final(session_p, hash, &hash_len);
1018 	} else {
1019 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1020 	}
1021 
1022 	if (rv != CKR_OK) {
1023 		/* free the signature key */
1024 		soft_cleanup_object(key);
1025 		free(key);
1026 		goto clean_exit;
1027 	}
1028 
1029 	/*
1030 	 * Prepare the DER encoding of the DigestInfo value by setting it to:
1031 	 *	<MECH>_DER_PREFIX || H
1032 	 *
1033 	 * See rsa_impl.c for more details.
1034 	 */
1035 	switch (session_p->digest.mech.mechanism) {
1036 	case CKM_MD5:
1037 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1038 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1039 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1040 		break;
1041 	case CKM_SHA_1:
1042 		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1043 		    &der_prefix);
1044 		(void) memcpy(der_data, der_prefix, der_len);
1045 		(void) memcpy(der_data + der_len, hash, hash_len);
1046 		der_data_len = der_len + hash_len;
1047 		break;
1048 	case CKM_SHA256:
1049 		(void) memcpy(der_data, SHA256_DER_PREFIX,
1050 		    SHA2_DER_PREFIX_Len);
1051 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1052 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1053 		break;
1054 	case CKM_SHA384:
1055 		(void) memcpy(der_data, SHA384_DER_PREFIX,
1056 		    SHA2_DER_PREFIX_Len);
1057 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1058 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1059 		break;
1060 	case CKM_SHA512:
1061 		(void) memcpy(der_data, SHA512_DER_PREFIX,
1062 		    SHA2_DER_PREFIX_Len);
1063 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1064 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1065 		break;
1066 	}
1067 
1068 	/*
1069 	 * Now, we are ready to sign the DER_ENCODED data
1070 	 * soft_rsa_sign_common() will free the signature key.
1071 	 */
1072 	rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
1073 	    pSigned, pulSignedLen, mechanism);
1074 
1075 clean_exit:
1076 	(void) pthread_mutex_lock(&session_p->session_mutex);
1077 	/* soft_digest_common() has freed the digest context */
1078 	session_p->digest.flags = 0;
1079 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1080 
1081 clean1:
1082 	return (rv);
1083 }
1084 
1085 
1086 CK_RV
soft_rsa_digest_verify_common(soft_session_t * session_p,CK_BYTE_PTR pData,CK_ULONG ulDataLen,CK_BYTE_PTR pSigned,CK_ULONG ulSignedLen,CK_MECHANISM_TYPE mechanism,boolean_t Final)1087 soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1088     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1089     CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1090 {
1091 
1092 	CK_RV rv = CKR_OK;
1093 	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space for all mechs */
1094 	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
1095 	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1096 	CK_ULONG der_data_len;
1097 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1098 	soft_object_t *key = rsa_ctx->key;
1099 	CK_ULONG der_len;
1100 	CK_BYTE_PTR der_prefix;
1101 
1102 	if (Final) {
1103 		rv = soft_digest_final(session_p, hash, &hash_len);
1104 	} else {
1105 		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1106 	}
1107 
1108 	if (rv != CKR_OK) {
1109 		/* free the verification key */
1110 		soft_cleanup_object(key);
1111 		free(key);
1112 		goto clean_exit;
1113 	}
1114 
1115 	/*
1116 	 * Prepare the DER encoding of the DigestInfo value as follows:
1117 	 * MD5:		MD5_DER_PREFIX || H
1118 	 * SHA-1:	SHA1_DER_PREFIX || H
1119 	 * SHA2:	SHA2_DER_PREFIX || H
1120 	 *
1121 	 * See rsa_impl.c for more details.
1122 	 */
1123 	switch (session_p->digest.mech.mechanism) {
1124 	case CKM_MD5:
1125 		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1126 		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1127 		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1128 		break;
1129 	case CKM_SHA_1:
1130 		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1131 		    &der_prefix);
1132 		(void) memcpy(der_data, der_prefix, der_len);
1133 		(void) memcpy(der_data + der_len, hash, hash_len);
1134 		der_data_len = der_len + hash_len;
1135 		break;
1136 	case CKM_SHA256:
1137 		(void) memcpy(der_data, SHA256_DER_PREFIX,
1138 		    SHA2_DER_PREFIX_Len);
1139 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1140 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1141 		break;
1142 	case CKM_SHA384:
1143 		(void) memcpy(der_data, SHA384_DER_PREFIX,
1144 		    SHA2_DER_PREFIX_Len);
1145 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1146 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1147 		break;
1148 	case CKM_SHA512:
1149 		(void) memcpy(der_data, SHA512_DER_PREFIX,
1150 		    SHA2_DER_PREFIX_Len);
1151 		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1152 		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1153 		break;
1154 	}
1155 
1156 	/*
1157 	 * Now, we are ready to verify the DER_ENCODED data using signature.
1158 	 * soft_rsa_verify_common() will free the verification key.
1159 	 */
1160 	rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
1161 	    pSigned, ulSignedLen, mechanism);
1162 
1163 clean_exit:
1164 	(void) pthread_mutex_lock(&session_p->session_mutex);
1165 	/* soft_digest_common() has freed the digest context */
1166 	session_p->digest.flags = 0;
1167 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1168 
1169 	return (rv);
1170 
1171 }
1172 
1173 
1174 CK_RV
soft_rsa_verify_recover(soft_session_t * session_p,CK_BYTE_PTR pSignature,CK_ULONG ulSignatureLen,CK_BYTE_PTR pData,CK_ULONG_PTR pulDataLen)1175 soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
1176     CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
1177 {
1178 
1179 	CK_RV rv = CKR_OK;
1180 	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1181 	CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
1182 	soft_object_t *key = rsa_ctx->key;
1183 	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1184 	uint32_t modulus_len = sizeof (modulus);
1185 	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1186 
1187 	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
1188 	if (rv != CKR_OK) {
1189 		goto clean_exit;
1190 	}
1191 
1192 	if (ulSignatureLen != (CK_ULONG)modulus_len) {
1193 		rv = CKR_SIGNATURE_LEN_RANGE;
1194 		goto clean_exit;
1195 	}
1196 
1197 	/*
1198 	 * Perform RSA decryption with the signer's RSA public key
1199 	 * for verification process.
1200 	 */
1201 	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
1202 	if (rv == CKR_OK) {
1203 		switch (mechanism) {
1204 
1205 		case CKM_RSA_PKCS:
1206 		{
1207 			/*
1208 			 * Strip off the encoded padding bytes in front of the
1209 			 * recovered data.
1210 			 */
1211 			size_t data_len = modulus_len;
1212 
1213 			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1214 			if (rv != CKR_OK) {
1215 				goto clean_exit;
1216 			}
1217 
1218 			/*
1219 			 * If application asks for the length of the output
1220 			 * buffer?
1221 			 */
1222 			if (pData == NULL) {
1223 				*pulDataLen = data_len;
1224 				rv = CKR_OK;
1225 				goto clean1;
1226 			}
1227 
1228 			/* Is the application-supplied buffer large enough? */
1229 			if (*pulDataLen < (CK_ULONG)data_len) {
1230 				*pulDataLen = data_len;
1231 				rv = CKR_BUFFER_TOO_SMALL;
1232 				goto clean1;
1233 			}
1234 
1235 			(void) memcpy(pData,
1236 			    &plain_data[modulus_len - data_len], data_len);
1237 			*pulDataLen = data_len;
1238 
1239 			break;
1240 		}
1241 
1242 		case CKM_RSA_X_509:
1243 			/*
1244 			 * If application asks for the length of the output
1245 			 * buffer?
1246 			 */
1247 			if (pData == NULL) {
1248 				*pulDataLen = modulus_len;
1249 				rv = CKR_OK;
1250 				goto clean1;
1251 			}
1252 
1253 			/* Is the application-supplied buffer large enough? */
1254 			if (*pulDataLen < (CK_ULONG)modulus_len) {
1255 				*pulDataLen = modulus_len;
1256 				rv = CKR_BUFFER_TOO_SMALL;
1257 				goto clean1;
1258 			}
1259 
1260 			(void) memcpy(pData, plain_data, modulus_len);
1261 			*pulDataLen = modulus_len;
1262 
1263 			break;
1264 		}
1265 	}
1266 
1267 clean_exit:
1268 	(void) pthread_mutex_lock(&session_p->session_mutex);
1269 	free(session_p->verify.context);
1270 	session_p->verify.context = NULL;
1271 	(void) pthread_mutex_unlock(&session_p->session_mutex);
1272 	soft_cleanup_object(key);
1273 	free(key);
1274 
1275 clean1:
1276 	return (rv);
1277 }
1278