xref: /illumos-gate/usr/src/uts/common/crypto/io/rsa.c (revision bdfc6d18da790deeec2e0eb09c625902defe2498)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * RSA provider for the Kernel Cryptographic Framework (KCF)
31  */
32 
33 #include <sys/types.h>
34 #include <sys/systm.h>
35 #include <sys/modctl.h>
36 #include <sys/cmn_err.h>
37 #include <sys/ddi.h>
38 #include <sys/crypto/spi.h>
39 #include <sys/sysmacros.h>
40 #include <sys/strsun.h>
41 #include <sys/md5.h>
42 #include <sys/sha1.h>
43 #include <sys/random.h>
44 #include "rsa_impl.h"
45 
46 extern struct mod_ops mod_cryptoops;
47 
48 /*
49  * Module linkage information for the kernel.
50  */
51 static struct modlcrypto modlcrypto = {
52 	&mod_cryptoops,
53 	"RSA Kernel SW Provider %I%"
54 };
55 
56 static struct modlinkage modlinkage = {
57 	MODREV_1,
58 	(void *)&modlcrypto,
59 	NULL
60 };
61 
62 /*
63  * CSPI information (entry points, provider info, etc.)
64  */
65 typedef enum rsa_mech_type {
66 	RSA_PKCS_MECH_INFO_TYPE,	/* SUN_CKM_RSA_PKCS */
67 	RSA_X_509_MECH_INFO_TYPE,	/* SUN_CKM_RSA_X_509 */
68 	MD5_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_MD5_RSA_PKCS */
69 	SHA1_RSA_PKCS_MECH_INFO_TYPE	/* SUN_SHA1_RSA_PKCS */
70 } rsa_mech_type_t;
71 
72 /*
73  * Context for RSA_PKCS and RSA_X_509 mechanisms.
74  */
75 typedef struct rsa_ctx {
76 	rsa_mech_type_t	mech_type;
77 	crypto_key_t *key;
78 	size_t keychunk_size;
79 } rsa_ctx_t;
80 
81 /*
82  * Context for MD5_RSA_PKCS and SHA1_RSA_PKCS mechanisms.
83  */
84 typedef struct digest_rsa_ctx {
85 	rsa_mech_type_t	mech_type;
86 	crypto_key_t *key;
87 	size_t keychunk_size;
88 	union {
89 		MD5_CTX md5ctx;
90 		SHA1_CTX sha1ctx;
91 	} dctx_u;
92 } digest_rsa_ctx_t;
93 
94 #define	md5_ctx		dctx_u.md5ctx
95 #define	sha1_ctx	dctx_u.sha1ctx
96 
97 /*
98  * Mechanism info structure passed to KCF during registration.
99  */
100 static crypto_mech_info_t rsa_mech_info_tab[] = {
101 	/* RSA_PKCS */
102 	{SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
103 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
104 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
105 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
106 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
107 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
108 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
109 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
110 
111 	/* RSA_X_509 */
112 	{SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
113 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
114 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
115 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
116 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
117 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
118 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
119 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
120 
121 	/* MD5_RSA_PKCS */
122 	{SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
123 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
124 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
125 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
126 
127 	/* SHA1_RSA_PKCS */
128 	{SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
129 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
130 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
131 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
132 };
133 
134 #define	RSA_VALID_MECH(mech)					\
135 	(((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||		\
136 	(mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||		\
137 	(mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||	\
138 	(mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
139 
140 /* operations are in-place if the output buffer is NULL */
141 #define	RSA_ARG_INPLACE(input, output)				\
142 	if ((output) == NULL)					\
143 		(output) = (input);
144 
145 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
146 
147 static crypto_control_ops_t rsa_control_ops = {
148 	rsa_provider_status
149 };
150 
151 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
152     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
153 static int rsa_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
154     crypto_req_handle_t);
155 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
156     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
157     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
158 static int rsa_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
159     crypto_req_handle_t);
160 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
161     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
162     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
163 
164 /*
165  * The RSA mechanisms do not have multiple-part cipher operations.
166  * So, the update and final routines are set to NULL.
167  */
168 static crypto_cipher_ops_t rsa_cipher_ops = {
169 	rsa_common_init,
170 	rsa_encrypt,
171 	NULL,
172 	NULL,
173 	rsa_encrypt_atomic,
174 	rsa_common_init,
175 	rsa_decrypt,
176 	NULL,
177 	NULL,
178 	rsa_decrypt_atomic
179 };
180 
181 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
182     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
183 static int rsa_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
184     crypto_req_handle_t);
185 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
186     crypto_req_handle_t);
187 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
188     crypto_req_handle_t);
189 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
190     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
191     crypto_spi_ctx_template_t, crypto_req_handle_t);
192 
193 /*
194  * We use the same routine for sign_init and sign_recover_init fields
195  * as they do the same thing. Same holds for sign and sign_recover fields,
196  * and sign_atomic and sign_recover_atomic fields.
197  */
198 static crypto_sign_ops_t rsa_sign_ops = {
199 	rsa_sign_verify_common_init,
200 	rsa_sign,
201 	rsa_sign_update,
202 	rsa_sign_final,
203 	rsa_sign_atomic,
204 	rsa_sign_verify_common_init,
205 	rsa_sign,
206 	rsa_sign_atomic
207 };
208 
209 static int rsa_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
210     crypto_req_handle_t);
211 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
212     crypto_req_handle_t);
213 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
214     crypto_req_handle_t);
215 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
216     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
217     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
218 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
219     crypto_data_t *, crypto_req_handle_t);
220 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
221     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
222     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
223     crypto_req_handle_t);
224 
225 /*
226  * We use the same routine (rsa_sign_verify_common_init) for verify_init
227  * and verify_recover_init fields as they do the same thing.
228  */
229 static crypto_verify_ops_t rsa_verify_ops = {
230 	rsa_sign_verify_common_init,
231 	rsa_verify,
232 	rsa_verify_update,
233 	rsa_verify_final,
234 	rsa_verify_atomic,
235 	rsa_sign_verify_common_init,
236 	rsa_verify_recover,
237 	rsa_verify_recover_atomic
238 };
239 
240 static int rsa_free_context(crypto_ctx_t *);
241 
242 static crypto_ctx_ops_t rsa_ctx_ops = {
243 	NULL,
244 	rsa_free_context
245 };
246 
247 static crypto_ops_t rsa_crypto_ops = {
248 	&rsa_control_ops,
249 	NULL,
250 	&rsa_cipher_ops,
251 	NULL,
252 	&rsa_sign_ops,
253 	&rsa_verify_ops,
254 	NULL,
255 	NULL,
256 	NULL,
257 	NULL,
258 	NULL,
259 	NULL,
260 	NULL,
261 	&rsa_ctx_ops
262 };
263 
264 static crypto_provider_info_t rsa_prov_info = {
265 	CRYPTO_SPI_VERSION_1,
266 	"RSA Software Provider",
267 	CRYPTO_SW_PROVIDER,
268 	{&modlinkage},
269 	NULL,
270 	&rsa_crypto_ops,
271 	sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
272 	rsa_mech_info_tab
273 };
274 
275 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
276     crypto_data_t *, crypto_data_t *, int);
277 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
278     crypto_data_t *, crypto_data_t *, int);
279 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
280     crypto_data_t *, crypto_data_t *, int);
281 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
282     crypto_data_t *, crypto_data_t *, int);
283 static int compare_data(crypto_data_t *, uchar_t *);
284 
285 /* EXPORT DELETE START */
286 
287 static int core_rsa_encrypt(crypto_key_t *, uchar_t *,
288     int, uchar_t *, int, int);
289 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int,
290     uchar_t *, int);
291 
292 /* EXPORT DELETE END */
293 
294 static crypto_kcf_provider_handle_t rsa_prov_handle = NULL;
295 
296 int
297 _init(void)
298 {
299 	int ret;
300 
301 	/*
302 	 * Register with KCF. If the registration fails, return error.
303 	 */
304 	if ((ret = crypto_register_provider(&rsa_prov_info,
305 	    &rsa_prov_handle)) != CRYPTO_SUCCESS) {
306 		cmn_err(CE_WARN, "rsa _init: crypto_register_provider()"
307 		    "failed (0x%x)", ret);
308 		return (EACCES);
309 	}
310 
311 	if ((ret = mod_install(&modlinkage)) != 0) {
312 		int rv;
313 
314 		ASSERT(rsa_prov_handle != NULL);
315 		/* We should not return if the unregister returns busy. */
316 		while ((rv = crypto_unregister_provider(rsa_prov_handle))
317 		    == CRYPTO_BUSY) {
318 			cmn_err(CE_WARN, "rsa _init: "
319 			    "crypto_unregister_provider() "
320 			    "failed (0x%x). Retrying.", rv);
321 			/* wait 10 seconds and try again. */
322 			delay(10 * drv_usectohz(1000000));
323 		}
324 	}
325 
326 	return (ret);
327 }
328 
329 int
330 _fini(void)
331 {
332 	int ret;
333 
334 	/*
335 	 * Unregister from KCF if previous registration succeeded.
336 	 */
337 	if (rsa_prov_handle != NULL) {
338 		if ((ret = crypto_unregister_provider(rsa_prov_handle)) !=
339 		    CRYPTO_SUCCESS) {
340 			cmn_err(CE_WARN, "rsa _fini: "
341 			    "crypto_unregister_provider() "
342 			    "failed (0x%x)", ret);
343 			return (EBUSY);
344 		}
345 		rsa_prov_handle = NULL;
346 	}
347 
348 	return (mod_remove(&modlinkage));
349 }
350 
351 int
352 _info(struct modinfo *modinfop)
353 {
354 	return (mod_info(&modlinkage, modinfop));
355 }
356 
357 /* ARGSUSED */
358 static void
359 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
360 {
361 	*status = CRYPTO_PROVIDER_READY;
362 }
363 
364 /*
365  * Utility routine to look up a attribute of type, 'type',
366  * in the key.
367  */
368 static int
369 get_key_attr(crypto_key_t *key, crypto_attr_type_t type,
370     uchar_t **value, ssize_t *value_len)
371 {
372 	int i;
373 
374 	ASSERT(key->ck_format == CRYPTO_KEY_ATTR_LIST);
375 	for (i = 0; i < key->ck_count; i++) {
376 		if (key->ck_attrs[i].oa_type == type) {
377 			*value = (uchar_t *)key->ck_attrs[i].oa_value;
378 			*value_len = key->ck_attrs[i].oa_value_len;
379 			return (CRYPTO_SUCCESS);
380 		}
381 	}
382 
383 	return (CRYPTO_FAILED);
384 }
385 
386 static int
387 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
388 {
389 	int rv = CRYPTO_FAILED;
390 
391 /* EXPORT DELETE START */
392 
393 	uchar_t *modulus;
394 	ssize_t modulus_len; /* In bytes */
395 
396 	if (!RSA_VALID_MECH(mechanism))
397 		return (CRYPTO_MECHANISM_INVALID);
398 
399 	/*
400 	 * We only support RSA keys that are passed as a list of
401 	 * object attributes.
402 	 */
403 	if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
404 		return (CRYPTO_KEY_TYPE_INCONSISTENT);
405 	}
406 
407 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
408 	    &modulus_len)) != CRYPTO_SUCCESS) {
409 		return (rv);
410 	}
411 	if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
412 	    modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
413 		return (CRYPTO_KEY_SIZE_RANGE);
414 
415 /* EXPORT DELETE END */
416 
417 	return (rv);
418 }
419 
420 void
421 kmemset(uint8_t *buf, char pattern, size_t len)
422 {
423 	int i = 0;
424 
425 	while (i < len)
426 		buf[i++] = pattern;
427 }
428 
429 /*
430  * This function guarantees to return non-zero random numbers.
431  * This is needed as the /dev/urandom kernel interface,
432  * random_get_pseudo_bytes(), may return zeros.
433  */
434 int
435 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
436 {
437 	int rv;
438 	size_t ebc = 0; /* count of extra bytes in extrarand */
439 	size_t i = 0;
440 	uint8_t extrarand[32];
441 	size_t extrarand_len;
442 
443 	if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
444 		return (rv);
445 
446 	/*
447 	 * Walk through the returned random numbers pointed by ran_out,
448 	 * and look for any random number which is zero.
449 	 * If we find zero, call random_get_pseudo_bytes() to generate
450 	 * another 32 random numbers pool. Replace any zeros in ran_out[]
451 	 * from the random number in pool.
452 	 */
453 	while (i < ran_len) {
454 		if (ran_out[i] != 0) {
455 			i++;
456 			continue;
457 		}
458 
459 		/*
460 		 * Note that it is 'while' so we are guaranteed a
461 		 * non-zero value on exit.
462 		 */
463 		if (ebc == 0) {
464 			/* refresh extrarand */
465 			extrarand_len = sizeof (extrarand);
466 			if ((rv = random_get_pseudo_bytes(extrarand,
467 			    extrarand_len)) != 0) {
468 				return (rv);
469 			}
470 
471 			ebc = extrarand_len;
472 		}
473 		/* Replace zero with byte from extrarand. */
474 		-- ebc;
475 
476 		/*
477 		 * The new random byte zero/non-zero will be checked in
478 		 * the next pass through the loop.
479 		 */
480 		ran_out[i] = extrarand[ebc];
481 	}
482 
483 	return (CRYPTO_SUCCESS);
484 }
485 
486 typedef enum cmd_type {
487 	COPY_FROM_DATA,
488 	COPY_TO_DATA,
489 	COMPARE_TO_DATA,
490 	MD5_DIGEST_DATA,
491 	SHA1_DIGEST_DATA
492 } cmd_type_t;
493 
494 /*
495  * Utility routine to apply the command, 'cmd', to the
496  * data in the uio structure.
497  */
498 static int
499 process_uio_data(crypto_data_t *data, uchar_t *buf, int len,
500     cmd_type_t cmd, void *digest_ctx)
501 {
502 	uio_t *uiop = data->cd_uio;
503 	off_t offset = data->cd_offset;
504 	size_t length = len;
505 	uint_t vec_idx;
506 	size_t cur_len;
507 	uchar_t *datap;
508 
509 	ASSERT(data->cd_format == CRYPTO_DATA_UIO);
510 	if (uiop->uio_segflg != UIO_SYSSPACE) {
511 		return (CRYPTO_ARGUMENTS_BAD);
512 	}
513 
514 	/*
515 	 * Jump to the first iovec containing data to be
516 	 * processed.
517 	 */
518 	for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
519 	    offset >= uiop->uio_iov[vec_idx].iov_len;
520 	    offset -= uiop->uio_iov[vec_idx++].iov_len);
521 
522 	if (vec_idx == uiop->uio_iovcnt) {
523 		/*
524 		 * The caller specified an offset that is larger than
525 		 * the total size of the buffers it provided.
526 		 */
527 		return (CRYPTO_DATA_LEN_RANGE);
528 	}
529 
530 	while (vec_idx < uiop->uio_iovcnt && length > 0) {
531 		cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
532 		    offset, length);
533 
534 		datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
535 		    offset);
536 		switch (cmd) {
537 		case COPY_FROM_DATA:
538 			bcopy(datap, buf, cur_len);
539 			buf += cur_len;
540 			break;
541 		case COPY_TO_DATA:
542 			bcopy(buf, datap, cur_len);
543 			buf += cur_len;
544 			break;
545 		case COMPARE_TO_DATA:
546 			if (bcmp(datap, buf, cur_len))
547 				return (CRYPTO_SIGNATURE_INVALID);
548 			buf += cur_len;
549 			break;
550 		case MD5_DIGEST_DATA:
551 			MD5Update(digest_ctx, datap, cur_len);
552 			break;
553 		case SHA1_DIGEST_DATA:
554 			SHA1Update(digest_ctx, datap, cur_len);
555 			break;
556 		}
557 
558 		length -= cur_len;
559 		vec_idx++;
560 		offset = 0;
561 	}
562 
563 	if (vec_idx == uiop->uio_iovcnt && length > 0) {
564 		/*
565 		 * The end of the specified iovec's was reached but
566 		 * the length requested could not be processed.
567 		 */
568 		switch (cmd) {
569 		case COPY_TO_DATA:
570 			data->cd_length = len;
571 			return (CRYPTO_BUFFER_TOO_SMALL);
572 		default:
573 			return (CRYPTO_DATA_LEN_RANGE);
574 		}
575 	}
576 
577 	return (CRYPTO_SUCCESS);
578 }
579 
580 /*
581  * Utility routine to apply the command, 'cmd', to the
582  * data in the mblk structure.
583  */
584 static int
585 process_mblk_data(crypto_data_t *data, uchar_t *buf, int len,
586     cmd_type_t cmd, void *digest_ctx)
587 {
588 	off_t offset = data->cd_offset;
589 	size_t length = len;
590 	mblk_t *mp;
591 	size_t cur_len;
592 	uchar_t *datap;
593 
594 	ASSERT(data->cd_format == CRYPTO_DATA_MBLK);
595 	/*
596 	 * Jump to the first mblk_t containing data to be processed.
597 	 */
598 	for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
599 	    offset -= MBLKL(mp), mp = mp->b_cont);
600 	if (mp == NULL) {
601 		/*
602 		 * The caller specified an offset that is larger
603 		 * than the total size of the buffers it provided.
604 		 */
605 		return (CRYPTO_DATA_LEN_RANGE);
606 	}
607 
608 	/*
609 	 * Now do the processing on the mblk chain.
610 	 */
611 	while (mp != NULL && length > 0) {
612 		cur_len = MIN(MBLKL(mp) - offset, length);
613 
614 		datap = (uchar_t *)(mp->b_rptr + offset);
615 		switch (cmd) {
616 		case COPY_FROM_DATA:
617 			bcopy(datap, buf, cur_len);
618 			buf += cur_len;
619 			break;
620 		case COPY_TO_DATA:
621 			bcopy(buf, datap, cur_len);
622 			buf += cur_len;
623 			break;
624 		case COMPARE_TO_DATA:
625 			if (bcmp(datap, buf, cur_len))
626 				return (CRYPTO_SIGNATURE_INVALID);
627 			buf += cur_len;
628 			break;
629 		case MD5_DIGEST_DATA:
630 			MD5Update(digest_ctx, datap, cur_len);
631 			break;
632 		case SHA1_DIGEST_DATA:
633 			SHA1Update(digest_ctx, datap, cur_len);
634 			break;
635 		}
636 
637 		length -= cur_len;
638 		offset = 0;
639 		mp = mp->b_cont;
640 	}
641 
642 	if (mp == NULL && length > 0) {
643 		/*
644 		 * The end of the mblk was reached but the length
645 		 * requested could not be processed.
646 		 */
647 		switch (cmd) {
648 		case COPY_TO_DATA:
649 			data->cd_length = len;
650 			return (CRYPTO_BUFFER_TOO_SMALL);
651 		default:
652 			return (CRYPTO_DATA_LEN_RANGE);
653 		}
654 	}
655 
656 	return (CRYPTO_SUCCESS);
657 }
658 
659 static int
660 compare_data(crypto_data_t *data, uchar_t *buf)
661 {
662 	int len;
663 	uchar_t *dptr;
664 
665 	len = data->cd_length;
666 	switch (data->cd_format) {
667 	case CRYPTO_DATA_RAW:
668 		dptr = (uchar_t *)(data->cd_raw.iov_base +
669 		    data->cd_offset);
670 
671 		return (bcmp(dptr, buf, len));
672 
673 	case CRYPTO_DATA_UIO:
674 		return (process_uio_data(data, buf, len,
675 		    COMPARE_TO_DATA, NULL));
676 
677 	case CRYPTO_DATA_MBLK:
678 		return (process_mblk_data(data, buf, len,
679 		    COMPARE_TO_DATA, NULL));
680 	}
681 
682 	return (CRYPTO_FAILED);
683 }
684 
685 /*
686  * Utility routine to copy a buffer to a crypto_data structure.
687  */
688 static int
689 put_output_data(uchar_t *buf, crypto_data_t *output, int len)
690 {
691 	switch (output->cd_format) {
692 	case CRYPTO_DATA_RAW:
693 		if (output->cd_raw.iov_len < len) {
694 			output->cd_length = len;
695 			return (CRYPTO_BUFFER_TOO_SMALL);
696 		}
697 		bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
698 		    output->cd_offset), len);
699 		break;
700 
701 	case CRYPTO_DATA_UIO:
702 		return (process_uio_data(output, buf, len, COPY_TO_DATA, NULL));
703 
704 	case CRYPTO_DATA_MBLK:
705 		return (process_mblk_data(output, buf, len,
706 		    COPY_TO_DATA, NULL));
707 
708 	default:
709 		return (CRYPTO_ARGUMENTS_BAD);
710 	}
711 
712 	return (CRYPTO_SUCCESS);
713 }
714 
715 /*
716  * Utility routine to get data from a crypto_data structure.
717  *
718  * '*dptr' contains a pointer to a buffer on return. 'buf'
719  * is allocated by the caller and is ignored for CRYPTO_DATA_RAW case.
720  */
721 static int
722 get_input_data(crypto_data_t *input, uchar_t **dptr, uchar_t *buf)
723 {
724 	int rv;
725 
726 	switch (input->cd_format) {
727 	case CRYPTO_DATA_RAW:
728 		if (input->cd_raw.iov_len < input->cd_length)
729 			return (CRYPTO_ARGUMENTS_BAD);
730 		*dptr = (uchar_t *)(input->cd_raw.iov_base +
731 		    input->cd_offset);
732 		break;
733 
734 	case CRYPTO_DATA_UIO:
735 		if ((rv = process_uio_data(input, buf, input->cd_length,
736 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
737 			return (rv);
738 		*dptr = buf;
739 		break;
740 
741 	case CRYPTO_DATA_MBLK:
742 		if ((rv = process_mblk_data(input, buf, input->cd_length,
743 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
744 			return (rv);
745 		*dptr = buf;
746 		break;
747 
748 	default:
749 		return (CRYPTO_ARGUMENTS_BAD);
750 	}
751 
752 	return (CRYPTO_SUCCESS);
753 }
754 
755 static int
756 copy_key_to_ctx(crypto_key_t *in_key, rsa_ctx_t *ctx, int kmflag)
757 {
758 	int i, count;
759 	size_t len;
760 	caddr_t attr_val;
761 	crypto_object_attribute_t *k_attrs = NULL;
762 
763 	ASSERT(in_key->ck_format == CRYPTO_KEY_ATTR_LIST);
764 
765 	count = in_key->ck_count;
766 	/* figure out how much memory to allocate for everything */
767 	len = sizeof (crypto_key_t) +
768 	    count * sizeof (crypto_object_attribute_t);
769 	for (i = 0; i < count; i++) {
770 		len += roundup(in_key->ck_attrs[i].oa_value_len,
771 		    sizeof (caddr_t));
772 	}
773 
774 	/* one big allocation for everything */
775 	ctx->key = kmem_alloc(len, kmflag);
776 	if (ctx->key == NULL)
777 		return (CRYPTO_HOST_MEMORY);
778 	k_attrs = (crypto_object_attribute_t *)((caddr_t)(ctx->key) +
779 	    sizeof (crypto_key_t));
780 
781 	attr_val = (caddr_t)k_attrs +
782 	    count * sizeof (crypto_object_attribute_t);
783 	for (i = 0; i < count; i++) {
784 		k_attrs[i].oa_type = in_key->ck_attrs[i].oa_type;
785 		bcopy(in_key->ck_attrs[i].oa_value, attr_val,
786 		    in_key->ck_attrs[i].oa_value_len);
787 		k_attrs[i].oa_value = attr_val;
788 		k_attrs[i].oa_value_len = in_key->ck_attrs[i].oa_value_len;
789 		attr_val += roundup(k_attrs[i].oa_value_len, sizeof (caddr_t));
790 	}
791 
792 	ctx->keychunk_size = len;	/* save the size to be freed */
793 	ctx->key->ck_format = CRYPTO_KEY_ATTR_LIST;
794 	ctx->key->ck_count = count;
795 	ctx->key->ck_attrs = k_attrs;
796 
797 	return (CRYPTO_SUCCESS);
798 }
799 
800 /* ARGSUSED */
801 static int
802 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
803     crypto_key_t *key, crypto_spi_ctx_template_t template,
804     crypto_req_handle_t req)
805 {
806 	int rv;
807 	int kmflag;
808 	rsa_ctx_t *ctxp;
809 
810 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
811 		return (rv);
812 
813 	/*
814 	 * Allocate a RSA context.
815 	 */
816 	kmflag = crypto_kmflag(req);
817 	if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
818 		return (CRYPTO_HOST_MEMORY);
819 
820 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
821 		kmem_free(ctxp, sizeof (rsa_ctx_t));
822 		return (rv);
823 	}
824 	ctxp->mech_type = mechanism->cm_type;
825 
826 	ctx->cc_provider_private = ctxp;
827 
828 	return (CRYPTO_SUCCESS);
829 }
830 
831 /* ARGSUSED */
832 static int
833 rsa_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
834     crypto_data_t *ciphertext, crypto_req_handle_t req)
835 {
836 	int rv;
837 	rsa_ctx_t *ctxp;
838 
839 	ASSERT(ctx->cc_provider_private != NULL);
840 	ctxp = ctx->cc_provider_private;
841 
842 	RSA_ARG_INPLACE(plaintext, ciphertext);
843 
844 	/*
845 	 * Note on the KM_SLEEP flag passed to the routine below -
846 	 * rsa_encrypt() is a single-part encryption routine which is
847 	 * currently usable only by /dev/crypto. Since /dev/crypto calls are
848 	 * always synchronous, we can safely pass KM_SLEEP here.
849 	 */
850 	rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
851 	    ciphertext, KM_SLEEP);
852 
853 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
854 		(void) rsa_free_context(ctx);
855 
856 	return (rv);
857 }
858 
859 /* ARGSUSED */
860 static int
861 rsa_encrypt_atomic(crypto_provider_handle_t provider,
862     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
863     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
864     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
865 {
866 	int rv;
867 
868 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
869 		return (rv);
870 	RSA_ARG_INPLACE(plaintext, ciphertext);
871 
872 	return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
873 	    ciphertext, crypto_kmflag(req)));
874 }
875 
876 static int
877 rsa_free_context(crypto_ctx_t *ctx)
878 {
879 	rsa_ctx_t *ctxp = ctx->cc_provider_private;
880 
881 	if (ctxp != NULL) {
882 		bzero(ctxp->key, ctxp->keychunk_size);
883 		kmem_free(ctxp->key, ctxp->keychunk_size);
884 
885 		if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
886 		    ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
887 			kmem_free(ctxp, sizeof (rsa_ctx_t));
888 		else
889 			kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
890 
891 		ctx->cc_provider_private = NULL;
892 	}
893 
894 	return (CRYPTO_SUCCESS);
895 }
896 
897 static int
898 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
899     crypto_data_t *plaintext, crypto_data_t *ciphertext, int kmflag)
900 {
901 	int rv = CRYPTO_FAILED;
902 
903 /* EXPORT DELETE START */
904 
905 	int plen;
906 	uchar_t *ptptr;
907 	uchar_t *modulus;
908 	ssize_t modulus_len;
909 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
910 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
911 	uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
912 
913 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
914 	    &modulus_len)) != CRYPTO_SUCCESS) {
915 		return (rv);
916 	}
917 
918 	plen = plaintext->cd_length;
919 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
920 		if (plen > (modulus_len - MIN_PKCS1_PADLEN))
921 			return (CRYPTO_DATA_LEN_RANGE);
922 	} else {
923 		if (plen > modulus_len)
924 			return (CRYPTO_DATA_LEN_RANGE);
925 	}
926 
927 	/*
928 	 * Output buf len must not be less than RSA modulus size.
929 	 */
930 	if (ciphertext->cd_length < modulus_len) {
931 		ciphertext->cd_length = modulus_len;
932 		return (CRYPTO_BUFFER_TOO_SMALL);
933 	}
934 
935 	ASSERT(plaintext->cd_length <= sizeof (tmp_data));
936 	if ((rv = get_input_data(plaintext, &ptptr, tmp_data))
937 	    != CRYPTO_SUCCESS)
938 		return (rv);
939 
940 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
941 		rv = soft_encrypt_rsa_pkcs_encode(ptptr, plen,
942 		    plain_data, modulus_len);
943 
944 		if (rv != CRYPTO_SUCCESS)
945 			return (rv);
946 	} else {
947 		bzero(plain_data, modulus_len - plen);
948 		bcopy(ptptr, &plain_data[modulus_len - plen], plen);
949 	}
950 
951 	rv = core_rsa_encrypt(key, plain_data, modulus_len,
952 	    cipher_data, kmflag, 1);
953 	if (rv == CRYPTO_SUCCESS) {
954 		/* copy out to ciphertext */
955 		if ((rv = put_output_data(cipher_data,
956 		    ciphertext, modulus_len)) != CRYPTO_SUCCESS)
957 			return (rv);
958 
959 		ciphertext->cd_length = modulus_len;
960 	}
961 
962 /* EXPORT DELETE END */
963 
964 	return (rv);
965 }
966 
967 /* EXPORT DELETE START */
968 
969 static int
970 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
971     int in_len, uchar_t *out, int kmflag, int is_public)
972 {
973 	int rv;
974 	uchar_t *expo, *modulus;
975 	ssize_t	expo_len;
976 	ssize_t modulus_len;
977 	BIGNUM msg;
978 	RSAkey *rsakey;
979 
980 	if (is_public) {
981 		if ((rv = get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT, &expo,
982 		    &expo_len)) != CRYPTO_SUCCESS)
983 			return (rv);
984 	} else {
985 		/*
986 		 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
987 		 * RSA secret key. See the comments in core_rsa_decrypt
988 		 * routine which calls this routine with a private key.
989 		 */
990 		if ((rv = get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT, &expo,
991 		    &expo_len)) != CRYPTO_SUCCESS)
992 			return (rv);
993 	}
994 
995 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
996 	    &modulus_len)) != CRYPTO_SUCCESS) {
997 		return (rv);
998 	}
999 
1000 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1001 	if (rsakey == NULL)
1002 		return (CRYPTO_HOST_MEMORY);
1003 
1004 	/* psize and qsize for RSA_key_init is in bits. */
1005 	if (RSA_key_init(rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
1006 		rv = CRYPTO_HOST_MEMORY;
1007 		goto clean1;
1008 	}
1009 
1010 	/* Size for big_init is in (32-bit) words. */
1011 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1012 	    (int)sizeof (uint32_t)) != BIG_OK) {
1013 		rv = CRYPTO_HOST_MEMORY;
1014 		goto clean2;
1015 	}
1016 
1017 	/* Convert octet string exponent to big integer format. */
1018 	bytestring2bignum(&(rsakey->e), expo, expo_len);
1019 
1020 	/* Convert octet string modulus to big integer format. */
1021 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1022 
1023 	/* Convert octet string input data to big integer format. */
1024 	bytestring2bignum(&msg, in, in_len);
1025 
1026 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1027 		rv = CRYPTO_DATA_LEN_RANGE;
1028 		goto clean3;
1029 	}
1030 
1031 	/* Perform RSA computation on big integer input data. */
1032 	if (big_modexp(&msg, &msg, &(rsakey->e), &(rsakey->n), NULL)
1033 	    != BIG_OK) {
1034 		rv = CRYPTO_HOST_MEMORY;
1035 		goto clean3;
1036 	}
1037 
1038 	/* Convert the big integer output data to octet string. */
1039 	bignum2bytestring(out, &msg, modulus_len);
1040 
1041 	/*
1042 	 * Should not free modulus and expo as both are just pointers
1043 	 * to an attribute value buffer from the caller.
1044 	 */
1045 clean3:
1046 	big_finish(&msg);
1047 clean2:
1048 	RSA_key_finish(rsakey);
1049 clean1:
1050 	kmem_free(rsakey, sizeof (RSAkey));
1051 
1052 	return (rv);
1053 }
1054 
1055 /* EXPORT DELETE END */
1056 
1057 /* ARGSUSED */
1058 static int
1059 rsa_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
1060     crypto_data_t *plaintext, crypto_req_handle_t req)
1061 {
1062 	int rv;
1063 	rsa_ctx_t *ctxp;
1064 
1065 	ASSERT(ctx->cc_provider_private != NULL);
1066 	ctxp = ctx->cc_provider_private;
1067 
1068 	RSA_ARG_INPLACE(ciphertext, plaintext);
1069 
1070 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1071 	rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
1072 	    ciphertext, plaintext, KM_SLEEP);
1073 
1074 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1075 		(void) rsa_free_context(ctx);
1076 
1077 	return (rv);
1078 }
1079 
1080 /* ARGSUSED */
1081 static int
1082 rsa_decrypt_atomic(crypto_provider_handle_t provider,
1083     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1084     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
1085     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1086 {
1087 	int rv;
1088 
1089 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1090 		return (rv);
1091 	RSA_ARG_INPLACE(ciphertext, plaintext);
1092 
1093 	return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
1094 	    plaintext, crypto_kmflag(req)));
1095 }
1096 
1097 static int
1098 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1099     crypto_data_t *ciphertext, crypto_data_t *plaintext, int kmflag)
1100 {
1101 	int rv = CRYPTO_FAILED;
1102 
1103 /* EXPORT DELETE START */
1104 
1105 	int plain_len;
1106 	uchar_t *ctptr;
1107 	uchar_t *modulus;
1108 	ssize_t modulus_len;
1109 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1110 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1111 
1112 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1113 	    &modulus_len)) != CRYPTO_SUCCESS) {
1114 		return (rv);
1115 	}
1116 
1117 	/*
1118 	 * Ciphertext length must be equal to RSA modulus size.
1119 	 */
1120 	if (ciphertext->cd_length != modulus_len)
1121 		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
1122 
1123 	ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
1124 	if ((rv = get_input_data(ciphertext, &ctptr, tmp_data))
1125 	    != CRYPTO_SUCCESS)
1126 		return (rv);
1127 
1128 	rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data, kmflag);
1129 	if (rv == CRYPTO_SUCCESS) {
1130 		plain_len = modulus_len;
1131 
1132 		if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1133 			/* Strip off the PKCS block formatting data. */
1134 			rv = soft_decrypt_rsa_pkcs_decode(plain_data,
1135 			    &plain_len);
1136 			if (rv != CRYPTO_SUCCESS)
1137 				return (rv);
1138 		}
1139 
1140 		if (plain_len > plaintext->cd_length) {
1141 			plaintext->cd_length = plain_len;
1142 			return (CRYPTO_BUFFER_TOO_SMALL);
1143 		}
1144 
1145 		if ((rv = put_output_data(plain_data + modulus_len - plain_len,
1146 		    plaintext, plain_len)) != CRYPTO_SUCCESS)
1147 			return (rv);
1148 
1149 		plaintext->cd_length = plain_len;
1150 	}
1151 
1152 /* EXPORT DELETE END */
1153 
1154 	return (rv);
1155 }
1156 
1157 /* EXPORT DELETE START */
1158 
1159 static int
1160 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len,
1161     uchar_t *out, int kmflag)
1162 {
1163 	int rv;
1164 	uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
1165 	ssize_t modulus_len;
1166 	ssize_t	prime1_len, prime2_len;
1167 	ssize_t	expo1_len, expo2_len, coef_len;
1168 	BIGNUM msg;
1169 	RSAkey *rsakey;
1170 
1171 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1172 	    &modulus_len)) != CRYPTO_SUCCESS) {
1173 		return (rv);
1174 	}
1175 
1176 	/*
1177 	 * The following attributes are not required to be
1178 	 * present in a RSA secret key. If any of them is not present
1179 	 * we call the encrypt routine with a flag indicating use of
1180 	 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
1181 	 * a required attribute for a RSA secret key.
1182 	 */
1183 	if ((get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
1184 		!= CRYPTO_SUCCESS) ||
1185 	    (get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
1186 		!= CRYPTO_SUCCESS) ||
1187 	    (get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
1188 		!= CRYPTO_SUCCESS) ||
1189 	    (get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
1190 		!= CRYPTO_SUCCESS) ||
1191 	    (get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
1192 		!= CRYPTO_SUCCESS)) {
1193 		return (core_rsa_encrypt(key, in, in_len, out, kmflag, 0));
1194 	}
1195 
1196 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1197 	if (rsakey == NULL)
1198 		return (CRYPTO_HOST_MEMORY);
1199 
1200 	/* psize and qsize for RSA_key_init is in bits. */
1201 	if (RSA_key_init(rsakey, prime2_len * 8, prime1_len * 8) != BIG_OK) {
1202 		rv = CRYPTO_HOST_MEMORY;
1203 		goto clean1;
1204 	}
1205 
1206 	/* Size for big_init is in (32-bit) words. */
1207 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1208 	    (int)sizeof (uint32_t)) != BIG_OK) {
1209 		rv = CRYPTO_HOST_MEMORY;
1210 		goto clean2;
1211 	}
1212 
1213 	/* Convert octet string input data to big integer format. */
1214 	bytestring2bignum(&msg, in, in_len);
1215 
1216 	/* Convert octet string modulus to big integer format. */
1217 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1218 
1219 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1220 		rv = CRYPTO_DATA_LEN_RANGE;
1221 		goto clean3;
1222 	}
1223 
1224 	/* Convert the rest of private key attributes to big integer format. */
1225 	bytestring2bignum(&(rsakey->dmodpminus1), expo2, expo2_len);
1226 	bytestring2bignum(&(rsakey->dmodqminus1), expo1, expo1_len);
1227 	bytestring2bignum(&(rsakey->p), prime2, prime2_len);
1228 	bytestring2bignum(&(rsakey->q), prime1, prime1_len);
1229 	bytestring2bignum(&(rsakey->pinvmodq), coef, coef_len);
1230 
1231 	if ((big_cmp_abs(&(rsakey->dmodpminus1), &(rsakey->p)) > 0) ||
1232 	    (big_cmp_abs(&(rsakey->dmodqminus1), &(rsakey->q)) > 0) ||
1233 	    (big_cmp_abs(&(rsakey->pinvmodq), &(rsakey->q)) > 0)) {
1234 		rv = CRYPTO_KEY_SIZE_RANGE;
1235 		goto clean3;
1236 	}
1237 
1238 	/* Perform RSA computation on big integer input data. */
1239 	if (big_modexp_crt(&msg, &msg, &(rsakey->dmodpminus1),
1240 	    &(rsakey->dmodqminus1), &(rsakey->p), &(rsakey->q),
1241 	    &(rsakey->pinvmodq), NULL, NULL) != BIG_OK) {
1242 		rv = CRYPTO_HOST_MEMORY;
1243 		goto clean3;
1244 	}
1245 
1246 	/* Convert the big integer output data to octet string. */
1247 	bignum2bytestring(out, &msg, modulus_len);
1248 
1249 	/*
1250 	 * Should not free modulus and friends as they are just pointers
1251 	 * to an attribute value buffer from the caller.
1252 	 */
1253 clean3:
1254 	big_finish(&msg);
1255 clean2:
1256 	RSA_key_finish(rsakey);
1257 clean1:
1258 	kmem_free(rsakey, sizeof (RSAkey));
1259 
1260 	return (rv);
1261 }
1262 
1263 /* EXPORT DELETE END */
1264 
1265 /* ARGSUSED */
1266 static int
1267 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
1268     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
1269     crypto_req_handle_t req)
1270 {
1271 	int rv;
1272 	int kmflag;
1273 	rsa_ctx_t *ctxp;
1274 	digest_rsa_ctx_t *dctxp;
1275 
1276 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1277 		return (rv);
1278 
1279 	/*
1280 	 * Allocate a RSA context.
1281 	 */
1282 	kmflag = crypto_kmflag(req);
1283 	switch (mechanism->cm_type) {
1284 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1285 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1286 		dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
1287 		ctxp = (rsa_ctx_t *)dctxp;
1288 		break;
1289 	default:
1290 		ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
1291 		break;
1292 	}
1293 
1294 	if (ctxp == NULL)
1295 		return (CRYPTO_HOST_MEMORY);
1296 
1297 	ctxp->mech_type = mechanism->cm_type;
1298 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
1299 		switch (mechanism->cm_type) {
1300 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1301 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1302 			kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
1303 			break;
1304 		default:
1305 			kmem_free(ctxp, sizeof (rsa_ctx_t));
1306 			break;
1307 		}
1308 		return (rv);
1309 	}
1310 
1311 	switch (mechanism->cm_type) {
1312 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1313 		MD5Init(&(dctxp->md5_ctx));
1314 		break;
1315 
1316 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1317 		SHA1Init(&(dctxp->sha1_ctx));
1318 		break;
1319 	}
1320 
1321 	ctx->cc_provider_private = ctxp;
1322 
1323 	return (CRYPTO_SUCCESS);
1324 }
1325 
1326 #define	SHA1_DIGEST_SIZE 20
1327 #define	MD5_DIGEST_SIZE 16
1328 
1329 #define	INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)	\
1330 	(data).cd_format = CRYPTO_DATA_RAW;		\
1331 	(data).cd_offset = 0;				\
1332 	(data).cd_raw.iov_base = (char *)base;		\
1333 	(data).cd_raw.iov_len = len;			\
1334 	(data).cd_length = cd_len;
1335 
1336 #define	DO_UPDATE	0x01
1337 #define	DO_FINAL	0x02
1338 #define	DO_MD5		0x04
1339 #define	DO_SHA1		0x08
1340 #define	DO_SIGN		0x10
1341 #define	DO_VERIFY	0x20
1342 
1343 static int
1344 digest_data(crypto_data_t *data, void *dctx, uchar_t *digest,
1345     uchar_t flag)
1346 {
1347 	int rv, dlen;
1348 	uchar_t *dptr;
1349 	cmd_type_t cmd;
1350 
1351 	ASSERT(flag & DO_MD5 || flag & DO_SHA1);
1352 	if (data == NULL) {
1353 		ASSERT((flag & DO_UPDATE) == 0);
1354 		goto dofinal;
1355 	}
1356 
1357 	dlen = data->cd_length;
1358 
1359 	switch (data->cd_format) {
1360 	case CRYPTO_DATA_RAW:
1361 		dptr = (uchar_t *)(data->cd_raw.iov_base +
1362 		    data->cd_offset);
1363 
1364 		if (flag & DO_MD5) {
1365 			if (flag & DO_UPDATE)
1366 				MD5Update(dctx, dptr, dlen);
1367 		} else {
1368 			if (flag & DO_UPDATE)
1369 				SHA1Update(dctx, dptr, dlen);
1370 		}
1371 		break;
1372 
1373 	case CRYPTO_DATA_UIO:
1374 		cmd = ((flag & DO_MD5) ? MD5_DIGEST_DATA : SHA1_DIGEST_DATA);
1375 		if (flag & DO_UPDATE) {
1376 			rv = process_uio_data(data, NULL, dlen, cmd, dctx);
1377 			if (rv != CRYPTO_SUCCESS)
1378 				return (rv);
1379 		}
1380 		break;
1381 
1382 	case CRYPTO_DATA_MBLK:
1383 		cmd = ((flag & DO_MD5) ? MD5_DIGEST_DATA : SHA1_DIGEST_DATA);
1384 		if (flag & DO_UPDATE) {
1385 			rv = process_mblk_data(data, NULL, dlen, cmd, dctx);
1386 			if (rv != CRYPTO_SUCCESS)
1387 				return (rv);
1388 		}
1389 		break;
1390 	}
1391 
1392 dofinal:
1393 	if (flag & DO_FINAL) {
1394 		if (flag & DO_MD5)
1395 			MD5Final(digest, dctx);
1396 		else
1397 			SHA1Final(digest, dctx);
1398 	}
1399 
1400 	return (CRYPTO_SUCCESS);
1401 }
1402 
1403 static int
1404 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
1405     crypto_data_t *signature, int kmflag, uchar_t flag)
1406 {
1407 	int rv = CRYPTO_FAILED;
1408 
1409 /* EXPORT DELETE START */
1410 
1411 	int dlen;
1412 	uchar_t digest[SHA1_DIGEST_SIZE];
1413 	/* The der_data size is enough for MD5 also */
1414 	uchar_t der_data[SHA1_DIGEST_SIZE + SHA1_DER_PREFIX_Len];
1415 	ulong_t der_data_len;
1416 	crypto_data_t der_cd;
1417 	rsa_mech_type_t mech_type;
1418 
1419 	ASSERT(flag & DO_SIGN || flag & DO_VERIFY);
1420 	ASSERT(data != NULL || (flag & DO_FINAL));
1421 
1422 	mech_type = ctxp->mech_type;
1423 	if (!(mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||
1424 	    mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE))
1425 		return (CRYPTO_MECHANISM_INVALID);
1426 
1427 	/*
1428 	 * We need to do the BUFFER_TOO_SMALL check before digesting
1429 	 * the data. No check is needed for verify as signature is not
1430 	 * an output argument for verify.
1431 	 */
1432 	if (flag & DO_SIGN) {
1433 		uchar_t *modulus;
1434 		ssize_t modulus_len;
1435 
1436 		if ((rv = get_key_attr(ctxp->key, SUN_CKA_MODULUS, &modulus,
1437 		    &modulus_len)) != CRYPTO_SUCCESS) {
1438 			return (rv);
1439 		}
1440 
1441 		if (signature->cd_length < modulus_len) {
1442 			signature->cd_length = modulus_len;
1443 			return (CRYPTO_BUFFER_TOO_SMALL);
1444 		}
1445 	}
1446 
1447 	dlen = ((mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE) ?
1448 	    MD5_DIGEST_SIZE : SHA1_DIGEST_SIZE);
1449 
1450 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1451 		rv = digest_data(data, &(ctxp->md5_ctx),
1452 		    digest, flag | DO_MD5);
1453 	else
1454 		rv = digest_data(data, &(ctxp->sha1_ctx),
1455 		    digest, flag | DO_SHA1);
1456 	if (rv != CRYPTO_SUCCESS)
1457 		return (rv);
1458 
1459 	/*
1460 	 * Prepare the DER encoding of the DigestInfo value as follows:
1461 	 * MD5:		MD5_DER_PREFIX || H
1462 	 * SHA-1:	SHA1_DER_PREFIX || H
1463 	 *
1464 	 * See rsa_impl.c for more details.
1465 	 */
1466 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE) {
1467 		bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1468 		bcopy(digest, der_data + MD5_DER_PREFIX_Len, dlen);
1469 		der_data_len = MD5_DER_PREFIX_Len + dlen;
1470 	} else {
1471 		bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1472 		bcopy(digest, der_data + SHA1_DER_PREFIX_Len, dlen);
1473 		der_data_len = SHA1_DER_PREFIX_Len + dlen;
1474 	}
1475 
1476 	INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1477 	/*
1478 	 * Now, we are ready to sign or verify the DER_ENCODED data.
1479 	 */
1480 	if (flag & DO_SIGN)
1481 		rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1482 		    signature, kmflag);
1483 	else
1484 		rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1485 		    signature, kmflag);
1486 
1487 /* EXPORT DELETE END */
1488 
1489 	return (rv);
1490 }
1491 
1492 static int
1493 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1494     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1495 {
1496 	int rv = CRYPTO_FAILED;
1497 
1498 /* EXPORT DELETE START */
1499 
1500 	int dlen;
1501 	uchar_t *dataptr, *modulus;
1502 	ssize_t modulus_len;
1503 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1504 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1505 	uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1506 
1507 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1508 	    &modulus_len)) != CRYPTO_SUCCESS) {
1509 		return (rv);
1510 	}
1511 
1512 	dlen = data->cd_length;
1513 	switch (mech_type) {
1514 	case RSA_PKCS_MECH_INFO_TYPE:
1515 		if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1516 			return (CRYPTO_DATA_LEN_RANGE);
1517 		break;
1518 	case RSA_X_509_MECH_INFO_TYPE:
1519 		if (dlen > modulus_len)
1520 			return (CRYPTO_DATA_LEN_RANGE);
1521 		break;
1522 	}
1523 
1524 	if (signature->cd_length < modulus_len) {
1525 		signature->cd_length = modulus_len;
1526 		return (CRYPTO_BUFFER_TOO_SMALL);
1527 	}
1528 
1529 	ASSERT(data->cd_length <= sizeof (tmp_data));
1530 	if ((rv = get_input_data(data, &dataptr, tmp_data))
1531 	    != CRYPTO_SUCCESS)
1532 		return (rv);
1533 
1534 	switch (mech_type) {
1535 	case RSA_PKCS_MECH_INFO_TYPE:
1536 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1537 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1538 		/*
1539 		 * Add PKCS padding to the input data to format a block
1540 		 * type "01" encryption block.
1541 		 */
1542 		rv = soft_sign_rsa_pkcs_encode(dataptr, dlen, plain_data,
1543 		    modulus_len);
1544 		if (rv != CRYPTO_SUCCESS)
1545 			return (rv);
1546 
1547 		break;
1548 
1549 	case RSA_X_509_MECH_INFO_TYPE:
1550 		bzero(plain_data, modulus_len - dlen);
1551 		bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1552 		break;
1553 	}
1554 
1555 	rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data,
1556 	    kmflag);
1557 	if (rv == CRYPTO_SUCCESS) {
1558 		/* copy out to signature */
1559 		if ((rv = put_output_data(signed_data,
1560 		    signature, modulus_len)) != CRYPTO_SUCCESS)
1561 			return (rv);
1562 
1563 		signature->cd_length = modulus_len;
1564 	}
1565 
1566 /* EXPORT DELETE END */
1567 
1568 	return (rv);
1569 }
1570 
1571 /* ARGSUSED */
1572 static int
1573 rsa_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1574     crypto_req_handle_t req)
1575 {
1576 	int rv;
1577 	rsa_ctx_t *ctxp;
1578 
1579 	ASSERT(ctx->cc_provider_private != NULL);
1580 	ctxp = ctx->cc_provider_private;
1581 
1582 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1583 	switch (ctxp->mech_type) {
1584 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1585 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1586 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1587 		    signature, KM_SLEEP, DO_SIGN | DO_UPDATE | DO_FINAL);
1588 		break;
1589 	default:
1590 		rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1591 		    signature, KM_SLEEP);
1592 		break;
1593 	}
1594 
1595 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1596 		(void) rsa_free_context(ctx);
1597 
1598 	return (rv);
1599 }
1600 
1601 /* ARGSUSED */
1602 static int
1603 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1604 {
1605 	int rv;
1606 	digest_rsa_ctx_t *ctxp;
1607 	rsa_mech_type_t mech_type;
1608 
1609 	ASSERT(ctx->cc_provider_private != NULL);
1610 	ctxp = ctx->cc_provider_private;
1611 	mech_type = ctxp->mech_type;
1612 
1613 	if (!(mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||
1614 	    mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE))
1615 		return (CRYPTO_MECHANISM_INVALID);
1616 
1617 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1618 		rv = digest_data(data, &(ctxp->md5_ctx),
1619 		    NULL, DO_MD5 | DO_UPDATE);
1620 	else
1621 		rv = digest_data(data, &(ctxp->sha1_ctx),
1622 		    NULL, DO_SHA1 | DO_UPDATE);
1623 	return (rv);
1624 }
1625 
1626 static int
1627 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1628     crypto_req_handle_t req)
1629 {
1630 	int rv;
1631 	digest_rsa_ctx_t *ctxp;
1632 
1633 	ASSERT(ctx->cc_provider_private != NULL);
1634 	ctxp = ctx->cc_provider_private;
1635 
1636 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1637 	    crypto_kmflag(req), DO_SIGN | DO_FINAL);
1638 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1639 		(void) rsa_free_context(ctx);
1640 
1641 	return (rv);
1642 }
1643 
1644 /* ARGSUSED */
1645 static int
1646 rsa_sign_atomic(crypto_provider_handle_t provider,
1647     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1648     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1649     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1650 {
1651 	int rv;
1652 	digest_rsa_ctx_t dctx;
1653 
1654 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1655 		return (rv);
1656 
1657 	switch (mechanism->cm_type) {
1658 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1659 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1660 		dctx.mech_type = mechanism->cm_type;
1661 		dctx.key = key;
1662 		if (mechanism->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1663 			MD5Init(&(dctx.md5_ctx));
1664 		else
1665 			SHA1Init(&(dctx.sha1_ctx));
1666 		rv = rsa_digest_svrfy_common(&dctx, data,
1667 		    signature, crypto_kmflag(req),
1668 		    DO_SIGN | DO_UPDATE | DO_FINAL);
1669 		break;
1670 	default:
1671 		rv = rsa_sign_common(mechanism->cm_type, key, data,
1672 		    signature, crypto_kmflag(req));
1673 		break;
1674 	}
1675 
1676 	return (rv);
1677 }
1678 
1679 static int
1680 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1681     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1682 {
1683 	int rv = CRYPTO_FAILED;
1684 
1685 /* EXPORT DELETE START */
1686 
1687 	uchar_t *sigptr, *modulus;
1688 	ssize_t modulus_len;
1689 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1690 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1691 
1692 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1693 	    &modulus_len)) != CRYPTO_SUCCESS) {
1694 		return (rv);
1695 	}
1696 
1697 	if (signature->cd_length != modulus_len)
1698 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1699 
1700 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1701 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
1702 	    != CRYPTO_SUCCESS)
1703 		return (rv);
1704 
1705 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
1706 	if (rv != CRYPTO_SUCCESS)
1707 		return (rv);
1708 
1709 	switch (mech_type) {
1710 	case RSA_PKCS_MECH_INFO_TYPE:
1711 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1712 	case SHA1_RSA_PKCS_MECH_INFO_TYPE: {
1713 		int data_len = modulus_len;
1714 
1715 		/*
1716 		 * Strip off the encoded padding bytes in front of the
1717 		 * recovered data, then compare the recovered data with
1718 		 * the original data.
1719 		 */
1720 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1721 		if (rv != CRYPTO_SUCCESS)
1722 			break;
1723 
1724 		if (data_len != data->cd_length) {
1725 			rv = CRYPTO_SIGNATURE_LEN_RANGE;
1726 			break;
1727 		}
1728 
1729 		if (compare_data(data, (plain_data + modulus_len
1730 		    - data_len)) != 0)
1731 			rv = CRYPTO_SIGNATURE_INVALID;
1732 		break;
1733 	}
1734 
1735 	case RSA_X_509_MECH_INFO_TYPE:
1736 		if (compare_data(data, (plain_data + modulus_len
1737 		    - data->cd_length)) != 0)
1738 			rv = CRYPTO_SIGNATURE_INVALID;
1739 		break;
1740 
1741 	default:
1742 		break;
1743 	}
1744 
1745 /* EXPORT DELETE END */
1746 
1747 	return (rv);
1748 }
1749 
1750 /* ARGSUSED */
1751 static int
1752 rsa_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1753     crypto_req_handle_t req)
1754 {
1755 	int rv;
1756 	rsa_ctx_t *ctxp;
1757 
1758 	ASSERT(ctx->cc_provider_private != NULL);
1759 	ctxp = ctx->cc_provider_private;
1760 
1761 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1762 	switch (ctxp->mech_type) {
1763 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1764 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1765 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1766 		    signature, KM_SLEEP, DO_VERIFY | DO_UPDATE | DO_FINAL);
1767 		break;
1768 	default:
1769 		rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1770 		    signature, KM_SLEEP);
1771 		break;
1772 	}
1773 
1774 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1775 		(void) rsa_free_context(ctx);
1776 
1777 	return (rv);
1778 }
1779 
1780 /* ARGSUSED */
1781 static int
1782 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1783     crypto_req_handle_t req)
1784 {
1785 	int rv;
1786 	digest_rsa_ctx_t *ctxp;
1787 	rsa_mech_type_t mech_type;
1788 
1789 	ASSERT(ctx->cc_provider_private != NULL);
1790 	ctxp = ctx->cc_provider_private;
1791 	mech_type = ctxp->mech_type;
1792 
1793 	if (!(mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||
1794 	    mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE))
1795 		return (CRYPTO_MECHANISM_INVALID);
1796 
1797 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1798 		rv = digest_data(data, &(ctxp->md5_ctx),
1799 		    NULL, DO_MD5 | DO_UPDATE);
1800 	else
1801 		rv = digest_data(data, &(ctxp->sha1_ctx),
1802 		    NULL, DO_SHA1 | DO_UPDATE);
1803 	return (rv);
1804 }
1805 
1806 static int
1807 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1808     crypto_req_handle_t req)
1809 {
1810 	int rv;
1811 	digest_rsa_ctx_t *ctxp;
1812 
1813 	ASSERT(ctx->cc_provider_private != NULL);
1814 	ctxp = ctx->cc_provider_private;
1815 
1816 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1817 	    crypto_kmflag(req), DO_VERIFY | DO_FINAL);
1818 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1819 		(void) rsa_free_context(ctx);
1820 
1821 	return (rv);
1822 }
1823 
1824 
1825 /* ARGSUSED */
1826 static int
1827 rsa_verify_atomic(crypto_provider_handle_t provider,
1828     crypto_session_id_t session_id,
1829     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1830     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1831     crypto_req_handle_t req)
1832 {
1833 	int rv;
1834 	digest_rsa_ctx_t dctx;
1835 
1836 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1837 		return (rv);
1838 
1839 	switch (mechanism->cm_type) {
1840 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1841 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1842 		dctx.mech_type = mechanism->cm_type;
1843 		dctx.key = key;
1844 		if (mechanism->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1845 			MD5Init(&(dctx.md5_ctx));
1846 		else
1847 			SHA1Init(&(dctx.sha1_ctx));
1848 		rv = rsa_digest_svrfy_common(&dctx, data,
1849 		    signature, crypto_kmflag(req),
1850 		    DO_VERIFY | DO_UPDATE | DO_FINAL);
1851 		break;
1852 	default:
1853 		rv = rsa_verify_common(mechanism->cm_type, key, data,
1854 		    signature, crypto_kmflag(req));
1855 		break;
1856 	}
1857 
1858 	return (rv);
1859 }
1860 
1861 static int
1862 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1863     crypto_data_t *signature, crypto_data_t *data, int kmflag)
1864 {
1865 	int rv = CRYPTO_FAILED;
1866 
1867 /* EXPORT DELETE START */
1868 
1869 	int data_len;
1870 	uchar_t *sigptr, *modulus;
1871 	ssize_t modulus_len;
1872 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1873 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1874 
1875 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1876 	    &modulus_len)) != CRYPTO_SUCCESS) {
1877 		return (rv);
1878 	}
1879 
1880 	if (signature->cd_length != modulus_len)
1881 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1882 
1883 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1884 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
1885 	    != CRYPTO_SUCCESS)
1886 		return (rv);
1887 
1888 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
1889 	if (rv != CRYPTO_SUCCESS)
1890 		return (rv);
1891 
1892 	data_len = modulus_len;
1893 
1894 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1895 		/*
1896 		 * Strip off the encoded padding bytes in front of the
1897 		 * recovered data, then compare the recovered data with
1898 		 * the original data.
1899 		 */
1900 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1901 		if (rv != CRYPTO_SUCCESS)
1902 			return (rv);
1903 	}
1904 
1905 	if (data->cd_length < data_len) {
1906 		data->cd_length = data_len;
1907 		return (CRYPTO_BUFFER_TOO_SMALL);
1908 	}
1909 
1910 	if ((rv = put_output_data(plain_data + modulus_len - data_len,
1911 	    data, data_len)) != CRYPTO_SUCCESS)
1912 		return (rv);
1913 	data->cd_length = data_len;
1914 
1915 /* EXPORT DELETE END */
1916 
1917 	return (rv);
1918 }
1919 
1920 /* ARGSUSED */
1921 static int
1922 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1923     crypto_data_t *data, crypto_req_handle_t req)
1924 {
1925 	int rv;
1926 	rsa_ctx_t *ctxp;
1927 
1928 	ASSERT(ctx->cc_provider_private != NULL);
1929 	ctxp = ctx->cc_provider_private;
1930 
1931 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1932 	rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1933 	    signature, data, KM_SLEEP);
1934 
1935 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1936 		(void) rsa_free_context(ctx);
1937 
1938 	return (rv);
1939 }
1940 
1941 /* ARGSUSED */
1942 static int
1943 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1944     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1945     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1946     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1947 {
1948 	int rv;
1949 
1950 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1951 		return (rv);
1952 
1953 	return (rsa_verify_recover_common(mechanism->cm_type, key,
1954 	    signature, data, crypto_kmflag(req)));
1955 }
1956