xref: /titanic_41/usr/src/uts/common/crypto/io/rsa.c (revision b02e9a2d4d2071d770e5aa9ae8f83f2bbe1f2ced)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * RSA provider for the Kernel Cryptographic Framework (KCF)
30  */
31 
32 #include <sys/types.h>
33 #include <sys/systm.h>
34 #include <sys/modctl.h>
35 #include <sys/cmn_err.h>
36 #include <sys/ddi.h>
37 #include <sys/crypto/spi.h>
38 #include <sys/sysmacros.h>
39 #include <sys/strsun.h>
40 #include <sys/md5.h>
41 #include <sys/sha1.h>
42 #include <sys/sha2.h>
43 #include <sys/random.h>
44 #include "rsa_impl.h"
45 
46 extern struct mod_ops mod_cryptoops;
47 
48 /*
49  * Module linkage information for the kernel.
50  */
51 static struct modlcrypto modlcrypto = {
52 	&mod_cryptoops,
53 	"RSA Kernel SW Provider"
54 };
55 
56 static struct modlinkage modlinkage = {
57 	MODREV_1,
58 	(void *)&modlcrypto,
59 	NULL
60 };
61 
62 /*
63  * CSPI information (entry points, provider info, etc.)
64  */
65 typedef enum rsa_mech_type {
66 	RSA_PKCS_MECH_INFO_TYPE,	/* SUN_CKM_RSA_PKCS */
67 	RSA_X_509_MECH_INFO_TYPE,	/* SUN_CKM_RSA_X_509 */
68 	MD5_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_MD5_RSA_PKCS */
69 	SHA1_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA1_RSA_PKCS */
70 	SHA256_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA256_RSA_PKCS */
71 	SHA384_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA384_RSA_PKCS */
72 	SHA512_RSA_PKCS_MECH_INFO_TYPE	/* SUN_SHA512_RSA_PKCS */
73 } rsa_mech_type_t;
74 
75 /*
76  * Context for RSA_PKCS and RSA_X_509 mechanisms.
77  */
78 typedef struct rsa_ctx {
79 	rsa_mech_type_t	mech_type;
80 	crypto_key_t *key;
81 	size_t keychunk_size;
82 } rsa_ctx_t;
83 
84 /*
85  * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
86  */
87 typedef struct digest_rsa_ctx {
88 	rsa_mech_type_t	mech_type;
89 	crypto_key_t *key;
90 	size_t keychunk_size;
91 	union {
92 		MD5_CTX md5ctx;
93 		SHA1_CTX sha1ctx;
94 		SHA2_CTX sha2ctx;
95 	} dctx_u;
96 } digest_rsa_ctx_t;
97 
98 #define	md5_ctx		dctx_u.md5ctx
99 #define	sha1_ctx	dctx_u.sha1ctx
100 #define	sha2_ctx	dctx_u.sha2ctx
101 
102 /*
103  * Mechanism info structure passed to KCF during registration.
104  */
105 static crypto_mech_info_t rsa_mech_info_tab[] = {
106 	/* RSA_PKCS */
107 	{SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
108 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
109 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
110 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
111 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
112 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
113 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
114 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
115 
116 	/* RSA_X_509 */
117 	{SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
118 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
119 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
120 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
121 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
122 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
123 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
124 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
125 
126 	/* MD5_RSA_PKCS */
127 	{SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
128 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
129 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
130 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
131 
132 	/* SHA1_RSA_PKCS */
133 	{SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
134 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
135 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
136 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
137 
138 	/* SHA256_RSA_PKCS */
139 	{SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
140 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
141 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
142 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
143 
144 	/* SHA384_RSA_PKCS */
145 	{SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
146 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
147 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
148 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
149 
150 	/* SHA512_RSA_PKCS */
151 	{SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
152 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
153 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
154 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
155 
156 };
157 
158 #define	RSA_VALID_MECH(mech)					\
159 	(((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||		\
160 	(mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||		\
161 	(mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||	\
162 	(mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE ||	\
163 	(mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE ||	\
164 	(mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE ||	\
165 	(mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
166 
167 /* operations are in-place if the output buffer is NULL */
168 #define	RSA_ARG_INPLACE(input, output)				\
169 	if ((output) == NULL)					\
170 		(output) = (input);
171 
172 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
173 
174 static crypto_control_ops_t rsa_control_ops = {
175 	rsa_provider_status
176 };
177 
178 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
179     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
180 static int rsa_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
181     crypto_req_handle_t);
182 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
183     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
184     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
185 static int rsa_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
186     crypto_req_handle_t);
187 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
188     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
189     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
190 
191 /*
192  * The RSA mechanisms do not have multiple-part cipher operations.
193  * So, the update and final routines are set to NULL.
194  */
195 static crypto_cipher_ops_t rsa_cipher_ops = {
196 	rsa_common_init,
197 	rsa_encrypt,
198 	NULL,
199 	NULL,
200 	rsa_encrypt_atomic,
201 	rsa_common_init,
202 	rsa_decrypt,
203 	NULL,
204 	NULL,
205 	rsa_decrypt_atomic
206 };
207 
208 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
209     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
210 static int rsa_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
211     crypto_req_handle_t);
212 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
213     crypto_req_handle_t);
214 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
215     crypto_req_handle_t);
216 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
217     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
218     crypto_spi_ctx_template_t, crypto_req_handle_t);
219 
220 /*
221  * We use the same routine for sign_init and sign_recover_init fields
222  * as they do the same thing. Same holds for sign and sign_recover fields,
223  * and sign_atomic and sign_recover_atomic fields.
224  */
225 static crypto_sign_ops_t rsa_sign_ops = {
226 	rsa_sign_verify_common_init,
227 	rsa_sign,
228 	rsa_sign_update,
229 	rsa_sign_final,
230 	rsa_sign_atomic,
231 	rsa_sign_verify_common_init,
232 	rsa_sign,
233 	rsa_sign_atomic
234 };
235 
236 static int rsa_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
237     crypto_req_handle_t);
238 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
239     crypto_req_handle_t);
240 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
241     crypto_req_handle_t);
242 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
243     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
244     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
245 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
246     crypto_data_t *, crypto_req_handle_t);
247 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
248     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
249     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
250     crypto_req_handle_t);
251 
252 /*
253  * We use the same routine (rsa_sign_verify_common_init) for verify_init
254  * and verify_recover_init fields as they do the same thing.
255  */
256 static crypto_verify_ops_t rsa_verify_ops = {
257 	rsa_sign_verify_common_init,
258 	rsa_verify,
259 	rsa_verify_update,
260 	rsa_verify_final,
261 	rsa_verify_atomic,
262 	rsa_sign_verify_common_init,
263 	rsa_verify_recover,
264 	rsa_verify_recover_atomic
265 };
266 
267 static int rsa_free_context(crypto_ctx_t *);
268 
269 static crypto_ctx_ops_t rsa_ctx_ops = {
270 	NULL,
271 	rsa_free_context
272 };
273 
274 static crypto_ops_t rsa_crypto_ops = {
275 	&rsa_control_ops,
276 	NULL,
277 	&rsa_cipher_ops,
278 	NULL,
279 	&rsa_sign_ops,
280 	&rsa_verify_ops,
281 	NULL,
282 	NULL,
283 	NULL,
284 	NULL,
285 	NULL,
286 	NULL,
287 	NULL,
288 	&rsa_ctx_ops
289 };
290 
291 static crypto_provider_info_t rsa_prov_info = {
292 	CRYPTO_SPI_VERSION_1,
293 	"RSA Software Provider",
294 	CRYPTO_SW_PROVIDER,
295 	{&modlinkage},
296 	NULL,
297 	&rsa_crypto_ops,
298 	sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
299 	rsa_mech_info_tab
300 };
301 
302 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
303     crypto_data_t *, crypto_data_t *, int);
304 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
305     crypto_data_t *, crypto_data_t *, int);
306 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
307     crypto_data_t *, crypto_data_t *, int);
308 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
309     crypto_data_t *, crypto_data_t *, int);
310 static int compare_data(crypto_data_t *, uchar_t *);
311 
312 /* EXPORT DELETE START */
313 
314 static int core_rsa_encrypt(crypto_key_t *, uchar_t *,
315     int, uchar_t *, int, int);
316 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int,
317     uchar_t *, int);
318 
319 /* EXPORT DELETE END */
320 
321 static crypto_kcf_provider_handle_t rsa_prov_handle = NULL;
322 
323 int
324 _init(void)
325 {
326 	int ret;
327 
328 	/*
329 	 * Register with KCF. If the registration fails, return error.
330 	 */
331 	if ((ret = crypto_register_provider(&rsa_prov_info,
332 	    &rsa_prov_handle)) != CRYPTO_SUCCESS) {
333 		cmn_err(CE_WARN, "rsa _init: crypto_register_provider()"
334 		    "failed (0x%x)", ret);
335 		return (EACCES);
336 	}
337 
338 	if ((ret = mod_install(&modlinkage)) != 0) {
339 		int rv;
340 
341 		ASSERT(rsa_prov_handle != NULL);
342 		/* We should not return if the unregister returns busy. */
343 		while ((rv = crypto_unregister_provider(rsa_prov_handle))
344 		    == CRYPTO_BUSY) {
345 			cmn_err(CE_WARN, "rsa _init: "
346 			    "crypto_unregister_provider() "
347 			    "failed (0x%x). Retrying.", rv);
348 			/* wait 10 seconds and try again. */
349 			delay(10 * drv_usectohz(1000000));
350 		}
351 	}
352 
353 	return (ret);
354 }
355 
356 int
357 _fini(void)
358 {
359 	int ret;
360 
361 	/*
362 	 * Unregister from KCF if previous registration succeeded.
363 	 */
364 	if (rsa_prov_handle != NULL) {
365 		if ((ret = crypto_unregister_provider(rsa_prov_handle)) !=
366 		    CRYPTO_SUCCESS) {
367 			cmn_err(CE_WARN, "rsa _fini: "
368 			    "crypto_unregister_provider() "
369 			    "failed (0x%x)", ret);
370 			return (EBUSY);
371 		}
372 		rsa_prov_handle = NULL;
373 	}
374 
375 	return (mod_remove(&modlinkage));
376 }
377 
378 int
379 _info(struct modinfo *modinfop)
380 {
381 	return (mod_info(&modlinkage, modinfop));
382 }
383 
384 /* ARGSUSED */
385 static void
386 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
387 {
388 	*status = CRYPTO_PROVIDER_READY;
389 }
390 
391 /*
392  * Utility routine to look up a attribute of type, 'type',
393  * in the key.
394  */
395 static int
396 get_key_attr(crypto_key_t *key, crypto_attr_type_t type,
397     uchar_t **value, ssize_t *value_len)
398 {
399 	int i;
400 
401 	ASSERT(key->ck_format == CRYPTO_KEY_ATTR_LIST);
402 	for (i = 0; i < key->ck_count; i++) {
403 		if (key->ck_attrs[i].oa_type == type) {
404 			*value = (uchar_t *)key->ck_attrs[i].oa_value;
405 			*value_len = key->ck_attrs[i].oa_value_len;
406 			return (CRYPTO_SUCCESS);
407 		}
408 	}
409 
410 	return (CRYPTO_FAILED);
411 }
412 
413 static int
414 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
415 {
416 	int rv = CRYPTO_FAILED;
417 
418 /* EXPORT DELETE START */
419 
420 	uchar_t *modulus;
421 	ssize_t modulus_len; /* In bytes */
422 
423 	if (!RSA_VALID_MECH(mechanism))
424 		return (CRYPTO_MECHANISM_INVALID);
425 
426 	/*
427 	 * We only support RSA keys that are passed as a list of
428 	 * object attributes.
429 	 */
430 	if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
431 		return (CRYPTO_KEY_TYPE_INCONSISTENT);
432 	}
433 
434 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
435 	    &modulus_len)) != CRYPTO_SUCCESS) {
436 		return (rv);
437 	}
438 	if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
439 	    modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
440 		return (CRYPTO_KEY_SIZE_RANGE);
441 
442 /* EXPORT DELETE END */
443 
444 	return (rv);
445 }
446 
447 void
448 kmemset(uint8_t *buf, char pattern, size_t len)
449 {
450 	int i = 0;
451 
452 	while (i < len)
453 		buf[i++] = pattern;
454 }
455 
456 /*
457  * This function guarantees to return non-zero random numbers.
458  * This is needed as the /dev/urandom kernel interface,
459  * random_get_pseudo_bytes(), may return zeros.
460  */
461 int
462 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
463 {
464 	int rv;
465 	size_t ebc = 0; /* count of extra bytes in extrarand */
466 	size_t i = 0;
467 	uint8_t extrarand[32];
468 	size_t extrarand_len;
469 
470 	if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
471 		return (rv);
472 
473 	/*
474 	 * Walk through the returned random numbers pointed by ran_out,
475 	 * and look for any random number which is zero.
476 	 * If we find zero, call random_get_pseudo_bytes() to generate
477 	 * another 32 random numbers pool. Replace any zeros in ran_out[]
478 	 * from the random number in pool.
479 	 */
480 	while (i < ran_len) {
481 		if (ran_out[i] != 0) {
482 			i++;
483 			continue;
484 		}
485 
486 		/*
487 		 * Note that it is 'while' so we are guaranteed a
488 		 * non-zero value on exit.
489 		 */
490 		if (ebc == 0) {
491 			/* refresh extrarand */
492 			extrarand_len = sizeof (extrarand);
493 			if ((rv = random_get_pseudo_bytes(extrarand,
494 			    extrarand_len)) != 0) {
495 				return (rv);
496 			}
497 
498 			ebc = extrarand_len;
499 		}
500 		/* Replace zero with byte from extrarand. */
501 		-- ebc;
502 
503 		/*
504 		 * The new random byte zero/non-zero will be checked in
505 		 * the next pass through the loop.
506 		 */
507 		ran_out[i] = extrarand[ebc];
508 	}
509 
510 	return (CRYPTO_SUCCESS);
511 }
512 
513 typedef enum cmd_type {
514 	COPY_FROM_DATA,
515 	COPY_TO_DATA,
516 	COMPARE_TO_DATA,
517 	MD5_DIGEST_DATA,
518 	SHA1_DIGEST_DATA,
519 	SHA2_DIGEST_DATA
520 } cmd_type_t;
521 
522 /*
523  * Utility routine to apply the command, 'cmd', to the
524  * data in the uio structure.
525  */
526 static int
527 process_uio_data(crypto_data_t *data, uchar_t *buf, int len,
528     cmd_type_t cmd, void *digest_ctx)
529 {
530 	uio_t *uiop = data->cd_uio;
531 	off_t offset = data->cd_offset;
532 	size_t length = len;
533 	uint_t vec_idx;
534 	size_t cur_len;
535 	uchar_t *datap;
536 
537 	ASSERT(data->cd_format == CRYPTO_DATA_UIO);
538 	if (uiop->uio_segflg != UIO_SYSSPACE) {
539 		return (CRYPTO_ARGUMENTS_BAD);
540 	}
541 
542 	/*
543 	 * Jump to the first iovec containing data to be
544 	 * processed.
545 	 */
546 	for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
547 	    offset >= uiop->uio_iov[vec_idx].iov_len;
548 	    offset -= uiop->uio_iov[vec_idx++].iov_len)
549 		;
550 
551 	if (vec_idx == uiop->uio_iovcnt) {
552 		/*
553 		 * The caller specified an offset that is larger than
554 		 * the total size of the buffers it provided.
555 		 */
556 		return (CRYPTO_DATA_LEN_RANGE);
557 	}
558 
559 	while (vec_idx < uiop->uio_iovcnt && length > 0) {
560 		cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
561 		    offset, length);
562 
563 		datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
564 		    offset);
565 		switch (cmd) {
566 		case COPY_FROM_DATA:
567 			bcopy(datap, buf, cur_len);
568 			buf += cur_len;
569 			break;
570 		case COPY_TO_DATA:
571 			bcopy(buf, datap, cur_len);
572 			buf += cur_len;
573 			break;
574 		case COMPARE_TO_DATA:
575 			if (bcmp(datap, buf, cur_len))
576 				return (CRYPTO_SIGNATURE_INVALID);
577 			buf += cur_len;
578 			break;
579 		case MD5_DIGEST_DATA:
580 			MD5Update(digest_ctx, datap, cur_len);
581 			break;
582 		case SHA1_DIGEST_DATA:
583 			SHA1Update(digest_ctx, datap, cur_len);
584 			break;
585 		case SHA2_DIGEST_DATA:
586 			SHA2Update(digest_ctx, datap, cur_len);
587 			break;
588 		}
589 
590 		length -= cur_len;
591 		vec_idx++;
592 		offset = 0;
593 	}
594 
595 	if (vec_idx == uiop->uio_iovcnt && length > 0) {
596 		/*
597 		 * The end of the specified iovec's was reached but
598 		 * the length requested could not be processed.
599 		 */
600 		switch (cmd) {
601 		case COPY_TO_DATA:
602 			data->cd_length = len;
603 			return (CRYPTO_BUFFER_TOO_SMALL);
604 		default:
605 			return (CRYPTO_DATA_LEN_RANGE);
606 		}
607 	}
608 
609 	return (CRYPTO_SUCCESS);
610 }
611 
612 /*
613  * Utility routine to apply the command, 'cmd', to the
614  * data in the mblk structure.
615  */
616 static int
617 process_mblk_data(crypto_data_t *data, uchar_t *buf, int len,
618     cmd_type_t cmd, void *digest_ctx)
619 {
620 	off_t offset = data->cd_offset;
621 	size_t length = len;
622 	mblk_t *mp;
623 	size_t cur_len;
624 	uchar_t *datap;
625 
626 	ASSERT(data->cd_format == CRYPTO_DATA_MBLK);
627 	/*
628 	 * Jump to the first mblk_t containing data to be processed.
629 	 */
630 	for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
631 	    offset -= MBLKL(mp), mp = mp->b_cont)
632 		;
633 	if (mp == NULL) {
634 		/*
635 		 * The caller specified an offset that is larger
636 		 * than the total size of the buffers it provided.
637 		 */
638 		return (CRYPTO_DATA_LEN_RANGE);
639 	}
640 
641 	/*
642 	 * Now do the processing on the mblk chain.
643 	 */
644 	while (mp != NULL && length > 0) {
645 		cur_len = MIN(MBLKL(mp) - offset, length);
646 
647 		datap = (uchar_t *)(mp->b_rptr + offset);
648 		switch (cmd) {
649 		case COPY_FROM_DATA:
650 			bcopy(datap, buf, cur_len);
651 			buf += cur_len;
652 			break;
653 		case COPY_TO_DATA:
654 			bcopy(buf, datap, cur_len);
655 			buf += cur_len;
656 			break;
657 		case COMPARE_TO_DATA:
658 			if (bcmp(datap, buf, cur_len))
659 				return (CRYPTO_SIGNATURE_INVALID);
660 			buf += cur_len;
661 			break;
662 		case MD5_DIGEST_DATA:
663 			MD5Update(digest_ctx, datap, cur_len);
664 			break;
665 		case SHA1_DIGEST_DATA:
666 			SHA1Update(digest_ctx, datap, cur_len);
667 			break;
668 		case SHA2_DIGEST_DATA:
669 			SHA2Update(digest_ctx, datap, cur_len);
670 			break;
671 		}
672 
673 		length -= cur_len;
674 		offset = 0;
675 		mp = mp->b_cont;
676 	}
677 
678 	if (mp == NULL && length > 0) {
679 		/*
680 		 * The end of the mblk was reached but the length
681 		 * requested could not be processed.
682 		 */
683 		switch (cmd) {
684 		case COPY_TO_DATA:
685 			data->cd_length = len;
686 			return (CRYPTO_BUFFER_TOO_SMALL);
687 		default:
688 			return (CRYPTO_DATA_LEN_RANGE);
689 		}
690 	}
691 
692 	return (CRYPTO_SUCCESS);
693 }
694 
695 static int
696 compare_data(crypto_data_t *data, uchar_t *buf)
697 {
698 	int len;
699 	uchar_t *dptr;
700 
701 	len = data->cd_length;
702 	switch (data->cd_format) {
703 	case CRYPTO_DATA_RAW:
704 		dptr = (uchar_t *)(data->cd_raw.iov_base +
705 		    data->cd_offset);
706 
707 		return (bcmp(dptr, buf, len));
708 
709 	case CRYPTO_DATA_UIO:
710 		return (process_uio_data(data, buf, len,
711 		    COMPARE_TO_DATA, NULL));
712 
713 	case CRYPTO_DATA_MBLK:
714 		return (process_mblk_data(data, buf, len,
715 		    COMPARE_TO_DATA, NULL));
716 	}
717 
718 	return (CRYPTO_FAILED);
719 }
720 
721 /*
722  * Utility routine to copy a buffer to a crypto_data structure.
723  */
724 static int
725 put_output_data(uchar_t *buf, crypto_data_t *output, int len)
726 {
727 	switch (output->cd_format) {
728 	case CRYPTO_DATA_RAW:
729 		if (output->cd_raw.iov_len < len) {
730 			output->cd_length = len;
731 			return (CRYPTO_BUFFER_TOO_SMALL);
732 		}
733 		bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
734 		    output->cd_offset), len);
735 		break;
736 
737 	case CRYPTO_DATA_UIO:
738 		return (process_uio_data(output, buf, len, COPY_TO_DATA, NULL));
739 
740 	case CRYPTO_DATA_MBLK:
741 		return (process_mblk_data(output, buf, len,
742 		    COPY_TO_DATA, NULL));
743 
744 	default:
745 		return (CRYPTO_ARGUMENTS_BAD);
746 	}
747 
748 	return (CRYPTO_SUCCESS);
749 }
750 
751 /*
752  * Utility routine to get data from a crypto_data structure.
753  *
754  * '*dptr' contains a pointer to a buffer on return. 'buf'
755  * is allocated by the caller and is ignored for CRYPTO_DATA_RAW case.
756  */
757 static int
758 get_input_data(crypto_data_t *input, uchar_t **dptr, uchar_t *buf)
759 {
760 	int rv;
761 
762 	switch (input->cd_format) {
763 	case CRYPTO_DATA_RAW:
764 		if (input->cd_raw.iov_len < input->cd_length)
765 			return (CRYPTO_ARGUMENTS_BAD);
766 		*dptr = (uchar_t *)(input->cd_raw.iov_base +
767 		    input->cd_offset);
768 		break;
769 
770 	case CRYPTO_DATA_UIO:
771 		if ((rv = process_uio_data(input, buf, input->cd_length,
772 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
773 			return (rv);
774 		*dptr = buf;
775 		break;
776 
777 	case CRYPTO_DATA_MBLK:
778 		if ((rv = process_mblk_data(input, buf, input->cd_length,
779 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
780 			return (rv);
781 		*dptr = buf;
782 		break;
783 
784 	default:
785 		return (CRYPTO_ARGUMENTS_BAD);
786 	}
787 
788 	return (CRYPTO_SUCCESS);
789 }
790 
791 static int
792 copy_key_to_ctx(crypto_key_t *in_key, rsa_ctx_t *ctx, int kmflag)
793 {
794 	int i, count;
795 	size_t len;
796 	caddr_t attr_val;
797 	crypto_object_attribute_t *k_attrs = NULL;
798 
799 	ASSERT(in_key->ck_format == CRYPTO_KEY_ATTR_LIST);
800 
801 	count = in_key->ck_count;
802 	/* figure out how much memory to allocate for everything */
803 	len = sizeof (crypto_key_t) +
804 	    count * sizeof (crypto_object_attribute_t);
805 	for (i = 0; i < count; i++) {
806 		len += roundup(in_key->ck_attrs[i].oa_value_len,
807 		    sizeof (caddr_t));
808 	}
809 
810 	/* one big allocation for everything */
811 	ctx->key = kmem_alloc(len, kmflag);
812 	if (ctx->key == NULL)
813 		return (CRYPTO_HOST_MEMORY);
814 	k_attrs = (crypto_object_attribute_t *)((caddr_t)(ctx->key) +
815 	    sizeof (crypto_key_t));
816 
817 	attr_val = (caddr_t)k_attrs +
818 	    count * sizeof (crypto_object_attribute_t);
819 	for (i = 0; i < count; i++) {
820 		k_attrs[i].oa_type = in_key->ck_attrs[i].oa_type;
821 		bcopy(in_key->ck_attrs[i].oa_value, attr_val,
822 		    in_key->ck_attrs[i].oa_value_len);
823 		k_attrs[i].oa_value = attr_val;
824 		k_attrs[i].oa_value_len = in_key->ck_attrs[i].oa_value_len;
825 		attr_val += roundup(k_attrs[i].oa_value_len, sizeof (caddr_t));
826 	}
827 
828 	ctx->keychunk_size = len;	/* save the size to be freed */
829 	ctx->key->ck_format = CRYPTO_KEY_ATTR_LIST;
830 	ctx->key->ck_count = count;
831 	ctx->key->ck_attrs = k_attrs;
832 
833 	return (CRYPTO_SUCCESS);
834 }
835 
836 /* ARGSUSED */
837 static int
838 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
839     crypto_key_t *key, crypto_spi_ctx_template_t template,
840     crypto_req_handle_t req)
841 {
842 	int rv;
843 	int kmflag;
844 	rsa_ctx_t *ctxp;
845 
846 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
847 		return (rv);
848 
849 	/*
850 	 * Allocate a RSA context.
851 	 */
852 	kmflag = crypto_kmflag(req);
853 	if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
854 		return (CRYPTO_HOST_MEMORY);
855 
856 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
857 		kmem_free(ctxp, sizeof (rsa_ctx_t));
858 		return (rv);
859 	}
860 	ctxp->mech_type = mechanism->cm_type;
861 
862 	ctx->cc_provider_private = ctxp;
863 
864 	return (CRYPTO_SUCCESS);
865 }
866 
867 /* ARGSUSED */
868 static int
869 rsa_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
870     crypto_data_t *ciphertext, crypto_req_handle_t req)
871 {
872 	int rv;
873 	rsa_ctx_t *ctxp;
874 
875 	ASSERT(ctx->cc_provider_private != NULL);
876 	ctxp = ctx->cc_provider_private;
877 
878 	RSA_ARG_INPLACE(plaintext, ciphertext);
879 
880 	/*
881 	 * Note on the KM_SLEEP flag passed to the routine below -
882 	 * rsa_encrypt() is a single-part encryption routine which is
883 	 * currently usable only by /dev/crypto. Since /dev/crypto calls are
884 	 * always synchronous, we can safely pass KM_SLEEP here.
885 	 */
886 	rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
887 	    ciphertext, KM_SLEEP);
888 
889 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
890 		(void) rsa_free_context(ctx);
891 
892 	return (rv);
893 }
894 
895 /* ARGSUSED */
896 static int
897 rsa_encrypt_atomic(crypto_provider_handle_t provider,
898     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
899     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
900     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
901 {
902 	int rv;
903 
904 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
905 		return (rv);
906 	RSA_ARG_INPLACE(plaintext, ciphertext);
907 
908 	return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
909 	    ciphertext, crypto_kmflag(req)));
910 }
911 
912 static int
913 rsa_free_context(crypto_ctx_t *ctx)
914 {
915 	rsa_ctx_t *ctxp = ctx->cc_provider_private;
916 
917 	if (ctxp != NULL) {
918 		bzero(ctxp->key, ctxp->keychunk_size);
919 		kmem_free(ctxp->key, ctxp->keychunk_size);
920 
921 		if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
922 		    ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
923 			kmem_free(ctxp, sizeof (rsa_ctx_t));
924 		else
925 			kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
926 
927 		ctx->cc_provider_private = NULL;
928 	}
929 
930 	return (CRYPTO_SUCCESS);
931 }
932 
933 static int
934 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
935     crypto_data_t *plaintext, crypto_data_t *ciphertext, int kmflag)
936 {
937 	int rv = CRYPTO_FAILED;
938 
939 /* EXPORT DELETE START */
940 
941 	int plen;
942 	uchar_t *ptptr;
943 	uchar_t *modulus;
944 	ssize_t modulus_len;
945 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
946 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
947 	uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
948 
949 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
950 	    &modulus_len)) != CRYPTO_SUCCESS) {
951 		return (rv);
952 	}
953 
954 	plen = plaintext->cd_length;
955 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
956 		if (plen > (modulus_len - MIN_PKCS1_PADLEN))
957 			return (CRYPTO_DATA_LEN_RANGE);
958 	} else {
959 		if (plen > modulus_len)
960 			return (CRYPTO_DATA_LEN_RANGE);
961 	}
962 
963 	/*
964 	 * Output buf len must not be less than RSA modulus size.
965 	 */
966 	if (ciphertext->cd_length < modulus_len) {
967 		ciphertext->cd_length = modulus_len;
968 		return (CRYPTO_BUFFER_TOO_SMALL);
969 	}
970 
971 	ASSERT(plaintext->cd_length <= sizeof (tmp_data));
972 	if ((rv = get_input_data(plaintext, &ptptr, tmp_data))
973 	    != CRYPTO_SUCCESS)
974 		return (rv);
975 
976 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
977 		rv = soft_encrypt_rsa_pkcs_encode(ptptr, plen,
978 		    plain_data, modulus_len);
979 
980 		if (rv != CRYPTO_SUCCESS)
981 			return (rv);
982 	} else {
983 		bzero(plain_data, modulus_len - plen);
984 		bcopy(ptptr, &plain_data[modulus_len - plen], plen);
985 	}
986 
987 	rv = core_rsa_encrypt(key, plain_data, modulus_len,
988 	    cipher_data, kmflag, 1);
989 	if (rv == CRYPTO_SUCCESS) {
990 		/* copy out to ciphertext */
991 		if ((rv = put_output_data(cipher_data,
992 		    ciphertext, modulus_len)) != CRYPTO_SUCCESS)
993 			return (rv);
994 
995 		ciphertext->cd_length = modulus_len;
996 	}
997 
998 /* EXPORT DELETE END */
999 
1000 	return (rv);
1001 }
1002 
1003 /* EXPORT DELETE START */
1004 
1005 static int
1006 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
1007     int in_len, uchar_t *out, int kmflag, int is_public)
1008 {
1009 	int rv;
1010 	uchar_t *expo, *modulus;
1011 	ssize_t	expo_len;
1012 	ssize_t modulus_len;
1013 	BIGNUM msg;
1014 	RSAkey *rsakey;
1015 
1016 	if (is_public) {
1017 		if ((rv = get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT, &expo,
1018 		    &expo_len)) != CRYPTO_SUCCESS)
1019 			return (rv);
1020 	} else {
1021 		/*
1022 		 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
1023 		 * RSA secret key. See the comments in core_rsa_decrypt
1024 		 * routine which calls this routine with a private key.
1025 		 */
1026 		if ((rv = get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT, &expo,
1027 		    &expo_len)) != CRYPTO_SUCCESS)
1028 			return (rv);
1029 	}
1030 
1031 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1032 	    &modulus_len)) != CRYPTO_SUCCESS) {
1033 		return (rv);
1034 	}
1035 
1036 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1037 	if (rsakey == NULL)
1038 		return (CRYPTO_HOST_MEMORY);
1039 
1040 	/* psize and qsize for RSA_key_init is in bits. */
1041 	if (RSA_key_init(rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
1042 		rv = CRYPTO_HOST_MEMORY;
1043 		goto clean1;
1044 	}
1045 
1046 	/* Size for big_init is in (32-bit) words. */
1047 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1048 	    (int)sizeof (uint32_t)) != BIG_OK) {
1049 		rv = CRYPTO_HOST_MEMORY;
1050 		goto clean2;
1051 	}
1052 
1053 	/* Convert octet string exponent to big integer format. */
1054 	bytestring2bignum(&(rsakey->e), expo, expo_len);
1055 
1056 	/* Convert octet string modulus to big integer format. */
1057 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1058 
1059 	/* Convert octet string input data to big integer format. */
1060 	bytestring2bignum(&msg, in, in_len);
1061 
1062 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1063 		rv = CRYPTO_DATA_LEN_RANGE;
1064 		goto clean3;
1065 	}
1066 
1067 	/* Perform RSA computation on big integer input data. */
1068 	if (big_modexp(&msg, &msg, &(rsakey->e), &(rsakey->n), NULL)
1069 	    != BIG_OK) {
1070 		rv = CRYPTO_HOST_MEMORY;
1071 		goto clean3;
1072 	}
1073 
1074 	/* Convert the big integer output data to octet string. */
1075 	bignum2bytestring(out, &msg, modulus_len);
1076 
1077 	/*
1078 	 * Should not free modulus and expo as both are just pointers
1079 	 * to an attribute value buffer from the caller.
1080 	 */
1081 clean3:
1082 	big_finish(&msg);
1083 clean2:
1084 	RSA_key_finish(rsakey);
1085 clean1:
1086 	kmem_free(rsakey, sizeof (RSAkey));
1087 
1088 	return (rv);
1089 }
1090 
1091 /* EXPORT DELETE END */
1092 
1093 /* ARGSUSED */
1094 static int
1095 rsa_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
1096     crypto_data_t *plaintext, crypto_req_handle_t req)
1097 {
1098 	int rv;
1099 	rsa_ctx_t *ctxp;
1100 
1101 	ASSERT(ctx->cc_provider_private != NULL);
1102 	ctxp = ctx->cc_provider_private;
1103 
1104 	RSA_ARG_INPLACE(ciphertext, plaintext);
1105 
1106 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1107 	rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
1108 	    ciphertext, plaintext, KM_SLEEP);
1109 
1110 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1111 		(void) rsa_free_context(ctx);
1112 
1113 	return (rv);
1114 }
1115 
1116 /* ARGSUSED */
1117 static int
1118 rsa_decrypt_atomic(crypto_provider_handle_t provider,
1119     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1120     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
1121     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1122 {
1123 	int rv;
1124 
1125 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1126 		return (rv);
1127 	RSA_ARG_INPLACE(ciphertext, plaintext);
1128 
1129 	return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
1130 	    plaintext, crypto_kmflag(req)));
1131 }
1132 
1133 static int
1134 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1135     crypto_data_t *ciphertext, crypto_data_t *plaintext, int kmflag)
1136 {
1137 	int rv = CRYPTO_FAILED;
1138 
1139 /* EXPORT DELETE START */
1140 
1141 	int plain_len;
1142 	uchar_t *ctptr;
1143 	uchar_t *modulus;
1144 	ssize_t modulus_len;
1145 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1146 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1147 
1148 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1149 	    &modulus_len)) != CRYPTO_SUCCESS) {
1150 		return (rv);
1151 	}
1152 
1153 	/*
1154 	 * Ciphertext length must be equal to RSA modulus size.
1155 	 */
1156 	if (ciphertext->cd_length != modulus_len)
1157 		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
1158 
1159 	ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
1160 	if ((rv = get_input_data(ciphertext, &ctptr, tmp_data))
1161 	    != CRYPTO_SUCCESS)
1162 		return (rv);
1163 
1164 	rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data, kmflag);
1165 	if (rv == CRYPTO_SUCCESS) {
1166 		plain_len = modulus_len;
1167 
1168 		if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1169 			/* Strip off the PKCS block formatting data. */
1170 			rv = soft_decrypt_rsa_pkcs_decode(plain_data,
1171 			    &plain_len);
1172 			if (rv != CRYPTO_SUCCESS)
1173 				return (rv);
1174 		}
1175 
1176 		if (plain_len > plaintext->cd_length) {
1177 			plaintext->cd_length = plain_len;
1178 			return (CRYPTO_BUFFER_TOO_SMALL);
1179 		}
1180 
1181 		if ((rv = put_output_data(plain_data + modulus_len - plain_len,
1182 		    plaintext, plain_len)) != CRYPTO_SUCCESS)
1183 			return (rv);
1184 
1185 		plaintext->cd_length = plain_len;
1186 	}
1187 
1188 /* EXPORT DELETE END */
1189 
1190 	return (rv);
1191 }
1192 
1193 /* EXPORT DELETE START */
1194 
1195 static int
1196 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len,
1197     uchar_t *out, int kmflag)
1198 {
1199 	int rv;
1200 	uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
1201 	ssize_t modulus_len;
1202 	ssize_t	prime1_len, prime2_len;
1203 	ssize_t	expo1_len, expo2_len, coef_len;
1204 	BIGNUM msg;
1205 	RSAkey *rsakey;
1206 
1207 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1208 	    &modulus_len)) != CRYPTO_SUCCESS) {
1209 		return (rv);
1210 	}
1211 
1212 	/*
1213 	 * The following attributes are not required to be
1214 	 * present in a RSA secret key. If any of them is not present
1215 	 * we call the encrypt routine with a flag indicating use of
1216 	 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
1217 	 * a required attribute for a RSA secret key.
1218 	 */
1219 	if ((get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
1220 	    != CRYPTO_SUCCESS) ||
1221 	    (get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
1222 	    != CRYPTO_SUCCESS) ||
1223 	    (get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
1224 	    != CRYPTO_SUCCESS) ||
1225 	    (get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
1226 	    != CRYPTO_SUCCESS) ||
1227 	    (get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
1228 	    != CRYPTO_SUCCESS)) {
1229 		return (core_rsa_encrypt(key, in, in_len, out, kmflag, 0));
1230 	}
1231 
1232 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1233 	if (rsakey == NULL)
1234 		return (CRYPTO_HOST_MEMORY);
1235 
1236 	/* psize and qsize for RSA_key_init is in bits. */
1237 	if (RSA_key_init(rsakey, prime2_len * 8, prime1_len * 8) != BIG_OK) {
1238 		rv = CRYPTO_HOST_MEMORY;
1239 		goto clean1;
1240 	}
1241 
1242 	/* Size for big_init is in (32-bit) words. */
1243 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1244 	    (int)sizeof (uint32_t)) != BIG_OK) {
1245 		rv = CRYPTO_HOST_MEMORY;
1246 		goto clean2;
1247 	}
1248 
1249 	/* Convert octet string input data to big integer format. */
1250 	bytestring2bignum(&msg, in, in_len);
1251 
1252 	/* Convert octet string modulus to big integer format. */
1253 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1254 
1255 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1256 		rv = CRYPTO_DATA_LEN_RANGE;
1257 		goto clean3;
1258 	}
1259 
1260 	/* Convert the rest of private key attributes to big integer format. */
1261 	bytestring2bignum(&(rsakey->dmodpminus1), expo2, expo2_len);
1262 	bytestring2bignum(&(rsakey->dmodqminus1), expo1, expo1_len);
1263 	bytestring2bignum(&(rsakey->p), prime2, prime2_len);
1264 	bytestring2bignum(&(rsakey->q), prime1, prime1_len);
1265 	bytestring2bignum(&(rsakey->pinvmodq), coef, coef_len);
1266 
1267 	if ((big_cmp_abs(&(rsakey->dmodpminus1), &(rsakey->p)) > 0) ||
1268 	    (big_cmp_abs(&(rsakey->dmodqminus1), &(rsakey->q)) > 0) ||
1269 	    (big_cmp_abs(&(rsakey->pinvmodq), &(rsakey->q)) > 0)) {
1270 		rv = CRYPTO_KEY_SIZE_RANGE;
1271 		goto clean3;
1272 	}
1273 
1274 	/* Perform RSA computation on big integer input data. */
1275 	if (big_modexp_crt(&msg, &msg, &(rsakey->dmodpminus1),
1276 	    &(rsakey->dmodqminus1), &(rsakey->p), &(rsakey->q),
1277 	    &(rsakey->pinvmodq), NULL, NULL) != BIG_OK) {
1278 		rv = CRYPTO_HOST_MEMORY;
1279 		goto clean3;
1280 	}
1281 
1282 	/* Convert the big integer output data to octet string. */
1283 	bignum2bytestring(out, &msg, modulus_len);
1284 
1285 	/*
1286 	 * Should not free modulus and friends as they are just pointers
1287 	 * to an attribute value buffer from the caller.
1288 	 */
1289 clean3:
1290 	big_finish(&msg);
1291 clean2:
1292 	RSA_key_finish(rsakey);
1293 clean1:
1294 	kmem_free(rsakey, sizeof (RSAkey));
1295 
1296 	return (rv);
1297 }
1298 
1299 /* EXPORT DELETE END */
1300 
1301 /* ARGSUSED */
1302 static int
1303 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
1304     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
1305     crypto_req_handle_t req)
1306 {
1307 	int rv;
1308 	int kmflag;
1309 	rsa_ctx_t *ctxp;
1310 	digest_rsa_ctx_t *dctxp;
1311 
1312 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1313 		return (rv);
1314 
1315 	/*
1316 	 * Allocate a RSA context.
1317 	 */
1318 	kmflag = crypto_kmflag(req);
1319 	switch (mechanism->cm_type) {
1320 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1321 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1322 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1323 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1324 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1325 		dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
1326 		ctxp = (rsa_ctx_t *)dctxp;
1327 		break;
1328 	default:
1329 		ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
1330 		break;
1331 	}
1332 
1333 	if (ctxp == NULL)
1334 		return (CRYPTO_HOST_MEMORY);
1335 
1336 	ctxp->mech_type = mechanism->cm_type;
1337 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
1338 		switch (mechanism->cm_type) {
1339 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1340 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1341 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1342 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1343 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1344 			kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
1345 			break;
1346 		default:
1347 			kmem_free(ctxp, sizeof (rsa_ctx_t));
1348 			break;
1349 		}
1350 		return (rv);
1351 	}
1352 
1353 	switch (mechanism->cm_type) {
1354 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1355 		MD5Init(&(dctxp->md5_ctx));
1356 		break;
1357 
1358 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1359 		SHA1Init(&(dctxp->sha1_ctx));
1360 		break;
1361 
1362 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1363 		SHA2Init(SHA256, &(dctxp->sha2_ctx));
1364 		break;
1365 
1366 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1367 		SHA2Init(SHA384, &(dctxp->sha2_ctx));
1368 		break;
1369 
1370 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1371 		SHA2Init(SHA512, &(dctxp->sha2_ctx));
1372 		break;
1373 	}
1374 
1375 	ctx->cc_provider_private = ctxp;
1376 
1377 	return (CRYPTO_SUCCESS);
1378 }
1379 
1380 #define	SHA1_DIGEST_SIZE 20
1381 #define	MD5_DIGEST_SIZE 16
1382 
1383 #define	INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)	\
1384 	(data).cd_format = CRYPTO_DATA_RAW;		\
1385 	(data).cd_offset = 0;				\
1386 	(data).cd_raw.iov_base = (char *)base;		\
1387 	(data).cd_raw.iov_len = len;			\
1388 	(data).cd_length = cd_len;
1389 
1390 #define	DO_UPDATE	0x01
1391 #define	DO_FINAL	0x02
1392 #define	DO_MD5		0x04
1393 #define	DO_SHA1		0x08
1394 #define	DO_SIGN		0x10
1395 #define	DO_VERIFY	0x20
1396 #define	DO_SHA2		0x40
1397 
1398 static int
1399 digest_data(crypto_data_t *data, void *dctx, uchar_t *digest,
1400     uchar_t flag)
1401 {
1402 	int rv, dlen;
1403 	uchar_t *dptr;
1404 
1405 	ASSERT(flag & DO_MD5 || flag & DO_SHA1 || flag & DO_SHA2);
1406 	if (data == NULL) {
1407 		ASSERT((flag & DO_UPDATE) == 0);
1408 		goto dofinal;
1409 	}
1410 
1411 	dlen = data->cd_length;
1412 
1413 	if (flag & DO_UPDATE) {
1414 
1415 		switch (data->cd_format) {
1416 		case CRYPTO_DATA_RAW:
1417 			dptr = (uchar_t *)(data->cd_raw.iov_base +
1418 			    data->cd_offset);
1419 
1420 			if (flag & DO_MD5)
1421 				MD5Update(dctx, dptr, dlen);
1422 
1423 			else if (flag & DO_SHA1)
1424 				SHA1Update(dctx, dptr, dlen);
1425 
1426 			else
1427 				SHA2Update(dctx, dptr, dlen);
1428 
1429 		break;
1430 
1431 		case CRYPTO_DATA_UIO:
1432 			if (flag & DO_MD5)
1433 				rv = process_uio_data(data, NULL, dlen,
1434 				    MD5_DIGEST_DATA, dctx);
1435 
1436 			else if (flag & DO_SHA1)
1437 				rv = process_uio_data(data, NULL, dlen,
1438 				    SHA1_DIGEST_DATA, dctx);
1439 
1440 			else
1441 				rv = process_uio_data(data, NULL, dlen,
1442 				    SHA2_DIGEST_DATA, dctx);
1443 
1444 			if (rv != CRYPTO_SUCCESS)
1445 				return (rv);
1446 
1447 			break;
1448 
1449 		case CRYPTO_DATA_MBLK:
1450 			if (flag & DO_MD5)
1451 				rv = process_mblk_data(data, NULL, dlen,
1452 				    MD5_DIGEST_DATA, dctx);
1453 
1454 			else if (flag & DO_SHA1)
1455 				rv = process_mblk_data(data, NULL, dlen,
1456 				    SHA1_DIGEST_DATA, dctx);
1457 
1458 			else
1459 				rv = process_mblk_data(data, NULL, dlen,
1460 				    SHA2_DIGEST_DATA, dctx);
1461 
1462 			if (rv != CRYPTO_SUCCESS)
1463 				return (rv);
1464 
1465 			break;
1466 		}
1467 	}
1468 
1469 dofinal:
1470 	if (flag & DO_FINAL) {
1471 		if (flag & DO_MD5)
1472 			MD5Final(digest, dctx);
1473 
1474 		else if (flag & DO_SHA1)
1475 			SHA1Final(digest, dctx);
1476 
1477 		else
1478 			SHA2Final(digest, dctx);
1479 	}
1480 
1481 	return (CRYPTO_SUCCESS);
1482 }
1483 
1484 static int
1485 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
1486     crypto_data_t *signature, int kmflag, uchar_t flag)
1487 {
1488 	int rv = CRYPTO_FAILED;
1489 
1490 /* EXPORT DELETE START */
1491 
1492 	uchar_t digest[SHA512_DIGEST_LENGTH];
1493 	/* The der_data size is enough for MD5 also */
1494 	uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1495 	ulong_t der_data_len;
1496 	crypto_data_t der_cd;
1497 	rsa_mech_type_t mech_type;
1498 
1499 	ASSERT(flag & DO_SIGN || flag & DO_VERIFY);
1500 	ASSERT(data != NULL || (flag & DO_FINAL));
1501 
1502 	mech_type = ctxp->mech_type;
1503 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1504 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
1505 		return (CRYPTO_MECHANISM_INVALID);
1506 
1507 	/*
1508 	 * We need to do the BUFFER_TOO_SMALL check before digesting
1509 	 * the data. No check is needed for verify as signature is not
1510 	 * an output argument for verify.
1511 	 */
1512 	if (flag & DO_SIGN) {
1513 		uchar_t *modulus;
1514 		ssize_t modulus_len;
1515 
1516 		if ((rv = get_key_attr(ctxp->key, SUN_CKA_MODULUS, &modulus,
1517 		    &modulus_len)) != CRYPTO_SUCCESS) {
1518 			return (rv);
1519 		}
1520 
1521 		if (signature->cd_length < modulus_len) {
1522 			signature->cd_length = modulus_len;
1523 			return (CRYPTO_BUFFER_TOO_SMALL);
1524 		}
1525 	}
1526 
1527 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1528 		rv = digest_data(data, &(ctxp->md5_ctx),
1529 		    digest, flag | DO_MD5);
1530 
1531 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1532 		rv = digest_data(data, &(ctxp->sha1_ctx),
1533 		    digest, flag | DO_SHA1);
1534 
1535 	else
1536 		rv = digest_data(data, &(ctxp->sha2_ctx),
1537 		    digest, flag | DO_SHA2);
1538 
1539 	if (rv != CRYPTO_SUCCESS)
1540 		return (rv);
1541 
1542 
1543 	/*
1544 	 * Prepare the DER encoding of the DigestInfo value as follows:
1545 	 * MD5:		MD5_DER_PREFIX || H
1546 	 * SHA-1:	SHA1_DER_PREFIX || H
1547 	 *
1548 	 * See rsa_impl.c for more details.
1549 	 */
1550 	switch (mech_type) {
1551 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1552 		bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1553 		bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1554 		der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1555 		break;
1556 
1557 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1558 		bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1559 		bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1560 		    SHA1_DIGEST_SIZE);
1561 		der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1562 		break;
1563 
1564 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1565 		bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1566 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1567 		    SHA256_DIGEST_LENGTH);
1568 		der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1569 		break;
1570 
1571 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1572 		bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1573 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1574 		    SHA384_DIGEST_LENGTH);
1575 		der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1576 		break;
1577 
1578 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1579 		bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1580 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1581 		    SHA512_DIGEST_LENGTH);
1582 		der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1583 		break;
1584 	}
1585 
1586 	INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1587 	/*
1588 	 * Now, we are ready to sign or verify the DER_ENCODED data.
1589 	 */
1590 	if (flag & DO_SIGN)
1591 		rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1592 		    signature, kmflag);
1593 	else
1594 		rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1595 		    signature, kmflag);
1596 
1597 /* EXPORT DELETE END */
1598 
1599 	return (rv);
1600 }
1601 
1602 static int
1603 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1604     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1605 {
1606 	int rv = CRYPTO_FAILED;
1607 
1608 /* EXPORT DELETE START */
1609 
1610 	int dlen;
1611 	uchar_t *dataptr, *modulus;
1612 	ssize_t modulus_len;
1613 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1614 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1615 	uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1616 
1617 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1618 	    &modulus_len)) != CRYPTO_SUCCESS) {
1619 		return (rv);
1620 	}
1621 
1622 	dlen = data->cd_length;
1623 	switch (mech_type) {
1624 	case RSA_PKCS_MECH_INFO_TYPE:
1625 		if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1626 			return (CRYPTO_DATA_LEN_RANGE);
1627 		break;
1628 	case RSA_X_509_MECH_INFO_TYPE:
1629 		if (dlen > modulus_len)
1630 			return (CRYPTO_DATA_LEN_RANGE);
1631 		break;
1632 	}
1633 
1634 	if (signature->cd_length < modulus_len) {
1635 		signature->cd_length = modulus_len;
1636 		return (CRYPTO_BUFFER_TOO_SMALL);
1637 	}
1638 
1639 	ASSERT(data->cd_length <= sizeof (tmp_data));
1640 	if ((rv = get_input_data(data, &dataptr, tmp_data))
1641 	    != CRYPTO_SUCCESS)
1642 		return (rv);
1643 
1644 	switch (mech_type) {
1645 	case RSA_PKCS_MECH_INFO_TYPE:
1646 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1647 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1648 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1649 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1650 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1651 		/*
1652 		 * Add PKCS padding to the input data to format a block
1653 		 * type "01" encryption block.
1654 		 */
1655 		rv = soft_sign_rsa_pkcs_encode(dataptr, dlen, plain_data,
1656 		    modulus_len);
1657 		if (rv != CRYPTO_SUCCESS)
1658 			return (rv);
1659 
1660 		break;
1661 
1662 	case RSA_X_509_MECH_INFO_TYPE:
1663 		bzero(plain_data, modulus_len - dlen);
1664 		bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1665 		break;
1666 	}
1667 
1668 	rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data,
1669 	    kmflag);
1670 	if (rv == CRYPTO_SUCCESS) {
1671 		/* copy out to signature */
1672 		if ((rv = put_output_data(signed_data,
1673 		    signature, modulus_len)) != CRYPTO_SUCCESS)
1674 			return (rv);
1675 
1676 		signature->cd_length = modulus_len;
1677 	}
1678 
1679 /* EXPORT DELETE END */
1680 
1681 	return (rv);
1682 }
1683 
1684 /* ARGSUSED */
1685 static int
1686 rsa_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1687     crypto_req_handle_t req)
1688 {
1689 	int rv;
1690 	rsa_ctx_t *ctxp;
1691 
1692 	ASSERT(ctx->cc_provider_private != NULL);
1693 	ctxp = ctx->cc_provider_private;
1694 
1695 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1696 	switch (ctxp->mech_type) {
1697 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1698 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1699 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1700 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1701 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1702 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1703 		    signature, KM_SLEEP, DO_SIGN | DO_UPDATE | DO_FINAL);
1704 		break;
1705 	default:
1706 		rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1707 		    signature, KM_SLEEP);
1708 		break;
1709 	}
1710 
1711 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1712 		(void) rsa_free_context(ctx);
1713 
1714 	return (rv);
1715 }
1716 
1717 /* ARGSUSED */
1718 static int
1719 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1720 {
1721 	int rv;
1722 	digest_rsa_ctx_t *ctxp;
1723 	rsa_mech_type_t mech_type;
1724 
1725 	ASSERT(ctx->cc_provider_private != NULL);
1726 	ctxp = ctx->cc_provider_private;
1727 	mech_type = ctxp->mech_type;
1728 
1729 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1730 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
1731 		return (CRYPTO_MECHANISM_INVALID);
1732 
1733 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1734 		rv = digest_data(data, &(ctxp->md5_ctx),
1735 		    NULL, DO_MD5 | DO_UPDATE);
1736 
1737 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1738 		rv = digest_data(data, &(ctxp->sha1_ctx),
1739 		    NULL, DO_SHA1 | DO_UPDATE);
1740 
1741 	else
1742 		rv = digest_data(data, &(ctxp->sha2_ctx),
1743 		    NULL, DO_SHA2 | DO_UPDATE);
1744 
1745 	return (rv);
1746 }
1747 
1748 static int
1749 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1750     crypto_req_handle_t req)
1751 {
1752 	int rv;
1753 	digest_rsa_ctx_t *ctxp;
1754 
1755 	ASSERT(ctx->cc_provider_private != NULL);
1756 	ctxp = ctx->cc_provider_private;
1757 
1758 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1759 	    crypto_kmflag(req), DO_SIGN | DO_FINAL);
1760 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1761 		(void) rsa_free_context(ctx);
1762 
1763 	return (rv);
1764 }
1765 
1766 /* ARGSUSED */
1767 static int
1768 rsa_sign_atomic(crypto_provider_handle_t provider,
1769     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1770     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1771     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1772 {
1773 	int rv;
1774 	digest_rsa_ctx_t dctx;
1775 
1776 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1777 		return (rv);
1778 
1779 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1780 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1781 		rv = rsa_sign_common(mechanism->cm_type, key, data,
1782 		    signature, crypto_kmflag(req));
1783 
1784 	else {
1785 		dctx.mech_type = mechanism->cm_type;
1786 		dctx.key = key;
1787 		switch (mechanism->cm_type) {
1788 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1789 			MD5Init(&(dctx.md5_ctx));
1790 			break;
1791 
1792 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1793 			SHA1Init(&(dctx.sha1_ctx));
1794 			break;
1795 
1796 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1797 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1798 			break;
1799 
1800 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1801 			SHA2Init(SHA384, &(dctx.sha2_ctx));
1802 			break;
1803 
1804 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1805 			SHA2Init(SHA512, &(dctx.sha2_ctx));
1806 			break;
1807 		}
1808 
1809 		rv = rsa_digest_svrfy_common(&dctx, data, signature,
1810 		    crypto_kmflag(req), DO_SIGN | DO_UPDATE | DO_FINAL);
1811 	}
1812 
1813 	return (rv);
1814 }
1815 
1816 static int
1817 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1818     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1819 {
1820 	int rv = CRYPTO_FAILED;
1821 
1822 /* EXPORT DELETE START */
1823 
1824 	uchar_t *sigptr, *modulus;
1825 	ssize_t modulus_len;
1826 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1827 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1828 
1829 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1830 	    &modulus_len)) != CRYPTO_SUCCESS) {
1831 		return (rv);
1832 	}
1833 
1834 	if (signature->cd_length != modulus_len)
1835 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1836 
1837 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1838 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
1839 	    != CRYPTO_SUCCESS)
1840 		return (rv);
1841 
1842 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
1843 	if (rv != CRYPTO_SUCCESS)
1844 		return (rv);
1845 
1846 	if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1847 		if (compare_data(data, (plain_data + modulus_len
1848 		    - data->cd_length)) != 0)
1849 			rv = CRYPTO_SIGNATURE_INVALID;
1850 
1851 	} else {
1852 		int data_len = modulus_len;
1853 
1854 		/*
1855 		 * Strip off the encoded padding bytes in front of the
1856 		 * recovered data, then compare the recovered data with
1857 		 * the original data.
1858 		 */
1859 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1860 		if (rv != CRYPTO_SUCCESS)
1861 			return (rv);
1862 
1863 		if (data_len != data->cd_length)
1864 			return (CRYPTO_SIGNATURE_LEN_RANGE);
1865 
1866 		if (compare_data(data, (plain_data + modulus_len
1867 		    - data_len)) != 0)
1868 			rv = CRYPTO_SIGNATURE_INVALID;
1869 	}
1870 
1871 /* EXPORT DELETE END */
1872 
1873 	return (rv);
1874 }
1875 
1876 /* ARGSUSED */
1877 static int
1878 rsa_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1879     crypto_req_handle_t req)
1880 {
1881 	int rv;
1882 	rsa_ctx_t *ctxp;
1883 
1884 	ASSERT(ctx->cc_provider_private != NULL);
1885 	ctxp = ctx->cc_provider_private;
1886 
1887 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1888 	switch (ctxp->mech_type) {
1889 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1890 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1891 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1892 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1893 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1894 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1895 		    signature, KM_SLEEP, DO_VERIFY | DO_UPDATE | DO_FINAL);
1896 		break;
1897 	default:
1898 		rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1899 		    signature, KM_SLEEP);
1900 		break;
1901 	}
1902 
1903 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1904 		(void) rsa_free_context(ctx);
1905 
1906 	return (rv);
1907 }
1908 
1909 /* ARGSUSED */
1910 static int
1911 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1912     crypto_req_handle_t req)
1913 {
1914 	int rv;
1915 	digest_rsa_ctx_t *ctxp;
1916 
1917 	ASSERT(ctx->cc_provider_private != NULL);
1918 	ctxp = ctx->cc_provider_private;
1919 
1920 	switch (ctxp->mech_type) {
1921 
1922 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1923 		rv = digest_data(data, &(ctxp->md5_ctx),
1924 		    NULL, DO_MD5 | DO_UPDATE);
1925 		break;
1926 
1927 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1928 		rv = digest_data(data, &(ctxp->sha1_ctx),
1929 		    NULL, DO_SHA1 | DO_UPDATE);
1930 		break;
1931 
1932 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1933 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1934 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1935 		rv = digest_data(data, &(ctxp->sha2_ctx),
1936 		    NULL, DO_SHA2 | DO_UPDATE);
1937 		break;
1938 
1939 	default:
1940 		return (CRYPTO_MECHANISM_INVALID);
1941 	}
1942 
1943 	return (rv);
1944 }
1945 
1946 static int
1947 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1948     crypto_req_handle_t req)
1949 {
1950 	int rv;
1951 	digest_rsa_ctx_t *ctxp;
1952 
1953 	ASSERT(ctx->cc_provider_private != NULL);
1954 	ctxp = ctx->cc_provider_private;
1955 
1956 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1957 	    crypto_kmflag(req), DO_VERIFY | DO_FINAL);
1958 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1959 		(void) rsa_free_context(ctx);
1960 
1961 	return (rv);
1962 }
1963 
1964 
1965 /* ARGSUSED */
1966 static int
1967 rsa_verify_atomic(crypto_provider_handle_t provider,
1968     crypto_session_id_t session_id,
1969     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1970     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1971     crypto_req_handle_t req)
1972 {
1973 	int rv;
1974 	digest_rsa_ctx_t dctx;
1975 
1976 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1977 		return (rv);
1978 
1979 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1980 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1981 		rv = rsa_verify_common(mechanism->cm_type, key, data,
1982 		    signature, crypto_kmflag(req));
1983 
1984 	else {
1985 		dctx.mech_type = mechanism->cm_type;
1986 		dctx.key = key;
1987 
1988 		switch (mechanism->cm_type) {
1989 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1990 			MD5Init(&(dctx.md5_ctx));
1991 			break;
1992 
1993 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1994 			SHA1Init(&(dctx.sha1_ctx));
1995 			break;
1996 
1997 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1998 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1999 			break;
2000 
2001 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
2002 			SHA2Init(SHA384, &(dctx.sha2_ctx));
2003 			break;
2004 
2005 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
2006 			SHA2Init(SHA512, &(dctx.sha2_ctx));
2007 			break;
2008 		}
2009 
2010 		rv = rsa_digest_svrfy_common(&dctx, data,
2011 		    signature, crypto_kmflag(req),
2012 		    DO_VERIFY | DO_UPDATE | DO_FINAL);
2013 	}
2014 
2015 	return (rv);
2016 }
2017 
2018 static int
2019 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
2020     crypto_data_t *signature, crypto_data_t *data, int kmflag)
2021 {
2022 	int rv = CRYPTO_FAILED;
2023 
2024 /* EXPORT DELETE START */
2025 
2026 	int data_len;
2027 	uchar_t *sigptr, *modulus;
2028 	ssize_t modulus_len;
2029 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
2030 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
2031 
2032 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
2033 	    &modulus_len)) != CRYPTO_SUCCESS) {
2034 		return (rv);
2035 	}
2036 
2037 	if (signature->cd_length != modulus_len)
2038 		return (CRYPTO_SIGNATURE_LEN_RANGE);
2039 
2040 	ASSERT(signature->cd_length <= sizeof (tmp_data));
2041 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
2042 	    != CRYPTO_SUCCESS)
2043 		return (rv);
2044 
2045 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
2046 	if (rv != CRYPTO_SUCCESS)
2047 		return (rv);
2048 
2049 	data_len = modulus_len;
2050 
2051 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
2052 		/*
2053 		 * Strip off the encoded padding bytes in front of the
2054 		 * recovered data, then compare the recovered data with
2055 		 * the original data.
2056 		 */
2057 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
2058 		if (rv != CRYPTO_SUCCESS)
2059 			return (rv);
2060 	}
2061 
2062 	if (data->cd_length < data_len) {
2063 		data->cd_length = data_len;
2064 		return (CRYPTO_BUFFER_TOO_SMALL);
2065 	}
2066 
2067 	if ((rv = put_output_data(plain_data + modulus_len - data_len,
2068 	    data, data_len)) != CRYPTO_SUCCESS)
2069 		return (rv);
2070 	data->cd_length = data_len;
2071 
2072 /* EXPORT DELETE END */
2073 
2074 	return (rv);
2075 }
2076 
2077 /* ARGSUSED */
2078 static int
2079 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
2080     crypto_data_t *data, crypto_req_handle_t req)
2081 {
2082 	int rv;
2083 	rsa_ctx_t *ctxp;
2084 
2085 	ASSERT(ctx->cc_provider_private != NULL);
2086 	ctxp = ctx->cc_provider_private;
2087 
2088 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
2089 	rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
2090 	    signature, data, KM_SLEEP);
2091 
2092 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
2093 		(void) rsa_free_context(ctx);
2094 
2095 	return (rv);
2096 }
2097 
2098 /* ARGSUSED */
2099 static int
2100 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
2101     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
2102     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
2103     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
2104 {
2105 	int rv;
2106 
2107 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
2108 		return (rv);
2109 
2110 	return (rsa_verify_recover_common(mechanism->cm_type, key,
2111 	    signature, data, crypto_kmflag(req)));
2112 }
2113