xref: /illumos-gate/usr/src/uts/common/crypto/io/rsa.c (revision b4a44cf294b47530f872176be732538fdf3536a0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * RSA provider for the Kernel Cryptographic Framework (KCF)
28  */
29 
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/modctl.h>
33 #include <sys/cmn_err.h>
34 #include <sys/ddi.h>
35 #include <sys/crypto/spi.h>
36 #include <sys/sysmacros.h>
37 #include <sys/strsun.h>
38 #include <sys/md5.h>
39 #include <sys/sha1.h>
40 #define	_SHA2_IMPL
41 #include <sys/sha2.h>
42 #include <sys/random.h>
43 #include <sys/crypto/impl.h>
44 #include <sha1/sha1_impl.h>
45 #include <sha2/sha2_impl.h>
46 #include <padding/padding.h>
47 #include <rsa/rsa_impl.h>
48 
49 extern struct mod_ops mod_cryptoops;
50 
51 /*
52  * Module linkage information for the kernel.
53  */
54 static struct modlcrypto modlcrypto = {
55 	&mod_cryptoops,
56 	"RSA Kernel SW Provider"
57 };
58 
59 static struct modlinkage modlinkage = {
60 	MODREV_1,
61 	(void *)&modlcrypto,
62 	NULL
63 };
64 
65 /*
66  * CSPI information (entry points, provider info, etc.)
67  */
68 typedef enum rsa_mech_type {
69 	RSA_PKCS_MECH_INFO_TYPE,	/* SUN_CKM_RSA_PKCS */
70 	RSA_X_509_MECH_INFO_TYPE,	/* SUN_CKM_RSA_X_509 */
71 	MD5_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_MD5_RSA_PKCS */
72 	SHA1_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA1_RSA_PKCS */
73 	SHA256_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA256_RSA_PKCS */
74 	SHA384_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA384_RSA_PKCS */
75 	SHA512_RSA_PKCS_MECH_INFO_TYPE	/* SUN_SHA512_RSA_PKCS */
76 } rsa_mech_type_t;
77 
78 /*
79  * Context for RSA_PKCS and RSA_X_509 mechanisms.
80  */
81 typedef struct rsa_ctx {
82 	rsa_mech_type_t	mech_type;
83 	crypto_key_t *key;
84 	size_t keychunk_size;
85 } rsa_ctx_t;
86 
87 /*
88  * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
89  */
90 typedef struct digest_rsa_ctx {
91 	rsa_mech_type_t	mech_type;
92 	crypto_key_t *key;
93 	size_t keychunk_size;
94 	union {
95 		MD5_CTX md5ctx;
96 		SHA1_CTX sha1ctx;
97 		SHA2_CTX sha2ctx;
98 	} dctx_u;
99 } digest_rsa_ctx_t;
100 
101 #define	md5_ctx		dctx_u.md5ctx
102 #define	sha1_ctx	dctx_u.sha1ctx
103 #define	sha2_ctx	dctx_u.sha2ctx
104 
105 /*
106  * Mechanism info structure passed to KCF during registration.
107  */
108 static crypto_mech_info_t rsa_mech_info_tab[] = {
109 	/* RSA_PKCS */
110 	{SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
111 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
112 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
113 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
114 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
115 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
116 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
117 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
118 
119 	/* RSA_X_509 */
120 	{SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
121 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
122 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
123 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
124 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
125 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
126 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
127 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
128 
129 	/* MD5_RSA_PKCS */
130 	{SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
131 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
132 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
133 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
134 
135 	/* SHA1_RSA_PKCS */
136 	{SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
137 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
138 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
139 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
140 
141 	/* SHA256_RSA_PKCS */
142 	{SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
143 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
144 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
145 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
146 
147 	/* SHA384_RSA_PKCS */
148 	{SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
149 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
150 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
151 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
152 
153 	/* SHA512_RSA_PKCS */
154 	{SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
155 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
156 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
157 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
158 
159 };
160 
161 #define	RSA_VALID_MECH(mech)					\
162 	(((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||		\
163 	(mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||		\
164 	(mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||	\
165 	(mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE ||	\
166 	(mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE ||	\
167 	(mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE ||	\
168 	(mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
169 
170 /* operations are in-place if the output buffer is NULL */
171 #define	RSA_ARG_INPLACE(input, output)				\
172 	if ((output) == NULL)					\
173 		(output) = (input);
174 
175 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
176 
177 static crypto_control_ops_t rsa_control_ops = {
178 	rsa_provider_status
179 };
180 
181 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
182     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
183 static int rsaprov_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
184     crypto_req_handle_t);
185 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
186     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
187     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
188 static int rsaprov_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
189     crypto_req_handle_t);
190 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
191     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
192     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
193 
194 /*
195  * The RSA mechanisms do not have multiple-part cipher operations.
196  * So, the update and final routines are set to NULL.
197  */
198 static crypto_cipher_ops_t rsa_cipher_ops = {
199 	rsa_common_init,
200 	rsaprov_encrypt,
201 	NULL,
202 	NULL,
203 	rsa_encrypt_atomic,
204 	rsa_common_init,
205 	rsaprov_decrypt,
206 	NULL,
207 	NULL,
208 	rsa_decrypt_atomic
209 };
210 
211 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
212     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
213 static int rsaprov_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
214     crypto_req_handle_t);
215 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
216     crypto_req_handle_t);
217 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
218     crypto_req_handle_t);
219 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
220     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
221     crypto_spi_ctx_template_t, crypto_req_handle_t);
222 
223 /*
224  * We use the same routine for sign_init and sign_recover_init fields
225  * as they do the same thing. Same holds for sign and sign_recover fields,
226  * and sign_atomic and sign_recover_atomic fields.
227  */
228 static crypto_sign_ops_t rsa_sign_ops = {
229 	rsa_sign_verify_common_init,
230 	rsaprov_sign,
231 	rsa_sign_update,
232 	rsa_sign_final,
233 	rsa_sign_atomic,
234 	rsa_sign_verify_common_init,
235 	rsaprov_sign,
236 	rsa_sign_atomic
237 };
238 
239 static int rsaprov_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
240     crypto_req_handle_t);
241 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
242     crypto_req_handle_t);
243 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
244     crypto_req_handle_t);
245 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
246     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
247     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
248 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
249     crypto_data_t *, crypto_req_handle_t);
250 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
251     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
252     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
253     crypto_req_handle_t);
254 
255 /*
256  * We use the same routine (rsa_sign_verify_common_init) for verify_init
257  * and verify_recover_init fields as they do the same thing.
258  */
259 static crypto_verify_ops_t rsa_verify_ops = {
260 	rsa_sign_verify_common_init,
261 	rsaprov_verify,
262 	rsa_verify_update,
263 	rsa_verify_final,
264 	rsa_verify_atomic,
265 	rsa_sign_verify_common_init,
266 	rsa_verify_recover,
267 	rsa_verify_recover_atomic
268 };
269 
270 static int rsa_free_context(crypto_ctx_t *);
271 
272 static crypto_ctx_ops_t rsa_ctx_ops = {
273 	NULL,
274 	rsa_free_context
275 };
276 
277 static crypto_ops_t rsa_crypto_ops = {
278 	&rsa_control_ops,
279 	NULL,
280 	&rsa_cipher_ops,
281 	NULL,
282 	&rsa_sign_ops,
283 	&rsa_verify_ops,
284 	NULL,
285 	NULL,
286 	NULL,
287 	NULL,
288 	NULL,
289 	NULL,
290 	NULL,
291 	&rsa_ctx_ops,
292 	NULL,
293 	NULL,
294 	NULL,
295 };
296 
297 static crypto_provider_info_t rsa_prov_info = {
298 	CRYPTO_SPI_VERSION_4,
299 	"RSA Software Provider",
300 	CRYPTO_SW_PROVIDER,
301 	{&modlinkage},
302 	NULL,
303 	&rsa_crypto_ops,
304 	sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
305 	rsa_mech_info_tab
306 };
307 
308 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
309     crypto_data_t *, crypto_data_t *);
310 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
311     crypto_data_t *, crypto_data_t *);
312 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
313     crypto_data_t *, crypto_data_t *);
314 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
315     crypto_data_t *, crypto_data_t *);
316 static int compare_data(crypto_data_t *, uchar_t *);
317 
318 static int core_rsa_encrypt(crypto_key_t *, uchar_t *, int, uchar_t *, int);
319 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int, uchar_t *);
320 
321 static crypto_kcf_provider_handle_t rsa_prov_handle = 0;
322 
323 int
_init(void)324 _init(void)
325 {
326 	int ret;
327 
328 	if ((ret = mod_install(&modlinkage)) != 0)
329 		return (ret);
330 
331 	/* Register with KCF.  If the registration fails, remove the module. */
332 	if (crypto_register_provider(&rsa_prov_info, &rsa_prov_handle)) {
333 		(void) mod_remove(&modlinkage);
334 		return (EACCES);
335 	}
336 
337 	return (0);
338 }
339 
340 int
_fini(void)341 _fini(void)
342 {
343 	/* Unregister from KCF if module is registered */
344 	if (rsa_prov_handle != 0) {
345 		if (crypto_unregister_provider(rsa_prov_handle))
346 			return (EBUSY);
347 
348 		rsa_prov_handle = 0;
349 	}
350 
351 	return (mod_remove(&modlinkage));
352 }
353 
354 int
_info(struct modinfo * modinfop)355 _info(struct modinfo *modinfop)
356 {
357 	return (mod_info(&modlinkage, modinfop));
358 }
359 
360 /* ARGSUSED */
361 static void
rsa_provider_status(crypto_provider_handle_t provider,uint_t * status)362 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
363 {
364 	*status = CRYPTO_PROVIDER_READY;
365 }
366 
367 static int
check_mech_and_key(crypto_mechanism_t * mechanism,crypto_key_t * key)368 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
369 {
370 	int rv = CRYPTO_FAILED;
371 
372 	uchar_t *modulus;
373 	ssize_t modulus_len; /* In bytes */
374 
375 	if (!RSA_VALID_MECH(mechanism))
376 		return (CRYPTO_MECHANISM_INVALID);
377 
378 	/*
379 	 * We only support RSA keys that are passed as a list of
380 	 * object attributes.
381 	 */
382 	if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
383 		return (CRYPTO_KEY_TYPE_INCONSISTENT);
384 	}
385 
386 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
387 	    &modulus_len)) != CRYPTO_SUCCESS) {
388 		return (rv);
389 	}
390 	if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
391 	    modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
392 		return (CRYPTO_KEY_SIZE_RANGE);
393 
394 	return (rv);
395 }
396 
397 void
kmemset(uint8_t * buf,char pattern,size_t len)398 kmemset(uint8_t *buf, char pattern, size_t len)
399 {
400 	int i = 0;
401 
402 	while (i < len)
403 		buf[i++] = pattern;
404 }
405 
406 /*
407  * This function guarantees to return non-zero random numbers.
408  * This is needed as the /dev/urandom kernel interface,
409  * random_get_pseudo_bytes(), may return zeros.
410  */
411 int
knzero_random_generator(uint8_t * ran_out,size_t ran_len)412 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
413 {
414 	int rv;
415 	size_t ebc = 0; /* count of extra bytes in extrarand */
416 	size_t i = 0;
417 	uint8_t extrarand[32];
418 	size_t extrarand_len;
419 
420 	if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
421 		return (rv);
422 
423 	/*
424 	 * Walk through the returned random numbers pointed by ran_out,
425 	 * and look for any random number which is zero.
426 	 * If we find zero, call random_get_pseudo_bytes() to generate
427 	 * another 32 random numbers pool. Replace any zeros in ran_out[]
428 	 * from the random number in pool.
429 	 */
430 	while (i < ran_len) {
431 		if (ran_out[i] != 0) {
432 			i++;
433 			continue;
434 		}
435 
436 		/*
437 		 * Note that it is 'while' so we are guaranteed a
438 		 * non-zero value on exit.
439 		 */
440 		if (ebc == 0) {
441 			/* refresh extrarand */
442 			extrarand_len = sizeof (extrarand);
443 			if ((rv = random_get_pseudo_bytes(extrarand,
444 			    extrarand_len)) != 0) {
445 				return (rv);
446 			}
447 
448 			ebc = extrarand_len;
449 		}
450 		/* Replace zero with byte from extrarand. */
451 		-- ebc;
452 
453 		/*
454 		 * The new random byte zero/non-zero will be checked in
455 		 * the next pass through the loop.
456 		 */
457 		ran_out[i] = extrarand[ebc];
458 	}
459 
460 	return (CRYPTO_SUCCESS);
461 }
462 
463 static int
compare_data(crypto_data_t * data,uchar_t * buf)464 compare_data(crypto_data_t *data, uchar_t *buf)
465 {
466 	int len;
467 	uchar_t *dptr;
468 
469 	len = data->cd_length;
470 	switch (data->cd_format) {
471 	case CRYPTO_DATA_RAW:
472 		dptr = (uchar_t *)(data->cd_raw.iov_base +
473 		    data->cd_offset);
474 
475 		return (bcmp(dptr, buf, len));
476 
477 	case CRYPTO_DATA_UIO:
478 		return (crypto_uio_data(data, buf, len,
479 		    COMPARE_TO_DATA, NULL, NULL));
480 
481 	case CRYPTO_DATA_MBLK:
482 		return (crypto_mblk_data(data, buf, len,
483 		    COMPARE_TO_DATA, NULL, NULL));
484 	}
485 
486 	return (CRYPTO_FAILED);
487 }
488 
489 /* ARGSUSED */
490 static int
rsa_common_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t template,crypto_req_handle_t req)491 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
492     crypto_key_t *key, crypto_spi_ctx_template_t template,
493     crypto_req_handle_t req)
494 {
495 	int rv;
496 	int kmflag;
497 	rsa_ctx_t *ctxp;
498 
499 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
500 		return (rv);
501 
502 	/*
503 	 * Allocate a RSA context.
504 	 */
505 	kmflag = crypto_kmflag(req);
506 	if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
507 		return (CRYPTO_HOST_MEMORY);
508 
509 	if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
510 	    kmflag)) != CRYPTO_SUCCESS) {
511 		kmem_free(ctxp, sizeof (rsa_ctx_t));
512 		return (rv);
513 	}
514 	ctxp->mech_type = mechanism->cm_type;
515 
516 	ctx->cc_provider_private = ctxp;
517 
518 	return (CRYPTO_SUCCESS);
519 }
520 
521 /* ARGSUSED */
522 static int
rsaprov_encrypt(crypto_ctx_t * ctx,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_req_handle_t req)523 rsaprov_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
524     crypto_data_t *ciphertext, crypto_req_handle_t req)
525 {
526 	int rv;
527 	rsa_ctx_t *ctxp;
528 
529 	ASSERT(ctx->cc_provider_private != NULL);
530 	ctxp = ctx->cc_provider_private;
531 
532 	RSA_ARG_INPLACE(plaintext, ciphertext);
533 
534 	/*
535 	 * Note on the KM_SLEEP flag passed to the routine below -
536 	 * rsaprov_encrypt() is a single-part encryption routine which is
537 	 * currently usable only by /dev/crypto. Since /dev/crypto calls are
538 	 * always synchronous, we can safely pass KM_SLEEP here.
539 	 */
540 	rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
541 	    ciphertext);
542 
543 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
544 		(void) rsa_free_context(ctx);
545 
546 	return (rv);
547 }
548 
549 /* ARGSUSED */
550 static int
rsa_encrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_spi_ctx_template_t template,crypto_req_handle_t req)551 rsa_encrypt_atomic(crypto_provider_handle_t provider,
552     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
553     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
554     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
555 {
556 	int rv;
557 
558 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
559 		return (rv);
560 	RSA_ARG_INPLACE(plaintext, ciphertext);
561 
562 	return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
563 	    ciphertext));
564 }
565 
566 static int
rsa_free_context(crypto_ctx_t * ctx)567 rsa_free_context(crypto_ctx_t *ctx)
568 {
569 	rsa_ctx_t *ctxp = ctx->cc_provider_private;
570 
571 	if (ctxp != NULL) {
572 		bzero(ctxp->key, ctxp->keychunk_size);
573 		kmem_free(ctxp->key, ctxp->keychunk_size);
574 
575 		if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
576 		    ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
577 			kmem_free(ctxp, sizeof (rsa_ctx_t));
578 		else
579 			kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
580 
581 		ctx->cc_provider_private = NULL;
582 	}
583 
584 	return (CRYPTO_SUCCESS);
585 }
586 
587 static int
rsa_encrypt_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * plaintext,crypto_data_t * ciphertext)588 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
589     crypto_data_t *plaintext, crypto_data_t *ciphertext)
590 {
591 	int rv = CRYPTO_FAILED;
592 
593 	int plen;
594 	uchar_t *ptptr;
595 	uchar_t *modulus;
596 	ssize_t modulus_len;
597 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
598 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
599 	uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
600 
601 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
602 	    &modulus_len)) != CRYPTO_SUCCESS) {
603 		return (rv);
604 	}
605 
606 	plen = plaintext->cd_length;
607 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
608 		if (plen > (modulus_len - MIN_PKCS1_PADLEN))
609 			return (CRYPTO_DATA_LEN_RANGE);
610 	} else {
611 		if (plen > modulus_len)
612 			return (CRYPTO_DATA_LEN_RANGE);
613 	}
614 
615 	/*
616 	 * Output buf len must not be less than RSA modulus size.
617 	 */
618 	if (ciphertext->cd_length < modulus_len) {
619 		ciphertext->cd_length = modulus_len;
620 		return (CRYPTO_BUFFER_TOO_SMALL);
621 	}
622 
623 	ASSERT(plaintext->cd_length <= sizeof (tmp_data));
624 	if ((rv = crypto_get_input_data(plaintext, &ptptr, tmp_data))
625 	    != CRYPTO_SUCCESS)
626 		return (rv);
627 
628 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
629 		rv = pkcs1_encode(PKCS1_ENCRYPT, ptptr, plen,
630 		    plain_data, modulus_len);
631 
632 		if (rv != CRYPTO_SUCCESS)
633 			return (rv);
634 	} else {
635 		bzero(plain_data, modulus_len - plen);
636 		bcopy(ptptr, &plain_data[modulus_len - plen], plen);
637 	}
638 
639 	rv = core_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
640 	if (rv == CRYPTO_SUCCESS) {
641 		/* copy out to ciphertext */
642 		if ((rv = crypto_put_output_data(cipher_data,
643 		    ciphertext, modulus_len)) != CRYPTO_SUCCESS)
644 			return (rv);
645 
646 		ciphertext->cd_length = modulus_len;
647 	}
648 
649 	return (rv);
650 }
651 
652 static int
core_rsa_encrypt(crypto_key_t * key,uchar_t * in,int in_len,uchar_t * out,int is_public)653 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
654     int in_len, uchar_t *out, int is_public)
655 {
656 	int rv;
657 	uchar_t *expo, *modulus;
658 	ssize_t	expo_len;
659 	ssize_t modulus_len;
660 	RSAbytekey k;
661 
662 	if (is_public) {
663 		if ((rv = crypto_get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT,
664 		    &expo, &expo_len)) != CRYPTO_SUCCESS)
665 			return (rv);
666 	} else {
667 		/*
668 		 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
669 		 * RSA secret key. See the comments in core_rsa_decrypt
670 		 * routine which calls this routine with a private key.
671 		 */
672 		if ((rv = crypto_get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT,
673 		    &expo, &expo_len)) != CRYPTO_SUCCESS)
674 			return (rv);
675 	}
676 
677 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
678 	    &modulus_len)) != CRYPTO_SUCCESS) {
679 		return (rv);
680 	}
681 
682 	k.modulus = modulus;
683 	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
684 	k.pubexpo = expo;
685 	k.pubexpo_bytes = expo_len;
686 	k.rfunc = NULL;
687 
688 	rv = rsa_encrypt(&k, in, in_len, out);
689 
690 	return (rv);
691 }
692 
693 /* ARGSUSED */
694 static int
rsaprov_decrypt(crypto_ctx_t * ctx,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_req_handle_t req)695 rsaprov_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
696     crypto_data_t *plaintext, crypto_req_handle_t req)
697 {
698 	int rv;
699 	rsa_ctx_t *ctxp;
700 
701 	ASSERT(ctx->cc_provider_private != NULL);
702 	ctxp = ctx->cc_provider_private;
703 
704 	RSA_ARG_INPLACE(ciphertext, plaintext);
705 
706 	/* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
707 	rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
708 	    ciphertext, plaintext);
709 
710 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
711 		(void) rsa_free_context(ctx);
712 
713 	return (rv);
714 }
715 
716 /* ARGSUSED */
717 static int
rsa_decrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_spi_ctx_template_t template,crypto_req_handle_t req)718 rsa_decrypt_atomic(crypto_provider_handle_t provider,
719     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
720     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
721     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
722 {
723 	int rv;
724 
725 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
726 		return (rv);
727 	RSA_ARG_INPLACE(ciphertext, plaintext);
728 
729 	return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
730 	    plaintext));
731 }
732 
733 static int
rsa_decrypt_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * ciphertext,crypto_data_t * plaintext)734 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
735     crypto_data_t *ciphertext, crypto_data_t *plaintext)
736 {
737 	int rv = CRYPTO_FAILED;
738 
739 	size_t plain_len;
740 	uchar_t *ctptr;
741 	uchar_t *modulus;
742 	ssize_t modulus_len;
743 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
744 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
745 
746 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
747 	    &modulus_len)) != CRYPTO_SUCCESS) {
748 		return (rv);
749 	}
750 
751 	/*
752 	 * Ciphertext length must be equal to RSA modulus size.
753 	 */
754 	if (ciphertext->cd_length != modulus_len)
755 		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
756 
757 	ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
758 	if ((rv = crypto_get_input_data(ciphertext, &ctptr, tmp_data))
759 	    != CRYPTO_SUCCESS)
760 		return (rv);
761 
762 	rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data);
763 	if (rv == CRYPTO_SUCCESS) {
764 		plain_len = modulus_len;
765 
766 		if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
767 			/* Strip off the PKCS block formatting data. */
768 			rv = pkcs1_decode(PKCS1_DECRYPT, plain_data,
769 			    &plain_len);
770 			if (rv != CRYPTO_SUCCESS)
771 				return (rv);
772 		}
773 
774 		if (plain_len > plaintext->cd_length) {
775 			plaintext->cd_length = plain_len;
776 			return (CRYPTO_BUFFER_TOO_SMALL);
777 		}
778 
779 		if ((rv = crypto_put_output_data(
780 		    plain_data + modulus_len - plain_len,
781 		    plaintext, plain_len)) != CRYPTO_SUCCESS)
782 			return (rv);
783 
784 		plaintext->cd_length = plain_len;
785 	}
786 
787 	return (rv);
788 }
789 
790 static int
core_rsa_decrypt(crypto_key_t * key,uchar_t * in,int in_len,uchar_t * out)791 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len, uchar_t *out)
792 {
793 	int rv;
794 	uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
795 	ssize_t modulus_len;
796 	ssize_t	prime1_len, prime2_len;
797 	ssize_t	expo1_len, expo2_len, coef_len;
798 	RSAbytekey k;
799 
800 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
801 	    &modulus_len)) != CRYPTO_SUCCESS) {
802 		return (rv);
803 	}
804 
805 	/*
806 	 * The following attributes are not required to be
807 	 * present in a RSA secret key. If any of them is not present
808 	 * we call the encrypt routine with a flag indicating use of
809 	 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
810 	 * a required attribute for a RSA secret key.
811 	 */
812 	if ((crypto_get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
813 	    != CRYPTO_SUCCESS) ||
814 	    (crypto_get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
815 	    != CRYPTO_SUCCESS) ||
816 	    (crypto_get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
817 	    != CRYPTO_SUCCESS) ||
818 	    (crypto_get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
819 	    != CRYPTO_SUCCESS) ||
820 	    (crypto_get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
821 	    != CRYPTO_SUCCESS)) {
822 		return (core_rsa_encrypt(key, in, in_len, out, 0));
823 	}
824 
825 	k.modulus = modulus;
826 	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
827 	k.prime1 = prime1;
828 	k.prime1_bytes = prime1_len;
829 	k.prime2 = prime2;
830 	k.prime2_bytes = prime2_len;
831 	k.expo1 = expo1;
832 	k.expo1_bytes = expo1_len;
833 	k.expo2 = expo2;
834 	k.expo2_bytes = expo2_len;
835 	k.coeff = coef;
836 	k.coeff_bytes = coef_len;
837 	k.rfunc = NULL;
838 
839 	rv = rsa_decrypt(&k, in, in_len, out);
840 
841 	return (rv);
842 }
843 
844 /* ARGSUSED */
845 static int
rsa_sign_verify_common_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)846 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
847     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
848     crypto_req_handle_t req)
849 {
850 	int rv;
851 	int kmflag;
852 	rsa_ctx_t *ctxp;
853 	digest_rsa_ctx_t *dctxp;
854 
855 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
856 		return (rv);
857 
858 	/*
859 	 * Allocate a RSA context.
860 	 */
861 	kmflag = crypto_kmflag(req);
862 	switch (mechanism->cm_type) {
863 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
864 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
865 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
866 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
867 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
868 		dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
869 		ctxp = (rsa_ctx_t *)dctxp;
870 		break;
871 	default:
872 		ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
873 		break;
874 	}
875 
876 	if (ctxp == NULL)
877 		return (CRYPTO_HOST_MEMORY);
878 
879 	ctxp->mech_type = mechanism->cm_type;
880 	if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
881 	    kmflag)) != CRYPTO_SUCCESS) {
882 		switch (mechanism->cm_type) {
883 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
884 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
885 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
886 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
887 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
888 			kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
889 			break;
890 		default:
891 			kmem_free(ctxp, sizeof (rsa_ctx_t));
892 			break;
893 		}
894 		return (rv);
895 	}
896 
897 	switch (mechanism->cm_type) {
898 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
899 		MD5Init(&(dctxp->md5_ctx));
900 		break;
901 
902 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
903 		SHA1Init(&(dctxp->sha1_ctx));
904 		break;
905 
906 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
907 		SHA2Init(SHA256, &(dctxp->sha2_ctx));
908 		break;
909 
910 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
911 		SHA2Init(SHA384, &(dctxp->sha2_ctx));
912 		break;
913 
914 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
915 		SHA2Init(SHA512, &(dctxp->sha2_ctx));
916 		break;
917 	}
918 
919 	ctx->cc_provider_private = ctxp;
920 
921 	return (CRYPTO_SUCCESS);
922 }
923 
924 #define	SHA1_DIGEST_SIZE 20
925 #define	MD5_DIGEST_SIZE 16
926 
927 #define	INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)	\
928 	(data).cd_format = CRYPTO_DATA_RAW;		\
929 	(data).cd_offset = 0;				\
930 	(data).cd_raw.iov_base = (char *)base;		\
931 	(data).cd_raw.iov_len = len;			\
932 	(data).cd_length = cd_len;
933 
934 static int
rsa_digest_svrfy_common(digest_rsa_ctx_t * ctxp,crypto_data_t * data,crypto_data_t * signature,uchar_t flag)935 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
936     crypto_data_t *signature, uchar_t flag)
937 {
938 	int rv = CRYPTO_FAILED;
939 
940 	uchar_t digest[SHA512_DIGEST_LENGTH];
941 	/* The der_data size is enough for MD5 also */
942 	uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
943 	ulong_t der_data_len;
944 	crypto_data_t der_cd;
945 	rsa_mech_type_t mech_type;
946 
947 	ASSERT(flag & CRYPTO_DO_SIGN || flag & CRYPTO_DO_VERIFY);
948 	ASSERT(data != NULL || (flag & CRYPTO_DO_FINAL));
949 
950 	mech_type = ctxp->mech_type;
951 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
952 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
953 		return (CRYPTO_MECHANISM_INVALID);
954 
955 	/*
956 	 * We need to do the BUFFER_TOO_SMALL check before digesting
957 	 * the data. No check is needed for verify as signature is not
958 	 * an output argument for verify.
959 	 */
960 	if (flag & CRYPTO_DO_SIGN) {
961 		uchar_t *modulus;
962 		ssize_t modulus_len;
963 
964 		if ((rv = crypto_get_key_attr(ctxp->key, SUN_CKA_MODULUS,
965 		    &modulus, &modulus_len)) != CRYPTO_SUCCESS) {
966 			return (rv);
967 		}
968 
969 		if (signature->cd_length < modulus_len) {
970 			signature->cd_length = modulus_len;
971 			return (CRYPTO_BUFFER_TOO_SMALL);
972 		}
973 	}
974 
975 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
976 		rv = crypto_digest_data(data, &(ctxp->md5_ctx),
977 		    digest, MD5Update, MD5Final, flag | CRYPTO_DO_MD5);
978 
979 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
980 		rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
981 		    digest, SHA1Update, SHA1Final,  flag | CRYPTO_DO_SHA1);
982 
983 	else
984 		rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
985 		    digest, SHA2Update, SHA2Final, flag | CRYPTO_DO_SHA2);
986 
987 	if (rv != CRYPTO_SUCCESS)
988 		return (rv);
989 
990 
991 	/*
992 	 * Prepare the DER encoding of the DigestInfo value as follows:
993 	 * MD5:		MD5_DER_PREFIX || H
994 	 * SHA-1:	SHA1_DER_PREFIX || H
995 	 *
996 	 * See rsa_impl.c for more details.
997 	 */
998 	switch (mech_type) {
999 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1000 		bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1001 		bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1002 		der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1003 		break;
1004 
1005 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1006 		bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1007 		bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1008 		    SHA1_DIGEST_SIZE);
1009 		der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1010 		break;
1011 
1012 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1013 		bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1014 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1015 		    SHA256_DIGEST_LENGTH);
1016 		der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1017 		break;
1018 
1019 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1020 		bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1021 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1022 		    SHA384_DIGEST_LENGTH);
1023 		der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1024 		break;
1025 
1026 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1027 		bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1028 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1029 		    SHA512_DIGEST_LENGTH);
1030 		der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1031 		break;
1032 	}
1033 
1034 	INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1035 	/*
1036 	 * Now, we are ready to sign or verify the DER_ENCODED data.
1037 	 */
1038 	if (flag & CRYPTO_DO_SIGN)
1039 		rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1040 		    signature);
1041 	else
1042 		rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1043 		    signature);
1044 
1045 	return (rv);
1046 }
1047 
1048 static int
rsa_sign_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature)1049 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1050     crypto_data_t *data, crypto_data_t *signature)
1051 {
1052 	int rv = CRYPTO_FAILED;
1053 
1054 	int dlen;
1055 	uchar_t *dataptr, *modulus;
1056 	ssize_t modulus_len;
1057 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1058 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1059 	uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1060 
1061 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1062 	    &modulus_len)) != CRYPTO_SUCCESS) {
1063 		return (rv);
1064 	}
1065 
1066 	dlen = data->cd_length;
1067 	switch (mech_type) {
1068 	case RSA_PKCS_MECH_INFO_TYPE:
1069 		if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1070 			return (CRYPTO_DATA_LEN_RANGE);
1071 		break;
1072 	case RSA_X_509_MECH_INFO_TYPE:
1073 		if (dlen > modulus_len)
1074 			return (CRYPTO_DATA_LEN_RANGE);
1075 		break;
1076 	}
1077 
1078 	if (signature->cd_length < modulus_len) {
1079 		signature->cd_length = modulus_len;
1080 		return (CRYPTO_BUFFER_TOO_SMALL);
1081 	}
1082 
1083 	ASSERT(data->cd_length <= sizeof (tmp_data));
1084 	if ((rv = crypto_get_input_data(data, &dataptr, tmp_data))
1085 	    != CRYPTO_SUCCESS)
1086 		return (rv);
1087 
1088 	switch (mech_type) {
1089 	case RSA_PKCS_MECH_INFO_TYPE:
1090 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1091 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1092 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1093 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1094 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1095 		/*
1096 		 * Add PKCS padding to the input data to format a block
1097 		 * type "01" encryption block.
1098 		 */
1099 		rv = pkcs1_encode(PKCS1_SIGN, dataptr, dlen, plain_data,
1100 		    modulus_len);
1101 		if (rv != CRYPTO_SUCCESS)
1102 			return (rv);
1103 
1104 		break;
1105 
1106 	case RSA_X_509_MECH_INFO_TYPE:
1107 		bzero(plain_data, modulus_len - dlen);
1108 		bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1109 		break;
1110 	}
1111 
1112 	rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data);
1113 	if (rv == CRYPTO_SUCCESS) {
1114 		/* copy out to signature */
1115 		if ((rv = crypto_put_output_data(signed_data,
1116 		    signature, modulus_len)) != CRYPTO_SUCCESS)
1117 			return (rv);
1118 
1119 		signature->cd_length = modulus_len;
1120 	}
1121 
1122 	return (rv);
1123 }
1124 
1125 /* ARGSUSED */
1126 static int
rsaprov_sign(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)1127 rsaprov_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1128     crypto_req_handle_t req)
1129 {
1130 	int rv;
1131 	rsa_ctx_t *ctxp;
1132 
1133 	ASSERT(ctx->cc_provider_private != NULL);
1134 	ctxp = ctx->cc_provider_private;
1135 
1136 	/* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1137 	switch (ctxp->mech_type) {
1138 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1139 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1140 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1141 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1142 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1143 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1144 		    signature, CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE |
1145 		    CRYPTO_DO_FINAL);
1146 		break;
1147 	default:
1148 		rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1149 		    signature);
1150 		break;
1151 	}
1152 
1153 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1154 		(void) rsa_free_context(ctx);
1155 
1156 	return (rv);
1157 }
1158 
1159 /* ARGSUSED */
1160 static int
rsa_sign_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)1161 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1162 {
1163 	int rv;
1164 	digest_rsa_ctx_t *ctxp;
1165 	rsa_mech_type_t mech_type;
1166 
1167 	ASSERT(ctx->cc_provider_private != NULL);
1168 	ctxp = ctx->cc_provider_private;
1169 	mech_type = ctxp->mech_type;
1170 
1171 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1172 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
1173 		return (CRYPTO_MECHANISM_INVALID);
1174 
1175 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1176 		rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1177 		    NULL, MD5Update, MD5Final,
1178 		    CRYPTO_DO_MD5 | CRYPTO_DO_UPDATE);
1179 
1180 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1181 		rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1182 		    NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1183 		    CRYPTO_DO_UPDATE);
1184 
1185 	else
1186 		rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1187 		    NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1188 		    CRYPTO_DO_UPDATE);
1189 
1190 	return (rv);
1191 }
1192 
1193 /* ARGSUSED2 */
1194 static int
rsa_sign_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)1195 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1196     crypto_req_handle_t req)
1197 {
1198 	int rv;
1199 	digest_rsa_ctx_t *ctxp;
1200 
1201 	ASSERT(ctx->cc_provider_private != NULL);
1202 	ctxp = ctx->cc_provider_private;
1203 
1204 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1205 	    CRYPTO_DO_SIGN | CRYPTO_DO_FINAL);
1206 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1207 		(void) rsa_free_context(ctx);
1208 
1209 	return (rv);
1210 }
1211 
1212 /* ARGSUSED */
1213 static int
rsa_sign_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1214 rsa_sign_atomic(crypto_provider_handle_t provider,
1215     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1216     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1217     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1218 {
1219 	int rv;
1220 	digest_rsa_ctx_t dctx;
1221 
1222 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1223 		return (rv);
1224 
1225 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1226 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1227 		rv = rsa_sign_common(mechanism->cm_type, key, data,
1228 		    signature);
1229 
1230 	else {
1231 		dctx.mech_type = mechanism->cm_type;
1232 		dctx.key = key;
1233 		switch (mechanism->cm_type) {
1234 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1235 			MD5Init(&(dctx.md5_ctx));
1236 			break;
1237 
1238 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1239 			SHA1Init(&(dctx.sha1_ctx));
1240 			break;
1241 
1242 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1243 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1244 			break;
1245 
1246 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1247 			SHA2Init(SHA384, &(dctx.sha2_ctx));
1248 			break;
1249 
1250 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1251 			SHA2Init(SHA512, &(dctx.sha2_ctx));
1252 			break;
1253 		}
1254 
1255 		rv = rsa_digest_svrfy_common(&dctx, data, signature,
1256 		    CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1257 	}
1258 
1259 	return (rv);
1260 }
1261 
1262 static int
rsa_verify_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature)1263 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1264     crypto_data_t *data, crypto_data_t *signature)
1265 {
1266 	int rv = CRYPTO_FAILED;
1267 
1268 	uchar_t *sigptr, *modulus;
1269 	ssize_t modulus_len;
1270 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1271 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1272 
1273 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1274 	    &modulus_len)) != CRYPTO_SUCCESS) {
1275 		return (rv);
1276 	}
1277 
1278 	if (signature->cd_length != modulus_len)
1279 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1280 
1281 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1282 	if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1283 	    != CRYPTO_SUCCESS)
1284 		return (rv);
1285 
1286 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1287 	if (rv != CRYPTO_SUCCESS)
1288 		return (rv);
1289 
1290 	if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1291 		if (compare_data(data, (plain_data + modulus_len
1292 		    - data->cd_length)) != 0)
1293 			rv = CRYPTO_SIGNATURE_INVALID;
1294 
1295 	} else {
1296 		size_t data_len = modulus_len;
1297 
1298 		/*
1299 		 * Strip off the encoded padding bytes in front of the
1300 		 * recovered data, then compare the recovered data with
1301 		 * the original data.
1302 		 */
1303 		rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1304 		if (rv != CRYPTO_SUCCESS)
1305 			return (rv);
1306 
1307 		if (data_len != data->cd_length)
1308 			return (CRYPTO_SIGNATURE_LEN_RANGE);
1309 
1310 		if (compare_data(data, (plain_data + modulus_len
1311 		    - data_len)) != 0)
1312 			rv = CRYPTO_SIGNATURE_INVALID;
1313 	}
1314 
1315 	return (rv);
1316 }
1317 
1318 /* ARGSUSED */
1319 static int
rsaprov_verify(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)1320 rsaprov_verify(crypto_ctx_t *ctx, crypto_data_t *data,
1321     crypto_data_t *signature, crypto_req_handle_t req)
1322 {
1323 	int rv;
1324 	rsa_ctx_t *ctxp;
1325 
1326 	ASSERT(ctx->cc_provider_private != NULL);
1327 	ctxp = ctx->cc_provider_private;
1328 
1329 	/* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1330 	switch (ctxp->mech_type) {
1331 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1332 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1333 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1334 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1335 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1336 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1337 		    signature, CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE |
1338 		    CRYPTO_DO_FINAL);
1339 		break;
1340 	default:
1341 		rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1342 		    signature);
1343 		break;
1344 	}
1345 
1346 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1347 		(void) rsa_free_context(ctx);
1348 
1349 	return (rv);
1350 }
1351 
1352 /* ARGSUSED */
1353 static int
rsa_verify_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)1354 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1355     crypto_req_handle_t req)
1356 {
1357 	int rv;
1358 	digest_rsa_ctx_t *ctxp;
1359 
1360 	ASSERT(ctx->cc_provider_private != NULL);
1361 	ctxp = ctx->cc_provider_private;
1362 
1363 	switch (ctxp->mech_type) {
1364 
1365 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1366 		rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1367 		    NULL, MD5Update, MD5Final, CRYPTO_DO_MD5 |
1368 		    CRYPTO_DO_UPDATE);
1369 		break;
1370 
1371 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1372 		rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1373 		    NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1374 		    CRYPTO_DO_UPDATE);
1375 		break;
1376 
1377 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1378 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1379 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1380 		rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1381 		    NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1382 		    CRYPTO_DO_UPDATE);
1383 		break;
1384 
1385 	default:
1386 		return (CRYPTO_MECHANISM_INVALID);
1387 	}
1388 
1389 	return (rv);
1390 }
1391 
1392 /* ARGSUSED2 */
1393 static int
rsa_verify_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)1394 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1395     crypto_req_handle_t req)
1396 {
1397 	int rv;
1398 	digest_rsa_ctx_t *ctxp;
1399 
1400 	ASSERT(ctx->cc_provider_private != NULL);
1401 	ctxp = ctx->cc_provider_private;
1402 
1403 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1404 	    CRYPTO_DO_VERIFY | CRYPTO_DO_FINAL);
1405 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1406 		(void) rsa_free_context(ctx);
1407 
1408 	return (rv);
1409 }
1410 
1411 
1412 /* ARGSUSED */
1413 static int
rsa_verify_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1414 rsa_verify_atomic(crypto_provider_handle_t provider,
1415     crypto_session_id_t session_id,
1416     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1417     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1418     crypto_req_handle_t req)
1419 {
1420 	int rv;
1421 	digest_rsa_ctx_t dctx;
1422 
1423 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1424 		return (rv);
1425 
1426 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1427 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1428 		rv = rsa_verify_common(mechanism->cm_type, key, data,
1429 		    signature);
1430 
1431 	else {
1432 		dctx.mech_type = mechanism->cm_type;
1433 		dctx.key = key;
1434 
1435 		switch (mechanism->cm_type) {
1436 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1437 			MD5Init(&(dctx.md5_ctx));
1438 			break;
1439 
1440 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1441 			SHA1Init(&(dctx.sha1_ctx));
1442 			break;
1443 
1444 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1445 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1446 			break;
1447 
1448 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1449 			SHA2Init(SHA384, &(dctx.sha2_ctx));
1450 			break;
1451 
1452 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1453 			SHA2Init(SHA512, &(dctx.sha2_ctx));
1454 			break;
1455 		}
1456 
1457 		rv = rsa_digest_svrfy_common(&dctx, data, signature,
1458 		    CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1459 	}
1460 
1461 	return (rv);
1462 }
1463 
1464 static int
rsa_verify_recover_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * signature,crypto_data_t * data)1465 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1466     crypto_data_t *signature, crypto_data_t *data)
1467 {
1468 	int rv = CRYPTO_FAILED;
1469 
1470 	size_t data_len;
1471 	uchar_t *sigptr, *modulus;
1472 	ssize_t modulus_len;
1473 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1474 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1475 
1476 	if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1477 	    &modulus_len)) != CRYPTO_SUCCESS) {
1478 		return (rv);
1479 	}
1480 
1481 	if (signature->cd_length != modulus_len)
1482 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1483 
1484 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1485 	if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1486 	    != CRYPTO_SUCCESS)
1487 		return (rv);
1488 
1489 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1490 	if (rv != CRYPTO_SUCCESS)
1491 		return (rv);
1492 
1493 	data_len = modulus_len;
1494 
1495 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1496 		/*
1497 		 * Strip off the encoded padding bytes in front of the
1498 		 * recovered data, then compare the recovered data with
1499 		 * the original data.
1500 		 */
1501 		rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1502 		if (rv != CRYPTO_SUCCESS)
1503 			return (rv);
1504 	}
1505 
1506 	if (data->cd_length < data_len) {
1507 		data->cd_length = data_len;
1508 		return (CRYPTO_BUFFER_TOO_SMALL);
1509 	}
1510 
1511 	if ((rv = crypto_put_output_data(plain_data + modulus_len - data_len,
1512 	    data, data_len)) != CRYPTO_SUCCESS)
1513 		return (rv);
1514 	data->cd_length = data_len;
1515 
1516 	return (rv);
1517 }
1518 
1519 /* ARGSUSED */
1520 static int
rsa_verify_recover(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_data_t * data,crypto_req_handle_t req)1521 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1522     crypto_data_t *data, crypto_req_handle_t req)
1523 {
1524 	int rv;
1525 	rsa_ctx_t *ctxp;
1526 
1527 	ASSERT(ctx->cc_provider_private != NULL);
1528 	ctxp = ctx->cc_provider_private;
1529 
1530 	/* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1531 	rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1532 	    signature, data);
1533 
1534 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1535 		(void) rsa_free_context(ctx);
1536 
1537 	return (rv);
1538 }
1539 
1540 /* ARGSUSED */
1541 static int
rsa_verify_recover_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * signature,crypto_data_t * data,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1542 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1543     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1544     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1545     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1546 {
1547 	int rv;
1548 
1549 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1550 		return (rv);
1551 
1552 	return (rsa_verify_recover_common(mechanism->cm_type, key,
1553 	    signature, data));
1554 }
1555