xref: /titanic_44/usr/src/common/crypto/rsa/rsa_impl.c (revision 56b2bdd1f04d465cfe4a95b88ae5cba5884154e4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * This file contains RSA helper routines common to
28  * the PKCS11 soft token code and the kernel RSA code.
29  */
30 
31 #include <sys/types.h>
32 #include <bignum.h>
33 
34 #ifdef _KERNEL
35 #include <sys/param.h>
36 #else
37 #include <strings.h>
38 #include <cryptoutil.h>
39 #endif
40 
41 #include <sys/crypto/common.h>
42 #include "rsa_impl.h"
43 
44 /*
45  * DER encoding T of the DigestInfo values for MD5, SHA1, and SHA2
46  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
47  *
48  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
49  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
50  * SHA-256: (0x)30 31 30 0d 06 09 60 86 48 01 65 03 04 02 01 05 00 04 20 || H.
51  * SHA-384: (0x)30 41 30 0d 06 09 60 86 48 01 65 03 04 02 02 05 00 04 30 || H.
52  * SHA-512: (0x)30 51 30 0d 06 09 60 86 48 01 65 03 04 02 03 05 00 04 40 || H.
53  *
54  * Where H is the digested output from MD5 or SHA1. We define the constant
55  * byte array (the prefix) here and use it rather than doing the DER
56  * encoding of the OID in a separate routine.
57  */
58 const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
59     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
60     0x04, 0x10};
61 
62 const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
63     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
64 
65 const CK_BYTE SHA1_DER_PREFIX_OID[SHA1_DER_PREFIX_OID_Len] = {0x30, 0x1f, 0x30,
66     0x07, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x04, 0x14};
67 
68 const CK_BYTE SHA256_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x31, 0x30, 0x0d,
69     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
70     0x00, 0x04, 0x20};
71 
72 const CK_BYTE SHA384_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x41, 0x30, 0x0d,
73     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05,
74     0x00, 0x04, 0x30};
75 
76 const CK_BYTE SHA512_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x51, 0x30, 0x0d,
77     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05,
78     0x00, 0x04, 0x40};
79 
80 const CK_BYTE DEFAULT_PUB_EXPO[DEFAULT_PUB_EXPO_Len] = { 0x01, 0x00, 0x01 };
81 
82 
83 static CK_RV
84 convert_rv(BIG_ERR_CODE err)
85 {
86 	switch (err) {
87 
88 	case BIG_OK:
89 		return (CKR_OK);
90 
91 	case BIG_NO_MEM:
92 		return (CKR_HOST_MEMORY);
93 
94 	case BIG_NO_RANDOM:
95 		return (CKR_DEVICE_ERROR);
96 
97 	case BIG_INVALID_ARGS:
98 		return (CKR_ARGUMENTS_BAD);
99 
100 	case BIG_DIV_BY_0:
101 	default:
102 		return (CKR_GENERAL_ERROR);
103 	}
104 }
105 
106 /* psize and qsize are in bits */
107 static BIG_ERR_CODE
108 RSA_key_init(RSAkey *key, int psize, int qsize)
109 {
110 	BIG_ERR_CODE err = BIG_OK;
111 
112 /* EXPORT DELETE START */
113 
114 	int plen, qlen, nlen;
115 
116 	plen = BITLEN2BIGNUMLEN(psize);
117 	qlen = BITLEN2BIGNUMLEN(qsize);
118 	nlen = plen + qlen;
119 	key->size = psize + qsize;
120 	if ((err = big_init(&(key->p), plen)) != BIG_OK)
121 		return (err);
122 	if ((err = big_init(&(key->q), qlen)) != BIG_OK)
123 		goto ret1;
124 	if ((err = big_init(&(key->n), nlen)) != BIG_OK)
125 		goto ret2;
126 	if ((err = big_init(&(key->d), nlen)) != BIG_OK)
127 		goto ret3;
128 	if ((err = big_init(&(key->e), nlen)) != BIG_OK)
129 		goto ret4;
130 	if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
131 		goto ret5;
132 	if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
133 		goto ret6;
134 	if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
135 		goto ret7;
136 	if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
137 		goto ret8;
138 	if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
139 		goto ret9;
140 	if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
141 		goto ret10;
142 
143 	return (BIG_OK);
144 
145 ret10:
146 	big_finish(&(key->q_rr));
147 ret9:
148 	big_finish(&(key->p_rr));
149 ret8:
150 	big_finish(&(key->pinvmodq));
151 ret7:
152 	big_finish(&(key->dmodqminus1));
153 ret6:
154 	big_finish(&(key->dmodpminus1));
155 ret5:
156 	big_finish(&(key->e));
157 ret4:
158 	big_finish(&(key->d));
159 ret3:
160 	big_finish(&(key->n));
161 ret2:
162 	big_finish(&(key->q));
163 ret1:
164 	big_finish(&(key->p));
165 
166 /* EXPORT DELETE END */
167 
168 	return (err);
169 }
170 
171 static void
172 RSA_key_finish(RSAkey *key)
173 {
174 
175 /* EXPORT DELETE START */
176 
177 	big_finish(&(key->n_rr));
178 	big_finish(&(key->q_rr));
179 	big_finish(&(key->p_rr));
180 	big_finish(&(key->pinvmodq));
181 	big_finish(&(key->dmodqminus1));
182 	big_finish(&(key->dmodpminus1));
183 	big_finish(&(key->e));
184 	big_finish(&(key->d));
185 	big_finish(&(key->n));
186 	big_finish(&(key->q));
187 	big_finish(&(key->p));
188 
189 /* EXPORT DELETE END */
190 
191 }
192 
193 /*
194  * Generate RSA key
195  */
196 static CK_RV
197 generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM *pubexp,
198     int (*rfunc)(void *, size_t))
199 {
200 	CK_RV		rv = CKR_OK;
201 
202 /* EXPORT DELETE START */
203 
204 	int		(*rf)(void *, size_t);
205 	BIGNUM		a, b, c, d, e, f, g, h;
206 	int		len, keylen, size;
207 	BIG_ERR_CODE	brv = BIG_OK;
208 
209 	size = psize + qsize;
210 	keylen = BITLEN2BIGNUMLEN(size);
211 	len = keylen * 2 + 1;
212 	key->size = size;
213 
214 	/*
215 	 * Note: It is not really necessary to compute e, it is in pubexp:
216 	 * 	(void) big_copy(&(key->e), pubexp);
217 	 */
218 
219 	a.malloced = 0;
220 	b.malloced = 0;
221 	c.malloced = 0;
222 	d.malloced = 0;
223 	e.malloced = 0;
224 	f.malloced = 0;
225 	g.malloced = 0;
226 	h.malloced = 0;
227 
228 	if ((big_init(&a, len) != BIG_OK) ||
229 	    (big_init(&b, len) != BIG_OK) ||
230 	    (big_init(&c, len) != BIG_OK) ||
231 	    (big_init(&d, len) != BIG_OK) ||
232 	    (big_init(&e, len) != BIG_OK) ||
233 	    (big_init(&f, len) != BIG_OK) ||
234 	    (big_init(&g, len) != BIG_OK) ||
235 	    (big_init(&h, len) != BIG_OK)) {
236 		big_finish(&h);
237 		big_finish(&g);
238 		big_finish(&f);
239 		big_finish(&e);
240 		big_finish(&d);
241 		big_finish(&c);
242 		big_finish(&b);
243 		big_finish(&a);
244 
245 		return (CKR_HOST_MEMORY);
246 	}
247 
248 	rf = rfunc;
249 	if (rf == NULL) {
250 #ifdef _KERNEL
251 		rf = (int (*)(void *, size_t))random_get_pseudo_bytes;
252 #else
253 		rf = pkcs11_get_urandom;
254 #endif
255 	}
256 
257 nextp:
258 	if ((brv = big_random(&a, psize, rf)) != BIG_OK) {
259 		goto ret;
260 	}
261 
262 	if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
263 		goto ret;
264 	}
265 	/* b now contains the potential prime p */
266 
267 	(void) big_sub_pos(&a, &b, &big_One);
268 	if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
269 		goto ret;
270 	}
271 	if (big_cmp_abs(&f, &big_One) != 0) {
272 		goto nextp;
273 	}
274 
275 	if ((brv = big_random(&c, qsize, rf)) != BIG_OK) {
276 		goto ret;
277 	}
278 
279 nextq:
280 	(void) big_add(&a, &c, &big_Two);
281 
282 	if (big_bitlength(&a) != qsize) {
283 		goto nextp;
284 	}
285 	if (big_cmp_abs(&a, &b) == 0) {
286 		goto nextp;
287 	}
288 	if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
289 		goto ret;
290 	}
291 	/* c now contains the potential prime q */
292 
293 	if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
294 		goto ret;
295 	}
296 	if (big_bitlength(&g) != size) {
297 		goto nextp;
298 	}
299 	/* g now contains the potential modulus n */
300 
301 	(void) big_sub_pos(&a, &b, &big_One);
302 	(void) big_sub_pos(&d, &c, &big_One);
303 
304 	if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
305 		goto ret;
306 	}
307 	if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
308 		goto ret;
309 	}
310 	if (big_cmp_abs(&f, &big_One) != 0) {
311 		goto nextq;
312 	} else {
313 		(void) big_copy(&e, pubexp);
314 	}
315 	if (d.sign == -1) {
316 		if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
317 			goto ret;
318 		}
319 	}
320 	(void) big_copy(&(key->p), &b);
321 	(void) big_copy(&(key->q), &c);
322 	(void) big_copy(&(key->n), &g);
323 	(void) big_copy(&(key->d), &d);
324 	(void) big_copy(&(key->e), &e);
325 
326 	if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
327 		goto ret;
328 	}
329 	if (f.sign == -1) {
330 		if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
331 			goto ret;
332 		}
333 	}
334 	(void) big_copy(&(key->pinvmodq), &f);
335 
336 	(void) big_sub(&a, &b, &big_One);
337 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
338 		goto ret;
339 	}
340 	(void) big_copy(&(key->dmodpminus1), &f);
341 	(void) big_sub(&a, &c, &big_One);
342 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
343 		goto ret;
344 	}
345 	(void) big_copy(&(key->dmodqminus1), &f);
346 
347 	/* pairwise consistency check:  decrypt and encrypt restores value */
348 	if ((brv = big_random(&h, size, rf)) != BIG_OK) {
349 		goto ret;
350 	}
351 	if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
352 		goto ret;
353 	}
354 	if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
355 		goto ret;
356 	}
357 
358 	if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
359 		goto ret;
360 	}
361 
362 	if (big_cmp_abs(&b, &h) != 0) {
363 		/* this should not happen */
364 		rv = generate_rsa_key(key, psize, qsize, pubexp, rf);
365 		goto ret1;
366 	} else {
367 		brv = BIG_OK;
368 	}
369 
370 ret:
371 	rv = convert_rv(brv);
372 ret1:
373 	big_finish(&h);
374 	big_finish(&g);
375 	big_finish(&f);
376 	big_finish(&e);
377 	big_finish(&d);
378 	big_finish(&c);
379 	big_finish(&b);
380 	big_finish(&a);
381 
382 /* EXPORT DELETE END */
383 
384 	return (rv);
385 }
386 
387 CK_RV
388 rsa_genkey_pair(RSAbytekey *bkey)
389 {
390 	/*
391 	 * NOTE:  Whomever originally wrote this function swapped p and q.
392 	 * This table shows the mapping between name convention used here
393 	 * versus what is used in most texts that describe RSA key generation.
394 	 *	This function:			Standard convention:
395 	 *	--------------			--------------------
396 	 *	modulus, n			-same-
397 	 *	prime 1, q			prime 1, p
398 	 *	prime 2, p			prime 2, q
399 	 *	private exponent, d		-same-
400 	 *	public exponent, e		-same-
401 	 *	exponent 1, d mod (q-1)		d mod (p-1)
402 	 *	exponent 2, d mod (p-1)		d mod (q-1)
403 	 *	coefficient, p^-1 mod q		q^-1 mod p
404 	 *
405 	 * Also notice the struct member for coefficient is named .pinvmodq
406 	 * rather than .qinvmodp, reflecting the switch.
407 	 *
408 	 * The code here wasn't unswapped, because "it works".  Further,
409 	 * p and q are interchangeable as long as exponent 1 and 2 and
410 	 * the coefficient are kept straight too.  This note is here to
411 	 * make the reader aware of the switcheroo.
412 	 */
413 	CK_RV	rv = CKR_OK;
414 
415 /* EXPORT DELETE START */
416 
417 	BIGNUM	public_exponent = {0};
418 	RSAkey	rsakey;
419 	uint32_t modulus_bytes;
420 
421 	if (bkey == NULL)
422 		return (CKR_ARGUMENTS_BAD);
423 
424 	/* Must have modulus bits set */
425 	if (bkey->modulus_bits == 0)
426 		return (CKR_ARGUMENTS_BAD);
427 
428 	/* Must have public exponent set */
429 	if (bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
430 		return (CKR_ARGUMENTS_BAD);
431 
432 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
433 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
434 
435 	/* Modulus length needs to be between min key size and max key size. */
436 	if ((modulus_bytes < MIN_RSA_KEYLENGTH_IN_BYTES) ||
437 	    (modulus_bytes > MAX_RSA_KEYLENGTH_IN_BYTES)) {
438 		return (CKR_KEY_SIZE_RANGE);
439 	}
440 
441 	/*
442 	 * Initialize the RSA key.
443 	 */
444 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
445 	    BIG_OK) {
446 		return (CKR_HOST_MEMORY);
447 	}
448 
449 	/* Create a public exponent in bignum format. */
450 	if (big_init(&public_exponent,
451 	    CHARLEN2BIGNUMLEN(bkey->pubexpo_bytes)) != BIG_OK) {
452 		rv = CKR_HOST_MEMORY;
453 		goto clean1;
454 	}
455 	bytestring2bignum(&public_exponent, bkey->pubexpo, bkey->pubexpo_bytes);
456 
457 	/* Generate RSA key pair. */
458 	if ((rv = generate_rsa_key(&rsakey,
459 	    modulus_bytes * 4, modulus_bytes * 4, &public_exponent,
460 	    bkey->rfunc)) != CKR_OK) {
461 		big_finish(&public_exponent);
462 		goto clean1;
463 	}
464 	big_finish(&public_exponent);
465 
466 	/* modulus_bytes = rsakey.n.len * (int)sizeof (BIG_CHUNK_TYPE); */
467 	bignum2bytestring(bkey->modulus, &(rsakey.n), modulus_bytes);
468 
469 	bkey->privexpo_bytes = rsakey.d.len * (int)sizeof (BIG_CHUNK_TYPE);
470 	bignum2bytestring(bkey->privexpo, &(rsakey.d), bkey->privexpo_bytes);
471 
472 	bkey->pubexpo_bytes = rsakey.e.len * (int)sizeof (BIG_CHUNK_TYPE);
473 	bignum2bytestring(bkey->pubexpo, &(rsakey.e), bkey->pubexpo_bytes);
474 
475 	bkey->prime1_bytes = rsakey.q.len * (int)sizeof (BIG_CHUNK_TYPE);
476 	bignum2bytestring(bkey->prime1, &(rsakey.q), bkey->prime1_bytes);
477 
478 	bkey->prime2_bytes = rsakey.p.len * (int)sizeof (BIG_CHUNK_TYPE);
479 	bignum2bytestring(bkey->prime2, &(rsakey.p), bkey->prime2_bytes);
480 
481 	bkey->expo1_bytes =
482 	    rsakey.dmodqminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
483 	bignum2bytestring(bkey->expo1, &(rsakey.dmodqminus1),
484 	    bkey->expo1_bytes);
485 
486 	bkey->expo2_bytes =
487 	    rsakey.dmodpminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
488 	bignum2bytestring(bkey->expo2,
489 	    &(rsakey.dmodpminus1), bkey->expo2_bytes);
490 
491 	bkey->coeff_bytes =
492 	    rsakey.pinvmodq.len * (int)sizeof (BIG_CHUNK_TYPE);
493 	bignum2bytestring(bkey->coeff, &(rsakey.pinvmodq), bkey->coeff_bytes);
494 
495 clean1:
496 	RSA_key_finish(&rsakey);
497 
498 /* EXPORT DELETE END */
499 
500 	return (rv);
501 }
502 
503 /*
504  * RSA encrypt operation
505  */
506 CK_RV
507 rsa_encrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
508 {
509 	CK_RV rv = CKR_OK;
510 
511 /* EXPORT DELETE START */
512 
513 	BIGNUM msg;
514 	RSAkey rsakey;
515 	uint32_t modulus_bytes;
516 
517 	if (bkey == NULL)
518 		return (CKR_ARGUMENTS_BAD);
519 
520 	/* Must have modulus and public exponent set */
521 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
522 	    bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
523 		return (CKR_ARGUMENTS_BAD);
524 
525 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
526 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
527 
528 	if (bkey->pubexpo_bytes > modulus_bytes) {
529 		return (CKR_KEY_SIZE_RANGE);
530 	}
531 
532 	/* psize and qsize for RSA_key_init is in bits. */
533 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
534 	    BIG_OK) {
535 		return (CKR_HOST_MEMORY);
536 	}
537 
538 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
539 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
540 		rv = CKR_HOST_MEMORY;
541 		goto clean2;
542 	}
543 	bytestring2bignum(&msg, in, in_len);
544 
545 	/* Convert public exponent and modulus to big integer format. */
546 	bytestring2bignum(&(rsakey.e), bkey->pubexpo, bkey->pubexpo_bytes);
547 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
548 
549 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
550 		rv = CKR_DATA_LEN_RANGE;
551 		goto clean3;
552 	}
553 
554 	/* Perform RSA computation on big integer input data. */
555 	if (big_modexp(&msg, &msg, &(rsakey.e), &(rsakey.n), NULL) !=
556 	    BIG_OK) {
557 		rv = CKR_HOST_MEMORY;
558 		goto clean3;
559 	}
560 
561 	/* Convert the big integer output data to octet string. */
562 	bignum2bytestring(out, &msg, modulus_bytes);
563 
564 clean3:
565 	big_finish(&msg);
566 clean2:
567 	RSA_key_finish(&rsakey);
568 
569 /* EXPORT DELETE END */
570 
571 	return (rv);
572 }
573 
574 /*
575  * RSA decrypt operation
576  */
577 CK_RV
578 rsa_decrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
579 {
580 	CK_RV rv = CKR_OK;
581 
582 /* EXPORT DELETE START */
583 
584 	BIGNUM msg;
585 	RSAkey rsakey;
586 	uint32_t modulus_bytes;
587 
588 	if (bkey == NULL)
589 		return (CKR_ARGUMENTS_BAD);
590 
591 	/* Must have modulus, prime1, prime2, expo1, expo2, and coeff set */
592 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
593 	    bkey->prime1_bytes == 0 || bkey->prime1 == NULL ||
594 	    bkey->prime2_bytes == 0 || bkey->prime2 == NULL ||
595 	    bkey->expo1_bytes == 0 || bkey->expo1 == NULL ||
596 	    bkey->expo2_bytes == 0 || bkey->expo2 == NULL ||
597 	    bkey->coeff_bytes == 0 || bkey->coeff == NULL)
598 		return (CKR_ARGUMENTS_BAD);
599 
600 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
601 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
602 
603 	/* psize and qsize for RSA_key_init is in bits. */
604 	if (RSA_key_init(&rsakey, CRYPTO_BYTES2BITS(bkey->prime2_bytes),
605 	    CRYPTO_BYTES2BITS(bkey->prime1_bytes)) != BIG_OK) {
606 		return (CKR_HOST_MEMORY);
607 	}
608 
609 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
610 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
611 		rv = CKR_HOST_MEMORY;
612 		goto clean3;
613 	}
614 	/* Convert octet string input data to big integer format. */
615 	bytestring2bignum(&msg, in, in_len);
616 
617 	/* Convert octet string modulus to big integer format. */
618 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
619 
620 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
621 		rv = CKR_DATA_LEN_RANGE;
622 		goto clean4;
623 	}
624 
625 	/* Convert the rest of private key attributes to big integer format. */
626 	bytestring2bignum(&(rsakey.q), bkey->prime1, bkey->prime1_bytes);
627 	bytestring2bignum(&(rsakey.p), bkey->prime2, bkey->prime2_bytes);
628 	bytestring2bignum(&(rsakey.dmodqminus1),
629 	    bkey->expo1, bkey->expo1_bytes);
630 	bytestring2bignum(&(rsakey.dmodpminus1),
631 	    bkey->expo2, bkey->expo2_bytes);
632 	bytestring2bignum(&(rsakey.pinvmodq),
633 	    bkey->coeff, bkey->coeff_bytes);
634 
635 	if ((big_cmp_abs(&(rsakey.dmodpminus1), &(rsakey.p)) > 0) ||
636 	    (big_cmp_abs(&(rsakey.dmodqminus1), &(rsakey.q)) > 0) ||
637 	    (big_cmp_abs(&(rsakey.pinvmodq), &(rsakey.q)) > 0)) {
638 		rv = CKR_KEY_SIZE_RANGE;
639 		goto clean4;
640 	}
641 
642 	/* Perform RSA computation on big integer input data. */
643 	if (big_modexp_crt(&msg, &msg, &(rsakey.dmodpminus1),
644 	    &(rsakey.dmodqminus1), &(rsakey.p), &(rsakey.q),
645 	    &(rsakey.pinvmodq), NULL, NULL) != BIG_OK) {
646 		rv = CKR_HOST_MEMORY;
647 		goto clean4;
648 	}
649 
650 	/* Convert the big integer output data to octet string. */
651 	bignum2bytestring(out, &msg, modulus_bytes);
652 
653 clean4:
654 	big_finish(&msg);
655 clean3:
656 	RSA_key_finish(&rsakey);
657 
658 /* EXPORT DELETE END */
659 
660 	return (rv);
661 }
662