xref: /illumos-gate/usr/src/common/crypto/rsa/rsa_impl.c (revision 15c07adc1c7b828006b5e3c4d528b92229d6bd23)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * This file contains RSA helper routines common to
28  * the PKCS11 soft token code and the kernel RSA code.
29  */
30 
31 #include <sys/types.h>
32 #include <bignum.h>
33 
34 #ifdef _KERNEL
35 #include <sys/param.h>
36 #else
37 #include <strings.h>
38 #include <cryptoutil.h>
39 #endif
40 
41 #include <sys/crypto/common.h>
42 #include "rsa_impl.h"
43 
44 /*
45  * DER encoding T of the DigestInfo values for MD5, SHA1, and SHA2
46  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
47  *
48  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
49  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
50  * SHA-256: (0x)30 31 30 0d 06 09 60 86 48 01 65 03 04 02 01 05 00 04 20 || H.
51  * SHA-384: (0x)30 41 30 0d 06 09 60 86 48 01 65 03 04 02 02 05 00 04 30 || H.
52  * SHA-512: (0x)30 51 30 0d 06 09 60 86 48 01 65 03 04 02 03 05 00 04 40 || H.
53  *
54  * Where H is the digested output from MD5 or SHA1. We define the constant
55  * byte array (the prefix) here and use it rather than doing the DER
56  * encoding of the OID in a separate routine.
57  */
58 const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
59     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
60     0x04, 0x10};
61 
62 const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
63     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
64 
65 const CK_BYTE SHA1_DER_PREFIX_OID[SHA1_DER_PREFIX_OID_Len] = {0x30, 0x1f, 0x30,
66     0x07, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x04, 0x14};
67 
68 const CK_BYTE SHA256_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x31, 0x30, 0x0d,
69     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
70     0x00, 0x04, 0x20};
71 
72 const CK_BYTE SHA384_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x41, 0x30, 0x0d,
73     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05,
74     0x00, 0x04, 0x30};
75 
76 const CK_BYTE SHA512_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x51, 0x30, 0x0d,
77     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05,
78     0x00, 0x04, 0x40};
79 
80 const CK_BYTE DEFAULT_PUB_EXPO[DEFAULT_PUB_EXPO_Len] = { 0x01, 0x00, 0x01 };
81 
82 
83 static CK_RV
84 convert_rv(BIG_ERR_CODE err)
85 {
86 	switch (err) {
87 
88 	case BIG_OK:
89 		return (CKR_OK);
90 
91 	case BIG_NO_MEM:
92 		return (CKR_HOST_MEMORY);
93 
94 	case BIG_NO_RANDOM:
95 		return (CKR_DEVICE_ERROR);
96 
97 	case BIG_INVALID_ARGS:
98 		return (CKR_ARGUMENTS_BAD);
99 
100 	case BIG_DIV_BY_0:
101 	default:
102 		return (CKR_GENERAL_ERROR);
103 	}
104 }
105 
106 /* psize and qsize are in bits */
107 static BIG_ERR_CODE
108 RSA_key_init(RSAkey *key, int psize, int qsize)
109 {
110 	BIG_ERR_CODE err = BIG_OK;
111 
112 	int plen, qlen, nlen;
113 
114 	plen = BITLEN2BIGNUMLEN(psize);
115 	qlen = BITLEN2BIGNUMLEN(qsize);
116 	nlen = plen + qlen;
117 	key->size = psize + qsize;
118 	if ((err = big_init(&(key->p), plen)) != BIG_OK)
119 		return (err);
120 	if ((err = big_init(&(key->q), qlen)) != BIG_OK)
121 		goto ret1;
122 	if ((err = big_init(&(key->n), nlen)) != BIG_OK)
123 		goto ret2;
124 	if ((err = big_init(&(key->d), nlen)) != BIG_OK)
125 		goto ret3;
126 	if ((err = big_init(&(key->e), nlen)) != BIG_OK)
127 		goto ret4;
128 	if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
129 		goto ret5;
130 	if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
131 		goto ret6;
132 	if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
133 		goto ret7;
134 	if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
135 		goto ret8;
136 	if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
137 		goto ret9;
138 	if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
139 		goto ret10;
140 
141 	return (BIG_OK);
142 
143 ret10:
144 	big_finish(&(key->q_rr));
145 ret9:
146 	big_finish(&(key->p_rr));
147 ret8:
148 	big_finish(&(key->pinvmodq));
149 ret7:
150 	big_finish(&(key->dmodqminus1));
151 ret6:
152 	big_finish(&(key->dmodpminus1));
153 ret5:
154 	big_finish(&(key->e));
155 ret4:
156 	big_finish(&(key->d));
157 ret3:
158 	big_finish(&(key->n));
159 ret2:
160 	big_finish(&(key->q));
161 ret1:
162 	big_finish(&(key->p));
163 
164 	return (err);
165 }
166 
167 static void
168 RSA_key_finish(RSAkey *key)
169 {
170 	big_finish(&(key->n_rr));
171 	big_finish(&(key->q_rr));
172 	big_finish(&(key->p_rr));
173 	big_finish(&(key->pinvmodq));
174 	big_finish(&(key->dmodqminus1));
175 	big_finish(&(key->dmodpminus1));
176 	big_finish(&(key->e));
177 	big_finish(&(key->d));
178 	big_finish(&(key->n));
179 	big_finish(&(key->q));
180 	big_finish(&(key->p));
181 }
182 
183 /*
184  * Generate RSA key
185  */
186 static CK_RV
187 generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM *pubexp,
188     int (*rfunc)(void *, size_t))
189 {
190 	CK_RV		rv = CKR_OK;
191 
192 	int		(*rf)(void *, size_t);
193 	BIGNUM		a, b, c, d, e, f, g, h;
194 	int		len, keylen, size;
195 	BIG_ERR_CODE	brv = BIG_OK;
196 
197 	size = psize + qsize;
198 	keylen = BITLEN2BIGNUMLEN(size);
199 	len = keylen * 2 + 1;
200 	key->size = size;
201 
202 	/*
203 	 * Note: It is not really necessary to compute e, it is in pubexp:
204 	 * 	(void) big_copy(&(key->e), pubexp);
205 	 */
206 
207 	a.malloced = 0;
208 	b.malloced = 0;
209 	c.malloced = 0;
210 	d.malloced = 0;
211 	e.malloced = 0;
212 	f.malloced = 0;
213 	g.malloced = 0;
214 	h.malloced = 0;
215 
216 	if ((big_init(&a, len) != BIG_OK) ||
217 	    (big_init(&b, len) != BIG_OK) ||
218 	    (big_init(&c, len) != BIG_OK) ||
219 	    (big_init(&d, len) != BIG_OK) ||
220 	    (big_init(&e, len) != BIG_OK) ||
221 	    (big_init(&f, len) != BIG_OK) ||
222 	    (big_init(&g, len) != BIG_OK) ||
223 	    (big_init(&h, len) != BIG_OK)) {
224 		big_finish(&h);
225 		big_finish(&g);
226 		big_finish(&f);
227 		big_finish(&e);
228 		big_finish(&d);
229 		big_finish(&c);
230 		big_finish(&b);
231 		big_finish(&a);
232 
233 		return (CKR_HOST_MEMORY);
234 	}
235 
236 	rf = rfunc;
237 	if (rf == NULL) {
238 #ifdef _KERNEL
239 		rf = (int (*)(void *, size_t))random_get_pseudo_bytes;
240 #else
241 		rf = pkcs11_get_urandom;
242 #endif
243 	}
244 
245 nextp:
246 	if ((brv = big_random(&a, psize, rf)) != BIG_OK) {
247 		goto ret;
248 	}
249 
250 	if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
251 		goto ret;
252 	}
253 	/* b now contains the potential prime p */
254 
255 	(void) big_sub_pos(&a, &b, &big_One);
256 	if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
257 		goto ret;
258 	}
259 	if (big_cmp_abs(&f, &big_One) != 0) {
260 		goto nextp;
261 	}
262 
263 	if ((brv = big_random(&c, qsize, rf)) != BIG_OK) {
264 		goto ret;
265 	}
266 
267 nextq:
268 	(void) big_add(&a, &c, &big_Two);
269 
270 	if (big_bitlength(&a) != qsize) {
271 		goto nextp;
272 	}
273 	if (big_cmp_abs(&a, &b) == 0) {
274 		goto nextp;
275 	}
276 	if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
277 		goto ret;
278 	}
279 	/* c now contains the potential prime q */
280 
281 	if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
282 		goto ret;
283 	}
284 	if (big_bitlength(&g) != size) {
285 		goto nextp;
286 	}
287 	/* g now contains the potential modulus n */
288 
289 	(void) big_sub_pos(&a, &b, &big_One);
290 	(void) big_sub_pos(&d, &c, &big_One);
291 
292 	if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
293 		goto ret;
294 	}
295 	if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
296 		goto ret;
297 	}
298 	if (big_cmp_abs(&f, &big_One) != 0) {
299 		goto nextq;
300 	} else {
301 		(void) big_copy(&e, pubexp);
302 	}
303 	if (d.sign == -1) {
304 		if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
305 			goto ret;
306 		}
307 	}
308 	(void) big_copy(&(key->p), &b);
309 	(void) big_copy(&(key->q), &c);
310 	(void) big_copy(&(key->n), &g);
311 	(void) big_copy(&(key->d), &d);
312 	(void) big_copy(&(key->e), &e);
313 
314 	if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
315 		goto ret;
316 	}
317 	if (f.sign == -1) {
318 		if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
319 			goto ret;
320 		}
321 	}
322 	(void) big_copy(&(key->pinvmodq), &f);
323 
324 	(void) big_sub(&a, &b, &big_One);
325 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
326 		goto ret;
327 	}
328 	(void) big_copy(&(key->dmodpminus1), &f);
329 	(void) big_sub(&a, &c, &big_One);
330 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
331 		goto ret;
332 	}
333 	(void) big_copy(&(key->dmodqminus1), &f);
334 
335 	/* pairwise consistency check:  decrypt and encrypt restores value */
336 	if ((brv = big_random(&h, size, rf)) != BIG_OK) {
337 		goto ret;
338 	}
339 	if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
340 		goto ret;
341 	}
342 	if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
343 		goto ret;
344 	}
345 
346 	if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
347 		goto ret;
348 	}
349 
350 	if (big_cmp_abs(&b, &h) != 0) {
351 		/* this should not happen */
352 		rv = generate_rsa_key(key, psize, qsize, pubexp, rf);
353 		goto ret1;
354 	} else {
355 		brv = BIG_OK;
356 	}
357 
358 ret:
359 	rv = convert_rv(brv);
360 ret1:
361 	big_finish(&h);
362 	big_finish(&g);
363 	big_finish(&f);
364 	big_finish(&e);
365 	big_finish(&d);
366 	big_finish(&c);
367 	big_finish(&b);
368 	big_finish(&a);
369 
370 	return (rv);
371 }
372 
373 CK_RV
374 rsa_genkey_pair(RSAbytekey *bkey)
375 {
376 	/*
377 	 * NOTE:  Whomever originally wrote this function swapped p and q.
378 	 * This table shows the mapping between name convention used here
379 	 * versus what is used in most texts that describe RSA key generation.
380 	 *	This function:			Standard convention:
381 	 *	--------------			--------------------
382 	 *	modulus, n			-same-
383 	 *	prime 1, q			prime 1, p
384 	 *	prime 2, p			prime 2, q
385 	 *	private exponent, d		-same-
386 	 *	public exponent, e		-same-
387 	 *	exponent 1, d mod (q-1)		d mod (p-1)
388 	 *	exponent 2, d mod (p-1)		d mod (q-1)
389 	 *	coefficient, p^-1 mod q		q^-1 mod p
390 	 *
391 	 * Also notice the struct member for coefficient is named .pinvmodq
392 	 * rather than .qinvmodp, reflecting the switch.
393 	 *
394 	 * The code here wasn't unswapped, because "it works".  Further,
395 	 * p and q are interchangeable as long as exponent 1 and 2 and
396 	 * the coefficient are kept straight too.  This note is here to
397 	 * make the reader aware of the switcheroo.
398 	 */
399 	CK_RV	rv = CKR_OK;
400 
401 	BIGNUM	public_exponent = {0};
402 	RSAkey	rsakey;
403 	uint32_t modulus_bytes;
404 
405 	if (bkey == NULL)
406 		return (CKR_ARGUMENTS_BAD);
407 
408 	/* Must have modulus bits set */
409 	if (bkey->modulus_bits == 0)
410 		return (CKR_ARGUMENTS_BAD);
411 
412 	/* Must have public exponent set */
413 	if (bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
414 		return (CKR_ARGUMENTS_BAD);
415 
416 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
417 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
418 
419 	/* Modulus length needs to be between min key size and max key size. */
420 	if ((modulus_bytes < MIN_RSA_KEYLENGTH_IN_BYTES) ||
421 	    (modulus_bytes > MAX_RSA_KEYLENGTH_IN_BYTES)) {
422 		return (CKR_KEY_SIZE_RANGE);
423 	}
424 
425 	/*
426 	 * Initialize the RSA key.
427 	 */
428 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
429 	    BIG_OK) {
430 		return (CKR_HOST_MEMORY);
431 	}
432 
433 	/* Create a public exponent in bignum format. */
434 	if (big_init(&public_exponent,
435 	    CHARLEN2BIGNUMLEN(bkey->pubexpo_bytes)) != BIG_OK) {
436 		rv = CKR_HOST_MEMORY;
437 		goto clean1;
438 	}
439 	bytestring2bignum(&public_exponent, bkey->pubexpo, bkey->pubexpo_bytes);
440 
441 	/* Generate RSA key pair. */
442 	if ((rv = generate_rsa_key(&rsakey,
443 	    modulus_bytes * 4, modulus_bytes * 4, &public_exponent,
444 	    bkey->rfunc)) != CKR_OK) {
445 		big_finish(&public_exponent);
446 		goto clean1;
447 	}
448 	big_finish(&public_exponent);
449 
450 	/* modulus_bytes = rsakey.n.len * (int)sizeof (BIG_CHUNK_TYPE); */
451 	bignum2bytestring(bkey->modulus, &(rsakey.n), modulus_bytes);
452 
453 	bkey->privexpo_bytes = rsakey.d.len * (int)sizeof (BIG_CHUNK_TYPE);
454 	bignum2bytestring(bkey->privexpo, &(rsakey.d), bkey->privexpo_bytes);
455 
456 	bkey->pubexpo_bytes = rsakey.e.len * (int)sizeof (BIG_CHUNK_TYPE);
457 	bignum2bytestring(bkey->pubexpo, &(rsakey.e), bkey->pubexpo_bytes);
458 
459 	bkey->prime1_bytes = rsakey.q.len * (int)sizeof (BIG_CHUNK_TYPE);
460 	bignum2bytestring(bkey->prime1, &(rsakey.q), bkey->prime1_bytes);
461 
462 	bkey->prime2_bytes = rsakey.p.len * (int)sizeof (BIG_CHUNK_TYPE);
463 	bignum2bytestring(bkey->prime2, &(rsakey.p), bkey->prime2_bytes);
464 
465 	bkey->expo1_bytes =
466 	    rsakey.dmodqminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
467 	bignum2bytestring(bkey->expo1, &(rsakey.dmodqminus1),
468 	    bkey->expo1_bytes);
469 
470 	bkey->expo2_bytes =
471 	    rsakey.dmodpminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
472 	bignum2bytestring(bkey->expo2,
473 	    &(rsakey.dmodpminus1), bkey->expo2_bytes);
474 
475 	bkey->coeff_bytes =
476 	    rsakey.pinvmodq.len * (int)sizeof (BIG_CHUNK_TYPE);
477 	bignum2bytestring(bkey->coeff, &(rsakey.pinvmodq), bkey->coeff_bytes);
478 
479 clean1:
480 	RSA_key_finish(&rsakey);
481 
482 	return (rv);
483 }
484 
485 /*
486  * RSA encrypt operation
487  */
488 CK_RV
489 rsa_encrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
490 {
491 	CK_RV rv = CKR_OK;
492 
493 	BIGNUM msg;
494 	RSAkey rsakey;
495 	uint32_t modulus_bytes;
496 
497 	if (bkey == NULL)
498 		return (CKR_ARGUMENTS_BAD);
499 
500 	/* Must have modulus and public exponent set */
501 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
502 	    bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
503 		return (CKR_ARGUMENTS_BAD);
504 
505 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
506 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
507 
508 	if (bkey->pubexpo_bytes > modulus_bytes) {
509 		return (CKR_KEY_SIZE_RANGE);
510 	}
511 
512 	/* psize and qsize for RSA_key_init is in bits. */
513 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
514 	    BIG_OK) {
515 		return (CKR_HOST_MEMORY);
516 	}
517 
518 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
519 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
520 		rv = CKR_HOST_MEMORY;
521 		goto clean2;
522 	}
523 	bytestring2bignum(&msg, in, in_len);
524 
525 	/* Convert public exponent and modulus to big integer format. */
526 	bytestring2bignum(&(rsakey.e), bkey->pubexpo, bkey->pubexpo_bytes);
527 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
528 
529 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
530 		rv = CKR_DATA_LEN_RANGE;
531 		goto clean3;
532 	}
533 
534 	/* Perform RSA computation on big integer input data. */
535 	if (big_modexp(&msg, &msg, &(rsakey.e), &(rsakey.n), NULL) !=
536 	    BIG_OK) {
537 		rv = CKR_HOST_MEMORY;
538 		goto clean3;
539 	}
540 
541 	/* Convert the big integer output data to octet string. */
542 	bignum2bytestring(out, &msg, modulus_bytes);
543 
544 clean3:
545 	big_finish(&msg);
546 clean2:
547 	RSA_key_finish(&rsakey);
548 
549 	return (rv);
550 }
551 
552 /*
553  * RSA decrypt operation
554  */
555 CK_RV
556 rsa_decrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
557 {
558 	CK_RV rv = CKR_OK;
559 
560 	BIGNUM msg;
561 	RSAkey rsakey;
562 	uint32_t modulus_bytes;
563 
564 	if (bkey == NULL)
565 		return (CKR_ARGUMENTS_BAD);
566 
567 	/* Must have modulus, prime1, prime2, expo1, expo2, and coeff set */
568 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
569 	    bkey->prime1_bytes == 0 || bkey->prime1 == NULL ||
570 	    bkey->prime2_bytes == 0 || bkey->prime2 == NULL ||
571 	    bkey->expo1_bytes == 0 || bkey->expo1 == NULL ||
572 	    bkey->expo2_bytes == 0 || bkey->expo2 == NULL ||
573 	    bkey->coeff_bytes == 0 || bkey->coeff == NULL)
574 		return (CKR_ARGUMENTS_BAD);
575 
576 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
577 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
578 
579 	/* psize and qsize for RSA_key_init is in bits. */
580 	if (RSA_key_init(&rsakey, CRYPTO_BYTES2BITS(bkey->prime2_bytes),
581 	    CRYPTO_BYTES2BITS(bkey->prime1_bytes)) != BIG_OK) {
582 		return (CKR_HOST_MEMORY);
583 	}
584 
585 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
586 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
587 		rv = CKR_HOST_MEMORY;
588 		goto clean3;
589 	}
590 	/* Convert octet string input data to big integer format. */
591 	bytestring2bignum(&msg, in, in_len);
592 
593 	/* Convert octet string modulus to big integer format. */
594 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
595 
596 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
597 		rv = CKR_DATA_LEN_RANGE;
598 		goto clean4;
599 	}
600 
601 	/* Convert the rest of private key attributes to big integer format. */
602 	bytestring2bignum(&(rsakey.q), bkey->prime1, bkey->prime1_bytes);
603 	bytestring2bignum(&(rsakey.p), bkey->prime2, bkey->prime2_bytes);
604 	bytestring2bignum(&(rsakey.dmodqminus1),
605 	    bkey->expo1, bkey->expo1_bytes);
606 	bytestring2bignum(&(rsakey.dmodpminus1),
607 	    bkey->expo2, bkey->expo2_bytes);
608 	bytestring2bignum(&(rsakey.pinvmodq),
609 	    bkey->coeff, bkey->coeff_bytes);
610 
611 	if ((big_cmp_abs(&(rsakey.dmodpminus1), &(rsakey.p)) > 0) ||
612 	    (big_cmp_abs(&(rsakey.dmodqminus1), &(rsakey.q)) > 0) ||
613 	    (big_cmp_abs(&(rsakey.pinvmodq), &(rsakey.q)) > 0)) {
614 		rv = CKR_KEY_SIZE_RANGE;
615 		goto clean4;
616 	}
617 
618 	/* Perform RSA computation on big integer input data. */
619 	if (big_modexp_crt(&msg, &msg, &(rsakey.dmodpminus1),
620 	    &(rsakey.dmodqminus1), &(rsakey.p), &(rsakey.q),
621 	    &(rsakey.pinvmodq), NULL, NULL) != BIG_OK) {
622 		rv = CKR_HOST_MEMORY;
623 		goto clean4;
624 	}
625 
626 	/* Convert the big integer output data to octet string. */
627 	bignum2bytestring(out, &msg, modulus_bytes);
628 
629 clean4:
630 	big_finish(&msg);
631 clean3:
632 	RSA_key_finish(&rsakey);
633 
634 	return (rv);
635 }
636