xref: /titanic_52/usr/src/common/crypto/rsa/rsa_impl.c (revision 694c35faa87b858ecdadfe4fc592615f4eefbb07)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5b60f2a0bSfr41279  * Common Development and Distribution License (the "License").
6b60f2a0bSfr41279  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21*726fad2aSDina K Nimeh 
227c478bd9Sstevel@tonic-gate /*
23*726fad2aSDina K Nimeh  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate /*
277c478bd9Sstevel@tonic-gate  * This file contains RSA helper routines common to
287c478bd9Sstevel@tonic-gate  * the PKCS11 soft token code and the kernel RSA code.
297c478bd9Sstevel@tonic-gate  */
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate #include <sys/types.h>
32*726fad2aSDina K Nimeh #include <bignum.h>
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate #ifdef _KERNEL
357c478bd9Sstevel@tonic-gate #include <sys/param.h>
367c478bd9Sstevel@tonic-gate #else
377c478bd9Sstevel@tonic-gate #include <strings.h>
387b79d846SDina K Nimeh #include <cryptoutil.h>
397c478bd9Sstevel@tonic-gate #endif
407c478bd9Sstevel@tonic-gate 
41*726fad2aSDina K Nimeh #include <sys/crypto/common.h>
42*726fad2aSDina K Nimeh #include "rsa_impl.h"
43*726fad2aSDina K Nimeh 
447c478bd9Sstevel@tonic-gate /*
45f66d273dSizick  * DER encoding T of the DigestInfo values for MD5, SHA1, and SHA2
467c478bd9Sstevel@tonic-gate  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
477c478bd9Sstevel@tonic-gate  *
487c478bd9Sstevel@tonic-gate  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
497c478bd9Sstevel@tonic-gate  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
50f66d273dSizick  * SHA-256: (0x)30 31 30 0d 06 09 60 86 48 01 65 03 04 02 01 05 00 04 20 || H.
51f66d273dSizick  * SHA-384: (0x)30 41 30 0d 06 09 60 86 48 01 65 03 04 02 02 05 00 04 30 || H.
52f66d273dSizick  * SHA-512: (0x)30 51 30 0d 06 09 60 86 48 01 65 03 04 02 03 05 00 04 40 || H.
537c478bd9Sstevel@tonic-gate  *
547c478bd9Sstevel@tonic-gate  * Where H is the digested output from MD5 or SHA1. We define the constant
557c478bd9Sstevel@tonic-gate  * byte array (the prefix) here and use it rather than doing the DER
567c478bd9Sstevel@tonic-gate  * encoding of the OID in a separate routine.
577c478bd9Sstevel@tonic-gate  */
587c478bd9Sstevel@tonic-gate const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
597c478bd9Sstevel@tonic-gate     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
607c478bd9Sstevel@tonic-gate     0x04, 0x10};
617c478bd9Sstevel@tonic-gate 
627c478bd9Sstevel@tonic-gate const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
637c478bd9Sstevel@tonic-gate     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
647c478bd9Sstevel@tonic-gate 
6560722cc8Sizick const CK_BYTE SHA1_DER_PREFIX_OID[SHA1_DER_PREFIX_OID_Len] = {0x30, 0x1f, 0x30,
6660722cc8Sizick     0x07, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x04, 0x14};
6760722cc8Sizick 
68f66d273dSizick const CK_BYTE SHA256_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x31, 0x30, 0x0d,
69f66d273dSizick     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
70f66d273dSizick     0x00, 0x04, 0x20};
71f66d273dSizick 
72f66d273dSizick const CK_BYTE SHA384_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x41, 0x30, 0x0d,
73f66d273dSizick     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05,
74f66d273dSizick     0x00, 0x04, 0x30};
75f66d273dSizick 
76f66d273dSizick const CK_BYTE SHA512_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x51, 0x30, 0x0d,
77f66d273dSizick     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05,
78f66d273dSizick     0x00, 0x04, 0x40};
79f66d273dSizick 
8020d58091SDina K Nimeh const CK_BYTE DEFAULT_PUB_EXPO[DEFAULT_PUB_EXPO_Len] = { 0x01, 0x00, 0x01 };
81b60f2a0bSfr41279 
82*726fad2aSDina K Nimeh 
83*726fad2aSDina K Nimeh static CK_RV
84*726fad2aSDina K Nimeh convert_rv(BIG_ERR_CODE err)
85*726fad2aSDina K Nimeh {
86*726fad2aSDina K Nimeh 	switch (err) {
87*726fad2aSDina K Nimeh 
88*726fad2aSDina K Nimeh 	case BIG_OK:
89*726fad2aSDina K Nimeh 		return (CKR_OK);
90*726fad2aSDina K Nimeh 
91*726fad2aSDina K Nimeh 	case BIG_NO_MEM:
92*726fad2aSDina K Nimeh 		return (CKR_HOST_MEMORY);
93*726fad2aSDina K Nimeh 
94*726fad2aSDina K Nimeh 	case BIG_NO_RANDOM:
95*726fad2aSDina K Nimeh 		return (CKR_DEVICE_ERROR);
96*726fad2aSDina K Nimeh 
97*726fad2aSDina K Nimeh 	case BIG_INVALID_ARGS:
98*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
99*726fad2aSDina K Nimeh 
100*726fad2aSDina K Nimeh 	case BIG_DIV_BY_0:
101*726fad2aSDina K Nimeh 	default:
102*726fad2aSDina K Nimeh 		return (CKR_GENERAL_ERROR);
103*726fad2aSDina K Nimeh 	}
104*726fad2aSDina K Nimeh }
105*726fad2aSDina K Nimeh 
106b60f2a0bSfr41279 /* psize and qsize are in bits */
107*726fad2aSDina K Nimeh static BIG_ERR_CODE
1087c478bd9Sstevel@tonic-gate RSA_key_init(RSAkey *key, int psize, int qsize)
1097c478bd9Sstevel@tonic-gate {
1107c478bd9Sstevel@tonic-gate 	BIG_ERR_CODE err = BIG_OK;
1117c478bd9Sstevel@tonic-gate 
1127c478bd9Sstevel@tonic-gate 	int plen, qlen, nlen;
1137c478bd9Sstevel@tonic-gate 
114b60f2a0bSfr41279 	plen = BITLEN2BIGNUMLEN(psize);
115b60f2a0bSfr41279 	qlen = BITLEN2BIGNUMLEN(qsize);
1167c478bd9Sstevel@tonic-gate 	nlen = plen + qlen;
1177c478bd9Sstevel@tonic-gate 	key->size = psize + qsize;
1187c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->p), plen)) != BIG_OK)
1197c478bd9Sstevel@tonic-gate 		return (err);
1207c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->q), qlen)) != BIG_OK)
1217c478bd9Sstevel@tonic-gate 		goto ret1;
1227c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->n), nlen)) != BIG_OK)
1237c478bd9Sstevel@tonic-gate 		goto ret2;
1247c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->d), nlen)) != BIG_OK)
1257c478bd9Sstevel@tonic-gate 		goto ret3;
1267c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->e), nlen)) != BIG_OK)
1277c478bd9Sstevel@tonic-gate 		goto ret4;
1287c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
1297c478bd9Sstevel@tonic-gate 		goto ret5;
1307c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
1317c478bd9Sstevel@tonic-gate 		goto ret6;
1327c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
1337c478bd9Sstevel@tonic-gate 		goto ret7;
1347c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
1357c478bd9Sstevel@tonic-gate 		goto ret8;
1367c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
1377c478bd9Sstevel@tonic-gate 		goto ret9;
1387c478bd9Sstevel@tonic-gate 	if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
1397c478bd9Sstevel@tonic-gate 		goto ret10;
1407c478bd9Sstevel@tonic-gate 
1417c478bd9Sstevel@tonic-gate 	return (BIG_OK);
1427c478bd9Sstevel@tonic-gate 
1437c478bd9Sstevel@tonic-gate ret10:
1447c478bd9Sstevel@tonic-gate 	big_finish(&(key->q_rr));
1457c478bd9Sstevel@tonic-gate ret9:
1467c478bd9Sstevel@tonic-gate 	big_finish(&(key->p_rr));
1477c478bd9Sstevel@tonic-gate ret8:
1487c478bd9Sstevel@tonic-gate 	big_finish(&(key->pinvmodq));
1497c478bd9Sstevel@tonic-gate ret7:
1507c478bd9Sstevel@tonic-gate 	big_finish(&(key->dmodqminus1));
1517c478bd9Sstevel@tonic-gate ret6:
1527c478bd9Sstevel@tonic-gate 	big_finish(&(key->dmodpminus1));
1537c478bd9Sstevel@tonic-gate ret5:
1547c478bd9Sstevel@tonic-gate 	big_finish(&(key->e));
1557c478bd9Sstevel@tonic-gate ret4:
1567c478bd9Sstevel@tonic-gate 	big_finish(&(key->d));
1577c478bd9Sstevel@tonic-gate ret3:
1587c478bd9Sstevel@tonic-gate 	big_finish(&(key->n));
1597c478bd9Sstevel@tonic-gate ret2:
1607c478bd9Sstevel@tonic-gate 	big_finish(&(key->q));
1617c478bd9Sstevel@tonic-gate ret1:
1627c478bd9Sstevel@tonic-gate 	big_finish(&(key->p));
1637c478bd9Sstevel@tonic-gate 
1647c478bd9Sstevel@tonic-gate 	return (err);
1657c478bd9Sstevel@tonic-gate }
1667c478bd9Sstevel@tonic-gate 
167*726fad2aSDina K Nimeh static void
1687c478bd9Sstevel@tonic-gate RSA_key_finish(RSAkey *key)
1697c478bd9Sstevel@tonic-gate {
1707c478bd9Sstevel@tonic-gate 	big_finish(&(key->n_rr));
1717c478bd9Sstevel@tonic-gate 	big_finish(&(key->q_rr));
1727c478bd9Sstevel@tonic-gate 	big_finish(&(key->p_rr));
1737c478bd9Sstevel@tonic-gate 	big_finish(&(key->pinvmodq));
1747c478bd9Sstevel@tonic-gate 	big_finish(&(key->dmodqminus1));
1757c478bd9Sstevel@tonic-gate 	big_finish(&(key->dmodpminus1));
1767c478bd9Sstevel@tonic-gate 	big_finish(&(key->e));
1777c478bd9Sstevel@tonic-gate 	big_finish(&(key->d));
1787c478bd9Sstevel@tonic-gate 	big_finish(&(key->n));
1797c478bd9Sstevel@tonic-gate 	big_finish(&(key->q));
1807c478bd9Sstevel@tonic-gate 	big_finish(&(key->p));
1817c478bd9Sstevel@tonic-gate }
1827c478bd9Sstevel@tonic-gate 
1837c478bd9Sstevel@tonic-gate /*
184*726fad2aSDina K Nimeh  * Generate RSA key
1857c478bd9Sstevel@tonic-gate  */
186*726fad2aSDina K Nimeh static CK_RV
187*726fad2aSDina K Nimeh generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM *pubexp,
188*726fad2aSDina K Nimeh     int (*rfunc)(void *, size_t))
1897c478bd9Sstevel@tonic-gate {
190*726fad2aSDina K Nimeh 	CK_RV		rv = CKR_OK;
1917c478bd9Sstevel@tonic-gate 
192*726fad2aSDina K Nimeh 	int		(*rf)(void *, size_t);
193*726fad2aSDina K Nimeh 	BIGNUM		a, b, c, d, e, f, g, h;
194*726fad2aSDina K Nimeh 	int		len, keylen, size;
195*726fad2aSDina K Nimeh 	BIG_ERR_CODE	brv = BIG_OK;
1967c478bd9Sstevel@tonic-gate 
197*726fad2aSDina K Nimeh 	size = psize + qsize;
198*726fad2aSDina K Nimeh 	keylen = BITLEN2BIGNUMLEN(size);
199*726fad2aSDina K Nimeh 	len = keylen * 2 + 1;
200*726fad2aSDina K Nimeh 	key->size = size;
201*726fad2aSDina K Nimeh 
202*726fad2aSDina K Nimeh 	/*
203*726fad2aSDina K Nimeh 	 * Note: It is not really necessary to compute e, it is in pubexp:
204*726fad2aSDina K Nimeh 	 * 	(void) big_copy(&(key->e), pubexp);
205*726fad2aSDina K Nimeh 	 */
206*726fad2aSDina K Nimeh 
207*726fad2aSDina K Nimeh 	a.malloced = 0;
208*726fad2aSDina K Nimeh 	b.malloced = 0;
209*726fad2aSDina K Nimeh 	c.malloced = 0;
210*726fad2aSDina K Nimeh 	d.malloced = 0;
211*726fad2aSDina K Nimeh 	e.malloced = 0;
212*726fad2aSDina K Nimeh 	f.malloced = 0;
213*726fad2aSDina K Nimeh 	g.malloced = 0;
214*726fad2aSDina K Nimeh 	h.malloced = 0;
215*726fad2aSDina K Nimeh 
216*726fad2aSDina K Nimeh 	if ((big_init(&a, len) != BIG_OK) ||
217*726fad2aSDina K Nimeh 	    (big_init(&b, len) != BIG_OK) ||
218*726fad2aSDina K Nimeh 	    (big_init(&c, len) != BIG_OK) ||
219*726fad2aSDina K Nimeh 	    (big_init(&d, len) != BIG_OK) ||
220*726fad2aSDina K Nimeh 	    (big_init(&e, len) != BIG_OK) ||
221*726fad2aSDina K Nimeh 	    (big_init(&f, len) != BIG_OK) ||
222*726fad2aSDina K Nimeh 	    (big_init(&g, len) != BIG_OK) ||
223*726fad2aSDina K Nimeh 	    (big_init(&h, len) != BIG_OK)) {
224*726fad2aSDina K Nimeh 		big_finish(&h);
225*726fad2aSDina K Nimeh 		big_finish(&g);
226*726fad2aSDina K Nimeh 		big_finish(&f);
227*726fad2aSDina K Nimeh 		big_finish(&e);
228*726fad2aSDina K Nimeh 		big_finish(&d);
229*726fad2aSDina K Nimeh 		big_finish(&c);
230*726fad2aSDina K Nimeh 		big_finish(&b);
231*726fad2aSDina K Nimeh 		big_finish(&a);
232*726fad2aSDina K Nimeh 
233*726fad2aSDina K Nimeh 		return (CKR_HOST_MEMORY);
2347c478bd9Sstevel@tonic-gate 	}
2357c478bd9Sstevel@tonic-gate 
236*726fad2aSDina K Nimeh 	rf = rfunc;
237*726fad2aSDina K Nimeh 	if (rf == NULL) {
2387c478bd9Sstevel@tonic-gate #ifdef _KERNEL
239*726fad2aSDina K Nimeh 		rf = (int (*)(void *, size_t))random_get_pseudo_bytes;
2407c478bd9Sstevel@tonic-gate #else
241*726fad2aSDina K Nimeh 		rf = pkcs11_get_urandom;
2427c478bd9Sstevel@tonic-gate #endif
243*726fad2aSDina K Nimeh 	}
244*726fad2aSDina K Nimeh 
245*726fad2aSDina K Nimeh nextp:
246*726fad2aSDina K Nimeh 	if ((brv = big_random(&a, psize, rf)) != BIG_OK) {
247*726fad2aSDina K Nimeh 		goto ret;
248*726fad2aSDina K Nimeh 	}
249*726fad2aSDina K Nimeh 
250*726fad2aSDina K Nimeh 	if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
251*726fad2aSDina K Nimeh 		goto ret;
252*726fad2aSDina K Nimeh 	}
253*726fad2aSDina K Nimeh 	/* b now contains the potential prime p */
254*726fad2aSDina K Nimeh 
255*726fad2aSDina K Nimeh 	(void) big_sub_pos(&a, &b, &big_One);
256*726fad2aSDina K Nimeh 	if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
257*726fad2aSDina K Nimeh 		goto ret;
258*726fad2aSDina K Nimeh 	}
259*726fad2aSDina K Nimeh 	if (big_cmp_abs(&f, &big_One) != 0) {
260*726fad2aSDina K Nimeh 		goto nextp;
261*726fad2aSDina K Nimeh 	}
262*726fad2aSDina K Nimeh 
263*726fad2aSDina K Nimeh 	if ((brv = big_random(&c, qsize, rf)) != BIG_OK) {
264*726fad2aSDina K Nimeh 		goto ret;
265*726fad2aSDina K Nimeh 	}
266*726fad2aSDina K Nimeh 
267*726fad2aSDina K Nimeh nextq:
268*726fad2aSDina K Nimeh 	(void) big_add(&a, &c, &big_Two);
269*726fad2aSDina K Nimeh 
270*726fad2aSDina K Nimeh 	if (big_bitlength(&a) != qsize) {
271*726fad2aSDina K Nimeh 		goto nextp;
272*726fad2aSDina K Nimeh 	}
273*726fad2aSDina K Nimeh 	if (big_cmp_abs(&a, &b) == 0) {
274*726fad2aSDina K Nimeh 		goto nextp;
275*726fad2aSDina K Nimeh 	}
276*726fad2aSDina K Nimeh 	if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
277*726fad2aSDina K Nimeh 		goto ret;
278*726fad2aSDina K Nimeh 	}
279*726fad2aSDina K Nimeh 	/* c now contains the potential prime q */
280*726fad2aSDina K Nimeh 
281*726fad2aSDina K Nimeh 	if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
282*726fad2aSDina K Nimeh 		goto ret;
283*726fad2aSDina K Nimeh 	}
284*726fad2aSDina K Nimeh 	if (big_bitlength(&g) != size) {
285*726fad2aSDina K Nimeh 		goto nextp;
286*726fad2aSDina K Nimeh 	}
287*726fad2aSDina K Nimeh 	/* g now contains the potential modulus n */
288*726fad2aSDina K Nimeh 
289*726fad2aSDina K Nimeh 	(void) big_sub_pos(&a, &b, &big_One);
290*726fad2aSDina K Nimeh 	(void) big_sub_pos(&d, &c, &big_One);
291*726fad2aSDina K Nimeh 
292*726fad2aSDina K Nimeh 	if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
293*726fad2aSDina K Nimeh 		goto ret;
294*726fad2aSDina K Nimeh 	}
295*726fad2aSDina K Nimeh 	if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
296*726fad2aSDina K Nimeh 		goto ret;
297*726fad2aSDina K Nimeh 	}
298*726fad2aSDina K Nimeh 	if (big_cmp_abs(&f, &big_One) != 0) {
299*726fad2aSDina K Nimeh 		goto nextq;
300*726fad2aSDina K Nimeh 	} else {
301*726fad2aSDina K Nimeh 		(void) big_copy(&e, pubexp);
302*726fad2aSDina K Nimeh 	}
303*726fad2aSDina K Nimeh 	if (d.sign == -1) {
304*726fad2aSDina K Nimeh 		if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
305*726fad2aSDina K Nimeh 			goto ret;
306*726fad2aSDina K Nimeh 		}
307*726fad2aSDina K Nimeh 	}
308*726fad2aSDina K Nimeh 	(void) big_copy(&(key->p), &b);
309*726fad2aSDina K Nimeh 	(void) big_copy(&(key->q), &c);
310*726fad2aSDina K Nimeh 	(void) big_copy(&(key->n), &g);
311*726fad2aSDina K Nimeh 	(void) big_copy(&(key->d), &d);
312*726fad2aSDina K Nimeh 	(void) big_copy(&(key->e), &e);
313*726fad2aSDina K Nimeh 
314*726fad2aSDina K Nimeh 	if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
315*726fad2aSDina K Nimeh 		goto ret;
316*726fad2aSDina K Nimeh 	}
317*726fad2aSDina K Nimeh 	if (f.sign == -1) {
318*726fad2aSDina K Nimeh 		if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
319*726fad2aSDina K Nimeh 			goto ret;
320*726fad2aSDina K Nimeh 		}
321*726fad2aSDina K Nimeh 	}
322*726fad2aSDina K Nimeh 	(void) big_copy(&(key->pinvmodq), &f);
323*726fad2aSDina K Nimeh 
324*726fad2aSDina K Nimeh 	(void) big_sub(&a, &b, &big_One);
325*726fad2aSDina K Nimeh 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
326*726fad2aSDina K Nimeh 		goto ret;
327*726fad2aSDina K Nimeh 	}
328*726fad2aSDina K Nimeh 	(void) big_copy(&(key->dmodpminus1), &f);
329*726fad2aSDina K Nimeh 	(void) big_sub(&a, &c, &big_One);
330*726fad2aSDina K Nimeh 	if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
331*726fad2aSDina K Nimeh 		goto ret;
332*726fad2aSDina K Nimeh 	}
333*726fad2aSDina K Nimeh 	(void) big_copy(&(key->dmodqminus1), &f);
334*726fad2aSDina K Nimeh 
335*726fad2aSDina K Nimeh 	/* pairwise consistency check:  decrypt and encrypt restores value */
336*726fad2aSDina K Nimeh 	if ((brv = big_random(&h, size, rf)) != BIG_OK) {
337*726fad2aSDina K Nimeh 		goto ret;
338*726fad2aSDina K Nimeh 	}
339*726fad2aSDina K Nimeh 	if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
340*726fad2aSDina K Nimeh 		goto ret;
341*726fad2aSDina K Nimeh 	}
342*726fad2aSDina K Nimeh 	if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
343*726fad2aSDina K Nimeh 		goto ret;
344*726fad2aSDina K Nimeh 	}
345*726fad2aSDina K Nimeh 
346*726fad2aSDina K Nimeh 	if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
347*726fad2aSDina K Nimeh 		goto ret;
348*726fad2aSDina K Nimeh 	}
349*726fad2aSDina K Nimeh 
350*726fad2aSDina K Nimeh 	if (big_cmp_abs(&b, &h) != 0) {
351*726fad2aSDina K Nimeh 		/* this should not happen */
352*726fad2aSDina K Nimeh 		rv = generate_rsa_key(key, psize, qsize, pubexp, rf);
353*726fad2aSDina K Nimeh 		goto ret1;
354*726fad2aSDina K Nimeh 	} else {
355*726fad2aSDina K Nimeh 		brv = BIG_OK;
356*726fad2aSDina K Nimeh 	}
357*726fad2aSDina K Nimeh 
358*726fad2aSDina K Nimeh ret:
359*726fad2aSDina K Nimeh 	rv = convert_rv(brv);
360*726fad2aSDina K Nimeh ret1:
361*726fad2aSDina K Nimeh 	big_finish(&h);
362*726fad2aSDina K Nimeh 	big_finish(&g);
363*726fad2aSDina K Nimeh 	big_finish(&f);
364*726fad2aSDina K Nimeh 	big_finish(&e);
365*726fad2aSDina K Nimeh 	big_finish(&d);
366*726fad2aSDina K Nimeh 	big_finish(&c);
367*726fad2aSDina K Nimeh 	big_finish(&b);
368*726fad2aSDina K Nimeh 	big_finish(&a);
369*726fad2aSDina K Nimeh 
3707c478bd9Sstevel@tonic-gate 	return (rv);
3717c478bd9Sstevel@tonic-gate }
3727c478bd9Sstevel@tonic-gate 
3737c478bd9Sstevel@tonic-gate CK_RV
374*726fad2aSDina K Nimeh rsa_genkey_pair(RSAbytekey *bkey)
3757c478bd9Sstevel@tonic-gate {
376*726fad2aSDina K Nimeh 	/*
377*726fad2aSDina K Nimeh 	 * NOTE:  Whomever originally wrote this function swapped p and q.
378*726fad2aSDina K Nimeh 	 * This table shows the mapping between name convention used here
379*726fad2aSDina K Nimeh 	 * versus what is used in most texts that describe RSA key generation.
380*726fad2aSDina K Nimeh 	 *	This function:			Standard convention:
381*726fad2aSDina K Nimeh 	 *	--------------			--------------------
382*726fad2aSDina K Nimeh 	 *	modulus, n			-same-
383*726fad2aSDina K Nimeh 	 *	prime 1, q			prime 1, p
384*726fad2aSDina K Nimeh 	 *	prime 2, p			prime 2, q
385*726fad2aSDina K Nimeh 	 *	private exponent, d		-same-
386*726fad2aSDina K Nimeh 	 *	public exponent, e		-same-
387*726fad2aSDina K Nimeh 	 *	exponent 1, d mod (q-1)		d mod (p-1)
388*726fad2aSDina K Nimeh 	 *	exponent 2, d mod (p-1)		d mod (q-1)
389*726fad2aSDina K Nimeh 	 *	coefficient, p^-1 mod q		q^-1 mod p
390*726fad2aSDina K Nimeh 	 *
391*726fad2aSDina K Nimeh 	 * Also notice the struct member for coefficient is named .pinvmodq
392*726fad2aSDina K Nimeh 	 * rather than .qinvmodp, reflecting the switch.
393*726fad2aSDina K Nimeh 	 *
394*726fad2aSDina K Nimeh 	 * The code here wasn't unswapped, because "it works".  Further,
395*726fad2aSDina K Nimeh 	 * p and q are interchangeable as long as exponent 1 and 2 and
396*726fad2aSDina K Nimeh 	 * the coefficient are kept straight too.  This note is here to
397*726fad2aSDina K Nimeh 	 * make the reader aware of the switcheroo.
398*726fad2aSDina K Nimeh 	 */
399*726fad2aSDina K Nimeh 	CK_RV	rv = CKR_OK;
4007c478bd9Sstevel@tonic-gate 
401*726fad2aSDina K Nimeh 	BIGNUM	public_exponent = {0};
402*726fad2aSDina K Nimeh 	RSAkey	rsakey;
403*726fad2aSDina K Nimeh 	uint32_t modulus_bytes;
4047c478bd9Sstevel@tonic-gate 
405*726fad2aSDina K Nimeh 	if (bkey == NULL)
406*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
4077c478bd9Sstevel@tonic-gate 
408*726fad2aSDina K Nimeh 	/* Must have modulus bits set */
409*726fad2aSDina K Nimeh 	if (bkey->modulus_bits == 0)
410*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
4117c478bd9Sstevel@tonic-gate 
412*726fad2aSDina K Nimeh 	/* Must have public exponent set */
413*726fad2aSDina K Nimeh 	if (bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
414*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
4157c478bd9Sstevel@tonic-gate 
416*726fad2aSDina K Nimeh 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
417*726fad2aSDina K Nimeh 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
4187c478bd9Sstevel@tonic-gate 
419*726fad2aSDina K Nimeh 	/* Modulus length needs to be between min key size and max key size. */
420*726fad2aSDina K Nimeh 	if ((modulus_bytes < MIN_RSA_KEYLENGTH_IN_BYTES) ||
421*726fad2aSDina K Nimeh 	    (modulus_bytes > MAX_RSA_KEYLENGTH_IN_BYTES)) {
422*726fad2aSDina K Nimeh 		return (CKR_KEY_SIZE_RANGE);
4237c478bd9Sstevel@tonic-gate 	}
4247c478bd9Sstevel@tonic-gate 
4257c478bd9Sstevel@tonic-gate 	/*
426*726fad2aSDina K Nimeh 	 * Initialize the RSA key.
427*726fad2aSDina K Nimeh 	 */
428*726fad2aSDina K Nimeh 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
429*726fad2aSDina K Nimeh 	    BIG_OK) {
430*726fad2aSDina K Nimeh 		return (CKR_HOST_MEMORY);
431*726fad2aSDina K Nimeh 	}
432*726fad2aSDina K Nimeh 
433*726fad2aSDina K Nimeh 	/* Create a public exponent in bignum format. */
434*726fad2aSDina K Nimeh 	if (big_init(&public_exponent,
435*726fad2aSDina K Nimeh 	    CHARLEN2BIGNUMLEN(bkey->pubexpo_bytes)) != BIG_OK) {
436*726fad2aSDina K Nimeh 		rv = CKR_HOST_MEMORY;
437*726fad2aSDina K Nimeh 		goto clean1;
438*726fad2aSDina K Nimeh 	}
439*726fad2aSDina K Nimeh 	bytestring2bignum(&public_exponent, bkey->pubexpo, bkey->pubexpo_bytes);
440*726fad2aSDina K Nimeh 
441*726fad2aSDina K Nimeh 	/* Generate RSA key pair. */
442*726fad2aSDina K Nimeh 	if ((rv = generate_rsa_key(&rsakey,
443*726fad2aSDina K Nimeh 	    modulus_bytes * 4, modulus_bytes * 4, &public_exponent,
444*726fad2aSDina K Nimeh 	    bkey->rfunc)) != CKR_OK) {
445*726fad2aSDina K Nimeh 		big_finish(&public_exponent);
446*726fad2aSDina K Nimeh 		goto clean1;
447*726fad2aSDina K Nimeh 	}
448*726fad2aSDina K Nimeh 	big_finish(&public_exponent);
449*726fad2aSDina K Nimeh 
450*726fad2aSDina K Nimeh 	/* modulus_bytes = rsakey.n.len * (int)sizeof (BIG_CHUNK_TYPE); */
451*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->modulus, &(rsakey.n), modulus_bytes);
452*726fad2aSDina K Nimeh 
453*726fad2aSDina K Nimeh 	bkey->privexpo_bytes = rsakey.d.len * (int)sizeof (BIG_CHUNK_TYPE);
454*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->privexpo, &(rsakey.d), bkey->privexpo_bytes);
455*726fad2aSDina K Nimeh 
456*726fad2aSDina K Nimeh 	bkey->pubexpo_bytes = rsakey.e.len * (int)sizeof (BIG_CHUNK_TYPE);
457*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->pubexpo, &(rsakey.e), bkey->pubexpo_bytes);
458*726fad2aSDina K Nimeh 
459*726fad2aSDina K Nimeh 	bkey->prime1_bytes = rsakey.q.len * (int)sizeof (BIG_CHUNK_TYPE);
460*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->prime1, &(rsakey.q), bkey->prime1_bytes);
461*726fad2aSDina K Nimeh 
462*726fad2aSDina K Nimeh 	bkey->prime2_bytes = rsakey.p.len * (int)sizeof (BIG_CHUNK_TYPE);
463*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->prime2, &(rsakey.p), bkey->prime2_bytes);
464*726fad2aSDina K Nimeh 
465*726fad2aSDina K Nimeh 	bkey->expo1_bytes =
466*726fad2aSDina K Nimeh 	    rsakey.dmodqminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
467*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->expo1, &(rsakey.dmodqminus1),
468*726fad2aSDina K Nimeh 	    bkey->expo1_bytes);
469*726fad2aSDina K Nimeh 
470*726fad2aSDina K Nimeh 	bkey->expo2_bytes =
471*726fad2aSDina K Nimeh 	    rsakey.dmodpminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
472*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->expo2,
473*726fad2aSDina K Nimeh 	    &(rsakey.dmodpminus1), bkey->expo2_bytes);
474*726fad2aSDina K Nimeh 
475*726fad2aSDina K Nimeh 	bkey->coeff_bytes =
476*726fad2aSDina K Nimeh 	    rsakey.pinvmodq.len * (int)sizeof (BIG_CHUNK_TYPE);
477*726fad2aSDina K Nimeh 	bignum2bytestring(bkey->coeff, &(rsakey.pinvmodq), bkey->coeff_bytes);
478*726fad2aSDina K Nimeh 
479*726fad2aSDina K Nimeh clean1:
480*726fad2aSDina K Nimeh 	RSA_key_finish(&rsakey);
481*726fad2aSDina K Nimeh 
482*726fad2aSDina K Nimeh 	return (rv);
483*726fad2aSDina K Nimeh }
484*726fad2aSDina K Nimeh 
485*726fad2aSDina K Nimeh /*
486*726fad2aSDina K Nimeh  * RSA encrypt operation
4877c478bd9Sstevel@tonic-gate  */
4887c478bd9Sstevel@tonic-gate CK_RV
489*726fad2aSDina K Nimeh rsa_encrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
4907c478bd9Sstevel@tonic-gate {
491*726fad2aSDina K Nimeh 	CK_RV rv = CKR_OK;
4927c478bd9Sstevel@tonic-gate 
493*726fad2aSDina K Nimeh 	BIGNUM msg;
494*726fad2aSDina K Nimeh 	RSAkey rsakey;
495*726fad2aSDina K Nimeh 	uint32_t modulus_bytes;
4967c478bd9Sstevel@tonic-gate 
497*726fad2aSDina K Nimeh 	if (bkey == NULL)
498*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
499*726fad2aSDina K Nimeh 
500*726fad2aSDina K Nimeh 	/* Must have modulus and public exponent set */
501*726fad2aSDina K Nimeh 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
502*726fad2aSDina K Nimeh 	    bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
503*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
504*726fad2aSDina K Nimeh 
505*726fad2aSDina K Nimeh 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
506*726fad2aSDina K Nimeh 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
507*726fad2aSDina K Nimeh 
508*726fad2aSDina K Nimeh 	if (bkey->pubexpo_bytes > modulus_bytes) {
509*726fad2aSDina K Nimeh 		return (CKR_KEY_SIZE_RANGE);
5107c478bd9Sstevel@tonic-gate 	}
5117c478bd9Sstevel@tonic-gate 
512*726fad2aSDina K Nimeh 	/* psize and qsize for RSA_key_init is in bits. */
513*726fad2aSDina K Nimeh 	if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
514*726fad2aSDina K Nimeh 	    BIG_OK) {
515*726fad2aSDina K Nimeh 		return (CKR_HOST_MEMORY);
516*726fad2aSDina K Nimeh 	}
517*726fad2aSDina K Nimeh 
518*726fad2aSDina K Nimeh 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
519*726fad2aSDina K Nimeh 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
520*726fad2aSDina K Nimeh 		rv = CKR_HOST_MEMORY;
521*726fad2aSDina K Nimeh 		goto clean2;
522*726fad2aSDina K Nimeh 	}
523*726fad2aSDina K Nimeh 	bytestring2bignum(&msg, in, in_len);
524*726fad2aSDina K Nimeh 
525*726fad2aSDina K Nimeh 	/* Convert public exponent and modulus to big integer format. */
526*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.e), bkey->pubexpo, bkey->pubexpo_bytes);
527*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
528*726fad2aSDina K Nimeh 
529*726fad2aSDina K Nimeh 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
530*726fad2aSDina K Nimeh 		rv = CKR_DATA_LEN_RANGE;
531*726fad2aSDina K Nimeh 		goto clean3;
532*726fad2aSDina K Nimeh 	}
533*726fad2aSDina K Nimeh 
534*726fad2aSDina K Nimeh 	/* Perform RSA computation on big integer input data. */
535*726fad2aSDina K Nimeh 	if (big_modexp(&msg, &msg, &(rsakey.e), &(rsakey.n), NULL) !=
536*726fad2aSDina K Nimeh 	    BIG_OK) {
537*726fad2aSDina K Nimeh 		rv = CKR_HOST_MEMORY;
538*726fad2aSDina K Nimeh 		goto clean3;
539*726fad2aSDina K Nimeh 	}
540*726fad2aSDina K Nimeh 
541*726fad2aSDina K Nimeh 	/* Convert the big integer output data to octet string. */
542*726fad2aSDina K Nimeh 	bignum2bytestring(out, &msg, modulus_bytes);
543*726fad2aSDina K Nimeh 
544*726fad2aSDina K Nimeh clean3:
545*726fad2aSDina K Nimeh 	big_finish(&msg);
546*726fad2aSDina K Nimeh clean2:
547*726fad2aSDina K Nimeh 	RSA_key_finish(&rsakey);
5487c478bd9Sstevel@tonic-gate 
549*726fad2aSDina K Nimeh 	return (rv);
5507c478bd9Sstevel@tonic-gate }
5517c478bd9Sstevel@tonic-gate 
552*726fad2aSDina K Nimeh /*
553*726fad2aSDina K Nimeh  * RSA decrypt operation
554*726fad2aSDina K Nimeh  */
5557c478bd9Sstevel@tonic-gate CK_RV
556*726fad2aSDina K Nimeh rsa_decrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
5577c478bd9Sstevel@tonic-gate {
558*726fad2aSDina K Nimeh 	CK_RV rv = CKR_OK;
5597c478bd9Sstevel@tonic-gate 
560*726fad2aSDina K Nimeh 	BIGNUM msg;
561*726fad2aSDina K Nimeh 	RSAkey rsakey;
562*726fad2aSDina K Nimeh 	uint32_t modulus_bytes;
5637c478bd9Sstevel@tonic-gate 
564*726fad2aSDina K Nimeh 	if (bkey == NULL)
565*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
5667c478bd9Sstevel@tonic-gate 
567*726fad2aSDina K Nimeh 	/* Must have modulus, prime1, prime2, expo1, expo2, and coeff set */
568*726fad2aSDina K Nimeh 	if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
569*726fad2aSDina K Nimeh 	    bkey->prime1_bytes == 0 || bkey->prime1 == NULL ||
570*726fad2aSDina K Nimeh 	    bkey->prime2_bytes == 0 || bkey->prime2 == NULL ||
571*726fad2aSDina K Nimeh 	    bkey->expo1_bytes == 0 || bkey->expo1 == NULL ||
572*726fad2aSDina K Nimeh 	    bkey->expo2_bytes == 0 || bkey->expo2 == NULL ||
573*726fad2aSDina K Nimeh 	    bkey->coeff_bytes == 0 || bkey->coeff == NULL)
574*726fad2aSDina K Nimeh 		return (CKR_ARGUMENTS_BAD);
575*726fad2aSDina K Nimeh 
576*726fad2aSDina K Nimeh 	/* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
577*726fad2aSDina K Nimeh 	modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
578*726fad2aSDina K Nimeh 
579*726fad2aSDina K Nimeh 	/* psize and qsize for RSA_key_init is in bits. */
580*726fad2aSDina K Nimeh 	if (RSA_key_init(&rsakey, CRYPTO_BYTES2BITS(bkey->prime2_bytes),
581*726fad2aSDina K Nimeh 	    CRYPTO_BYTES2BITS(bkey->prime1_bytes)) != BIG_OK) {
582*726fad2aSDina K Nimeh 		return (CKR_HOST_MEMORY);
5837c478bd9Sstevel@tonic-gate 	}
584*726fad2aSDina K Nimeh 
585*726fad2aSDina K Nimeh 	/* Size for big_init is in BIG_CHUNK_TYPE words. */
586*726fad2aSDina K Nimeh 	if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
587*726fad2aSDina K Nimeh 		rv = CKR_HOST_MEMORY;
588*726fad2aSDina K Nimeh 		goto clean3;
5897c478bd9Sstevel@tonic-gate 	}
590*726fad2aSDina K Nimeh 	/* Convert octet string input data to big integer format. */
591*726fad2aSDina K Nimeh 	bytestring2bignum(&msg, in, in_len);
592*726fad2aSDina K Nimeh 
593*726fad2aSDina K Nimeh 	/* Convert octet string modulus to big integer format. */
594*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
595*726fad2aSDina K Nimeh 
596*726fad2aSDina K Nimeh 	if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
597*726fad2aSDina K Nimeh 		rv = CKR_DATA_LEN_RANGE;
598*726fad2aSDina K Nimeh 		goto clean4;
599*726fad2aSDina K Nimeh 	}
600*726fad2aSDina K Nimeh 
601*726fad2aSDina K Nimeh 	/* Convert the rest of private key attributes to big integer format. */
602*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.q), bkey->prime1, bkey->prime1_bytes);
603*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.p), bkey->prime2, bkey->prime2_bytes);
604*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.dmodqminus1),
605*726fad2aSDina K Nimeh 	    bkey->expo1, bkey->expo1_bytes);
606*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.dmodpminus1),
607*726fad2aSDina K Nimeh 	    bkey->expo2, bkey->expo2_bytes);
608*726fad2aSDina K Nimeh 	bytestring2bignum(&(rsakey.pinvmodq),
609*726fad2aSDina K Nimeh 	    bkey->coeff, bkey->coeff_bytes);
610*726fad2aSDina K Nimeh 
611*726fad2aSDina K Nimeh 	if ((big_cmp_abs(&(rsakey.dmodpminus1), &(rsakey.p)) > 0) ||
612*726fad2aSDina K Nimeh 	    (big_cmp_abs(&(rsakey.dmodqminus1), &(rsakey.q)) > 0) ||
613*726fad2aSDina K Nimeh 	    (big_cmp_abs(&(rsakey.pinvmodq), &(rsakey.q)) > 0)) {
614*726fad2aSDina K Nimeh 		rv = CKR_KEY_SIZE_RANGE;
615*726fad2aSDina K Nimeh 		goto clean4;
616*726fad2aSDina K Nimeh 	}
617*726fad2aSDina K Nimeh 
618*726fad2aSDina K Nimeh 	/* Perform RSA computation on big integer input data. */
619*726fad2aSDina K Nimeh 	if (big_modexp_crt(&msg, &msg, &(rsakey.dmodpminus1),
620*726fad2aSDina K Nimeh 	    &(rsakey.dmodqminus1), &(rsakey.p), &(rsakey.q),
621*726fad2aSDina K Nimeh 	    &(rsakey.pinvmodq), NULL, NULL) != BIG_OK) {
622*726fad2aSDina K Nimeh 		rv = CKR_HOST_MEMORY;
623*726fad2aSDina K Nimeh 		goto clean4;
624*726fad2aSDina K Nimeh 	}
625*726fad2aSDina K Nimeh 
626*726fad2aSDina K Nimeh 	/* Convert the big integer output data to octet string. */
627*726fad2aSDina K Nimeh 	bignum2bytestring(out, &msg, modulus_bytes);
628*726fad2aSDina K Nimeh 
629*726fad2aSDina K Nimeh clean4:
630*726fad2aSDina K Nimeh 	big_finish(&msg);
631*726fad2aSDina K Nimeh clean3:
632*726fad2aSDina K Nimeh 	RSA_key_finish(&rsakey);
6337c478bd9Sstevel@tonic-gate 
634*726fad2aSDina K Nimeh 	return (rv);
6357c478bd9Sstevel@tonic-gate }
636