xref: /titanic_51/usr/src/common/crypto/rsa/rsa_impl.c (revision 5d54f3d8999eac1762fe0a8c7177d20f1f201fae)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * This file contains RSA helper routines common to
31  * the PKCS11 soft token code and the kernel RSA code.
32  */
33 
34 #include <sys/types.h>
35 #include "rsa_impl.h"
36 
37 #ifdef _KERNEL
38 #include <sys/param.h>
39 #else
40 #include <strings.h>
41 #include "softRandom.h"
42 #endif
43 
44 /*
45  * DER encoding T of the DigestInfo values for MD5, SHA1, and SHA2
46  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
47  *
48  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
49  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
50  * SHA-256: (0x)30 31 30 0d 06 09 60 86 48 01 65 03 04 02 01 05 00 04 20 || H.
51  * SHA-384: (0x)30 41 30 0d 06 09 60 86 48 01 65 03 04 02 02 05 00 04 30 || H.
52  * SHA-512: (0x)30 51 30 0d 06 09 60 86 48 01 65 03 04 02 03 05 00 04 40 || H.
53  *
54  * Where H is the digested output from MD5 or SHA1. We define the constant
55  * byte array (the prefix) here and use it rather than doing the DER
56  * encoding of the OID in a separate routine.
57  */
58 const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
59     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
60     0x04, 0x10};
61 
62 const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
63     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
64 
65 const CK_BYTE SHA256_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x31, 0x30, 0x0d,
66     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
67     0x00, 0x04, 0x20};
68 
69 const CK_BYTE SHA384_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x41, 0x30, 0x0d,
70     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05,
71     0x00, 0x04, 0x30};
72 
73 const CK_BYTE SHA512_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x51, 0x30, 0x0d,
74     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05,
75     0x00, 0x04, 0x40};
76 
77 BIG_ERR_CODE
78 RSA_key_init(RSAkey *key, int psize, int qsize)
79 {
80 	BIG_ERR_CODE err = BIG_OK;
81 
82 /* EXPORT DELETE START */
83 
84 	int plen, qlen, nlen;
85 
86 	plen = (psize + 31) / 32;
87 	qlen = (qsize + 31) / 32;
88 	nlen = plen + qlen;
89 	key->size = psize + qsize;
90 	if ((err = big_init(&(key->p), plen)) != BIG_OK)
91 		return (err);
92 	if ((err = big_init(&(key->q), qlen)) != BIG_OK)
93 		goto ret1;
94 	if ((err = big_init(&(key->n), nlen)) != BIG_OK)
95 		goto ret2;
96 	if ((err = big_init(&(key->d), nlen)) != BIG_OK)
97 		goto ret3;
98 	if ((err = big_init(&(key->e), nlen)) != BIG_OK)
99 		goto ret4;
100 	if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
101 		goto ret5;
102 	if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
103 		goto ret6;
104 	if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
105 		goto ret7;
106 	if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
107 		goto ret8;
108 	if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
109 		goto ret9;
110 	if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
111 		goto ret10;
112 
113 	return (BIG_OK);
114 
115 ret10:
116 	big_finish(&(key->q_rr));
117 ret9:
118 	big_finish(&(key->p_rr));
119 ret8:
120 	big_finish(&(key->pinvmodq));
121 ret7:
122 	big_finish(&(key->dmodqminus1));
123 ret6:
124 	big_finish(&(key->dmodpminus1));
125 ret5:
126 	big_finish(&(key->e));
127 ret4:
128 	big_finish(&(key->d));
129 ret3:
130 	big_finish(&(key->n));
131 ret2:
132 	big_finish(&(key->q));
133 ret1:
134 	big_finish(&(key->p));
135 
136 /* EXPORT DELETE END */
137 
138 	return (err);
139 }
140 
141 
142 void
143 RSA_key_finish(RSAkey *key)
144 {
145 
146 /* EXPORT DELETE START */
147 
148 	big_finish(&(key->n_rr));
149 	big_finish(&(key->q_rr));
150 	big_finish(&(key->p_rr));
151 	big_finish(&(key->pinvmodq));
152 	big_finish(&(key->dmodqminus1));
153 	big_finish(&(key->dmodpminus1));
154 	big_finish(&(key->e));
155 	big_finish(&(key->d));
156 	big_finish(&(key->n));
157 	big_finish(&(key->q));
158 	big_finish(&(key->p));
159 
160 /* EXPORT DELETE END */
161 
162 }
163 
164 
165 /*
166  * To create a block type "02" encryption block for RSA PKCS encryption
167  * process.
168  *
169  * The RSA PKCS Padding before encryption is in the following format:
170  * +------+--------------------+----+-----------------------------+
171  * |0x0002| 8 bytes or more RN |0x00|       DATA                  |
172  * +------+--------------------+----+-----------------------------+
173  *
174  */
175 CK_RV
176 soft_encrypt_rsa_pkcs_encode(uint8_t *databuf,
177     size_t datalen, uint8_t *padbuf, size_t padbuflen)
178 {
179 
180 /* EXPORT DELETE START */
181 
182 	size_t	padlen;
183 	CK_RV	rv;
184 
185 	padlen = padbuflen - datalen;
186 	if (padlen < MIN_PKCS1_PADLEN) {
187 		return (CKR_DATA_LEN_RANGE);
188 	}
189 
190 	/* Pad with 0x0002+non-zero pseudorandom numbers+0x00. */
191 	padbuf[0] = 0x00;
192 	padbuf[1] = 0x02;
193 #ifdef _KERNEL
194 	rv = knzero_random_generator(padbuf + 2, padbuflen - 3);
195 #else
196 	rv = soft_nzero_random_generator(padbuf + 2, padbuflen - 3);
197 #endif
198 	if (rv != CKR_OK) {
199 		return (rv);
200 	}
201 	padbuf[padlen - 1] = 0x00;
202 
203 	bcopy(databuf, padbuf + padlen, datalen);
204 
205 /* EXPORT DELETE END */
206 
207 	return (CKR_OK);
208 }
209 
210 
211 /*
212  * The RSA PKCS Padding after decryption is in the following format:
213  * +------+--------------------+----+-----------------------------+
214  * |0x0002| 8 bytes or more RN |0x00|       DATA                  |
215  * +------+--------------------+----+-----------------------------+
216  *
217  * 'padbuf' points to the recovered message which is the modulus
218  * length. As a result, 'plen' is changed to hold the actual data length.
219  */
220 CK_RV
221 soft_decrypt_rsa_pkcs_decode(uint8_t *padbuf, int *plen)
222 {
223 
224 /* EXPORT DELETE START */
225 
226 	int	i;
227 
228 	/* Check to see if the recovered data is padded is 0x0002. */
229 	if (padbuf[0] != 0x00 || padbuf[1] != 0x02) {
230 		return (CKR_ENCRYPTED_DATA_INVALID);
231 	}
232 
233 	/* Remove all the random bits up to 0x00 (= NULL char) */
234 	for (i = 2; (*plen - i) > 0; i++) {
235 		if (padbuf[i] == 0x00) {
236 			i++;
237 			if (i < MIN_PKCS1_PADLEN) {
238 				return (CKR_ENCRYPTED_DATA_INVALID);
239 			}
240 			*plen -= i;
241 
242 			return (CKR_OK);
243 		}
244 	}
245 
246 /* EXPORT DELETE END */
247 
248 	return (CKR_ENCRYPTED_DATA_INVALID);
249 }
250 
251 /*
252  * To create a block type "01" block for RSA PKCS signature process.
253  *
254  * The RSA PKCS Padding before Signing is in the following format:
255  * +------+--------------+----+-----------------------------+
256  * |0x0001| 0xFFFF.......|0x00|          DATA               |
257  * +------+--------------+----+-----------------------------+
258  */
259 CK_RV
260 soft_sign_rsa_pkcs_encode(uint8_t *pData, size_t dataLen, uint8_t *data,
261     size_t mbit_l)
262 {
263 
264 /* EXPORT DELETE START */
265 
266 	size_t	padlen;
267 
268 	padlen = mbit_l - dataLen;
269 	if (padlen < MIN_PKCS1_PADLEN) {
270 		return (CKR_DATA_LEN_RANGE);
271 	}
272 
273 	padlen -= 3;
274 	data[0] = 0x00;
275 	data[1] = 0x01;
276 #ifdef _KERNEL
277 	kmemset(data + 2, 0xFF, padlen);
278 #else
279 	(void) memset(data + 2, 0xFF, padlen);
280 #endif
281 	data[padlen + 2] = 0x00;
282 	bcopy(pData, data + padlen + 3, dataLen);
283 
284 /* EXPORT DELETE END */
285 
286 	return (CKR_OK);
287 }
288 
289 
290 CK_RV
291 soft_verify_rsa_pkcs_decode(uint8_t *data, int *mbit_l)
292 {
293 
294 /* EXPORT DELETE START */
295 
296 	int i;
297 
298 	/* Check to see if the padding of recovered data starts with 0x0001. */
299 	if ((data[0] != 0x00) || (data[1] != 0x01)) {
300 		return (CKR_SIGNATURE_INVALID);
301 	}
302 	/* Check to see if the recovered data is padded with 0xFFF...00. */
303 	for (i = 2; i < *mbit_l; i++) {
304 		if (data[i] == 0x00) {
305 			i++;
306 			if (i < MIN_PKCS1_PADLEN) {
307 				return (CKR_SIGNATURE_INVALID);
308 			}
309 			*mbit_l -= i;
310 
311 			return (CKR_OK);
312 		} else if (data[i] != 0xFF) {
313 			return (CKR_SIGNATURE_INVALID);
314 		}
315 	}
316 
317 /* EXPORT DELETE END */
318 
319 	return (CKR_SIGNATURE_INVALID);
320 }
321