xref: /titanic_52/usr/src/common/crypto/rsa/rsa_impl.c (revision bdfc6d18da790deeec2e0eb09c625902defe2498)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * This file contains RSA helper routines common to
31  * the PKCS11 soft token code and the kernel RSA code.
32  */
33 
34 #include <sys/types.h>
35 #include "rsa_impl.h"
36 
37 #ifdef _KERNEL
38 #include <sys/param.h>
39 #else
40 #include <strings.h>
41 #include "softRandom.h"
42 #endif
43 
44 /*
45  * DER encoding T of the DigestInfo values for MD5 and SHA1
46  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
47  *
48  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
49  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
50  *
51  * Where H is the digested output from MD5 or SHA1. We define the constant
52  * byte array (the prefix) here and use it rather than doing the DER
53  * encoding of the OID in a separate routine.
54  */
55 const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
56     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
57     0x04, 0x10};
58 
59 const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
60     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
61 
62 BIG_ERR_CODE
63 RSA_key_init(RSAkey *key, int psize, int qsize)
64 {
65 	BIG_ERR_CODE err = BIG_OK;
66 
67 /* EXPORT DELETE START */
68 
69 	int plen, qlen, nlen;
70 
71 	plen = (psize + 31) / 32;
72 	qlen = (qsize + 31) / 32;
73 	nlen = plen + qlen;
74 	key->size = psize + qsize;
75 	if ((err = big_init(&(key->p), plen)) != BIG_OK)
76 		return (err);
77 	if ((err = big_init(&(key->q), qlen)) != BIG_OK)
78 		goto ret1;
79 	if ((err = big_init(&(key->n), nlen)) != BIG_OK)
80 		goto ret2;
81 	if ((err = big_init(&(key->d), nlen)) != BIG_OK)
82 		goto ret3;
83 	if ((err = big_init(&(key->e), nlen)) != BIG_OK)
84 		goto ret4;
85 	if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
86 		goto ret5;
87 	if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
88 		goto ret6;
89 	if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
90 		goto ret7;
91 	if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
92 		goto ret8;
93 	if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
94 		goto ret9;
95 	if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
96 		goto ret10;
97 
98 	return (BIG_OK);
99 
100 ret10:
101 	big_finish(&(key->q_rr));
102 ret9:
103 	big_finish(&(key->p_rr));
104 ret8:
105 	big_finish(&(key->pinvmodq));
106 ret7:
107 	big_finish(&(key->dmodqminus1));
108 ret6:
109 	big_finish(&(key->dmodpminus1));
110 ret5:
111 	big_finish(&(key->e));
112 ret4:
113 	big_finish(&(key->d));
114 ret3:
115 	big_finish(&(key->n));
116 ret2:
117 	big_finish(&(key->q));
118 ret1:
119 	big_finish(&(key->p));
120 
121 /* EXPORT DELETE END */
122 
123 	return (err);
124 }
125 
126 
127 void
128 RSA_key_finish(RSAkey *key)
129 {
130 
131 /* EXPORT DELETE START */
132 
133 	big_finish(&(key->n_rr));
134 	big_finish(&(key->q_rr));
135 	big_finish(&(key->p_rr));
136 	big_finish(&(key->pinvmodq));
137 	big_finish(&(key->dmodqminus1));
138 	big_finish(&(key->dmodpminus1));
139 	big_finish(&(key->e));
140 	big_finish(&(key->d));
141 	big_finish(&(key->n));
142 	big_finish(&(key->q));
143 	big_finish(&(key->p));
144 
145 /* EXPORT DELETE END */
146 
147 }
148 
149 
150 /*
151  * To create a block type "02" encryption block for RSA PKCS encryption
152  * process.
153  *
154  * The RSA PKCS Padding before encryption is in the following format:
155  * +------+--------------------+----+-----------------------------+
156  * |0x0002| 8 bytes or more RN |0x00|       DATA                  |
157  * +------+--------------------+----+-----------------------------+
158  *
159  */
160 CK_RV
161 soft_encrypt_rsa_pkcs_encode(uint8_t *databuf,
162     size_t datalen, uint8_t *padbuf, size_t padbuflen)
163 {
164 
165 /* EXPORT DELETE START */
166 
167 	size_t	padlen;
168 	CK_RV	rv;
169 
170 	padlen = padbuflen - datalen;
171 	if (padlen < MIN_PKCS1_PADLEN) {
172 		return (CKR_DATA_LEN_RANGE);
173 	}
174 
175 	/* Pad with 0x0002+non-zero pseudorandom numbers+0x00. */
176 	padbuf[0] = 0x00;
177 	padbuf[1] = 0x02;
178 #ifdef _KERNEL
179 	rv = knzero_random_generator(padbuf + 2, padbuflen - 3);
180 #else
181 	rv = soft_nzero_random_generator(padbuf + 2, padbuflen - 3);
182 #endif
183 	if (rv != CKR_OK) {
184 		return (rv);
185 	}
186 	padbuf[padlen - 1] = 0x00;
187 
188 	bcopy(databuf, padbuf + padlen, datalen);
189 
190 /* EXPORT DELETE END */
191 
192 	return (CKR_OK);
193 }
194 
195 
196 /*
197  * The RSA PKCS Padding after decryption is in the following format:
198  * +------+--------------------+----+-----------------------------+
199  * |0x0002| 8 bytes or more RN |0x00|       DATA                  |
200  * +------+--------------------+----+-----------------------------+
201  *
202  * 'padbuf' points to the recovered message which is the modulus
203  * length. As a result, 'plen' is changed to hold the actual data length.
204  */
205 CK_RV
206 soft_decrypt_rsa_pkcs_decode(uint8_t *padbuf, int *plen)
207 {
208 
209 /* EXPORT DELETE START */
210 
211 	int	i;
212 
213 	/* Check to see if the recovered data is padded is 0x0002. */
214 	if (padbuf[0] != 0x00 || padbuf[1] != 0x02) {
215 		return (CKR_ENCRYPTED_DATA_INVALID);
216 	}
217 
218 	/* Remove all the random bits up to 0x00 (= NULL char) */
219 	for (i = 2; (*plen - i) > 0; i++) {
220 		if (padbuf[i] == 0x00) {
221 			i++;
222 			if (i < MIN_PKCS1_PADLEN) {
223 				return (CKR_ENCRYPTED_DATA_INVALID);
224 			}
225 			*plen -= i;
226 
227 			return (CKR_OK);
228 		}
229 	}
230 
231 /* EXPORT DELETE END */
232 
233 	return (CKR_ENCRYPTED_DATA_INVALID);
234 }
235 
236 /*
237  * To create a block type "01" block for RSA PKCS signature process.
238  *
239  * The RSA PKCS Padding before Signing is in the following format:
240  * +------+--------------+----+-----------------------------+
241  * |0x0001| 0xFFFF.......|0x00|          DATA               |
242  * +------+--------------+----+-----------------------------+
243  */
244 CK_RV
245 soft_sign_rsa_pkcs_encode(uint8_t *pData, size_t dataLen, uint8_t *data,
246     size_t mbit_l)
247 {
248 
249 /* EXPORT DELETE START */
250 
251 	size_t	padlen;
252 
253 	padlen = mbit_l - dataLen;
254 	if (padlen < MIN_PKCS1_PADLEN) {
255 		return (CKR_DATA_LEN_RANGE);
256 	}
257 
258 	padlen -= 3;
259 	data[0] = 0x00;
260 	data[1] = 0x01;
261 #ifdef _KERNEL
262 	kmemset(data + 2, 0xFF, padlen);
263 #else
264 	(void) memset(data + 2, 0xFF, padlen);
265 #endif
266 	data[padlen + 2] = 0x00;
267 	bcopy(pData, data + padlen + 3, dataLen);
268 
269 /* EXPORT DELETE END */
270 
271 	return (CKR_OK);
272 }
273 
274 
275 CK_RV
276 soft_verify_rsa_pkcs_decode(uint8_t *data, int *mbit_l)
277 {
278 
279 /* EXPORT DELETE START */
280 
281 	int i;
282 
283 	/* Check to see if the padding of recovered data starts with 0x0001. */
284 	if ((data[0] != 0x00) || (data[1] != 0x01)) {
285 		return (CKR_SIGNATURE_INVALID);
286 	}
287 	/* Check to see if the recovered data is padded with 0xFFF...00. */
288 	for (i = 2; i < *mbit_l; i++) {
289 		if (data[i] == 0x00) {
290 			i++;
291 			if (i < MIN_PKCS1_PADLEN) {
292 				return (CKR_SIGNATURE_INVALID);
293 			}
294 			*mbit_l -= i;
295 
296 			return (CKR_OK);
297 		} else if (data[i] != 0xFF) {
298 			return (CKR_SIGNATURE_INVALID);
299 		}
300 	}
301 
302 /* EXPORT DELETE END */
303 
304 	return (CKR_SIGNATURE_INVALID);
305 }
306