1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2023 RackTop Systems, Inc.
25 */
26
27 /*
28 * RSA provider for the Kernel Cryptographic Framework (KCF)
29 */
30
31 #include <sys/types.h>
32 #include <sys/systm.h>
33 #include <sys/modctl.h>
34 #include <sys/cmn_err.h>
35 #include <sys/ddi.h>
36 #include <sys/crypto/spi.h>
37 #include <sys/sysmacros.h>
38 #include <sys/strsun.h>
39 #include <sys/md5.h>
40 #include <sys/sha1.h>
41 #define _SHA2_IMPL
42 #include <sys/sha2.h>
43 #include <sys/random.h>
44 #include <sys/crypto/impl.h>
45 #include <sha1/sha1_impl.h>
46 #include <sha2/sha2_impl.h>
47 #include <padding/padding.h>
48 #include <rsa/rsa_impl.h>
49
50 extern struct mod_ops mod_cryptoops;
51
52 /*
53 * Module linkage information for the kernel.
54 */
55 static struct modlcrypto modlcrypto = {
56 &mod_cryptoops,
57 "RSA Kernel SW Provider"
58 };
59
60 static struct modlinkage modlinkage = {
61 MODREV_1,
62 (void *)&modlcrypto,
63 NULL
64 };
65
66 /*
67 * CSPI information (entry points, provider info, etc.)
68 */
69 typedef enum rsa_mech_type {
70 RSA_PKCS_MECH_INFO_TYPE, /* SUN_CKM_RSA_PKCS */
71 RSA_X_509_MECH_INFO_TYPE, /* SUN_CKM_RSA_X_509 */
72 MD5_RSA_PKCS_MECH_INFO_TYPE, /* SUN_MD5_RSA_PKCS */
73 SHA1_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA1_RSA_PKCS */
74 SHA256_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA256_RSA_PKCS */
75 SHA384_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA384_RSA_PKCS */
76 SHA512_RSA_PKCS_MECH_INFO_TYPE /* SUN_SHA512_RSA_PKCS */
77 } rsa_mech_type_t;
78
79 /*
80 * Context for RSA_PKCS and RSA_X_509 mechanisms.
81 */
82 typedef struct rsa_ctx {
83 rsa_mech_type_t mech_type;
84 crypto_key_t *key;
85 size_t keychunk_size;
86 } rsa_ctx_t;
87
88 /*
89 * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
90 */
91 typedef struct digest_rsa_ctx {
92 rsa_mech_type_t mech_type;
93 crypto_key_t *key;
94 size_t keychunk_size;
95 union {
96 MD5_CTX md5ctx;
97 SHA1_CTX sha1ctx;
98 SHA2_CTX sha2ctx;
99 } dctx_u;
100 } digest_rsa_ctx_t;
101
102 #define md5_ctx dctx_u.md5ctx
103 #define sha1_ctx dctx_u.sha1ctx
104 #define sha2_ctx dctx_u.sha2ctx
105
106 /*
107 * Mechanism info structure passed to KCF during registration.
108 */
109 static crypto_mech_info_t rsa_mech_info_tab[] = {
110 /* RSA_PKCS */
111 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
112 CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
113 CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
114 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
115 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
116 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
117 CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
118 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
119
120 /* RSA_X_509 */
121 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
122 CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
123 CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
124 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
125 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
126 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
127 CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
128 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
129
130 /* MD5_RSA_PKCS */
131 {SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
132 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
133 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
134 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
135
136 /* SHA1_RSA_PKCS */
137 {SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
138 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
139 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
140 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
141
142 /* SHA256_RSA_PKCS */
143 {SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
144 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
145 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
146 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
147
148 /* SHA384_RSA_PKCS */
149 {SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
150 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
151 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
152 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
153
154 /* SHA512_RSA_PKCS */
155 {SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
156 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
157 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
158 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
159
160 };
161
162 #define RSA_VALID_MECH(mech) \
163 (((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE || \
164 (mech)->cm_type == RSA_X_509_MECH_INFO_TYPE || \
165 (mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE || \
166 (mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE || \
167 (mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE || \
168 (mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE || \
169 (mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
170
171 /* operations are in-place if the output buffer is NULL */
172 #define RSA_ARG_INPLACE(input, output) \
173 if ((output) == NULL) \
174 (output) = (input);
175
176 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
177
178 static crypto_control_ops_t rsa_control_ops = {
179 rsa_provider_status
180 };
181
182 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
183 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
184 static int rsaprov_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
185 crypto_req_handle_t);
186 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
187 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
188 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
189 static int rsaprov_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
190 crypto_req_handle_t);
191 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
192 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
193 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
194
195 /*
196 * The RSA mechanisms do not have multiple-part cipher operations.
197 * So, the update and final routines are set to NULL.
198 */
199 static crypto_cipher_ops_t rsa_cipher_ops = {
200 rsa_common_init,
201 rsaprov_encrypt,
202 NULL,
203 NULL,
204 rsa_encrypt_atomic,
205 rsa_common_init,
206 rsaprov_decrypt,
207 NULL,
208 NULL,
209 rsa_decrypt_atomic
210 };
211
212 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
213 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
214 static int rsaprov_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
215 crypto_req_handle_t);
216 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
217 crypto_req_handle_t);
218 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
219 crypto_req_handle_t);
220 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
221 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
222 crypto_spi_ctx_template_t, crypto_req_handle_t);
223
224 /*
225 * We use the same routine for sign_init and sign_recover_init fields
226 * as they do the same thing. Same holds for sign and sign_recover fields,
227 * and sign_atomic and sign_recover_atomic fields.
228 */
229 static crypto_sign_ops_t rsa_sign_ops = {
230 rsa_sign_verify_common_init,
231 rsaprov_sign,
232 rsa_sign_update,
233 rsa_sign_final,
234 rsa_sign_atomic,
235 rsa_sign_verify_common_init,
236 rsaprov_sign,
237 rsa_sign_atomic
238 };
239
240 static int rsaprov_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
241 crypto_req_handle_t);
242 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
243 crypto_req_handle_t);
244 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
245 crypto_req_handle_t);
246 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
247 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
248 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
249 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
250 crypto_data_t *, crypto_req_handle_t);
251 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
252 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
253 crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
254 crypto_req_handle_t);
255
256 /*
257 * We use the same routine (rsa_sign_verify_common_init) for verify_init
258 * and verify_recover_init fields as they do the same thing.
259 */
260 static crypto_verify_ops_t rsa_verify_ops = {
261 rsa_sign_verify_common_init,
262 rsaprov_verify,
263 rsa_verify_update,
264 rsa_verify_final,
265 rsa_verify_atomic,
266 rsa_sign_verify_common_init,
267 rsa_verify_recover,
268 rsa_verify_recover_atomic
269 };
270
271 static int rsa_free_context(crypto_ctx_t *);
272
273 static crypto_ctx_ops_t rsa_ctx_ops = {
274 NULL,
275 rsa_free_context
276 };
277
278 static crypto_ops_t rsa_crypto_ops = {
279 &rsa_control_ops,
280 NULL,
281 &rsa_cipher_ops,
282 NULL,
283 &rsa_sign_ops,
284 &rsa_verify_ops,
285 NULL,
286 NULL,
287 NULL,
288 NULL,
289 NULL,
290 NULL,
291 NULL,
292 &rsa_ctx_ops,
293 NULL,
294 NULL,
295 NULL,
296 };
297
298 static crypto_provider_info_t rsa_prov_info = {
299 CRYPTO_SPI_VERSION_4,
300 "RSA Software Provider",
301 CRYPTO_SW_PROVIDER,
302 {&modlinkage},
303 NULL,
304 &rsa_crypto_ops,
305 sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
306 rsa_mech_info_tab
307 };
308
309 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
310 crypto_data_t *, crypto_data_t *);
311 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
312 crypto_data_t *, crypto_data_t *);
313 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
314 crypto_data_t *, crypto_data_t *);
315 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
316 crypto_data_t *, crypto_data_t *);
317 static int compare_data(crypto_data_t *, uchar_t *);
318
319 static int core_rsa_encrypt(crypto_key_t *, uchar_t *, int, uchar_t *, int);
320 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int, uchar_t *);
321
322 static crypto_kcf_provider_handle_t rsa_prov_handle = 0;
323
324 int
_init(void)325 _init(void)
326 {
327 int ret;
328
329 if ((ret = mod_install(&modlinkage)) != 0)
330 return (ret);
331
332 /* Register with KCF. If the registration fails, remove the module. */
333 if (crypto_register_provider(&rsa_prov_info, &rsa_prov_handle)) {
334 (void) mod_remove(&modlinkage);
335 return (EACCES);
336 }
337
338 return (0);
339 }
340
341 int
_fini(void)342 _fini(void)
343 {
344 /* Unregister from KCF if module is registered */
345 if (rsa_prov_handle != 0) {
346 if (crypto_unregister_provider(rsa_prov_handle))
347 return (EBUSY);
348
349 rsa_prov_handle = 0;
350 }
351
352 return (mod_remove(&modlinkage));
353 }
354
355 int
_info(struct modinfo * modinfop)356 _info(struct modinfo *modinfop)
357 {
358 return (mod_info(&modlinkage, modinfop));
359 }
360
361 /* ARGSUSED */
362 static void
rsa_provider_status(crypto_provider_handle_t provider,uint_t * status)363 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
364 {
365 *status = CRYPTO_PROVIDER_READY;
366 }
367
368 static int
check_mech_and_key(crypto_mechanism_t * mechanism,crypto_key_t * key)369 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
370 {
371 int rv = CRYPTO_FAILED;
372
373 uchar_t *modulus;
374 ssize_t modulus_len; /* In bytes */
375
376 if (!RSA_VALID_MECH(mechanism))
377 return (CRYPTO_MECHANISM_INVALID);
378
379 /*
380 * We only support RSA keys that are passed as a list of
381 * object attributes.
382 */
383 if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
384 return (CRYPTO_KEY_TYPE_INCONSISTENT);
385 }
386
387 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
388 &modulus_len)) != CRYPTO_SUCCESS) {
389 return (rv);
390 }
391 if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
392 modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
393 return (CRYPTO_KEY_SIZE_RANGE);
394
395 return (rv);
396 }
397
398 void
kmemset(uint8_t * buf,char pattern,size_t len)399 kmemset(uint8_t *buf, char pattern, size_t len)
400 {
401 int i = 0;
402
403 while (i < len)
404 buf[i++] = pattern;
405 }
406
407 /*
408 * This function guarantees to return non-zero random numbers.
409 * This is needed as the /dev/urandom kernel interface,
410 * random_get_pseudo_bytes(), may return zeros.
411 */
412 int
knzero_random_generator(uint8_t * ran_out,size_t ran_len)413 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
414 {
415 int rv;
416 size_t ebc = 0; /* count of extra bytes in extrarand */
417 size_t i = 0;
418 uint8_t extrarand[32];
419 size_t extrarand_len;
420
421 if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
422 return (rv);
423
424 /*
425 * Walk through the returned random numbers pointed by ran_out,
426 * and look for any random number which is zero.
427 * If we find zero, call random_get_pseudo_bytes() to generate
428 * another 32 random numbers pool. Replace any zeros in ran_out[]
429 * from the random number in pool.
430 */
431 while (i < ran_len) {
432 if (ran_out[i] != 0) {
433 i++;
434 continue;
435 }
436
437 /*
438 * Note that it is 'while' so we are guaranteed a
439 * non-zero value on exit.
440 */
441 if (ebc == 0) {
442 /* refresh extrarand */
443 extrarand_len = sizeof (extrarand);
444 if ((rv = random_get_pseudo_bytes(extrarand,
445 extrarand_len)) != 0) {
446 return (rv);
447 }
448
449 ebc = extrarand_len;
450 }
451 /* Replace zero with byte from extrarand. */
452 -- ebc;
453
454 /*
455 * The new random byte zero/non-zero will be checked in
456 * the next pass through the loop.
457 */
458 ran_out[i] = extrarand[ebc];
459 }
460
461 return (CRYPTO_SUCCESS);
462 }
463
464 static int
compare_data(crypto_data_t * data,uchar_t * buf)465 compare_data(crypto_data_t *data, uchar_t *buf)
466 {
467 return (crypto_compare_data(data, buf, data->cd_length));
468 }
469
470 /* ARGSUSED */
471 static int
rsa_common_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t template,crypto_req_handle_t req)472 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
473 crypto_key_t *key, crypto_spi_ctx_template_t template,
474 crypto_req_handle_t req)
475 {
476 int rv;
477 int kmflag;
478 rsa_ctx_t *ctxp;
479
480 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
481 return (rv);
482
483 /*
484 * Allocate a RSA context.
485 */
486 kmflag = crypto_kmflag(req);
487 if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
488 return (CRYPTO_HOST_MEMORY);
489
490 if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
491 kmflag)) != CRYPTO_SUCCESS) {
492 kmem_free(ctxp, sizeof (rsa_ctx_t));
493 return (rv);
494 }
495 ctxp->mech_type = mechanism->cm_type;
496
497 ctx->cc_provider_private = ctxp;
498
499 return (CRYPTO_SUCCESS);
500 }
501
502 /* ARGSUSED */
503 static int
rsaprov_encrypt(crypto_ctx_t * ctx,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_req_handle_t req)504 rsaprov_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
505 crypto_data_t *ciphertext, crypto_req_handle_t req)
506 {
507 int rv;
508 rsa_ctx_t *ctxp;
509
510 ASSERT(ctx->cc_provider_private != NULL);
511 ctxp = ctx->cc_provider_private;
512
513 RSA_ARG_INPLACE(plaintext, ciphertext);
514
515 /*
516 * Note on the KM_SLEEP flag passed to the routine below -
517 * rsaprov_encrypt() is a single-part encryption routine which is
518 * currently usable only by /dev/crypto. Since /dev/crypto calls are
519 * always synchronous, we can safely pass KM_SLEEP here.
520 */
521 rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
522 ciphertext);
523
524 if (rv != CRYPTO_BUFFER_TOO_SMALL)
525 (void) rsa_free_context(ctx);
526
527 return (rv);
528 }
529
530 /* ARGSUSED */
531 static int
rsa_encrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_spi_ctx_template_t template,crypto_req_handle_t req)532 rsa_encrypt_atomic(crypto_provider_handle_t provider,
533 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
534 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
535 crypto_spi_ctx_template_t template, crypto_req_handle_t req)
536 {
537 int rv;
538
539 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
540 return (rv);
541 RSA_ARG_INPLACE(plaintext, ciphertext);
542
543 return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
544 ciphertext));
545 }
546
547 static int
rsa_free_context(crypto_ctx_t * ctx)548 rsa_free_context(crypto_ctx_t *ctx)
549 {
550 rsa_ctx_t *ctxp = ctx->cc_provider_private;
551
552 if (ctxp != NULL) {
553 bzero(ctxp->key, ctxp->keychunk_size);
554 kmem_free(ctxp->key, ctxp->keychunk_size);
555
556 if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
557 ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
558 kmem_free(ctxp, sizeof (rsa_ctx_t));
559 else
560 kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
561
562 ctx->cc_provider_private = NULL;
563 }
564
565 return (CRYPTO_SUCCESS);
566 }
567
568 static int
rsa_encrypt_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * plaintext,crypto_data_t * ciphertext)569 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
570 crypto_data_t *plaintext, crypto_data_t *ciphertext)
571 {
572 int rv = CRYPTO_FAILED;
573
574 int plen;
575 uchar_t *ptptr;
576 uchar_t *modulus;
577 ssize_t modulus_len;
578 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
579 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
580 uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
581
582 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
583 &modulus_len)) != CRYPTO_SUCCESS) {
584 return (rv);
585 }
586
587 plen = plaintext->cd_length;
588 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
589 if (plen > (modulus_len - MIN_PKCS1_PADLEN))
590 return (CRYPTO_DATA_LEN_RANGE);
591 } else {
592 if (plen > modulus_len)
593 return (CRYPTO_DATA_LEN_RANGE);
594 }
595
596 /*
597 * Output buf len must not be less than RSA modulus size.
598 */
599 if (ciphertext->cd_length < modulus_len) {
600 ciphertext->cd_length = modulus_len;
601 return (CRYPTO_BUFFER_TOO_SMALL);
602 }
603
604 ASSERT(plaintext->cd_length <= sizeof (tmp_data));
605 if ((rv = crypto_get_input_data(plaintext, &ptptr, tmp_data))
606 != CRYPTO_SUCCESS)
607 return (rv);
608
609 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
610 rv = pkcs1_encode(PKCS1_ENCRYPT, ptptr, plen,
611 plain_data, modulus_len);
612
613 if (rv != CRYPTO_SUCCESS)
614 return (rv);
615 } else {
616 bzero(plain_data, modulus_len - plen);
617 bcopy(ptptr, &plain_data[modulus_len - plen], plen);
618 }
619
620 rv = core_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
621 if (rv == CRYPTO_SUCCESS) {
622 /* copy out to ciphertext */
623 if ((rv = crypto_put_output_data(cipher_data,
624 ciphertext, modulus_len)) != CRYPTO_SUCCESS)
625 return (rv);
626
627 ciphertext->cd_length = modulus_len;
628 }
629
630 return (rv);
631 }
632
633 static int
core_rsa_encrypt(crypto_key_t * key,uchar_t * in,int in_len,uchar_t * out,int is_public)634 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
635 int in_len, uchar_t *out, int is_public)
636 {
637 int rv;
638 uchar_t *expo, *modulus;
639 ssize_t expo_len;
640 ssize_t modulus_len;
641 RSAbytekey k;
642
643 if (is_public) {
644 if ((rv = crypto_get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT,
645 &expo, &expo_len)) != CRYPTO_SUCCESS)
646 return (rv);
647 } else {
648 /*
649 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
650 * RSA secret key. See the comments in core_rsa_decrypt
651 * routine which calls this routine with a private key.
652 */
653 if ((rv = crypto_get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT,
654 &expo, &expo_len)) != CRYPTO_SUCCESS)
655 return (rv);
656 }
657
658 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
659 &modulus_len)) != CRYPTO_SUCCESS) {
660 return (rv);
661 }
662
663 k.modulus = modulus;
664 k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
665 k.pubexpo = expo;
666 k.pubexpo_bytes = expo_len;
667 k.rfunc = NULL;
668
669 rv = rsa_encrypt(&k, in, in_len, out);
670
671 return (rv);
672 }
673
674 /* ARGSUSED */
675 static int
rsaprov_decrypt(crypto_ctx_t * ctx,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_req_handle_t req)676 rsaprov_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
677 crypto_data_t *plaintext, crypto_req_handle_t req)
678 {
679 int rv;
680 rsa_ctx_t *ctxp;
681
682 ASSERT(ctx->cc_provider_private != NULL);
683 ctxp = ctx->cc_provider_private;
684
685 RSA_ARG_INPLACE(ciphertext, plaintext);
686
687 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
688 rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
689 ciphertext, plaintext);
690
691 if (rv != CRYPTO_BUFFER_TOO_SMALL)
692 (void) rsa_free_context(ctx);
693
694 return (rv);
695 }
696
697 /* ARGSUSED */
698 static int
rsa_decrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_spi_ctx_template_t template,crypto_req_handle_t req)699 rsa_decrypt_atomic(crypto_provider_handle_t provider,
700 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
701 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
702 crypto_spi_ctx_template_t template, crypto_req_handle_t req)
703 {
704 int rv;
705
706 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
707 return (rv);
708 RSA_ARG_INPLACE(ciphertext, plaintext);
709
710 return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
711 plaintext));
712 }
713
714 static int
rsa_decrypt_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * ciphertext,crypto_data_t * plaintext)715 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
716 crypto_data_t *ciphertext, crypto_data_t *plaintext)
717 {
718 int rv = CRYPTO_FAILED;
719
720 size_t plain_len;
721 uchar_t *ctptr;
722 uchar_t *modulus;
723 ssize_t modulus_len;
724 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
725 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
726
727 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
728 &modulus_len)) != CRYPTO_SUCCESS) {
729 return (rv);
730 }
731
732 /*
733 * Ciphertext length must be equal to RSA modulus size.
734 */
735 if (ciphertext->cd_length != modulus_len)
736 return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
737
738 ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
739 if ((rv = crypto_get_input_data(ciphertext, &ctptr, tmp_data))
740 != CRYPTO_SUCCESS)
741 return (rv);
742
743 rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data);
744 if (rv == CRYPTO_SUCCESS) {
745 plain_len = modulus_len;
746
747 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
748 /* Strip off the PKCS block formatting data. */
749 rv = pkcs1_decode(PKCS1_DECRYPT, plain_data,
750 &plain_len);
751 if (rv != CRYPTO_SUCCESS)
752 return (rv);
753 }
754
755 if (plain_len > plaintext->cd_length) {
756 plaintext->cd_length = plain_len;
757 return (CRYPTO_BUFFER_TOO_SMALL);
758 }
759
760 if ((rv = crypto_put_output_data(
761 plain_data + modulus_len - plain_len,
762 plaintext, plain_len)) != CRYPTO_SUCCESS)
763 return (rv);
764
765 plaintext->cd_length = plain_len;
766 }
767
768 return (rv);
769 }
770
771 static int
core_rsa_decrypt(crypto_key_t * key,uchar_t * in,int in_len,uchar_t * out)772 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len, uchar_t *out)
773 {
774 int rv;
775 uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
776 ssize_t modulus_len;
777 ssize_t prime1_len, prime2_len;
778 ssize_t expo1_len, expo2_len, coef_len;
779 RSAbytekey k;
780
781 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
782 &modulus_len)) != CRYPTO_SUCCESS) {
783 return (rv);
784 }
785
786 /*
787 * The following attributes are not required to be
788 * present in a RSA secret key. If any of them is not present
789 * we call the encrypt routine with a flag indicating use of
790 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
791 * a required attribute for a RSA secret key.
792 */
793 if ((crypto_get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
794 != CRYPTO_SUCCESS) ||
795 (crypto_get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
796 != CRYPTO_SUCCESS) ||
797 (crypto_get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
798 != CRYPTO_SUCCESS) ||
799 (crypto_get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
800 != CRYPTO_SUCCESS) ||
801 (crypto_get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
802 != CRYPTO_SUCCESS)) {
803 return (core_rsa_encrypt(key, in, in_len, out, 0));
804 }
805
806 k.modulus = modulus;
807 k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
808 k.prime1 = prime1;
809 k.prime1_bytes = prime1_len;
810 k.prime2 = prime2;
811 k.prime2_bytes = prime2_len;
812 k.expo1 = expo1;
813 k.expo1_bytes = expo1_len;
814 k.expo2 = expo2;
815 k.expo2_bytes = expo2_len;
816 k.coeff = coef;
817 k.coeff_bytes = coef_len;
818 k.rfunc = NULL;
819
820 rv = rsa_decrypt(&k, in, in_len, out);
821
822 return (rv);
823 }
824
825 /* ARGSUSED */
826 static int
rsa_sign_verify_common_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)827 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
828 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
829 crypto_req_handle_t req)
830 {
831 int rv;
832 int kmflag;
833 rsa_ctx_t *ctxp;
834 digest_rsa_ctx_t *dctxp;
835
836 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
837 return (rv);
838
839 /*
840 * Allocate a RSA context.
841 */
842 kmflag = crypto_kmflag(req);
843 switch (mechanism->cm_type) {
844 case MD5_RSA_PKCS_MECH_INFO_TYPE:
845 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
846 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
847 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
848 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
849 dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
850 ctxp = (rsa_ctx_t *)dctxp;
851 break;
852 default:
853 ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
854 break;
855 }
856
857 if (ctxp == NULL)
858 return (CRYPTO_HOST_MEMORY);
859
860 ctxp->mech_type = mechanism->cm_type;
861 if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
862 kmflag)) != CRYPTO_SUCCESS) {
863 switch (mechanism->cm_type) {
864 case MD5_RSA_PKCS_MECH_INFO_TYPE:
865 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
866 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
867 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
868 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
869 kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
870 break;
871 default:
872 kmem_free(ctxp, sizeof (rsa_ctx_t));
873 break;
874 }
875 return (rv);
876 }
877
878 switch (mechanism->cm_type) {
879 case MD5_RSA_PKCS_MECH_INFO_TYPE:
880 MD5Init(&(dctxp->md5_ctx));
881 break;
882
883 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
884 SHA1Init(&(dctxp->sha1_ctx));
885 break;
886
887 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
888 SHA2Init(SHA256, &(dctxp->sha2_ctx));
889 break;
890
891 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
892 SHA2Init(SHA384, &(dctxp->sha2_ctx));
893 break;
894
895 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
896 SHA2Init(SHA512, &(dctxp->sha2_ctx));
897 break;
898 }
899
900 ctx->cc_provider_private = ctxp;
901
902 return (CRYPTO_SUCCESS);
903 }
904
905 #define SHA1_DIGEST_SIZE 20
906 #define MD5_DIGEST_SIZE 16
907
908 #define INIT_RAW_CRYPTO_DATA(data, base, len, cd_len) \
909 (data).cd_format = CRYPTO_DATA_RAW; \
910 (data).cd_offset = 0; \
911 (data).cd_raw.iov_base = (char *)base; \
912 (data).cd_raw.iov_len = len; \
913 (data).cd_length = cd_len;
914
915 static int
rsa_digest_svrfy_common(digest_rsa_ctx_t * ctxp,crypto_data_t * data,crypto_data_t * signature,uchar_t flag)916 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
917 crypto_data_t *signature, uchar_t flag)
918 {
919 int rv = CRYPTO_FAILED;
920
921 uchar_t digest[SHA512_DIGEST_LENGTH];
922 /* The der_data size is enough for MD5 also */
923 uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
924 ulong_t der_data_len;
925 crypto_data_t der_cd;
926 rsa_mech_type_t mech_type;
927
928 ASSERT(flag & CRYPTO_DO_SIGN || flag & CRYPTO_DO_VERIFY);
929 ASSERT(data != NULL || (flag & CRYPTO_DO_FINAL));
930
931 mech_type = ctxp->mech_type;
932 if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
933 mech_type == RSA_X_509_MECH_INFO_TYPE)
934 return (CRYPTO_MECHANISM_INVALID);
935
936 /*
937 * We need to do the BUFFER_TOO_SMALL check before digesting
938 * the data. No check is needed for verify as signature is not
939 * an output argument for verify.
940 */
941 if (flag & CRYPTO_DO_SIGN) {
942 uchar_t *modulus;
943 ssize_t modulus_len;
944
945 if ((rv = crypto_get_key_attr(ctxp->key, SUN_CKA_MODULUS,
946 &modulus, &modulus_len)) != CRYPTO_SUCCESS) {
947 return (rv);
948 }
949
950 if (signature->cd_length < modulus_len) {
951 signature->cd_length = modulus_len;
952 return (CRYPTO_BUFFER_TOO_SMALL);
953 }
954 }
955
956 if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
957 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
958 digest, MD5Update, MD5Final, flag | CRYPTO_DO_MD5);
959
960 else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
961 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
962 digest, SHA1Update, SHA1Final, flag | CRYPTO_DO_SHA1);
963
964 else
965 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
966 digest, SHA2Update, SHA2Final, flag | CRYPTO_DO_SHA2);
967
968 if (rv != CRYPTO_SUCCESS)
969 return (rv);
970
971
972 /*
973 * Prepare the DER encoding of the DigestInfo value as follows:
974 * MD5: MD5_DER_PREFIX || H
975 * SHA-1: SHA1_DER_PREFIX || H
976 *
977 * See rsa_impl.c for more details.
978 */
979 switch (mech_type) {
980 case MD5_RSA_PKCS_MECH_INFO_TYPE:
981 bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
982 bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
983 der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
984 break;
985
986 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
987 bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
988 bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
989 SHA1_DIGEST_SIZE);
990 der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
991 break;
992
993 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
994 bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
995 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
996 SHA256_DIGEST_LENGTH);
997 der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
998 break;
999
1000 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1001 bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1002 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1003 SHA384_DIGEST_LENGTH);
1004 der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1005 break;
1006
1007 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1008 bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1009 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1010 SHA512_DIGEST_LENGTH);
1011 der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1012 break;
1013 }
1014
1015 INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1016 /*
1017 * Now, we are ready to sign or verify the DER_ENCODED data.
1018 */
1019 if (flag & CRYPTO_DO_SIGN)
1020 rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1021 signature);
1022 else
1023 rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1024 signature);
1025
1026 return (rv);
1027 }
1028
1029 static int
rsa_sign_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature)1030 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1031 crypto_data_t *data, crypto_data_t *signature)
1032 {
1033 int rv = CRYPTO_FAILED;
1034
1035 int dlen;
1036 uchar_t *dataptr, *modulus;
1037 ssize_t modulus_len;
1038 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1039 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1040 uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1041
1042 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1043 &modulus_len)) != CRYPTO_SUCCESS) {
1044 return (rv);
1045 }
1046
1047 dlen = data->cd_length;
1048 switch (mech_type) {
1049 case RSA_PKCS_MECH_INFO_TYPE:
1050 if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1051 return (CRYPTO_DATA_LEN_RANGE);
1052 break;
1053 case RSA_X_509_MECH_INFO_TYPE:
1054 if (dlen > modulus_len)
1055 return (CRYPTO_DATA_LEN_RANGE);
1056 break;
1057 }
1058
1059 if (signature->cd_length < modulus_len) {
1060 signature->cd_length = modulus_len;
1061 return (CRYPTO_BUFFER_TOO_SMALL);
1062 }
1063
1064 ASSERT(data->cd_length <= sizeof (tmp_data));
1065 if ((rv = crypto_get_input_data(data, &dataptr, tmp_data))
1066 != CRYPTO_SUCCESS)
1067 return (rv);
1068
1069 switch (mech_type) {
1070 case RSA_PKCS_MECH_INFO_TYPE:
1071 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1072 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1073 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1074 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1075 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1076 /*
1077 * Add PKCS padding to the input data to format a block
1078 * type "01" encryption block.
1079 */
1080 rv = pkcs1_encode(PKCS1_SIGN, dataptr, dlen, plain_data,
1081 modulus_len);
1082 if (rv != CRYPTO_SUCCESS)
1083 return (rv);
1084
1085 break;
1086
1087 case RSA_X_509_MECH_INFO_TYPE:
1088 bzero(plain_data, modulus_len - dlen);
1089 bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1090 break;
1091 }
1092
1093 rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data);
1094 if (rv == CRYPTO_SUCCESS) {
1095 /* copy out to signature */
1096 if ((rv = crypto_put_output_data(signed_data,
1097 signature, modulus_len)) != CRYPTO_SUCCESS)
1098 return (rv);
1099
1100 signature->cd_length = modulus_len;
1101 }
1102
1103 return (rv);
1104 }
1105
1106 /* ARGSUSED */
1107 static int
rsaprov_sign(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)1108 rsaprov_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1109 crypto_req_handle_t req)
1110 {
1111 int rv;
1112 rsa_ctx_t *ctxp;
1113
1114 ASSERT(ctx->cc_provider_private != NULL);
1115 ctxp = ctx->cc_provider_private;
1116
1117 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1118 switch (ctxp->mech_type) {
1119 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1120 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1121 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1122 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1123 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1124 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1125 signature, CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE |
1126 CRYPTO_DO_FINAL);
1127 break;
1128 default:
1129 rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1130 signature);
1131 break;
1132 }
1133
1134 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1135 (void) rsa_free_context(ctx);
1136
1137 return (rv);
1138 }
1139
1140 /* ARGSUSED */
1141 static int
rsa_sign_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)1142 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1143 {
1144 int rv;
1145 digest_rsa_ctx_t *ctxp;
1146 rsa_mech_type_t mech_type;
1147
1148 ASSERT(ctx->cc_provider_private != NULL);
1149 ctxp = ctx->cc_provider_private;
1150 mech_type = ctxp->mech_type;
1151
1152 if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1153 mech_type == RSA_X_509_MECH_INFO_TYPE)
1154 return (CRYPTO_MECHANISM_INVALID);
1155
1156 if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1157 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1158 NULL, MD5Update, MD5Final,
1159 CRYPTO_DO_MD5 | CRYPTO_DO_UPDATE);
1160
1161 else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1162 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1163 NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1164 CRYPTO_DO_UPDATE);
1165
1166 else
1167 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1168 NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1169 CRYPTO_DO_UPDATE);
1170
1171 return (rv);
1172 }
1173
1174 /* ARGSUSED2 */
1175 static int
rsa_sign_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)1176 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1177 crypto_req_handle_t req)
1178 {
1179 int rv;
1180 digest_rsa_ctx_t *ctxp;
1181
1182 ASSERT(ctx->cc_provider_private != NULL);
1183 ctxp = ctx->cc_provider_private;
1184
1185 rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1186 CRYPTO_DO_SIGN | CRYPTO_DO_FINAL);
1187 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1188 (void) rsa_free_context(ctx);
1189
1190 return (rv);
1191 }
1192
1193 /* ARGSUSED */
1194 static int
rsa_sign_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1195 rsa_sign_atomic(crypto_provider_handle_t provider,
1196 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1197 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1198 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1199 {
1200 int rv;
1201 digest_rsa_ctx_t dctx;
1202
1203 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1204 return (rv);
1205
1206 if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1207 mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1208 rv = rsa_sign_common(mechanism->cm_type, key, data,
1209 signature);
1210
1211 else {
1212 dctx.mech_type = mechanism->cm_type;
1213 dctx.key = key;
1214 switch (mechanism->cm_type) {
1215 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1216 MD5Init(&(dctx.md5_ctx));
1217 break;
1218
1219 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1220 SHA1Init(&(dctx.sha1_ctx));
1221 break;
1222
1223 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1224 SHA2Init(SHA256, &(dctx.sha2_ctx));
1225 break;
1226
1227 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1228 SHA2Init(SHA384, &(dctx.sha2_ctx));
1229 break;
1230
1231 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1232 SHA2Init(SHA512, &(dctx.sha2_ctx));
1233 break;
1234 }
1235
1236 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1237 CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1238 }
1239
1240 return (rv);
1241 }
1242
1243 static int
rsa_verify_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature)1244 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1245 crypto_data_t *data, crypto_data_t *signature)
1246 {
1247 int rv = CRYPTO_FAILED;
1248
1249 uchar_t *sigptr, *modulus;
1250 ssize_t modulus_len;
1251 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1252 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1253
1254 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1255 &modulus_len)) != CRYPTO_SUCCESS) {
1256 return (rv);
1257 }
1258
1259 if (signature->cd_length != modulus_len)
1260 return (CRYPTO_SIGNATURE_LEN_RANGE);
1261
1262 ASSERT(signature->cd_length <= sizeof (tmp_data));
1263 if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1264 != CRYPTO_SUCCESS)
1265 return (rv);
1266
1267 rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1268 if (rv != CRYPTO_SUCCESS)
1269 return (rv);
1270
1271 if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1272 if (compare_data(data, (plain_data + modulus_len
1273 - data->cd_length)) != 0)
1274 rv = CRYPTO_SIGNATURE_INVALID;
1275
1276 } else {
1277 size_t data_len = modulus_len;
1278
1279 /*
1280 * Strip off the encoded padding bytes in front of the
1281 * recovered data, then compare the recovered data with
1282 * the original data.
1283 */
1284 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1285 if (rv != CRYPTO_SUCCESS)
1286 return (rv);
1287
1288 if (data_len != data->cd_length)
1289 return (CRYPTO_SIGNATURE_LEN_RANGE);
1290
1291 if (compare_data(data, (plain_data + modulus_len
1292 - data_len)) != 0)
1293 rv = CRYPTO_SIGNATURE_INVALID;
1294 }
1295
1296 return (rv);
1297 }
1298
1299 /* ARGSUSED */
1300 static int
rsaprov_verify(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)1301 rsaprov_verify(crypto_ctx_t *ctx, crypto_data_t *data,
1302 crypto_data_t *signature, crypto_req_handle_t req)
1303 {
1304 int rv;
1305 rsa_ctx_t *ctxp;
1306
1307 ASSERT(ctx->cc_provider_private != NULL);
1308 ctxp = ctx->cc_provider_private;
1309
1310 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1311 switch (ctxp->mech_type) {
1312 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1313 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1314 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1315 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1316 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1317 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1318 signature, CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE |
1319 CRYPTO_DO_FINAL);
1320 break;
1321 default:
1322 rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1323 signature);
1324 break;
1325 }
1326
1327 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1328 (void) rsa_free_context(ctx);
1329
1330 return (rv);
1331 }
1332
1333 /* ARGSUSED */
1334 static int
rsa_verify_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)1335 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1336 crypto_req_handle_t req)
1337 {
1338 int rv;
1339 digest_rsa_ctx_t *ctxp;
1340
1341 ASSERT(ctx->cc_provider_private != NULL);
1342 ctxp = ctx->cc_provider_private;
1343
1344 switch (ctxp->mech_type) {
1345
1346 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1347 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1348 NULL, MD5Update, MD5Final, CRYPTO_DO_MD5 |
1349 CRYPTO_DO_UPDATE);
1350 break;
1351
1352 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1353 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1354 NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1355 CRYPTO_DO_UPDATE);
1356 break;
1357
1358 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1359 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1360 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1361 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1362 NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1363 CRYPTO_DO_UPDATE);
1364 break;
1365
1366 default:
1367 return (CRYPTO_MECHANISM_INVALID);
1368 }
1369
1370 return (rv);
1371 }
1372
1373 /* ARGSUSED2 */
1374 static int
rsa_verify_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)1375 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1376 crypto_req_handle_t req)
1377 {
1378 int rv;
1379 digest_rsa_ctx_t *ctxp;
1380
1381 ASSERT(ctx->cc_provider_private != NULL);
1382 ctxp = ctx->cc_provider_private;
1383
1384 rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1385 CRYPTO_DO_VERIFY | CRYPTO_DO_FINAL);
1386 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1387 (void) rsa_free_context(ctx);
1388
1389 return (rv);
1390 }
1391
1392
1393 /* ARGSUSED */
1394 static int
rsa_verify_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1395 rsa_verify_atomic(crypto_provider_handle_t provider,
1396 crypto_session_id_t session_id,
1397 crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1398 crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1399 crypto_req_handle_t req)
1400 {
1401 int rv;
1402 digest_rsa_ctx_t dctx;
1403
1404 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1405 return (rv);
1406
1407 if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1408 mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1409 rv = rsa_verify_common(mechanism->cm_type, key, data,
1410 signature);
1411
1412 else {
1413 dctx.mech_type = mechanism->cm_type;
1414 dctx.key = key;
1415
1416 switch (mechanism->cm_type) {
1417 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1418 MD5Init(&(dctx.md5_ctx));
1419 break;
1420
1421 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1422 SHA1Init(&(dctx.sha1_ctx));
1423 break;
1424
1425 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1426 SHA2Init(SHA256, &(dctx.sha2_ctx));
1427 break;
1428
1429 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1430 SHA2Init(SHA384, &(dctx.sha2_ctx));
1431 break;
1432
1433 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1434 SHA2Init(SHA512, &(dctx.sha2_ctx));
1435 break;
1436 }
1437
1438 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1439 CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1440 }
1441
1442 return (rv);
1443 }
1444
1445 static int
rsa_verify_recover_common(rsa_mech_type_t mech_type,crypto_key_t * key,crypto_data_t * signature,crypto_data_t * data)1446 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1447 crypto_data_t *signature, crypto_data_t *data)
1448 {
1449 int rv = CRYPTO_FAILED;
1450
1451 size_t data_len;
1452 uchar_t *sigptr, *modulus;
1453 ssize_t modulus_len;
1454 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1455 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1456
1457 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1458 &modulus_len)) != CRYPTO_SUCCESS) {
1459 return (rv);
1460 }
1461
1462 if (signature->cd_length != modulus_len)
1463 return (CRYPTO_SIGNATURE_LEN_RANGE);
1464
1465 ASSERT(signature->cd_length <= sizeof (tmp_data));
1466 if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1467 != CRYPTO_SUCCESS)
1468 return (rv);
1469
1470 rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1471 if (rv != CRYPTO_SUCCESS)
1472 return (rv);
1473
1474 data_len = modulus_len;
1475
1476 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1477 /*
1478 * Strip off the encoded padding bytes in front of the
1479 * recovered data, then compare the recovered data with
1480 * the original data.
1481 */
1482 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1483 if (rv != CRYPTO_SUCCESS)
1484 return (rv);
1485 }
1486
1487 if (data->cd_length < data_len) {
1488 data->cd_length = data_len;
1489 return (CRYPTO_BUFFER_TOO_SMALL);
1490 }
1491
1492 if ((rv = crypto_put_output_data(plain_data + modulus_len - data_len,
1493 data, data_len)) != CRYPTO_SUCCESS)
1494 return (rv);
1495 data->cd_length = data_len;
1496
1497 return (rv);
1498 }
1499
1500 /* ARGSUSED */
1501 static int
rsa_verify_recover(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_data_t * data,crypto_req_handle_t req)1502 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1503 crypto_data_t *data, crypto_req_handle_t req)
1504 {
1505 int rv;
1506 rsa_ctx_t *ctxp;
1507
1508 ASSERT(ctx->cc_provider_private != NULL);
1509 ctxp = ctx->cc_provider_private;
1510
1511 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1512 rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1513 signature, data);
1514
1515 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1516 (void) rsa_free_context(ctx);
1517
1518 return (rv);
1519 }
1520
1521 /* ARGSUSED */
1522 static int
rsa_verify_recover_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * signature,crypto_data_t * data,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)1523 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1524 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1525 crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1526 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1527 {
1528 int rv;
1529
1530 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1531 return (rv);
1532
1533 return (rsa_verify_recover_common(mechanism->cm_type, key,
1534 signature, data));
1535 }
1536