xref: /freebsd/crypto/openssl/crypto/rsa/rsa_ameth.c (revision d93a896ef95946b0bf1219866fcb324b78543444)
1 /* crypto/rsa/rsa_ameth.c */
2 /*
3  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
4  * 2006.
5  */
6 /* ====================================================================
7  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  *
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in
18  *    the documentation and/or other materials provided with the
19  *    distribution.
20  *
21  * 3. All advertising materials mentioning features or use of this
22  *    software must display the following acknowledgment:
23  *    "This product includes software developed by the OpenSSL Project
24  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
25  *
26  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
27  *    endorse or promote products derived from this software without
28  *    prior written permission. For written permission, please contact
29  *    licensing@OpenSSL.org.
30  *
31  * 5. Products derived from this software may not be called "OpenSSL"
32  *    nor may "OpenSSL" appear in their names without prior written
33  *    permission of the OpenSSL Project.
34  *
35  * 6. Redistributions of any form whatsoever must retain the following
36  *    acknowledgment:
37  *    "This product includes software developed by the OpenSSL Project
38  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
39  *
40  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
41  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
43  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
44  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
45  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
49  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
51  * OF THE POSSIBILITY OF SUCH DAMAGE.
52  * ====================================================================
53  *
54  * This product includes cryptographic software written by Eric Young
55  * (eay@cryptsoft.com).  This product includes software written by Tim
56  * Hudson (tjh@cryptsoft.com).
57  *
58  */
59 
60 #include <stdio.h>
61 #include "cryptlib.h"
62 #include <openssl/asn1t.h>
63 #include <openssl/x509.h>
64 #include <openssl/rsa.h>
65 #include <openssl/bn.h>
66 #ifndef OPENSSL_NO_CMS
67 # include <openssl/cms.h>
68 #endif
69 #include "asn1_locl.h"
70 
71 #ifndef OPENSSL_NO_CMS
72 static int rsa_cms_sign(CMS_SignerInfo *si);
73 static int rsa_cms_verify(CMS_SignerInfo *si);
74 static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
75 static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
76 #endif
77 
78 static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
79 {
80     unsigned char *penc = NULL;
81     int penclen;
82     penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
83     if (penclen <= 0)
84         return 0;
85     if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_RSA),
86                                V_ASN1_NULL, NULL, penc, penclen))
87         return 1;
88 
89     OPENSSL_free(penc);
90     return 0;
91 }
92 
93 static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
94 {
95     const unsigned char *p;
96     int pklen;
97     RSA *rsa = NULL;
98     if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, NULL, pubkey))
99         return 0;
100     if (!(rsa = d2i_RSAPublicKey(NULL, &p, pklen))) {
101         RSAerr(RSA_F_RSA_PUB_DECODE, ERR_R_RSA_LIB);
102         return 0;
103     }
104     EVP_PKEY_assign_RSA(pkey, rsa);
105     return 1;
106 }
107 
108 static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
109 {
110     if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0
111         || BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
112         return 0;
113     return 1;
114 }
115 
116 static int old_rsa_priv_decode(EVP_PKEY *pkey,
117                                const unsigned char **pder, int derlen)
118 {
119     RSA *rsa;
120     if (!(rsa = d2i_RSAPrivateKey(NULL, pder, derlen))) {
121         RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB);
122         return 0;
123     }
124     EVP_PKEY_assign_RSA(pkey, rsa);
125     return 1;
126 }
127 
128 static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
129 {
130     return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
131 }
132 
133 static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
134 {
135     unsigned char *rk = NULL;
136     int rklen;
137     rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
138 
139     if (rklen <= 0) {
140         RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
141         return 0;
142     }
143 
144     if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_rsaEncryption), 0,
145                          V_ASN1_NULL, NULL, rk, rklen)) {
146         RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
147         return 0;
148     }
149 
150     return 1;
151 }
152 
153 static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8)
154 {
155     const unsigned char *p;
156     int pklen;
157     if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8))
158         return 0;
159     return old_rsa_priv_decode(pkey, &p, pklen);
160 }
161 
162 static int int_rsa_size(const EVP_PKEY *pkey)
163 {
164     return RSA_size(pkey->pkey.rsa);
165 }
166 
167 static int rsa_bits(const EVP_PKEY *pkey)
168 {
169     return BN_num_bits(pkey->pkey.rsa->n);
170 }
171 
172 static void int_rsa_free(EVP_PKEY *pkey)
173 {
174     RSA_free(pkey->pkey.rsa);
175 }
176 
177 static void update_buflen(const BIGNUM *b, size_t *pbuflen)
178 {
179     size_t i;
180     if (!b)
181         return;
182     if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
183         *pbuflen = i;
184 }
185 
186 static int do_rsa_print(BIO *bp, const RSA *x, int off, int priv)
187 {
188     char *str;
189     const char *s;
190     unsigned char *m = NULL;
191     int ret = 0, mod_len = 0;
192     size_t buf_len = 0;
193 
194     update_buflen(x->n, &buf_len);
195     update_buflen(x->e, &buf_len);
196 
197     if (priv) {
198         update_buflen(x->d, &buf_len);
199         update_buflen(x->p, &buf_len);
200         update_buflen(x->q, &buf_len);
201         update_buflen(x->dmp1, &buf_len);
202         update_buflen(x->dmq1, &buf_len);
203         update_buflen(x->iqmp, &buf_len);
204     }
205 
206     m = (unsigned char *)OPENSSL_malloc(buf_len + 10);
207     if (m == NULL) {
208         RSAerr(RSA_F_DO_RSA_PRINT, ERR_R_MALLOC_FAILURE);
209         goto err;
210     }
211 
212     if (x->n != NULL)
213         mod_len = BN_num_bits(x->n);
214 
215     if (!BIO_indent(bp, off, 128))
216         goto err;
217 
218     if (priv && x->d) {
219         if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len)
220             <= 0)
221             goto err;
222         str = "modulus:";
223         s = "publicExponent:";
224     } else {
225         if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len)
226             <= 0)
227             goto err;
228         str = "Modulus:";
229         s = "Exponent:";
230     }
231     if (!ASN1_bn_print(bp, str, x->n, m, off))
232         goto err;
233     if (!ASN1_bn_print(bp, s, x->e, m, off))
234         goto err;
235     if (priv) {
236         if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
237             goto err;
238         if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
239             goto err;
240         if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
241             goto err;
242         if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
243             goto err;
244         if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
245             goto err;
246         if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
247             goto err;
248     }
249     ret = 1;
250  err:
251     if (m != NULL)
252         OPENSSL_free(m);
253     return (ret);
254 }
255 
256 static int rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent,
257                          ASN1_PCTX *ctx)
258 {
259     return do_rsa_print(bp, pkey->pkey.rsa, indent, 0);
260 }
261 
262 static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
263                           ASN1_PCTX *ctx)
264 {
265     return do_rsa_print(bp, pkey->pkey.rsa, indent, 1);
266 }
267 
268 /* Given an MGF1 Algorithm ID decode to an Algorithm Identifier */
269 static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg)
270 {
271     const unsigned char *p;
272     int plen;
273     if (alg == NULL || alg->parameter == NULL)
274         return NULL;
275     if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
276         return NULL;
277     if (alg->parameter->type != V_ASN1_SEQUENCE)
278         return NULL;
279 
280     p = alg->parameter->value.sequence->data;
281     plen = alg->parameter->value.sequence->length;
282     return d2i_X509_ALGOR(NULL, &p, plen);
283 }
284 
285 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg,
286                                       X509_ALGOR **pmaskHash)
287 {
288     const unsigned char *p;
289     int plen;
290     RSA_PSS_PARAMS *pss;
291 
292     *pmaskHash = NULL;
293 
294     if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE)
295         return NULL;
296     p = alg->parameter->value.sequence->data;
297     plen = alg->parameter->value.sequence->length;
298     pss = d2i_RSA_PSS_PARAMS(NULL, &p, plen);
299 
300     if (!pss)
301         return NULL;
302 
303     *pmaskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
304 
305     return pss;
306 }
307 
308 static int rsa_pss_param_print(BIO *bp, RSA_PSS_PARAMS *pss,
309                                X509_ALGOR *maskHash, int indent)
310 {
311     int rv = 0;
312     if (!pss) {
313         if (BIO_puts(bp, " (INVALID PSS PARAMETERS)\n") <= 0)
314             return 0;
315         return 1;
316     }
317     if (BIO_puts(bp, "\n") <= 0)
318         goto err;
319     if (!BIO_indent(bp, indent, 128))
320         goto err;
321     if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
322         goto err;
323 
324     if (pss->hashAlgorithm) {
325         if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
326             goto err;
327     } else if (BIO_puts(bp, "sha1 (default)") <= 0)
328         goto err;
329 
330     if (BIO_puts(bp, "\n") <= 0)
331         goto err;
332 
333     if (!BIO_indent(bp, indent, 128))
334         goto err;
335 
336     if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
337         goto err;
338     if (pss->maskGenAlgorithm) {
339         if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
340             goto err;
341         if (BIO_puts(bp, " with ") <= 0)
342             goto err;
343         if (maskHash) {
344             if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
345                 goto err;
346         } else if (BIO_puts(bp, "INVALID") <= 0)
347             goto err;
348     } else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0)
349         goto err;
350     BIO_puts(bp, "\n");
351 
352     if (!BIO_indent(bp, indent, 128))
353         goto err;
354     if (BIO_puts(bp, "Salt Length: 0x") <= 0)
355         goto err;
356     if (pss->saltLength) {
357         if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
358             goto err;
359     } else if (BIO_puts(bp, "14 (default)") <= 0)
360         goto err;
361     BIO_puts(bp, "\n");
362 
363     if (!BIO_indent(bp, indent, 128))
364         goto err;
365     if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
366         goto err;
367     if (pss->trailerField) {
368         if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
369             goto err;
370     } else if (BIO_puts(bp, "BC (default)") <= 0)
371         goto err;
372     BIO_puts(bp, "\n");
373 
374     rv = 1;
375 
376  err:
377     return rv;
378 
379 }
380 
381 static int rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
382                          const ASN1_STRING *sig, int indent, ASN1_PCTX *pctx)
383 {
384     if (OBJ_obj2nid(sigalg->algorithm) == NID_rsassaPss) {
385         int rv;
386         RSA_PSS_PARAMS *pss;
387         X509_ALGOR *maskHash;
388         pss = rsa_pss_decode(sigalg, &maskHash);
389         rv = rsa_pss_param_print(bp, pss, maskHash, indent);
390         if (pss)
391             RSA_PSS_PARAMS_free(pss);
392         if (maskHash)
393             X509_ALGOR_free(maskHash);
394         if (!rv)
395             return 0;
396     } else if (!sig && BIO_puts(bp, "\n") <= 0)
397         return 0;
398     if (sig)
399         return X509_signature_dump(bp, sig, indent);
400     return 1;
401 }
402 
403 static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
404 {
405     X509_ALGOR *alg = NULL;
406     switch (op) {
407 
408     case ASN1_PKEY_CTRL_PKCS7_SIGN:
409         if (arg1 == 0)
410             PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
411         break;
412 
413     case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
414         if (arg1 == 0)
415             PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
416         break;
417 #ifndef OPENSSL_NO_CMS
418     case ASN1_PKEY_CTRL_CMS_SIGN:
419         if (arg1 == 0)
420             return rsa_cms_sign(arg2);
421         else if (arg1 == 1)
422             return rsa_cms_verify(arg2);
423         break;
424 
425     case ASN1_PKEY_CTRL_CMS_ENVELOPE:
426         if (arg1 == 0)
427             return rsa_cms_encrypt(arg2);
428         else if (arg1 == 1)
429             return rsa_cms_decrypt(arg2);
430         break;
431 
432     case ASN1_PKEY_CTRL_CMS_RI_TYPE:
433         *(int *)arg2 = CMS_RECIPINFO_TRANS;
434         return 1;
435 #endif
436 
437     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
438         *(int *)arg2 = NID_sha256;
439         return 1;
440 
441     default:
442         return -2;
443 
444     }
445 
446     if (alg)
447         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
448 
449     return 1;
450 
451 }
452 
453 /* allocate and set algorithm ID from EVP_MD, default SHA1 */
454 static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
455 {
456     if (EVP_MD_type(md) == NID_sha1)
457         return 1;
458     *palg = X509_ALGOR_new();
459     if (!*palg)
460         return 0;
461     X509_ALGOR_set_md(*palg, md);
462     return 1;
463 }
464 
465 /* Allocate and set MGF1 algorithm ID from EVP_MD */
466 static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
467 {
468     X509_ALGOR *algtmp = NULL;
469     ASN1_STRING *stmp = NULL;
470     *palg = NULL;
471     if (EVP_MD_type(mgf1md) == NID_sha1)
472         return 1;
473     /* need to embed algorithm ID inside another */
474     if (!rsa_md_to_algor(&algtmp, mgf1md))
475         goto err;
476     if (!ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp))
477          goto err;
478     *palg = X509_ALGOR_new();
479     if (!*palg)
480         goto err;
481     X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
482     stmp = NULL;
483  err:
484     if (stmp)
485         ASN1_STRING_free(stmp);
486     if (algtmp)
487         X509_ALGOR_free(algtmp);
488     if (*palg)
489         return 1;
490     return 0;
491 }
492 
493 /* convert algorithm ID to EVP_MD, default SHA1 */
494 static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg)
495 {
496     const EVP_MD *md;
497     if (!alg)
498         return EVP_sha1();
499     md = EVP_get_digestbyobj(alg->algorithm);
500     if (md == NULL)
501         RSAerr(RSA_F_RSA_ALGOR_TO_MD, RSA_R_UNKNOWN_DIGEST);
502     return md;
503 }
504 
505 /* convert MGF1 algorithm ID to EVP_MD, default SHA1 */
506 static const EVP_MD *rsa_mgf1_to_md(X509_ALGOR *alg, X509_ALGOR *maskHash)
507 {
508     const EVP_MD *md;
509     if (!alg)
510         return EVP_sha1();
511     /* Check mask and lookup mask hash algorithm */
512     if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) {
513         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_ALGORITHM);
514         return NULL;
515     }
516     if (!maskHash) {
517         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNSUPPORTED_MASK_PARAMETER);
518         return NULL;
519     }
520     md = EVP_get_digestbyobj(maskHash->algorithm);
521     if (md == NULL) {
522         RSAerr(RSA_F_RSA_MGF1_TO_MD, RSA_R_UNKNOWN_MASK_DIGEST);
523         return NULL;
524     }
525     return md;
526 }
527 
528 /*
529  * Convert EVP_PKEY_CTX is PSS mode into corresponding algorithm parameter,
530  * suitable for setting an AlgorithmIdentifier.
531  */
532 
533 static ASN1_STRING *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
534 {
535     const EVP_MD *sigmd, *mgf1md;
536     RSA_PSS_PARAMS *pss = NULL;
537     ASN1_STRING *os = NULL;
538     EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
539     int saltlen, rv = 0;
540     if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
541         goto err;
542     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
543         goto err;
544     if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
545         goto err;
546     if (saltlen == -1)
547         saltlen = EVP_MD_size(sigmd);
548     else if (saltlen == -2) {
549         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
550         if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
551             saltlen--;
552     }
553     pss = RSA_PSS_PARAMS_new();
554     if (!pss)
555         goto err;
556     if (saltlen != 20) {
557         pss->saltLength = ASN1_INTEGER_new();
558         if (!pss->saltLength)
559             goto err;
560         if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
561             goto err;
562     }
563     if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
564         goto err;
565     if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
566         goto err;
567     /* Finally create string with pss parameter encoding. */
568     if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os))
569          goto err;
570     rv = 1;
571  err:
572     if (pss)
573         RSA_PSS_PARAMS_free(pss);
574     if (rv)
575         return os;
576     if (os)
577         ASN1_STRING_free(os);
578     return NULL;
579 }
580 
581 /*
582  * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
583  * then the EVP_MD_CTX is setup and initalised. If it is NULL parameters are
584  * passed to pkctx instead.
585  */
586 
587 static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
588                           X509_ALGOR *sigalg, EVP_PKEY *pkey)
589 {
590     int rv = -1;
591     int saltlen;
592     const EVP_MD *mgf1md = NULL, *md = NULL;
593     RSA_PSS_PARAMS *pss;
594     X509_ALGOR *maskHash;
595     /* Sanity check: make sure it is PSS */
596     if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
597         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
598         return -1;
599     }
600     /* Decode PSS parameters */
601     pss = rsa_pss_decode(sigalg, &maskHash);
602 
603     if (pss == NULL) {
604         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_PSS_PARAMETERS);
605         goto err;
606     }
607     mgf1md = rsa_mgf1_to_md(pss->maskGenAlgorithm, maskHash);
608     if (!mgf1md)
609         goto err;
610     md = rsa_algor_to_md(pss->hashAlgorithm);
611     if (!md)
612         goto err;
613 
614     if (pss->saltLength) {
615         saltlen = ASN1_INTEGER_get(pss->saltLength);
616 
617         /*
618          * Could perform more salt length sanity checks but the main RSA
619          * routines will trap other invalid values anyway.
620          */
621         if (saltlen < 0) {
622             RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_SALT_LENGTH);
623             goto err;
624         }
625     } else
626         saltlen = 20;
627 
628     /*
629      * low-level routines support only trailer field 0xbc (value 1) and
630      * PKCS#1 says we should reject any other value anyway.
631      */
632     if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
633         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_TRAILER);
634         goto err;
635     }
636 
637     /* We have all parameters now set up context */
638 
639     if (pkey) {
640         if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
641             goto err;
642     } else {
643         const EVP_MD *checkmd;
644         if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
645             goto err;
646         if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
647             RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_DIGEST_DOES_NOT_MATCH);
648             goto err;
649         }
650     }
651 
652     if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
653         goto err;
654 
655     if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
656         goto err;
657 
658     if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
659         goto err;
660     /* Carry on */
661     rv = 1;
662 
663  err:
664     RSA_PSS_PARAMS_free(pss);
665     if (maskHash)
666         X509_ALGOR_free(maskHash);
667     return rv;
668 }
669 
670 #ifndef OPENSSL_NO_CMS
671 static int rsa_cms_verify(CMS_SignerInfo *si)
672 {
673     int nid, nid2;
674     X509_ALGOR *alg;
675     EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
676     CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
677     nid = OBJ_obj2nid(alg->algorithm);
678     if (nid == NID_rsaEncryption)
679         return 1;
680     if (nid == NID_rsassaPss)
681         return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
682     /* Workaround for some implementation that use a signature OID */
683     if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
684         if (nid2 == NID_rsaEncryption)
685             return 1;
686     }
687     return 0;
688 }
689 #endif
690 
691 /*
692  * Customised RSA item verification routine. This is called when a signature
693  * is encountered requiring special handling. We currently only handle PSS.
694  */
695 
696 static int rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
697                            X509_ALGOR *sigalg, ASN1_BIT_STRING *sig,
698                            EVP_PKEY *pkey)
699 {
700     /* Sanity check: make sure it is PSS */
701     if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
702         RSAerr(RSA_F_RSA_ITEM_VERIFY, RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
703         return -1;
704     }
705     if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
706         /* Carry on */
707         return 2;
708     }
709     return -1;
710 }
711 
712 #ifndef OPENSSL_NO_CMS
713 static int rsa_cms_sign(CMS_SignerInfo *si)
714 {
715     int pad_mode = RSA_PKCS1_PADDING;
716     X509_ALGOR *alg;
717     EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
718     ASN1_STRING *os = NULL;
719     CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
720     if (pkctx) {
721         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
722             return 0;
723     }
724     if (pad_mode == RSA_PKCS1_PADDING) {
725         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
726         return 1;
727     }
728     /* We don't support it */
729     if (pad_mode != RSA_PKCS1_PSS_PADDING)
730         return 0;
731     os = rsa_ctx_to_pss(pkctx);
732     if (!os)
733         return 0;
734     X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os);
735     return 1;
736 }
737 #endif
738 
739 static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
740                          X509_ALGOR *alg1, X509_ALGOR *alg2,
741                          ASN1_BIT_STRING *sig)
742 {
743     int pad_mode;
744     EVP_PKEY_CTX *pkctx = ctx->pctx;
745     if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
746         return 0;
747     if (pad_mode == RSA_PKCS1_PADDING)
748         return 2;
749     if (pad_mode == RSA_PKCS1_PSS_PADDING) {
750         ASN1_STRING *os1 = NULL;
751         os1 = rsa_ctx_to_pss(pkctx);
752         if (!os1)
753             return 0;
754         /* Duplicate parameters if we have to */
755         if (alg2) {
756             ASN1_STRING *os2 = ASN1_STRING_dup(os1);
757             if (!os2) {
758                 ASN1_STRING_free(os1);
759                 return 0;
760             }
761             X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_rsassaPss),
762                             V_ASN1_SEQUENCE, os2);
763         }
764         X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_rsassaPss),
765                         V_ASN1_SEQUENCE, os1);
766         return 3;
767     }
768     return 2;
769 }
770 
771 static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg,
772                                         X509_ALGOR **pmaskHash)
773 {
774     const unsigned char *p;
775     int plen;
776     RSA_OAEP_PARAMS *pss;
777 
778     *pmaskHash = NULL;
779 
780     if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE)
781         return NULL;
782     p = alg->parameter->value.sequence->data;
783     plen = alg->parameter->value.sequence->length;
784     pss = d2i_RSA_OAEP_PARAMS(NULL, &p, plen);
785 
786     if (!pss)
787         return NULL;
788 
789     *pmaskHash = rsa_mgf1_decode(pss->maskGenFunc);
790 
791     return pss;
792 }
793 
794 #ifndef OPENSSL_NO_CMS
795 static int rsa_cms_decrypt(CMS_RecipientInfo *ri)
796 {
797     EVP_PKEY_CTX *pkctx;
798     X509_ALGOR *cmsalg;
799     int nid;
800     int rv = -1;
801     unsigned char *label = NULL;
802     int labellen = 0;
803     const EVP_MD *mgf1md = NULL, *md = NULL;
804     RSA_OAEP_PARAMS *oaep;
805     X509_ALGOR *maskHash;
806     pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
807     if (!pkctx)
808         return 0;
809     if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
810         return -1;
811     nid = OBJ_obj2nid(cmsalg->algorithm);
812     if (nid == NID_rsaEncryption)
813         return 1;
814     if (nid != NID_rsaesOaep) {
815         RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
816         return -1;
817     }
818     /* Decode OAEP parameters */
819     oaep = rsa_oaep_decode(cmsalg, &maskHash);
820 
821     if (oaep == NULL) {
822         RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_OAEP_PARAMETERS);
823         goto err;
824     }
825 
826     mgf1md = rsa_mgf1_to_md(oaep->maskGenFunc, maskHash);
827     if (!mgf1md)
828         goto err;
829     md = rsa_algor_to_md(oaep->hashFunc);
830     if (!md)
831         goto err;
832 
833     if (oaep->pSourceFunc) {
834         X509_ALGOR *plab = oaep->pSourceFunc;
835         if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
836             RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_LABEL_SOURCE);
837             goto err;
838         }
839         if (plab->parameter->type != V_ASN1_OCTET_STRING) {
840             RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_LABEL);
841             goto err;
842         }
843 
844         label = plab->parameter->value.octet_string->data;
845         /* Stop label being freed when OAEP parameters are freed */
846         plab->parameter->value.octet_string->data = NULL;
847         labellen = plab->parameter->value.octet_string->length;
848     }
849 
850     if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
851         goto err;
852     if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
853         goto err;
854     if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
855         goto err;
856     if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
857         goto err;
858     /* Carry on */
859     rv = 1;
860 
861  err:
862     RSA_OAEP_PARAMS_free(oaep);
863     if (maskHash)
864         X509_ALGOR_free(maskHash);
865     return rv;
866 }
867 
868 static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
869 {
870     const EVP_MD *md, *mgf1md;
871     RSA_OAEP_PARAMS *oaep = NULL;
872     ASN1_STRING *os = NULL;
873     X509_ALGOR *alg;
874     EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
875     int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
876     unsigned char *label;
877     CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg);
878     if (pkctx) {
879         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
880             return 0;
881     }
882     if (pad_mode == RSA_PKCS1_PADDING) {
883         X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
884         return 1;
885     }
886     /* Not supported */
887     if (pad_mode != RSA_PKCS1_OAEP_PADDING)
888         return 0;
889     if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
890         goto err;
891     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
892         goto err;
893     labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
894     if (labellen < 0)
895         goto err;
896     oaep = RSA_OAEP_PARAMS_new();
897     if (!oaep)
898         goto err;
899     if (!rsa_md_to_algor(&oaep->hashFunc, md))
900         goto err;
901     if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
902         goto err;
903     if (labellen > 0) {
904         ASN1_OCTET_STRING *los = ASN1_OCTET_STRING_new();
905         oaep->pSourceFunc = X509_ALGOR_new();
906         if (!oaep->pSourceFunc)
907             goto err;
908         if (!los)
909             goto err;
910         if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
911             ASN1_OCTET_STRING_free(los);
912             goto err;
913         }
914         X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
915                         V_ASN1_OCTET_STRING, los);
916     }
917     /* create string with pss parameter encoding. */
918     if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os))
919          goto err;
920     X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
921     os = NULL;
922     rv = 1;
923  err:
924     if (oaep)
925         RSA_OAEP_PARAMS_free(oaep);
926     if (os)
927         ASN1_STRING_free(os);
928     return rv;
929 }
930 #endif
931 
932 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
933     {
934      EVP_PKEY_RSA,
935      EVP_PKEY_RSA,
936      ASN1_PKEY_SIGPARAM_NULL,
937 
938      "RSA",
939      "OpenSSL RSA method",
940 
941      rsa_pub_decode,
942      rsa_pub_encode,
943      rsa_pub_cmp,
944      rsa_pub_print,
945 
946      rsa_priv_decode,
947      rsa_priv_encode,
948      rsa_priv_print,
949 
950      int_rsa_size,
951      rsa_bits,
952 
953      0, 0, 0, 0, 0, 0,
954 
955      rsa_sig_print,
956      int_rsa_free,
957      rsa_pkey_ctrl,
958      old_rsa_priv_decode,
959      old_rsa_priv_encode,
960      rsa_item_verify,
961      rsa_item_sign},
962 
963     {
964      EVP_PKEY_RSA2,
965      EVP_PKEY_RSA,
966      ASN1_PKEY_ALIAS}
967 };
968