xref: /freebsd/crypto/openssl/crypto/rsa/rsa_pmeth.c (revision 273c26a3c3bea87a241d6879abd4f991db180bf0)
1 /* crypto/rsa/rsa_pmeth.c */
2 /*
3  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL project
4  * 2006.
5  */
6 /* ====================================================================
7  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  *
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in
18  *    the documentation and/or other materials provided with the
19  *    distribution.
20  *
21  * 3. All advertising materials mentioning features or use of this
22  *    software must display the following acknowledgment:
23  *    "This product includes software developed by the OpenSSL Project
24  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
25  *
26  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
27  *    endorse or promote products derived from this software without
28  *    prior written permission. For written permission, please contact
29  *    licensing@OpenSSL.org.
30  *
31  * 5. Products derived from this software may not be called "OpenSSL"
32  *    nor may "OpenSSL" appear in their names without prior written
33  *    permission of the OpenSSL Project.
34  *
35  * 6. Redistributions of any form whatsoever must retain the following
36  *    acknowledgment:
37  *    "This product includes software developed by the OpenSSL Project
38  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
39  *
40  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
41  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
43  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
44  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
45  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
49  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
51  * OF THE POSSIBILITY OF SUCH DAMAGE.
52  * ====================================================================
53  *
54  * This product includes cryptographic software written by Eric Young
55  * (eay@cryptsoft.com).  This product includes software written by Tim
56  * Hudson (tjh@cryptsoft.com).
57  *
58  */
59 
60 #include <stdio.h>
61 #include "cryptlib.h"
62 #include <openssl/asn1t.h>
63 #include <openssl/x509.h>
64 #include <openssl/rsa.h>
65 #include <openssl/bn.h>
66 #include <openssl/evp.h>
67 #include <openssl/x509v3.h>
68 #ifndef OPENSSL_NO_CMS
69 # include <openssl/cms.h>
70 #endif
71 #ifdef OPENSSL_FIPS
72 # include <openssl/fips.h>
73 #endif
74 #include "evp_locl.h"
75 #include "rsa_locl.h"
76 
77 /* RSA pkey context structure */
78 
79 typedef struct {
80     /* Key gen parameters */
81     int nbits;
82     BIGNUM *pub_exp;
83     /* Keygen callback info */
84     int gentmp[2];
85     /* RSA padding mode */
86     int pad_mode;
87     /* message digest */
88     const EVP_MD *md;
89     /* message digest for MGF1 */
90     const EVP_MD *mgf1md;
91     /* PSS salt length */
92     int saltlen;
93     /* Temp buffer */
94     unsigned char *tbuf;
95     /* OAEP label */
96     unsigned char *oaep_label;
97     size_t oaep_labellen;
98 } RSA_PKEY_CTX;
99 
100 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
101 {
102     RSA_PKEY_CTX *rctx;
103     rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
104     if (!rctx)
105         return 0;
106     rctx->nbits = 1024;
107     rctx->pub_exp = NULL;
108     rctx->pad_mode = RSA_PKCS1_PADDING;
109     rctx->md = NULL;
110     rctx->mgf1md = NULL;
111     rctx->tbuf = NULL;
112 
113     rctx->saltlen = -2;
114 
115     rctx->oaep_label = NULL;
116     rctx->oaep_labellen = 0;
117 
118     ctx->data = rctx;
119     ctx->keygen_info = rctx->gentmp;
120     ctx->keygen_info_count = 2;
121 
122     return 1;
123 }
124 
125 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
126 {
127     RSA_PKEY_CTX *dctx, *sctx;
128     if (!pkey_rsa_init(dst))
129         return 0;
130     sctx = src->data;
131     dctx = dst->data;
132     dctx->nbits = sctx->nbits;
133     if (sctx->pub_exp) {
134         dctx->pub_exp = BN_dup(sctx->pub_exp);
135         if (!dctx->pub_exp)
136             return 0;
137     }
138     dctx->pad_mode = sctx->pad_mode;
139     dctx->md = sctx->md;
140     dctx->mgf1md = sctx->mgf1md;
141     if (sctx->oaep_label) {
142         if (dctx->oaep_label)
143             OPENSSL_free(dctx->oaep_label);
144         dctx->oaep_label = BUF_memdup(sctx->oaep_label, sctx->oaep_labellen);
145         if (!dctx->oaep_label)
146             return 0;
147         dctx->oaep_labellen = sctx->oaep_labellen;
148     }
149     return 1;
150 }
151 
152 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
153 {
154     if (ctx->tbuf)
155         return 1;
156     ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
157     if (!ctx->tbuf)
158         return 0;
159     return 1;
160 }
161 
162 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
163 {
164     RSA_PKEY_CTX *rctx = ctx->data;
165     if (rctx) {
166         if (rctx->pub_exp)
167             BN_free(rctx->pub_exp);
168         if (rctx->tbuf)
169             OPENSSL_free(rctx->tbuf);
170         if (rctx->oaep_label)
171             OPENSSL_free(rctx->oaep_label);
172         OPENSSL_free(rctx);
173     }
174 }
175 
176 #ifdef OPENSSL_FIPS
177 /*
178  * FIP checker. Return value indicates status of context parameters: 1 :
179  * redirect to FIPS. 0 : don't redirect to FIPS. -1 : illegal operation in
180  * FIPS mode.
181  */
182 
183 static int pkey_fips_check_ctx(EVP_PKEY_CTX *ctx)
184 {
185     RSA_PKEY_CTX *rctx = ctx->data;
186     RSA *rsa = ctx->pkey->pkey.rsa;
187     int rv = -1;
188     if (!FIPS_mode())
189         return 0;
190     if (rsa->flags & RSA_FLAG_NON_FIPS_ALLOW)
191         rv = 0;
192     if (!(rsa->meth->flags & RSA_FLAG_FIPS_METHOD) && rv)
193         return -1;
194     if (rctx->md) {
195         const EVP_MD *fmd;
196         fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->md));
197         if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
198             return rv;
199     }
200     if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS)) {
201         const EVP_MD *fmd;
202         fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->mgf1md));
203         if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
204             return rv;
205     }
206     return 1;
207 }
208 #endif
209 
210 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
211                          size_t *siglen, const unsigned char *tbs,
212                          size_t tbslen)
213 {
214     int ret;
215     RSA_PKEY_CTX *rctx = ctx->data;
216     RSA *rsa = ctx->pkey->pkey.rsa;
217 
218 #ifdef OPENSSL_FIPS
219     ret = pkey_fips_check_ctx(ctx);
220     if (ret < 0) {
221         RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
222         return -1;
223     }
224 #endif
225 
226     if (rctx->md) {
227         if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
228             RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_INVALID_DIGEST_LENGTH);
229             return -1;
230         }
231 #ifdef OPENSSL_FIPS
232         if (ret > 0) {
233             unsigned int slen;
234             ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md,
235                                        rctx->pad_mode,
236                                        rctx->saltlen,
237                                        rctx->mgf1md, sig, &slen);
238             if (ret > 0)
239                 *siglen = slen;
240             else
241                 *siglen = 0;
242             return ret;
243         }
244 #endif
245 
246         if (EVP_MD_type(rctx->md) == NID_mdc2) {
247             unsigned int sltmp;
248             if (rctx->pad_mode != RSA_PKCS1_PADDING)
249                 return -1;
250             ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
251                                              tbs, tbslen, sig, &sltmp, rsa);
252 
253             if (ret <= 0)
254                 return ret;
255             ret = sltmp;
256         } else if (rctx->pad_mode == RSA_X931_PADDING) {
257             if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
258                 RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_KEY_SIZE_TOO_SMALL);
259                 return -1;
260             }
261             if (!setup_tbuf(rctx, ctx)) {
262                 RSAerr(RSA_F_PKEY_RSA_SIGN, ERR_R_MALLOC_FAILURE);
263                 return -1;
264             }
265             memcpy(rctx->tbuf, tbs, tbslen);
266             rctx->tbuf[tbslen] = RSA_X931_hash_id(EVP_MD_type(rctx->md));
267             ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
268                                       sig, rsa, RSA_X931_PADDING);
269         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
270             unsigned int sltmp;
271             ret = RSA_sign(EVP_MD_type(rctx->md),
272                            tbs, tbslen, sig, &sltmp, rsa);
273             if (ret <= 0)
274                 return ret;
275             ret = sltmp;
276         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
277             if (!setup_tbuf(rctx, ctx))
278                 return -1;
279             if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
280                                                 rctx->tbuf, tbs,
281                                                 rctx->md, rctx->mgf1md,
282                                                 rctx->saltlen))
283                 return -1;
284             ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
285                                       sig, rsa, RSA_NO_PADDING);
286         } else
287             return -1;
288     } else
289         ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
290                                   rctx->pad_mode);
291     if (ret < 0)
292         return ret;
293     *siglen = ret;
294     return 1;
295 }
296 
297 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
298                                   unsigned char *rout, size_t *routlen,
299                                   const unsigned char *sig, size_t siglen)
300 {
301     int ret;
302     RSA_PKEY_CTX *rctx = ctx->data;
303 
304     if (rctx->md) {
305         if (rctx->pad_mode == RSA_X931_PADDING) {
306             if (!setup_tbuf(rctx, ctx))
307                 return -1;
308             ret = RSA_public_decrypt(siglen, sig,
309                                      rctx->tbuf, ctx->pkey->pkey.rsa,
310                                      RSA_X931_PADDING);
311             if (ret < 1)
312                 return 0;
313             ret--;
314             if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
315                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
316                        RSA_R_ALGORITHM_MISMATCH);
317                 return 0;
318             }
319             if (ret != EVP_MD_size(rctx->md)) {
320                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
321                        RSA_R_INVALID_DIGEST_LENGTH);
322                 return 0;
323             }
324             if (rout)
325                 memcpy(rout, rctx->tbuf, ret);
326         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
327             size_t sltmp;
328             ret = int_rsa_verify(EVP_MD_type(rctx->md),
329                                  NULL, 0, rout, &sltmp,
330                                  sig, siglen, ctx->pkey->pkey.rsa);
331             if (ret <= 0)
332                 return 0;
333             ret = sltmp;
334         } else
335             return -1;
336     } else
337         ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
338                                  rctx->pad_mode);
339     if (ret < 0)
340         return ret;
341     *routlen = ret;
342     return 1;
343 }
344 
345 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
346                            const unsigned char *sig, size_t siglen,
347                            const unsigned char *tbs, size_t tbslen)
348 {
349     RSA_PKEY_CTX *rctx = ctx->data;
350     RSA *rsa = ctx->pkey->pkey.rsa;
351     size_t rslen;
352 #ifdef OPENSSL_FIPS
353     int rv;
354     rv = pkey_fips_check_ctx(ctx);
355     if (rv < 0) {
356         RSAerr(RSA_F_PKEY_RSA_VERIFY,
357                RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
358         return -1;
359     }
360 #endif
361     if (rctx->md) {
362 #ifdef OPENSSL_FIPS
363         if (rv > 0) {
364             return FIPS_rsa_verify_digest(rsa,
365                                           tbs, tbslen,
366                                           rctx->md,
367                                           rctx->pad_mode,
368                                           rctx->saltlen,
369                                           rctx->mgf1md, sig, siglen);
370 
371         }
372 #endif
373         if (rctx->pad_mode == RSA_PKCS1_PADDING)
374             return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
375                               sig, siglen, rsa);
376         if (rctx->pad_mode == RSA_X931_PADDING) {
377             if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig, siglen) <= 0)
378                 return 0;
379         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
380             int ret;
381             if (!setup_tbuf(rctx, ctx))
382                 return -1;
383             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
384                                      rsa, RSA_NO_PADDING);
385             if (ret <= 0)
386                 return 0;
387             ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
388                                             rctx->md, rctx->mgf1md,
389                                             rctx->tbuf, rctx->saltlen);
390             if (ret <= 0)
391                 return 0;
392             return 1;
393         } else
394             return -1;
395     } else {
396         if (!setup_tbuf(rctx, ctx))
397             return -1;
398         rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
399                                    rsa, rctx->pad_mode);
400         if (rslen == 0)
401             return 0;
402     }
403 
404     if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
405         return 0;
406 
407     return 1;
408 
409 }
410 
411 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
412                             unsigned char *out, size_t *outlen,
413                             const unsigned char *in, size_t inlen)
414 {
415     int ret;
416     RSA_PKEY_CTX *rctx = ctx->data;
417     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
418         int klen = RSA_size(ctx->pkey->pkey.rsa);
419         if (!setup_tbuf(rctx, ctx))
420             return -1;
421         if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
422                                              in, inlen,
423                                              rctx->oaep_label,
424                                              rctx->oaep_labellen,
425                                              rctx->md, rctx->mgf1md))
426             return -1;
427         ret = RSA_public_encrypt(klen, rctx->tbuf, out,
428                                  ctx->pkey->pkey.rsa, RSA_NO_PADDING);
429     } else
430         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
431                                  rctx->pad_mode);
432     if (ret < 0)
433         return ret;
434     *outlen = ret;
435     return 1;
436 }
437 
438 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
439                             unsigned char *out, size_t *outlen,
440                             const unsigned char *in, size_t inlen)
441 {
442     int ret;
443     RSA_PKEY_CTX *rctx = ctx->data;
444     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
445         int i;
446         if (!setup_tbuf(rctx, ctx))
447             return -1;
448         ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
449                                   ctx->pkey->pkey.rsa, RSA_NO_PADDING);
450         if (ret <= 0)
451             return ret;
452         for (i = 0; i < ret; i++) {
453             if (rctx->tbuf[i])
454                 break;
455         }
456         ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf + i,
457                                                 ret - i, ret,
458                                                 rctx->oaep_label,
459                                                 rctx->oaep_labellen,
460                                                 rctx->md, rctx->mgf1md);
461     } else
462         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
463                                   rctx->pad_mode);
464     if (ret < 0)
465         return ret;
466     *outlen = ret;
467     return 1;
468 }
469 
470 static int check_padding_md(const EVP_MD *md, int padding)
471 {
472     if (!md)
473         return 1;
474 
475     if (padding == RSA_NO_PADDING) {
476         RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
477         return 0;
478     }
479 
480     if (padding == RSA_X931_PADDING) {
481         if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
482             RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_X931_DIGEST);
483             return 0;
484         }
485         return 1;
486     }
487 
488     return 1;
489 }
490 
491 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
492 {
493     RSA_PKEY_CTX *rctx = ctx->data;
494     switch (type) {
495     case EVP_PKEY_CTRL_RSA_PADDING:
496         if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING)) {
497             if (!check_padding_md(rctx->md, p1))
498                 return 0;
499             if (p1 == RSA_PKCS1_PSS_PADDING) {
500                 if (!(ctx->operation &
501                       (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
502                     goto bad_pad;
503                 if (!rctx->md)
504                     rctx->md = EVP_sha1();
505             }
506             if (p1 == RSA_PKCS1_OAEP_PADDING) {
507                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
508                     goto bad_pad;
509                 if (!rctx->md)
510                     rctx->md = EVP_sha1();
511             }
512             rctx->pad_mode = p1;
513             return 1;
514         }
515  bad_pad:
516         RSAerr(RSA_F_PKEY_RSA_CTRL,
517                RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
518         return -2;
519 
520     case EVP_PKEY_CTRL_GET_RSA_PADDING:
521         *(int *)p2 = rctx->pad_mode;
522         return 1;
523 
524     case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
525     case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
526         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
527             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
528             return -2;
529         }
530         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
531             *(int *)p2 = rctx->saltlen;
532         else {
533             if (p1 < -2)
534                 return -2;
535             rctx->saltlen = p1;
536         }
537         return 1;
538 
539     case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
540         if (p1 < 256) {
541             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
542             return -2;
543         }
544         rctx->nbits = p1;
545         return 1;
546 
547     case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
548         if (p2 == NULL || !BN_is_odd((BIGNUM *)p2) || BN_is_one((BIGNUM *)p2)) {
549             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_BAD_E_VALUE);
550             return -2;
551         }
552         BN_free(rctx->pub_exp);
553         rctx->pub_exp = p2;
554         return 1;
555 
556     case EVP_PKEY_CTRL_RSA_OAEP_MD:
557     case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
558         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
559             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
560             return -2;
561         }
562         if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
563             *(const EVP_MD **)p2 = rctx->md;
564         else
565             rctx->md = p2;
566         return 1;
567 
568     case EVP_PKEY_CTRL_MD:
569         if (!check_padding_md(p2, rctx->pad_mode))
570             return 0;
571         rctx->md = p2;
572         return 1;
573 
574     case EVP_PKEY_CTRL_GET_MD:
575         *(const EVP_MD **)p2 = rctx->md;
576         return 1;
577 
578     case EVP_PKEY_CTRL_RSA_MGF1_MD:
579     case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
580         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
581             && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
582             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
583             return -2;
584         }
585         if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
586             if (rctx->mgf1md)
587                 *(const EVP_MD **)p2 = rctx->mgf1md;
588             else
589                 *(const EVP_MD **)p2 = rctx->md;
590         } else
591             rctx->mgf1md = p2;
592         return 1;
593 
594     case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
595         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
596             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
597             return -2;
598         }
599         if (rctx->oaep_label)
600             OPENSSL_free(rctx->oaep_label);
601         if (p2 && p1 > 0) {
602             rctx->oaep_label = p2;
603             rctx->oaep_labellen = p1;
604         } else {
605             rctx->oaep_label = NULL;
606             rctx->oaep_labellen = 0;
607         }
608         return 1;
609 
610     case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
611         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
612             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
613             return -2;
614         }
615         *(unsigned char **)p2 = rctx->oaep_label;
616         return rctx->oaep_labellen;
617 
618     case EVP_PKEY_CTRL_DIGESTINIT:
619     case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
620     case EVP_PKEY_CTRL_PKCS7_DECRYPT:
621     case EVP_PKEY_CTRL_PKCS7_SIGN:
622         return 1;
623 #ifndef OPENSSL_NO_CMS
624     case EVP_PKEY_CTRL_CMS_DECRYPT:
625     case EVP_PKEY_CTRL_CMS_ENCRYPT:
626     case EVP_PKEY_CTRL_CMS_SIGN:
627         return 1;
628 #endif
629     case EVP_PKEY_CTRL_PEER_KEY:
630         RSAerr(RSA_F_PKEY_RSA_CTRL,
631                RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
632         return -2;
633 
634     default:
635         return -2;
636 
637     }
638 }
639 
640 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
641                              const char *type, const char *value)
642 {
643     if (!value) {
644         RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
645         return 0;
646     }
647     if (!strcmp(type, "rsa_padding_mode")) {
648         int pm;
649         if (!strcmp(value, "pkcs1"))
650             pm = RSA_PKCS1_PADDING;
651         else if (!strcmp(value, "sslv23"))
652             pm = RSA_SSLV23_PADDING;
653         else if (!strcmp(value, "none"))
654             pm = RSA_NO_PADDING;
655         else if (!strcmp(value, "oeap"))
656             pm = RSA_PKCS1_OAEP_PADDING;
657         else if (!strcmp(value, "oaep"))
658             pm = RSA_PKCS1_OAEP_PADDING;
659         else if (!strcmp(value, "x931"))
660             pm = RSA_X931_PADDING;
661         else if (!strcmp(value, "pss"))
662             pm = RSA_PKCS1_PSS_PADDING;
663         else {
664             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_UNKNOWN_PADDING_TYPE);
665             return -2;
666         }
667         return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
668     }
669 
670     if (!strcmp(type, "rsa_pss_saltlen")) {
671         int saltlen;
672         saltlen = atoi(value);
673         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
674     }
675 
676     if (!strcmp(type, "rsa_keygen_bits")) {
677         int nbits;
678         nbits = atoi(value);
679         return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
680     }
681 
682     if (!strcmp(type, "rsa_keygen_pubexp")) {
683         int ret;
684         BIGNUM *pubexp = NULL;
685         if (!BN_asc2bn(&pubexp, value))
686             return 0;
687         ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
688         if (ret <= 0)
689             BN_free(pubexp);
690         return ret;
691     }
692 
693     if (!strcmp(type, "rsa_mgf1_md")) {
694         const EVP_MD *md;
695         if (!(md = EVP_get_digestbyname(value))) {
696             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
697             return 0;
698         }
699         return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
700     }
701 
702     if (!strcmp(type, "rsa_oaep_md")) {
703         const EVP_MD *md;
704         if (!(md = EVP_get_digestbyname(value))) {
705             RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_INVALID_DIGEST);
706             return 0;
707         }
708         return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
709     }
710     if (!strcmp(type, "rsa_oaep_label")) {
711         unsigned char *lab;
712         long lablen;
713         int ret;
714         lab = string_to_hex(value, &lablen);
715         if (!lab)
716             return 0;
717         ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
718         if (ret <= 0)
719             OPENSSL_free(lab);
720         return ret;
721     }
722 
723     return -2;
724 }
725 
726 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
727 {
728     RSA *rsa = NULL;
729     RSA_PKEY_CTX *rctx = ctx->data;
730     BN_GENCB *pcb, cb;
731     int ret;
732     if (!rctx->pub_exp) {
733         rctx->pub_exp = BN_new();
734         if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
735             return 0;
736     }
737     rsa = RSA_new();
738     if (!rsa)
739         return 0;
740     if (ctx->pkey_gencb) {
741         pcb = &cb;
742         evp_pkey_set_cb_translate(pcb, ctx);
743     } else
744         pcb = NULL;
745     ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
746     if (ret > 0)
747         EVP_PKEY_assign_RSA(pkey, rsa);
748     else
749         RSA_free(rsa);
750     return ret;
751 }
752 
753 const EVP_PKEY_METHOD rsa_pkey_meth = {
754     EVP_PKEY_RSA,
755     EVP_PKEY_FLAG_AUTOARGLEN,
756     pkey_rsa_init,
757     pkey_rsa_copy,
758     pkey_rsa_cleanup,
759 
760     0, 0,
761 
762     0,
763     pkey_rsa_keygen,
764 
765     0,
766     pkey_rsa_sign,
767 
768     0,
769     pkey_rsa_verify,
770 
771     0,
772     pkey_rsa_verifyrecover,
773 
774     0, 0, 0, 0,
775 
776     0,
777     pkey_rsa_encrypt,
778 
779     0,
780     pkey_rsa_decrypt,
781 
782     0, 0,
783 
784     pkey_rsa_ctrl,
785     pkey_rsa_ctrl_str
786 };
787