xref: /freebsd/crypto/openssl/crypto/rsa/rsa_pmeth.c (revision 06c3fb2749bda94cb5201f81ffdb8fa6c3161b2e)
1 /*
2  * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved.
3  *
4  * Licensed under the Apache License 2.0 (the "License").  You may not use
5  * this file except in compliance with the License.  You can obtain a copy
6  * in the file LICENSE in the source distribution or at
7  * https://www.openssl.org/source/license.html
8  */
9 
10 /*
11  * RSA low level APIs are deprecated for public use, but still ok for
12  * internal use.
13  */
14 #include "internal/deprecated.h"
15 
16 #include "internal/constant_time.h"
17 
18 #include <stdio.h>
19 #include "internal/cryptlib.h"
20 #include <openssl/asn1t.h>
21 #include <openssl/x509.h>
22 #include <openssl/rsa.h>
23 #include <openssl/bn.h>
24 #include <openssl/evp.h>
25 #include <openssl/x509v3.h>
26 #include <openssl/cms.h>
27 #include "crypto/evp.h"
28 #include "crypto/rsa.h"
29 #include "rsa_local.h"
30 
31 /* RSA pkey context structure */
32 
33 typedef struct {
34     /* Key gen parameters */
35     int nbits;
36     BIGNUM *pub_exp;
37     int primes;
38     /* Keygen callback info */
39     int gentmp[2];
40     /* RSA padding mode */
41     int pad_mode;
42     /* message digest */
43     const EVP_MD *md;
44     /* message digest for MGF1 */
45     const EVP_MD *mgf1md;
46     /* PSS salt length */
47     int saltlen;
48     /* Minimum salt length or -1 if no PSS parameter restriction */
49     int min_saltlen;
50     /* Temp buffer */
51     unsigned char *tbuf;
52     /* OAEP label */
53     unsigned char *oaep_label;
54     size_t oaep_labellen;
55 } RSA_PKEY_CTX;
56 
57 /* True if PSS parameters are restricted */
58 #define rsa_pss_restricted(rctx) (rctx->min_saltlen != -1)
59 
60 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
61 {
62     RSA_PKEY_CTX *rctx = OPENSSL_zalloc(sizeof(*rctx));
63 
64     if (rctx == NULL)
65         return 0;
66     rctx->nbits = 2048;
67     rctx->primes = RSA_DEFAULT_PRIME_NUM;
68     if (pkey_ctx_is_pss(ctx))
69         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
70     else
71         rctx->pad_mode = RSA_PKCS1_PADDING;
72     /* Maximum for sign, auto for verify */
73     rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
74     rctx->min_saltlen = -1;
75     ctx->data = rctx;
76     ctx->keygen_info = rctx->gentmp;
77     ctx->keygen_info_count = 2;
78 
79     return 1;
80 }
81 
82 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src)
83 {
84     RSA_PKEY_CTX *dctx, *sctx;
85 
86     if (!pkey_rsa_init(dst))
87         return 0;
88     sctx = src->data;
89     dctx = dst->data;
90     dctx->nbits = sctx->nbits;
91     if (sctx->pub_exp) {
92         dctx->pub_exp = BN_dup(sctx->pub_exp);
93         if (!dctx->pub_exp)
94             return 0;
95     }
96     dctx->pad_mode = sctx->pad_mode;
97     dctx->md = sctx->md;
98     dctx->mgf1md = sctx->mgf1md;
99     dctx->saltlen = sctx->saltlen;
100     if (sctx->oaep_label) {
101         OPENSSL_free(dctx->oaep_label);
102         dctx->oaep_label = OPENSSL_memdup(sctx->oaep_label, sctx->oaep_labellen);
103         if (!dctx->oaep_label)
104             return 0;
105         dctx->oaep_labellen = sctx->oaep_labellen;
106     }
107     return 1;
108 }
109 
110 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
111 {
112     if (ctx->tbuf != NULL)
113         return 1;
114     if ((ctx->tbuf =
115             OPENSSL_malloc(RSA_size(EVP_PKEY_get0_RSA(pk->pkey)))) == NULL) {
116         ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
117         return 0;
118     }
119     return 1;
120 }
121 
122 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
123 {
124     RSA_PKEY_CTX *rctx = ctx->data;
125     if (rctx) {
126         BN_free(rctx->pub_exp);
127         OPENSSL_free(rctx->tbuf);
128         OPENSSL_free(rctx->oaep_label);
129         OPENSSL_free(rctx);
130     }
131 }
132 
133 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig,
134                          size_t *siglen, const unsigned char *tbs,
135                          size_t tbslen)
136 {
137     int ret;
138     RSA_PKEY_CTX *rctx = ctx->data;
139     /*
140      * Discard const. Its marked as const because this may be a cached copy of
141      * the "real" key. These calls don't make any modifications that need to
142      * be reflected back in the "original" key.
143      */
144     RSA *rsa = (RSA *)EVP_PKEY_get0_RSA(ctx->pkey);
145 
146     if (rctx->md) {
147         if (tbslen != (size_t)EVP_MD_get_size(rctx->md)) {
148             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_DIGEST_LENGTH);
149             return -1;
150         }
151 
152         if (EVP_MD_get_type(rctx->md) == NID_mdc2) {
153             unsigned int sltmp;
154             if (rctx->pad_mode != RSA_PKCS1_PADDING)
155                 return -1;
156             ret = RSA_sign_ASN1_OCTET_STRING(0, tbs, tbslen, sig, &sltmp, rsa);
157 
158             if (ret <= 0)
159                 return ret;
160             ret = sltmp;
161         } else if (rctx->pad_mode == RSA_X931_PADDING) {
162             if ((size_t)RSA_size(rsa) < tbslen + 1) {
163                 ERR_raise(ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL);
164                 return -1;
165             }
166             if (!setup_tbuf(rctx, ctx)) {
167                 ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE);
168                 return -1;
169             }
170             memcpy(rctx->tbuf, tbs, tbslen);
171             rctx->tbuf[tbslen] = RSA_X931_hash_id(EVP_MD_get_type(rctx->md));
172             ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
173                                       sig, rsa, RSA_X931_PADDING);
174         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
175             unsigned int sltmp;
176             ret = RSA_sign(EVP_MD_get_type(rctx->md),
177                            tbs, tbslen, sig, &sltmp, rsa);
178             if (ret <= 0)
179                 return ret;
180             ret = sltmp;
181         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
182             if (!setup_tbuf(rctx, ctx))
183                 return -1;
184             if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
185                                                 rctx->tbuf, tbs,
186                                                 rctx->md, rctx->mgf1md,
187                                                 rctx->saltlen))
188                 return -1;
189             ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
190                                       sig, rsa, RSA_NO_PADDING);
191         } else {
192             return -1;
193         }
194     } else {
195         ret = RSA_private_encrypt(tbslen, tbs, sig, rsa, rctx->pad_mode);
196     }
197     if (ret < 0)
198         return ret;
199     *siglen = ret;
200     return 1;
201 }
202 
203 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
204                                   unsigned char *rout, size_t *routlen,
205                                   const unsigned char *sig, size_t siglen)
206 {
207     int ret;
208     RSA_PKEY_CTX *rctx = ctx->data;
209     /*
210      * Discard const. Its marked as const because this may be a cached copy of
211      * the "real" key. These calls don't make any modifications that need to
212      * be reflected back in the "original" key.
213      */
214     RSA *rsa = (RSA *)EVP_PKEY_get0_RSA(ctx->pkey);
215 
216     if (rctx->md) {
217         if (rctx->pad_mode == RSA_X931_PADDING) {
218             if (!setup_tbuf(rctx, ctx))
219                 return -1;
220             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
221                                      RSA_X931_PADDING);
222             if (ret < 1)
223                 return 0;
224             ret--;
225             if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_get_type(rctx->md))) {
226                 ERR_raise(ERR_LIB_RSA, RSA_R_ALGORITHM_MISMATCH);
227                 return 0;
228             }
229             if (ret != EVP_MD_get_size(rctx->md)) {
230                 ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_DIGEST_LENGTH);
231                 return 0;
232             }
233             if (rout)
234                 memcpy(rout, rctx->tbuf, ret);
235         } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
236             size_t sltmp;
237             ret = ossl_rsa_verify(EVP_MD_get_type(rctx->md),
238                                   NULL, 0, rout, &sltmp,
239                                   sig, siglen, rsa);
240             if (ret <= 0)
241                 return 0;
242             ret = sltmp;
243         } else {
244             return -1;
245         }
246     } else {
247         ret = RSA_public_decrypt(siglen, sig, rout, rsa, rctx->pad_mode);
248     }
249     if (ret < 0)
250         return ret;
251     *routlen = ret;
252     return 1;
253 }
254 
255 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
256                            const unsigned char *sig, size_t siglen,
257                            const unsigned char *tbs, size_t tbslen)
258 {
259     RSA_PKEY_CTX *rctx = ctx->data;
260     /*
261      * Discard const. Its marked as const because this may be a cached copy of
262      * the "real" key. These calls don't make any modifications that need to
263      * be reflected back in the "original" key.
264      */
265     RSA *rsa = (RSA *)EVP_PKEY_get0_RSA(ctx->pkey);
266     size_t rslen;
267 
268     if (rctx->md) {
269         if (rctx->pad_mode == RSA_PKCS1_PADDING)
270             return RSA_verify(EVP_MD_get_type(rctx->md), tbs, tbslen,
271                               sig, siglen, rsa);
272         if (tbslen != (size_t)EVP_MD_get_size(rctx->md)) {
273             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_DIGEST_LENGTH);
274             return -1;
275         }
276         if (rctx->pad_mode == RSA_X931_PADDING) {
277             if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig, siglen) <= 0)
278                 return 0;
279         } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
280             int ret;
281             if (!setup_tbuf(rctx, ctx))
282                 return -1;
283             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
284                                      rsa, RSA_NO_PADDING);
285             if (ret <= 0)
286                 return 0;
287             ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
288                                             rctx->md, rctx->mgf1md,
289                                             rctx->tbuf, rctx->saltlen);
290             if (ret <= 0)
291                 return 0;
292             return 1;
293         } else {
294             return -1;
295         }
296     } else {
297         if (!setup_tbuf(rctx, ctx))
298             return -1;
299         rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
300                                    rsa, rctx->pad_mode);
301         if (rslen == 0)
302             return 0;
303     }
304 
305     if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
306         return 0;
307 
308     return 1;
309 
310 }
311 
312 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
313                             unsigned char *out, size_t *outlen,
314                             const unsigned char *in, size_t inlen)
315 {
316     int ret;
317     RSA_PKEY_CTX *rctx = ctx->data;
318     /*
319      * Discard const. Its marked as const because this may be a cached copy of
320      * the "real" key. These calls don't make any modifications that need to
321      * be reflected back in the "original" key.
322      */
323     RSA *rsa = (RSA *)EVP_PKEY_get0_RSA(ctx->pkey);
324 
325     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
326         int klen = RSA_size(rsa);
327         if (!setup_tbuf(rctx, ctx))
328             return -1;
329         if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
330                                              in, inlen,
331                                              rctx->oaep_label,
332                                              rctx->oaep_labellen,
333                                              rctx->md, rctx->mgf1md))
334             return -1;
335         ret = RSA_public_encrypt(klen, rctx->tbuf, out, rsa, RSA_NO_PADDING);
336     } else {
337         ret = RSA_public_encrypt(inlen, in, out, rsa, rctx->pad_mode);
338     }
339     if (ret < 0)
340         return ret;
341     *outlen = ret;
342     return 1;
343 }
344 
345 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
346                             unsigned char *out, size_t *outlen,
347                             const unsigned char *in, size_t inlen)
348 {
349     int ret;
350     RSA_PKEY_CTX *rctx = ctx->data;
351     /*
352      * Discard const. Its marked as const because this may be a cached copy of
353      * the "real" key. These calls don't make any modifications that need to
354      * be reflected back in the "original" key.
355      */
356     RSA *rsa = (RSA *)EVP_PKEY_get0_RSA(ctx->pkey);
357 
358     if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
359         if (!setup_tbuf(rctx, ctx))
360             return -1;
361         ret = RSA_private_decrypt(inlen, in, rctx->tbuf, rsa, RSA_NO_PADDING);
362         if (ret <= 0)
363             return ret;
364         ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf,
365                                                 ret, ret,
366                                                 rctx->oaep_label,
367                                                 rctx->oaep_labellen,
368                                                 rctx->md, rctx->mgf1md);
369     } else {
370         ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode);
371     }
372     *outlen = constant_time_select_s(constant_time_msb_s(ret), *outlen, ret);
373     ret = constant_time_select_int(constant_time_msb(ret), ret, 1);
374     return ret;
375 }
376 
377 static int check_padding_md(const EVP_MD *md, int padding)
378 {
379     int mdnid;
380 
381     if (!md)
382         return 1;
383 
384     mdnid = EVP_MD_get_type(md);
385 
386     if (padding == RSA_NO_PADDING) {
387         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE);
388         return 0;
389     }
390 
391     if (padding == RSA_X931_PADDING) {
392         if (RSA_X931_hash_id(mdnid) == -1) {
393             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_X931_DIGEST);
394             return 0;
395         }
396     } else {
397         switch(mdnid) {
398         /* List of all supported RSA digests */
399         case NID_sha1:
400         case NID_sha224:
401         case NID_sha256:
402         case NID_sha384:
403         case NID_sha512:
404         case NID_sha512_224:
405         case NID_sha512_256:
406         case NID_md5:
407         case NID_md5_sha1:
408         case NID_md2:
409         case NID_md4:
410         case NID_mdc2:
411         case NID_ripemd160:
412         case NID_sha3_224:
413         case NID_sha3_256:
414         case NID_sha3_384:
415         case NID_sha3_512:
416             return 1;
417 
418         default:
419             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_DIGEST);
420             return 0;
421 
422         }
423     }
424 
425     return 1;
426 }
427 
428 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
429 {
430     RSA_PKEY_CTX *rctx = ctx->data;
431 
432     switch (type) {
433     case EVP_PKEY_CTRL_RSA_PADDING:
434         if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING)) {
435             if (!check_padding_md(rctx->md, p1))
436                 return 0;
437             if (p1 == RSA_PKCS1_PSS_PADDING) {
438                 if (!(ctx->operation &
439                       (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
440                     goto bad_pad;
441                 if (!rctx->md)
442                     rctx->md = EVP_sha1();
443             } else if (pkey_ctx_is_pss(ctx)) {
444                 goto bad_pad;
445             }
446             if (p1 == RSA_PKCS1_OAEP_PADDING) {
447                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
448                     goto bad_pad;
449                 if (!rctx->md)
450                     rctx->md = EVP_sha1();
451             }
452             rctx->pad_mode = p1;
453             return 1;
454         }
455  bad_pad:
456         ERR_raise(ERR_LIB_RSA, RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
457         return -2;
458 
459     case EVP_PKEY_CTRL_GET_RSA_PADDING:
460         *(int *)p2 = rctx->pad_mode;
461         return 1;
462 
463     case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
464     case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
465         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
466             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PSS_SALTLEN);
467             return -2;
468         }
469         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
470             *(int *)p2 = rctx->saltlen;
471         } else {
472             if (p1 < RSA_PSS_SALTLEN_MAX)
473                 return -2;
474             if (rsa_pss_restricted(rctx)) {
475                 if (p1 == RSA_PSS_SALTLEN_AUTO
476                     && ctx->operation == EVP_PKEY_OP_VERIFY) {
477                     ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PSS_SALTLEN);
478                     return -2;
479                 }
480                 if ((p1 == RSA_PSS_SALTLEN_DIGEST
481                      && rctx->min_saltlen > EVP_MD_get_size(rctx->md))
482                     || (p1 >= 0 && p1 < rctx->min_saltlen)) {
483                     ERR_raise(ERR_LIB_RSA, RSA_R_PSS_SALTLEN_TOO_SMALL);
484                     return 0;
485                 }
486             }
487             rctx->saltlen = p1;
488         }
489         return 1;
490 
491     case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
492         if (p1 < RSA_MIN_MODULUS_BITS) {
493             ERR_raise(ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL);
494             return -2;
495         }
496         rctx->nbits = p1;
497         return 1;
498 
499     case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
500         if (p2 == NULL || !BN_is_odd((BIGNUM *)p2) || BN_is_one((BIGNUM *)p2)) {
501             ERR_raise(ERR_LIB_RSA, RSA_R_BAD_E_VALUE);
502             return -2;
503         }
504         BN_free(rctx->pub_exp);
505         rctx->pub_exp = p2;
506         return 1;
507 
508     case EVP_PKEY_CTRL_RSA_KEYGEN_PRIMES:
509         if (p1 < RSA_DEFAULT_PRIME_NUM || p1 > RSA_MAX_PRIME_NUM) {
510             ERR_raise(ERR_LIB_RSA, RSA_R_KEY_PRIME_NUM_INVALID);
511             return -2;
512         }
513         rctx->primes = p1;
514         return 1;
515 
516     case EVP_PKEY_CTRL_RSA_OAEP_MD:
517     case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
518         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
519             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE);
520             return -2;
521         }
522         if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
523             *(const EVP_MD **)p2 = rctx->md;
524         else
525             rctx->md = p2;
526         return 1;
527 
528     case EVP_PKEY_CTRL_MD:
529         if (!check_padding_md(p2, rctx->pad_mode))
530             return 0;
531         if (rsa_pss_restricted(rctx)) {
532             if (EVP_MD_get_type(rctx->md) == EVP_MD_get_type(p2))
533                 return 1;
534             ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
535             return 0;
536         }
537         rctx->md = p2;
538         return 1;
539 
540     case EVP_PKEY_CTRL_GET_MD:
541         *(const EVP_MD **)p2 = rctx->md;
542         return 1;
543 
544     case EVP_PKEY_CTRL_RSA_MGF1_MD:
545     case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
546         if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
547             && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
548             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_MGF1_MD);
549             return -2;
550         }
551         if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
552             if (rctx->mgf1md)
553                 *(const EVP_MD **)p2 = rctx->mgf1md;
554             else
555                 *(const EVP_MD **)p2 = rctx->md;
556         } else {
557             if (rsa_pss_restricted(rctx)) {
558                 if (EVP_MD_get_type(rctx->mgf1md) == EVP_MD_get_type(p2))
559                     return 1;
560                 ERR_raise(ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
561                 return 0;
562             }
563             rctx->mgf1md = p2;
564         }
565         return 1;
566 
567     case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
568         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
569             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE);
570             return -2;
571         }
572         OPENSSL_free(rctx->oaep_label);
573         if (p2 && p1 > 0) {
574             rctx->oaep_label = p2;
575             rctx->oaep_labellen = p1;
576         } else {
577             rctx->oaep_label = NULL;
578             rctx->oaep_labellen = 0;
579         }
580         return 1;
581 
582     case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
583         if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
584             ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE);
585             return -2;
586         }
587         if (p2 == NULL) {
588             ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
589             return 0;
590         }
591         *(unsigned char **)p2 = rctx->oaep_label;
592         return rctx->oaep_labellen;
593 
594     case EVP_PKEY_CTRL_DIGESTINIT:
595     case EVP_PKEY_CTRL_PKCS7_SIGN:
596 #ifndef OPENSSL_NO_CMS
597     case EVP_PKEY_CTRL_CMS_SIGN:
598 #endif
599     return 1;
600 
601     case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
602     case EVP_PKEY_CTRL_PKCS7_DECRYPT:
603 #ifndef OPENSSL_NO_CMS
604     case EVP_PKEY_CTRL_CMS_DECRYPT:
605     case EVP_PKEY_CTRL_CMS_ENCRYPT:
606 #endif
607     if (!pkey_ctx_is_pss(ctx))
608         return 1;
609     /* fall through */
610     case EVP_PKEY_CTRL_PEER_KEY:
611         ERR_raise(ERR_LIB_RSA, RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
612         return -2;
613 
614     default:
615         return -2;
616 
617     }
618 }
619 
620 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
621                              const char *type, const char *value)
622 {
623     if (value == NULL) {
624         ERR_raise(ERR_LIB_RSA, RSA_R_VALUE_MISSING);
625         return 0;
626     }
627     if (strcmp(type, "rsa_padding_mode") == 0) {
628         int pm;
629 
630         if (strcmp(value, "pkcs1") == 0) {
631             pm = RSA_PKCS1_PADDING;
632         } else if (strcmp(value, "none") == 0) {
633             pm = RSA_NO_PADDING;
634         } else if (strcmp(value, "oeap") == 0) {
635             pm = RSA_PKCS1_OAEP_PADDING;
636         } else if (strcmp(value, "oaep") == 0) {
637             pm = RSA_PKCS1_OAEP_PADDING;
638         } else if (strcmp(value, "x931") == 0) {
639             pm = RSA_X931_PADDING;
640         } else if (strcmp(value, "pss") == 0) {
641             pm = RSA_PKCS1_PSS_PADDING;
642         } else {
643             ERR_raise(ERR_LIB_RSA, RSA_R_UNKNOWN_PADDING_TYPE);
644             return -2;
645         }
646         return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
647     }
648 
649     if (strcmp(type, "rsa_pss_saltlen") == 0) {
650         int saltlen;
651 
652         if (!strcmp(value, "digest"))
653             saltlen = RSA_PSS_SALTLEN_DIGEST;
654         else if (!strcmp(value, "max"))
655             saltlen = RSA_PSS_SALTLEN_MAX;
656         else if (!strcmp(value, "auto"))
657             saltlen = RSA_PSS_SALTLEN_AUTO;
658         else
659             saltlen = atoi(value);
660         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
661     }
662 
663     if (strcmp(type, "rsa_keygen_bits") == 0) {
664         int nbits = atoi(value);
665 
666         return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
667     }
668 
669     if (strcmp(type, "rsa_keygen_pubexp") == 0) {
670         int ret;
671 
672         BIGNUM *pubexp = NULL;
673         if (!BN_asc2bn(&pubexp, value))
674             return 0;
675         ret = EVP_PKEY_CTX_set1_rsa_keygen_pubexp(ctx, pubexp);
676         BN_free(pubexp);
677         return ret;
678     }
679 
680     if (strcmp(type, "rsa_keygen_primes") == 0) {
681         int nprimes = atoi(value);
682 
683         return EVP_PKEY_CTX_set_rsa_keygen_primes(ctx, nprimes);
684     }
685 
686     if (strcmp(type, "rsa_mgf1_md") == 0)
687         return EVP_PKEY_CTX_md(ctx,
688                                EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
689                                EVP_PKEY_CTRL_RSA_MGF1_MD, value);
690 
691     if (pkey_ctx_is_pss(ctx)) {
692 
693         if (strcmp(type, "rsa_pss_keygen_mgf1_md") == 0)
694             return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
695                                    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
696 
697         if (strcmp(type, "rsa_pss_keygen_md") == 0)
698             return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
699                                    EVP_PKEY_CTRL_MD, value);
700 
701         if (strcmp(type, "rsa_pss_keygen_saltlen") == 0) {
702             int saltlen = atoi(value);
703 
704             return EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, saltlen);
705         }
706     }
707 
708     if (strcmp(type, "rsa_oaep_md") == 0)
709         return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_TYPE_CRYPT,
710                                EVP_PKEY_CTRL_RSA_OAEP_MD, value);
711 
712     if (strcmp(type, "rsa_oaep_label") == 0) {
713         unsigned char *lab;
714         long lablen;
715         int ret;
716 
717         lab = OPENSSL_hexstr2buf(value, &lablen);
718         if (!lab)
719             return 0;
720         ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
721         if (ret <= 0)
722             OPENSSL_free(lab);
723         return ret;
724     }
725 
726     return -2;
727 }
728 
729 /* Set PSS parameters when generating a key, if necessary */
730 static int rsa_set_pss_param(RSA *rsa, EVP_PKEY_CTX *ctx)
731 {
732     RSA_PKEY_CTX *rctx = ctx->data;
733 
734     if (!pkey_ctx_is_pss(ctx))
735         return 1;
736     /* If all parameters are default values don't set pss */
737     if (rctx->md == NULL && rctx->mgf1md == NULL && rctx->saltlen == -2)
738         return 1;
739     rsa->pss = ossl_rsa_pss_params_create(rctx->md, rctx->mgf1md,
740                                           rctx->saltlen == -2
741                                           ? 0 : rctx->saltlen);
742     if (rsa->pss == NULL)
743         return 0;
744     return 1;
745 }
746 
747 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
748 {
749     RSA *rsa = NULL;
750     RSA_PKEY_CTX *rctx = ctx->data;
751     BN_GENCB *pcb;
752     int ret;
753 
754     if (rctx->pub_exp == NULL) {
755         rctx->pub_exp = BN_new();
756         if (rctx->pub_exp == NULL || !BN_set_word(rctx->pub_exp, RSA_F4))
757             return 0;
758     }
759     rsa = RSA_new();
760     if (rsa == NULL)
761         return 0;
762     if (ctx->pkey_gencb) {
763         pcb = BN_GENCB_new();
764         if (pcb == NULL) {
765             RSA_free(rsa);
766             return 0;
767         }
768         evp_pkey_set_cb_translate(pcb, ctx);
769     } else {
770         pcb = NULL;
771     }
772     ret = RSA_generate_multi_prime_key(rsa, rctx->nbits, rctx->primes,
773                                        rctx->pub_exp, pcb);
774     BN_GENCB_free(pcb);
775     if (ret > 0 && !rsa_set_pss_param(rsa, ctx)) {
776         RSA_free(rsa);
777         return 0;
778     }
779     if (ret > 0)
780         EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa);
781     else
782         RSA_free(rsa);
783     return ret;
784 }
785 
786 static const EVP_PKEY_METHOD rsa_pkey_meth = {
787     EVP_PKEY_RSA,
788     EVP_PKEY_FLAG_AUTOARGLEN,
789     pkey_rsa_init,
790     pkey_rsa_copy,
791     pkey_rsa_cleanup,
792 
793     0, 0,
794 
795     0,
796     pkey_rsa_keygen,
797 
798     0,
799     pkey_rsa_sign,
800 
801     0,
802     pkey_rsa_verify,
803 
804     0,
805     pkey_rsa_verifyrecover,
806 
807     0, 0, 0, 0,
808 
809     0,
810     pkey_rsa_encrypt,
811 
812     0,
813     pkey_rsa_decrypt,
814 
815     0, 0,
816 
817     pkey_rsa_ctrl,
818     pkey_rsa_ctrl_str
819 };
820 
821 const EVP_PKEY_METHOD *ossl_rsa_pkey_method(void)
822 {
823     return &rsa_pkey_meth;
824 }
825 
826 /*
827  * Called for PSS sign or verify initialisation: checks PSS parameter
828  * sanity and sets any restrictions on key usage.
829  */
830 
831 static int pkey_pss_init(EVP_PKEY_CTX *ctx)
832 {
833     const RSA *rsa;
834     RSA_PKEY_CTX *rctx = ctx->data;
835     const EVP_MD *md;
836     const EVP_MD *mgf1md;
837     int min_saltlen, max_saltlen;
838 
839     /* Should never happen */
840     if (!pkey_ctx_is_pss(ctx))
841         return 0;
842     rsa = EVP_PKEY_get0_RSA(ctx->pkey);
843     /* If no restrictions just return */
844     if (rsa->pss == NULL)
845         return 1;
846     /* Get and check parameters */
847     if (!ossl_rsa_pss_get_param(rsa->pss, &md, &mgf1md, &min_saltlen))
848         return 0;
849 
850     /* See if minimum salt length exceeds maximum possible */
851     max_saltlen = RSA_size(rsa) - EVP_MD_get_size(md);
852     if ((RSA_bits(rsa) & 0x7) == 1)
853         max_saltlen--;
854     if (min_saltlen > max_saltlen) {
855         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_SALT_LENGTH);
856         return 0;
857     }
858 
859     rctx->min_saltlen = min_saltlen;
860 
861     /*
862      * Set PSS restrictions as defaults: we can then block any attempt to
863      * use invalid values in pkey_rsa_ctrl
864      */
865 
866     rctx->md = md;
867     rctx->mgf1md = mgf1md;
868     rctx->saltlen = min_saltlen;
869 
870     return 1;
871 }
872 
873 static const EVP_PKEY_METHOD rsa_pss_pkey_meth = {
874     EVP_PKEY_RSA_PSS,
875     EVP_PKEY_FLAG_AUTOARGLEN,
876     pkey_rsa_init,
877     pkey_rsa_copy,
878     pkey_rsa_cleanup,
879 
880     0, 0,
881 
882     0,
883     pkey_rsa_keygen,
884 
885     pkey_pss_init,
886     pkey_rsa_sign,
887 
888     pkey_pss_init,
889     pkey_rsa_verify,
890 
891     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
892 
893     pkey_rsa_ctrl,
894     pkey_rsa_ctrl_str
895 };
896 
897 const EVP_PKEY_METHOD *ossl_rsa_pss_pkey_method(void)
898 {
899     return &rsa_pss_pkey_meth;
900 }
901